added object tracking

moved load of library

updated test

removed async

trying to retest

Retesting

added test back

fixed errors due to next button

removed try catch so errors occur

added try catch back

added in ignore

readded libraries

stops detection when photo is taken, stops erroring issue

added help text

added spinner and better mocked blazeface

moved img element back to correct place

updated for requested changes

updates for requested changes

added timeout for test

updated blazeface to pull from forked repo, and added changes based on accessibility feedback
This commit is contained in:
Alie Langston
2020-08-28 09:33:13 -04:00
parent 0cc38e2dc6
commit 526d6114f2
9 changed files with 311 additions and 14 deletions

54
package-lock.json generated
View File

@@ -4879,6 +4879,35 @@
"loader-utils": "^1.2.3"
}
},
"@tensorflow-models/blazeface": {
"version": "git+https://github.com/alangsto/blazeface.git#ae2ae0f29538ede33c404e2059608df4c1416bba",
"from": "git+https://github.com/alangsto/blazeface.git"
},
"@tensorflow/tfjs-converter": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.6.0.tgz",
"integrity": "sha512-jKu4rwBVAjQAH4+LiPcv0CIuj5uW4PDTb9HvlqcLm/43e7uAd7Qus74Dy82pwVOrGhv3BPD4/GZYrzOKxGbKEQ=="
},
"@tensorflow/tfjs-core": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.6.0.tgz",
"integrity": "sha512-b98jn1pjRuEDVNN6/ZQMFhyYV27ZIsG9CcHSMXq1ohX6ALQQB3mwgrMeC2TEVXFl6/L2vOD8W+txuBRGKHnvpg==",
"requires": {
"@types/offscreencanvas": "~2019.3.0",
"@types/seedrandom": "2.4.27",
"@types/webgl-ext": "0.0.30",
"@types/webgl2": "0.0.4",
"node-fetch": "~2.1.2",
"seedrandom": "2.4.3"
},
"dependencies": {
"node-fetch": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
"integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
}
}
},
"@testing-library/dom": {
"version": "7.21.5",
"resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-7.21.5.tgz",
@@ -5548,6 +5577,11 @@
"resolved": "https://registry.npmjs.org/@types/object-assign/-/object-assign-4.0.30.tgz",
"integrity": "sha1-iUk3HVqZ9Dge4PHfCpt6GH4H5lI="
},
"@types/offscreencanvas": {
"version": "2019.3.0",
"resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz",
"integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q=="
},
"@types/prettier": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.2.tgz",
@@ -5589,6 +5623,11 @@
"@types/react": "*"
}
},
"@types/seedrandom": {
"version": "2.4.27",
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz",
"integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE="
},
"@types/stack-utils": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz",
@@ -5609,6 +5648,16 @@
"resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.0.tgz",
"integrity": "sha1-DSUBJorY+ZYrdA04fEZU9fjiPlI="
},
"@types/webgl-ext": {
"version": "0.0.30",
"resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",
"integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg=="
},
"@types/webgl2": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz",
"integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw=="
},
"@types/yargs": {
"version": "13.0.7",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.7.tgz",
@@ -24537,6 +24586,11 @@
}
}
},
"seedrandom": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz",
"integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw="
},
"seek-bzip": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz",

View File

@@ -38,6 +38,9 @@
"@fortawesome/free-regular-svg-icons": "5.7.2",
"@fortawesome/free-solid-svg-icons": "5.8.2",
"@fortawesome/react-fontawesome": "0.1.11",
"@tensorflow-models/blazeface": "git+https://github.com/alangsto/blazeface.git",
"@tensorflow/tfjs-converter": "1.6.0",
"@tensorflow/tfjs-core": "1.6.0",
"babel-polyfill": "6.26.0",
"bowser": "^2.10.0",
"classnames": "2.2.6",

View File

@@ -1,7 +1,9 @@
import React from 'react';
import PropTypes from 'prop-types';
import * as blazeface from '@tensorflow-models/blazeface';
import CameraPhoto, { FACING_MODES } from 'jslib-html5-camera-photo';
import { injectIntl, intlShape } from '@edx/frontend-platform/i18n';
import { Form, Spinner } from '@edx/paragon';
import shutter from './data/camera-shutter.base64.json';
import messages from './IdVerification.messages';
@@ -11,9 +13,13 @@ class Camera extends React.Component {
super(props, context);
this.cameraPhoto = null;
this.videoRef = React.createRef();
this.canvasRef = React.createRef();
this.setDetection = this.setDetection.bind(this);
this.state = {
trackedObject: null,
dataUri: '',
videoHasLoaded: false,
shouldDetect: false,
isFinishedLoadingDetection: true,
};
}
@@ -26,6 +32,101 @@ class Camera extends React.Component {
this.cameraPhoto.stopCamera();
}
setDetection() {
this.setState(
{ shouldDetect: !this.state.shouldDetect },
() => {
if (this.state.shouldDetect) {
this.setState({ isFinishedLoadingDetection: false });
this.startDetection();
}
},
);
}
startDetection() {
setTimeout(() => {
if (this.state.videoHasLoaded) {
const loadModelPromise = blazeface.load();
Promise.all([loadModelPromise])
.then((values) => {
this.setState({ isFinishedLoadingDetection: true });
this.detectFromVideoFrame(values[0], this.videoRef.current);
});
} else {
this.setState({ isFinishedLoadingDetection: true });
this.setState({ shouldDetect: false });
// TODO: add error message
}
}, 1000);
}
detectFromVideoFrame = (model, video) => {
model.estimateFaces(video).then((predictions) => {
if (this.state.shouldDetect && !this.state.dataUri) {
this.showDetections(predictions);
requestAnimationFrame(() => {
this.detectFromVideoFrame(model, video);
});
}
});
};
showDetections = (predictions) => {
let canvasContext;
if (predictions.length > 0) {
canvasContext = this.canvasRef.current.getContext('2d');
canvasContext.clearRect(0, 0, canvasContext.canvas.width, canvasContext.canvas.height);
}
// predictions is an array of objects describing each detected face
predictions.forEach((prediction) => {
const xAdjustment = 70;
const yAdjustment = 55;
const start = [prediction.topLeft[0] - xAdjustment, prediction.topLeft[1] - yAdjustment];
const end = [prediction.bottomRight[0] - xAdjustment, prediction.bottomRight[1] - yAdjustment];
const size = [end[0] - start[0], end[1] - start[1]];
// landmarks is an array of points representing each facial landmark (i.e. right eye, left eye, nose, etc.)
const features = prediction.landmarks;
let isInPosition = true;
// for each of the landmarks, determine if it is in position
for (let j = 0; j < features.length; j++) {
const x = features[j][0] - xAdjustment;
const y = features[j][1] - yAdjustment;
if (this.props.isPortrait) {
isInPosition = isInPosition && this.isInRangeForPortrait(x, y);
} else {
isInPosition = isInPosition && this.isInRangeForID(x, y);
}
}
// draw a box depending on if all landmarks are in position
if (isInPosition) {
canvasContext.strokeStyle = '#00ffff';
canvasContext.lineWidth = 6;
canvasContext.strokeRect(start[0], start[1], size[0], size[1]);
} else {
canvasContext.fillStyle = 'rgba(255, 51, 0, 0.75)';
canvasContext.fillRect(start[0], start[1], size[0], size[1]);
}
});
}
isInRangeForPortrait(x, y) {
return x > 40 && x < 480 && y > 60 && y < 330;
}
isInRangeForID(x, y) {
return x > 60 && x < 360 && y > 150 && y < 250;
}
setVideoHasLoaded() {
this.setState({ videoHasLoaded: 'true' });
}
takePhoto() {
if (this.state.dataUri) {
return this.reset();
@@ -41,12 +142,15 @@ class Camera extends React.Component {
}
playShutterClick() {
const audio = new Audio('data:audio/mp3;base64,' + shutter.base64);
const audio = new Audio(`data:audio/mp3;base64,${shutter.base64}`);
audio.play();
}
reset() {
this.setState({ dataUri: '' });
if (this.state.shouldDetect) {
this.startDetection();
}
}
render() {
@@ -54,19 +158,39 @@ class Camera extends React.Component {
? 'do-transition camera-flash'
: 'camera-flash';
return (
<div className='camera-outer-wrapper shadow'>
<div className='camera-wrapper'>
<div className="camera-outer-wrapper shadow">
<Form.Group style={{ textAlign: 'left', padding: '0.5rem', marginBottom: '0.5rem' }} >
<Form.Check
id="videoDetection"
name="videoDetection"
label={this.props.intl.formatMessage(messages['id.verification.photo.enable.detection'])}
aria-describedby="videoDetectionHelpText"
checked={this.state.shouldDetect}
onChange={this.setDetection}
style={{ padding: '0rem', marginLeft: '1.25rem', float: this.state.isFinishedLoadingDetection ? 'none' : 'left' }}
/>
{!this.state.isFinishedLoadingDetection && <Spinner animation="border" variant="primary" style={{ marginLeft: '0.5rem' }} data-testid="spinner" />}
<Form.Text id="videoDetectionHelpText" data-testid="videoDetectionHelpText">
{this.props.isPortrait
? this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.portrait.help.text'])
: this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.id.help.text'])}
</Form.Text>
</Form.Group>
<div className="camera-wrapper">
<div className={cameraFlashClass} />
<video
ref={this.videoRef}
autoPlay={true}
className='camera-video'
data-testid="video"
autoPlay
className="camera-video"
onLoadedData={() => { this.setVideoHasLoaded(); }}
style={{ display: this.state.dataUri ? 'none' : 'block' }}
/>
<canvas ref={this.canvasRef} data-testid="detection-canvas" className="canvas-video" style={{ display: !this.state.shouldDetect || this.state.dataUri ? 'none' : 'block' }} height="375" width="500" />
<img
alt='imgCamera'
alt="imgCamera"
src={this.state.dataUri}
className='camera-video'
className="camera-video"
style={{ display: this.state.dataUri ? 'block' : 'none' }}
/>
</div>
@@ -76,7 +200,7 @@ class Camera extends React.Component {
'btn-outline-primary'
: 'btn-primary'
}`}
accessKey='c'
accessKey="c"
onClick={() => {
this.takePhoto();
}}
@@ -93,6 +217,7 @@ class Camera extends React.Component {
Camera.propTypes = {
intl: intlShape.isRequired,
onImageCapture: PropTypes.func.isRequired,
isPortrait: PropTypes.bool.isRequired,
};
export default injectIntl(Camera);

View File

@@ -81,6 +81,21 @@ const messages = defineMessages({
defaultMessage: 'Retake Photo',
description: 'Button to retake photo.',
},
'id.verification.photo.enable.detection': {
id: 'id.verification.photo.enable.detection',
defaultMessage: 'Enable Face Detection',
description: 'Text label for the checkbox to enable face detection.',
},
'id.verification.photo.enable.detection.portrait.help.text': {
id: 'id.verification.photo.enable.detection.portrait.help.text',
defaultMessage: 'If checked, a box will appear around your face. Your face can be seen clearly if the box around it is blue. If your face is not in a good position or undetectable, the box will be red.',
description: 'Help text that appears for enabling face detection on the portrait photo panel.',
},
'id.verification.photo.enable.detection.id.help.text': {
id: 'id.verification.photo.enable.detection.id.help.text',
defaultMessage: 'If checked, a box will appear around the face on your ID card. The face can be seen clearly if the box around it is blue. If the face is not in a good position or undetectable, the box will be red.',
description: 'Help text that appears for enabling face detection on the portrait photo panel.',
},
'id.verification.camera.access.title': {
id: 'id.verification.camera.access.title',
defaultMessage: 'Camera Permissions',

View File

@@ -58,7 +58,7 @@ function IdVerificationContextProvider({ children }) {
tracks.forEach(track => track.stop());
setMediaStream(null);
}
}
},
};
// Call verification status endpoint to check whether we can verify.

View File

@@ -13,7 +13,7 @@
}
}
.form-check {
padding: 0.5rem 0.5rem 1rem;
padding: 0.5rem 0.5rem 1rem;
.form-check-label {
margin-left: 0.5rem;
padding-top: 0.2rem;
@@ -52,6 +52,13 @@
width: 100%;
}
.canvas-video {
width: 100%;
position: absolute;
top: 0;
left: 0;
}
.camera-btn {
margin: 10px;
}
@@ -68,4 +75,4 @@
opacity: 0;
background: white;
}
}
}

View File

@@ -24,7 +24,7 @@ function TakeIdPhotoPanel(props) {
<p>
{props.intl.formatMessage(messages['id.verification.id.photo.instructions.camera'])}
</p>
<Camera onImageCapture={setIdPhotoFile} />
<Camera onImageCapture={setIdPhotoFile} isPortrait={false} />
</div>
<CameraHelp />
<div className="action-row" style={{ visibility: idPhotoFile ? 'unset' : 'hidden' }}>

View File

@@ -33,7 +33,7 @@ function TakePortraitPhotoPanel(props) {
<p>
{props.intl.formatMessage(messages['id.verification.portrait.photo.instructions.camera'])}
</p>
<Camera onImageCapture={setFacePhotoFile} />
<Camera onImageCapture={setFacePhotoFile} isPortrait />
</div>
) : (
<div>

View File

@@ -4,10 +4,12 @@ import { createMemoryHistory } from 'history';
import '@testing-library/jest-dom/extend-expect';
import { render, cleanup, screen, act, fireEvent } from '@testing-library/react';
import { injectIntl, IntlProvider } from '@edx/frontend-platform/i18n';
import * as blazeface from '@tensorflow-models/blazeface';
import { IdVerificationContext } from '../IdVerificationContext';
import Camera from '../Camera';
jest.mock('jslib-html5-camera-photo');
jest.mock('@tensorflow-models/blazeface');
window.HTMLMediaElement.prototype.play = () => {};
@@ -19,6 +21,13 @@ describe('SubmittedPanel', () => {
const defaultProps = {
intl: {},
onImageCapture: jest.fn(),
isPortrait: true,
};
const idProps = {
intl: {},
onImageCapture: jest.fn(),
isPortrait: false,
};
const contextValue = {};
@@ -42,4 +51,88 @@ describe('SubmittedPanel', () => {
fireEvent.click(button);
expect(defaultProps.onImageCapture).toHaveBeenCalled();
});
it('shows correct help text for portrait photo capture', async () => {
await act(async () => render((
<Router history={history}>
<IntlProvider locale="en">
<IdVerificationContext.Provider value={contextValue}>
<IntlCamera {...defaultProps} />
</IdVerificationContext.Provider>
</IntlProvider>
</Router>
)));
const helpText = screen.getByTestId('videoDetectionHelpText');
expect(helpText.textContent).toEqual(expect.stringContaining('Your face can be seen clearly'));
});
it('shows correct help text for id photo capture', async () => {
await act(async () => render((
<Router history={history}>
<IntlProvider locale="en">
<IdVerificationContext.Provider value={contextValue}>
<IntlCamera {...idProps} />
</IdVerificationContext.Provider>
</IntlProvider>
</Router>
)));
const helpText = screen.getByTestId('videoDetectionHelpText');
expect(helpText.textContent).toEqual(expect.stringContaining('The face can be seen clearly'));
});
it('shows spinner when loading face detection', async () => {
blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) });
await act(async () => render((
<Router history={history}>
<IntlProvider locale="en">
<IdVerificationContext.Provider value={contextValue}>
<IntlCamera {...defaultProps} />
</IdVerificationContext.Provider>
</IntlProvider>
</Router>
)));
await fireEvent.loadedData(screen.queryByTestId('video'));
const checkbox = await screen.findByLabelText('Enable Face Detection');
fireEvent.click(checkbox);
expect(screen.queryByTestId('spinner')).toBeDefined();
});
it('canvas is visible when detection is enabled', async () => {
blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) });
await act(async () => render((
<Router history={history}>
<IntlProvider locale="en">
<IdVerificationContext.Provider value={contextValue}>
<IntlCamera {...defaultProps} />
</IdVerificationContext.Provider>
</IntlProvider>
</Router>
)));
await fireEvent.loadedData(screen.queryByTestId('video'));
expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:none');
const checkbox = await screen.findByLabelText('Enable Face Detection');
await fireEvent.click(checkbox);
expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:block');
});
it('blazeface is called when detection is enabled', async () => {
blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) });
await act(async () => render((
<Router history={history}>
<IntlProvider locale="en">
<IdVerificationContext.Provider value={contextValue}>
<IntlCamera {...defaultProps} />
</IdVerificationContext.Provider>
</IntlProvider>
</Router>
)));
await fireEvent.loadedData(screen.queryByTestId('video'));
const checkbox = await screen.findByLabelText('Enable Face Detection');
await fireEvent.click(checkbox);
setTimeout(() => { expect(blazeface.load).toHaveBeenCalled(); }, 2000);
});
});