From 526d6114f2df99aa7807cad5a4ac283f1d97e672 Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Fri, 28 Aug 2020 09:33:13 -0400 Subject: [PATCH] added object tracking moved load of library updated test removed async trying to retest Retesting added test back fixed errors due to next button removed try catch so errors occur added try catch back added in ignore readded libraries stops detection when photo is taken, stops erroring issue added help text added spinner and better mocked blazeface moved img element back to correct place updated for requested changes updates for requested changes added timeout for test updated blazeface to pull from forked repo, and added changes based on accessibility feedback --- package-lock.json | 54 +++++++ package.json | 3 + src/id-verification/Camera.jsx | 143 ++++++++++++++++-- .../IdVerification.messages.js | 15 ++ src/id-verification/IdVerificationContext.jsx | 2 +- src/id-verification/_id-verification.scss | 11 +- .../panels/TakeIdPhotoPanel.jsx | 2 +- .../panels/TakePortraitPhotoPanel.jsx | 2 +- src/id-verification/tests/Camera.test.jsx | 93 ++++++++++++ 9 files changed, 311 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1fff837..b8e802d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4879,6 +4879,35 @@ "loader-utils": "^1.2.3" } }, + "@tensorflow-models/blazeface": { + "version": "git+https://github.com/alangsto/blazeface.git#ae2ae0f29538ede33c404e2059608df4c1416bba", + "from": "git+https://github.com/alangsto/blazeface.git" + }, + "@tensorflow/tfjs-converter": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.6.0.tgz", + "integrity": "sha512-jKu4rwBVAjQAH4+LiPcv0CIuj5uW4PDTb9HvlqcLm/43e7uAd7Qus74Dy82pwVOrGhv3BPD4/GZYrzOKxGbKEQ==" + }, + "@tensorflow/tfjs-core": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.6.0.tgz", + "integrity": "sha512-b98jn1pjRuEDVNN6/ZQMFhyYV27ZIsG9CcHSMXq1ohX6ALQQB3mwgrMeC2TEVXFl6/L2vOD8W+txuBRGKHnvpg==", + "requires": { + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "2.4.27", + "@types/webgl-ext": "0.0.30", + "@types/webgl2": "0.0.4", + "node-fetch": "~2.1.2", + "seedrandom": "2.4.3" + }, + "dependencies": { + "node-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" + } + } + }, "@testing-library/dom": { "version": "7.21.5", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-7.21.5.tgz", @@ -5548,6 +5577,11 @@ "resolved": "https://registry.npmjs.org/@types/object-assign/-/object-assign-4.0.30.tgz", "integrity": "sha1-iUk3HVqZ9Dge4PHfCpt6GH4H5lI=" }, + "@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" + }, "@types/prettier": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.2.tgz", @@ -5589,6 +5623,11 @@ "@types/react": "*" } }, + "@types/seedrandom": { + "version": "2.4.27", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", + "integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=" + }, "@types/stack-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", @@ -5609,6 +5648,16 @@ "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.0.tgz", "integrity": "sha1-DSUBJorY+ZYrdA04fEZU9fjiPlI=" }, + "@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" + }, + "@types/webgl2": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz", + "integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==" + }, "@types/yargs": { "version": "13.0.7", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.7.tgz", @@ -24537,6 +24586,11 @@ } } }, + "seedrandom": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", + "integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=" + }, "seek-bzip": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", diff --git a/package.json b/package.json index 8e7a53b..f04fe31 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,9 @@ "@fortawesome/free-regular-svg-icons": "5.7.2", "@fortawesome/free-solid-svg-icons": "5.8.2", "@fortawesome/react-fontawesome": "0.1.11", + "@tensorflow-models/blazeface": "git+https://github.com/alangsto/blazeface.git", + "@tensorflow/tfjs-converter": "1.6.0", + "@tensorflow/tfjs-core": "1.6.0", "babel-polyfill": "6.26.0", "bowser": "^2.10.0", "classnames": "2.2.6", diff --git a/src/id-verification/Camera.jsx b/src/id-verification/Camera.jsx index 015d2e5..1b55929 100644 --- a/src/id-verification/Camera.jsx +++ b/src/id-verification/Camera.jsx @@ -1,7 +1,9 @@ import React from 'react'; import PropTypes from 'prop-types'; +import * as blazeface from '@tensorflow-models/blazeface'; import CameraPhoto, { FACING_MODES } from 'jslib-html5-camera-photo'; import { injectIntl, intlShape } from '@edx/frontend-platform/i18n'; +import { Form, Spinner } from '@edx/paragon'; import shutter from './data/camera-shutter.base64.json'; import messages from './IdVerification.messages'; @@ -11,9 +13,13 @@ class Camera extends React.Component { super(props, context); this.cameraPhoto = null; this.videoRef = React.createRef(); + this.canvasRef = React.createRef(); + this.setDetection = this.setDetection.bind(this); this.state = { - trackedObject: null, dataUri: '', + videoHasLoaded: false, + shouldDetect: false, + isFinishedLoadingDetection: true, }; } @@ -26,6 +32,101 @@ class Camera extends React.Component { this.cameraPhoto.stopCamera(); } + setDetection() { + this.setState( + { shouldDetect: !this.state.shouldDetect }, + () => { + if (this.state.shouldDetect) { + this.setState({ isFinishedLoadingDetection: false }); + this.startDetection(); + } + }, + ); + } + + startDetection() { + setTimeout(() => { + if (this.state.videoHasLoaded) { + const loadModelPromise = blazeface.load(); + Promise.all([loadModelPromise]) + .then((values) => { + this.setState({ isFinishedLoadingDetection: true }); + this.detectFromVideoFrame(values[0], this.videoRef.current); + }); + } else { + this.setState({ isFinishedLoadingDetection: true }); + this.setState({ shouldDetect: false }); + // TODO: add error message + } + }, 1000); + } + + detectFromVideoFrame = (model, video) => { + model.estimateFaces(video).then((predictions) => { + if (this.state.shouldDetect && !this.state.dataUri) { + this.showDetections(predictions); + + requestAnimationFrame(() => { + this.detectFromVideoFrame(model, video); + }); + } + }); + }; + + showDetections = (predictions) => { + let canvasContext; + if (predictions.length > 0) { + canvasContext = this.canvasRef.current.getContext('2d'); + canvasContext.clearRect(0, 0, canvasContext.canvas.width, canvasContext.canvas.height); + } + // predictions is an array of objects describing each detected face + predictions.forEach((prediction) => { + const xAdjustment = 70; + const yAdjustment = 55; + const start = [prediction.topLeft[0] - xAdjustment, prediction.topLeft[1] - yAdjustment]; + const end = [prediction.bottomRight[0] - xAdjustment, prediction.bottomRight[1] - yAdjustment]; + const size = [end[0] - start[0], end[1] - start[1]]; + + // landmarks is an array of points representing each facial landmark (i.e. right eye, left eye, nose, etc.) + const features = prediction.landmarks; + let isInPosition = true; + + // for each of the landmarks, determine if it is in position + for (let j = 0; j < features.length; j++) { + const x = features[j][0] - xAdjustment; + const y = features[j][1] - yAdjustment; + + if (this.props.isPortrait) { + isInPosition = isInPosition && this.isInRangeForPortrait(x, y); + } else { + isInPosition = isInPosition && this.isInRangeForID(x, y); + } + } + + // draw a box depending on if all landmarks are in position + if (isInPosition) { + canvasContext.strokeStyle = '#00ffff'; + canvasContext.lineWidth = 6; + canvasContext.strokeRect(start[0], start[1], size[0], size[1]); + } else { + canvasContext.fillStyle = 'rgba(255, 51, 0, 0.75)'; + canvasContext.fillRect(start[0], start[1], size[0], size[1]); + } + }); + } + + isInRangeForPortrait(x, y) { + return x > 40 && x < 480 && y > 60 && y < 330; + } + + isInRangeForID(x, y) { + return x > 60 && x < 360 && y > 150 && y < 250; + } + + setVideoHasLoaded() { + this.setState({ videoHasLoaded: 'true' }); + } + takePhoto() { if (this.state.dataUri) { return this.reset(); @@ -41,12 +142,15 @@ class Camera extends React.Component { } playShutterClick() { - const audio = new Audio('data:audio/mp3;base64,' + shutter.base64); + const audio = new Audio(`data:audio/mp3;base64,${shutter.base64}`); audio.play(); } reset() { this.setState({ dataUri: '' }); + if (this.state.shouldDetect) { + this.startDetection(); + } } render() { @@ -54,19 +158,39 @@ class Camera extends React.Component { ? 'do-transition camera-flash' : 'camera-flash'; return ( -
-
+
+ + + {!this.state.isFinishedLoadingDetection && } + + {this.props.isPortrait + ? this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.portrait.help.text']) + : this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.id.help.text'])} + + +
@@ -76,7 +200,7 @@ class Camera extends React.Component { 'btn-outline-primary' : 'btn-primary' }`} - accessKey='c' + accessKey="c" onClick={() => { this.takePhoto(); }} @@ -93,6 +217,7 @@ class Camera extends React.Component { Camera.propTypes = { intl: intlShape.isRequired, onImageCapture: PropTypes.func.isRequired, + isPortrait: PropTypes.bool.isRequired, }; export default injectIntl(Camera); diff --git a/src/id-verification/IdVerification.messages.js b/src/id-verification/IdVerification.messages.js index 3958fd7..33119fa 100644 --- a/src/id-verification/IdVerification.messages.js +++ b/src/id-verification/IdVerification.messages.js @@ -81,6 +81,21 @@ const messages = defineMessages({ defaultMessage: 'Retake Photo', description: 'Button to retake photo.', }, + 'id.verification.photo.enable.detection': { + id: 'id.verification.photo.enable.detection', + defaultMessage: 'Enable Face Detection', + description: 'Text label for the checkbox to enable face detection.', + }, + 'id.verification.photo.enable.detection.portrait.help.text': { + id: 'id.verification.photo.enable.detection.portrait.help.text', + defaultMessage: 'If checked, a box will appear around your face. Your face can be seen clearly if the box around it is blue. If your face is not in a good position or undetectable, the box will be red.', + description: 'Help text that appears for enabling face detection on the portrait photo panel.', + }, + 'id.verification.photo.enable.detection.id.help.text': { + id: 'id.verification.photo.enable.detection.id.help.text', + defaultMessage: 'If checked, a box will appear around the face on your ID card. The face can be seen clearly if the box around it is blue. If the face is not in a good position or undetectable, the box will be red.', + description: 'Help text that appears for enabling face detection on the portrait photo panel.', + }, 'id.verification.camera.access.title': { id: 'id.verification.camera.access.title', defaultMessage: 'Camera Permissions', diff --git a/src/id-verification/IdVerificationContext.jsx b/src/id-verification/IdVerificationContext.jsx index 3375269..6a2f43d 100644 --- a/src/id-verification/IdVerificationContext.jsx +++ b/src/id-verification/IdVerificationContext.jsx @@ -58,7 +58,7 @@ function IdVerificationContextProvider({ children }) { tracks.forEach(track => track.stop()); setMediaStream(null); } - } + }, }; // Call verification status endpoint to check whether we can verify. diff --git a/src/id-verification/_id-verification.scss b/src/id-verification/_id-verification.scss index 3c88206..a028b0f 100644 --- a/src/id-verification/_id-verification.scss +++ b/src/id-verification/_id-verification.scss @@ -13,7 +13,7 @@ } } .form-check { - padding: 0.5rem 0.5rem 1rem; + padding: 0.5rem 0.5rem 1rem; .form-check-label { margin-left: 0.5rem; padding-top: 0.2rem; @@ -52,6 +52,13 @@ width: 100%; } + .canvas-video { + width: 100%; + position: absolute; + top: 0; + left: 0; + } + .camera-btn { margin: 10px; } @@ -68,4 +75,4 @@ opacity: 0; background: white; } -} \ No newline at end of file +} diff --git a/src/id-verification/panels/TakeIdPhotoPanel.jsx b/src/id-verification/panels/TakeIdPhotoPanel.jsx index f08078d..304cdd8 100644 --- a/src/id-verification/panels/TakeIdPhotoPanel.jsx +++ b/src/id-verification/panels/TakeIdPhotoPanel.jsx @@ -24,7 +24,7 @@ function TakeIdPhotoPanel(props) {

{props.intl.formatMessage(messages['id.verification.id.photo.instructions.camera'])}

- +
diff --git a/src/id-verification/panels/TakePortraitPhotoPanel.jsx b/src/id-verification/panels/TakePortraitPhotoPanel.jsx index e54f12b..d12683c 100644 --- a/src/id-verification/panels/TakePortraitPhotoPanel.jsx +++ b/src/id-verification/panels/TakePortraitPhotoPanel.jsx @@ -33,7 +33,7 @@ function TakePortraitPhotoPanel(props) {

{props.intl.formatMessage(messages['id.verification.portrait.photo.instructions.camera'])}

- +
) : (
diff --git a/src/id-verification/tests/Camera.test.jsx b/src/id-verification/tests/Camera.test.jsx index 8c5f7dc..fbe64ff 100644 --- a/src/id-verification/tests/Camera.test.jsx +++ b/src/id-verification/tests/Camera.test.jsx @@ -4,10 +4,12 @@ import { createMemoryHistory } from 'history'; import '@testing-library/jest-dom/extend-expect'; import { render, cleanup, screen, act, fireEvent } from '@testing-library/react'; import { injectIntl, IntlProvider } from '@edx/frontend-platform/i18n'; +import * as blazeface from '@tensorflow-models/blazeface'; import { IdVerificationContext } from '../IdVerificationContext'; import Camera from '../Camera'; jest.mock('jslib-html5-camera-photo'); +jest.mock('@tensorflow-models/blazeface'); window.HTMLMediaElement.prototype.play = () => {}; @@ -19,6 +21,13 @@ describe('SubmittedPanel', () => { const defaultProps = { intl: {}, onImageCapture: jest.fn(), + isPortrait: true, + }; + + const idProps = { + intl: {}, + onImageCapture: jest.fn(), + isPortrait: false, }; const contextValue = {}; @@ -42,4 +51,88 @@ describe('SubmittedPanel', () => { fireEvent.click(button); expect(defaultProps.onImageCapture).toHaveBeenCalled(); }); + + it('shows correct help text for portrait photo capture', async () => { + await act(async () => render(( + + + + + + + + ))); + const helpText = screen.getByTestId('videoDetectionHelpText'); + expect(helpText.textContent).toEqual(expect.stringContaining('Your face can be seen clearly')); + }); + + it('shows correct help text for id photo capture', async () => { + await act(async () => render(( + + + + + + + + ))); + const helpText = screen.getByTestId('videoDetectionHelpText'); + expect(helpText.textContent).toEqual(expect.stringContaining('The face can be seen clearly')); + }); + + it('shows spinner when loading face detection', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + fireEvent.click(checkbox); + expect(screen.queryByTestId('spinner')).toBeDefined(); + }); + + it('canvas is visible when detection is enabled', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:none'); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + await fireEvent.click(checkbox); + expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:block'); + }); + + it('blazeface is called when detection is enabled', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + await fireEvent.click(checkbox); + setTimeout(() => { expect(blazeface.load).toHaveBeenCalled(); }, 2000); + }); });