diff --git a/package-lock.json b/package-lock.json index 1fff837..b8e802d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4879,6 +4879,35 @@ "loader-utils": "^1.2.3" } }, + "@tensorflow-models/blazeface": { + "version": "git+https://github.com/alangsto/blazeface.git#ae2ae0f29538ede33c404e2059608df4c1416bba", + "from": "git+https://github.com/alangsto/blazeface.git" + }, + "@tensorflow/tfjs-converter": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.6.0.tgz", + "integrity": "sha512-jKu4rwBVAjQAH4+LiPcv0CIuj5uW4PDTb9HvlqcLm/43e7uAd7Qus74Dy82pwVOrGhv3BPD4/GZYrzOKxGbKEQ==" + }, + "@tensorflow/tfjs-core": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.6.0.tgz", + "integrity": "sha512-b98jn1pjRuEDVNN6/ZQMFhyYV27ZIsG9CcHSMXq1ohX6ALQQB3mwgrMeC2TEVXFl6/L2vOD8W+txuBRGKHnvpg==", + "requires": { + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "2.4.27", + "@types/webgl-ext": "0.0.30", + "@types/webgl2": "0.0.4", + "node-fetch": "~2.1.2", + "seedrandom": "2.4.3" + }, + "dependencies": { + "node-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" + } + } + }, "@testing-library/dom": { "version": "7.21.5", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-7.21.5.tgz", @@ -5548,6 +5577,11 @@ "resolved": "https://registry.npmjs.org/@types/object-assign/-/object-assign-4.0.30.tgz", "integrity": "sha1-iUk3HVqZ9Dge4PHfCpt6GH4H5lI=" }, + "@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" + }, "@types/prettier": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.2.tgz", @@ -5589,6 +5623,11 @@ "@types/react": "*" } }, + "@types/seedrandom": { + "version": "2.4.27", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", + "integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=" + }, "@types/stack-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", @@ -5609,6 +5648,16 @@ "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.0.tgz", "integrity": "sha1-DSUBJorY+ZYrdA04fEZU9fjiPlI=" }, + "@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" + }, + "@types/webgl2": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz", + "integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==" + }, "@types/yargs": { "version": "13.0.7", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.7.tgz", @@ -24537,6 +24586,11 @@ } } }, + "seedrandom": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", + "integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=" + }, "seek-bzip": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", diff --git a/package.json b/package.json index 8e7a53b..f04fe31 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,9 @@ "@fortawesome/free-regular-svg-icons": "5.7.2", "@fortawesome/free-solid-svg-icons": "5.8.2", "@fortawesome/react-fontawesome": "0.1.11", + "@tensorflow-models/blazeface": "git+https://github.com/alangsto/blazeface.git", + "@tensorflow/tfjs-converter": "1.6.0", + "@tensorflow/tfjs-core": "1.6.0", "babel-polyfill": "6.26.0", "bowser": "^2.10.0", "classnames": "2.2.6", diff --git a/src/id-verification/Camera.jsx b/src/id-verification/Camera.jsx index 015d2e5..1b55929 100644 --- a/src/id-verification/Camera.jsx +++ b/src/id-verification/Camera.jsx @@ -1,7 +1,9 @@ import React from 'react'; import PropTypes from 'prop-types'; +import * as blazeface from '@tensorflow-models/blazeface'; import CameraPhoto, { FACING_MODES } from 'jslib-html5-camera-photo'; import { injectIntl, intlShape } from '@edx/frontend-platform/i18n'; +import { Form, Spinner } from '@edx/paragon'; import shutter from './data/camera-shutter.base64.json'; import messages from './IdVerification.messages'; @@ -11,9 +13,13 @@ class Camera extends React.Component { super(props, context); this.cameraPhoto = null; this.videoRef = React.createRef(); + this.canvasRef = React.createRef(); + this.setDetection = this.setDetection.bind(this); this.state = { - trackedObject: null, dataUri: '', + videoHasLoaded: false, + shouldDetect: false, + isFinishedLoadingDetection: true, }; } @@ -26,6 +32,101 @@ class Camera extends React.Component { this.cameraPhoto.stopCamera(); } + setDetection() { + this.setState( + { shouldDetect: !this.state.shouldDetect }, + () => { + if (this.state.shouldDetect) { + this.setState({ isFinishedLoadingDetection: false }); + this.startDetection(); + } + }, + ); + } + + startDetection() { + setTimeout(() => { + if (this.state.videoHasLoaded) { + const loadModelPromise = blazeface.load(); + Promise.all([loadModelPromise]) + .then((values) => { + this.setState({ isFinishedLoadingDetection: true }); + this.detectFromVideoFrame(values[0], this.videoRef.current); + }); + } else { + this.setState({ isFinishedLoadingDetection: true }); + this.setState({ shouldDetect: false }); + // TODO: add error message + } + }, 1000); + } + + detectFromVideoFrame = (model, video) => { + model.estimateFaces(video).then((predictions) => { + if (this.state.shouldDetect && !this.state.dataUri) { + this.showDetections(predictions); + + requestAnimationFrame(() => { + this.detectFromVideoFrame(model, video); + }); + } + }); + }; + + showDetections = (predictions) => { + let canvasContext; + if (predictions.length > 0) { + canvasContext = this.canvasRef.current.getContext('2d'); + canvasContext.clearRect(0, 0, canvasContext.canvas.width, canvasContext.canvas.height); + } + // predictions is an array of objects describing each detected face + predictions.forEach((prediction) => { + const xAdjustment = 70; + const yAdjustment = 55; + const start = [prediction.topLeft[0] - xAdjustment, prediction.topLeft[1] - yAdjustment]; + const end = [prediction.bottomRight[0] - xAdjustment, prediction.bottomRight[1] - yAdjustment]; + const size = [end[0] - start[0], end[1] - start[1]]; + + // landmarks is an array of points representing each facial landmark (i.e. right eye, left eye, nose, etc.) + const features = prediction.landmarks; + let isInPosition = true; + + // for each of the landmarks, determine if it is in position + for (let j = 0; j < features.length; j++) { + const x = features[j][0] - xAdjustment; + const y = features[j][1] - yAdjustment; + + if (this.props.isPortrait) { + isInPosition = isInPosition && this.isInRangeForPortrait(x, y); + } else { + isInPosition = isInPosition && this.isInRangeForID(x, y); + } + } + + // draw a box depending on if all landmarks are in position + if (isInPosition) { + canvasContext.strokeStyle = '#00ffff'; + canvasContext.lineWidth = 6; + canvasContext.strokeRect(start[0], start[1], size[0], size[1]); + } else { + canvasContext.fillStyle = 'rgba(255, 51, 0, 0.75)'; + canvasContext.fillRect(start[0], start[1], size[0], size[1]); + } + }); + } + + isInRangeForPortrait(x, y) { + return x > 40 && x < 480 && y > 60 && y < 330; + } + + isInRangeForID(x, y) { + return x > 60 && x < 360 && y > 150 && y < 250; + } + + setVideoHasLoaded() { + this.setState({ videoHasLoaded: 'true' }); + } + takePhoto() { if (this.state.dataUri) { return this.reset(); @@ -41,12 +142,15 @@ class Camera extends React.Component { } playShutterClick() { - const audio = new Audio('data:audio/mp3;base64,' + shutter.base64); + const audio = new Audio(`data:audio/mp3;base64,${shutter.base64}`); audio.play(); } reset() { this.setState({ dataUri: '' }); + if (this.state.shouldDetect) { + this.startDetection(); + } } render() { @@ -54,19 +158,39 @@ class Camera extends React.Component { ? 'do-transition camera-flash' : 'camera-flash'; return ( -
-
+
+ + + {!this.state.isFinishedLoadingDetection && } + + {this.props.isPortrait + ? this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.portrait.help.text']) + : this.props.intl.formatMessage(messages['id.verification.photo.enable.detection.id.help.text'])} + + +
@@ -76,7 +200,7 @@ class Camera extends React.Component { 'btn-outline-primary' : 'btn-primary' }`} - accessKey='c' + accessKey="c" onClick={() => { this.takePhoto(); }} @@ -93,6 +217,7 @@ class Camera extends React.Component { Camera.propTypes = { intl: intlShape.isRequired, onImageCapture: PropTypes.func.isRequired, + isPortrait: PropTypes.bool.isRequired, }; export default injectIntl(Camera); diff --git a/src/id-verification/IdVerification.messages.js b/src/id-verification/IdVerification.messages.js index 3958fd7..33119fa 100644 --- a/src/id-verification/IdVerification.messages.js +++ b/src/id-verification/IdVerification.messages.js @@ -81,6 +81,21 @@ const messages = defineMessages({ defaultMessage: 'Retake Photo', description: 'Button to retake photo.', }, + 'id.verification.photo.enable.detection': { + id: 'id.verification.photo.enable.detection', + defaultMessage: 'Enable Face Detection', + description: 'Text label for the checkbox to enable face detection.', + }, + 'id.verification.photo.enable.detection.portrait.help.text': { + id: 'id.verification.photo.enable.detection.portrait.help.text', + defaultMessage: 'If checked, a box will appear around your face. Your face can be seen clearly if the box around it is blue. If your face is not in a good position or undetectable, the box will be red.', + description: 'Help text that appears for enabling face detection on the portrait photo panel.', + }, + 'id.verification.photo.enable.detection.id.help.text': { + id: 'id.verification.photo.enable.detection.id.help.text', + defaultMessage: 'If checked, a box will appear around the face on your ID card. The face can be seen clearly if the box around it is blue. If the face is not in a good position or undetectable, the box will be red.', + description: 'Help text that appears for enabling face detection on the portrait photo panel.', + }, 'id.verification.camera.access.title': { id: 'id.verification.camera.access.title', defaultMessage: 'Camera Permissions', diff --git a/src/id-verification/IdVerificationContext.jsx b/src/id-verification/IdVerificationContext.jsx index 3375269..6a2f43d 100644 --- a/src/id-verification/IdVerificationContext.jsx +++ b/src/id-verification/IdVerificationContext.jsx @@ -58,7 +58,7 @@ function IdVerificationContextProvider({ children }) { tracks.forEach(track => track.stop()); setMediaStream(null); } - } + }, }; // Call verification status endpoint to check whether we can verify. diff --git a/src/id-verification/_id-verification.scss b/src/id-verification/_id-verification.scss index 3c88206..a028b0f 100644 --- a/src/id-verification/_id-verification.scss +++ b/src/id-verification/_id-verification.scss @@ -13,7 +13,7 @@ } } .form-check { - padding: 0.5rem 0.5rem 1rem; + padding: 0.5rem 0.5rem 1rem; .form-check-label { margin-left: 0.5rem; padding-top: 0.2rem; @@ -52,6 +52,13 @@ width: 100%; } + .canvas-video { + width: 100%; + position: absolute; + top: 0; + left: 0; + } + .camera-btn { margin: 10px; } @@ -68,4 +75,4 @@ opacity: 0; background: white; } -} \ No newline at end of file +} diff --git a/src/id-verification/panels/TakeIdPhotoPanel.jsx b/src/id-verification/panels/TakeIdPhotoPanel.jsx index f08078d..304cdd8 100644 --- a/src/id-verification/panels/TakeIdPhotoPanel.jsx +++ b/src/id-verification/panels/TakeIdPhotoPanel.jsx @@ -24,7 +24,7 @@ function TakeIdPhotoPanel(props) {

{props.intl.formatMessage(messages['id.verification.id.photo.instructions.camera'])}

- +
diff --git a/src/id-verification/panels/TakePortraitPhotoPanel.jsx b/src/id-verification/panels/TakePortraitPhotoPanel.jsx index e54f12b..d12683c 100644 --- a/src/id-verification/panels/TakePortraitPhotoPanel.jsx +++ b/src/id-verification/panels/TakePortraitPhotoPanel.jsx @@ -33,7 +33,7 @@ function TakePortraitPhotoPanel(props) {

{props.intl.formatMessage(messages['id.verification.portrait.photo.instructions.camera'])}

- +
) : (
diff --git a/src/id-verification/tests/Camera.test.jsx b/src/id-verification/tests/Camera.test.jsx index 8c5f7dc..fbe64ff 100644 --- a/src/id-verification/tests/Camera.test.jsx +++ b/src/id-verification/tests/Camera.test.jsx @@ -4,10 +4,12 @@ import { createMemoryHistory } from 'history'; import '@testing-library/jest-dom/extend-expect'; import { render, cleanup, screen, act, fireEvent } from '@testing-library/react'; import { injectIntl, IntlProvider } from '@edx/frontend-platform/i18n'; +import * as blazeface from '@tensorflow-models/blazeface'; import { IdVerificationContext } from '../IdVerificationContext'; import Camera from '../Camera'; jest.mock('jslib-html5-camera-photo'); +jest.mock('@tensorflow-models/blazeface'); window.HTMLMediaElement.prototype.play = () => {}; @@ -19,6 +21,13 @@ describe('SubmittedPanel', () => { const defaultProps = { intl: {}, onImageCapture: jest.fn(), + isPortrait: true, + }; + + const idProps = { + intl: {}, + onImageCapture: jest.fn(), + isPortrait: false, }; const contextValue = {}; @@ -42,4 +51,88 @@ describe('SubmittedPanel', () => { fireEvent.click(button); expect(defaultProps.onImageCapture).toHaveBeenCalled(); }); + + it('shows correct help text for portrait photo capture', async () => { + await act(async () => render(( + + + + + + + + ))); + const helpText = screen.getByTestId('videoDetectionHelpText'); + expect(helpText.textContent).toEqual(expect.stringContaining('Your face can be seen clearly')); + }); + + it('shows correct help text for id photo capture', async () => { + await act(async () => render(( + + + + + + + + ))); + const helpText = screen.getByTestId('videoDetectionHelpText'); + expect(helpText.textContent).toEqual(expect.stringContaining('The face can be seen clearly')); + }); + + it('shows spinner when loading face detection', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + fireEvent.click(checkbox); + expect(screen.queryByTestId('spinner')).toBeDefined(); + }); + + it('canvas is visible when detection is enabled', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:none'); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + await fireEvent.click(checkbox); + expect(screen.queryByTestId('detection-canvas')).toHaveStyle('display:block'); + }); + + it('blazeface is called when detection is enabled', async () => { + blazeface.load = jest.fn().mockResolvedValue({ estimateFaces: jest.fn().mockResolvedValue([]) }); + + await act(async () => render(( + + + + + + + + ))); + + await fireEvent.loadedData(screen.queryByTestId('video')); + const checkbox = await screen.findByLabelText('Enable Face Detection'); + await fireEvent.click(checkbox); + setTimeout(() => { expect(blazeface.load).toHaveBeenCalled(); }, 2000); + }); });