Andal.LND/frontend-app-account
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #332 from edx/alangsto/add_realtime_feedback

Browse Source 
Add realtime feedback for screenreaders during face detection
This commit is contained in:
alangsto
2020-09-16 15:31:00 -04:00
committed by GitHub
parent 917152df22 1d01abc7da
commit 30e25b96bb
2 changed files with 157 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
src/id-verification/Camera.jsx
View File

@@ -20,14 +20,16 @@ class Camera extends React.Component {
videoHasLoaded: false,
shouldDetect: false,
isFinishedLoadingDetection: true,
shouldGiveFeedback: true,
feedback: '',
};
}
componentDidMount() {
this.cameraPhoto = new CameraPhoto(this.videoRef.current);
this.cameraPhoto.startCamera(
this.props.isPortrait ? FACING_MODES.USER : FACING_MODES.ENVIRONMENT,
{ width: 640, height: 480 }
this.props.isPortrait ? FACING_MODES.USER : FACING_MODES.ENVIRONMENT,
{ width: 640, height: 480 },
);
}
@@ -97,11 +99,14 @@ class Camera extends React.Component {
const x = features[j][0];
const y = features[j][1];
let isInRange;
if (this.props.isPortrait) {
isInPosition = isInPosition && this.isInRangeForPortrait(x, y);
isInRange = this.isInRangeForPortrait(x, y);
} else {
isInPosition = isInPosition && this.isInRangeForID(x, y);
isInRange = this.isInRangeForID(x, y);
}
// if it is not in range, give feedback depending on which feature is out of range
isInPosition = isInPosition && isInRange;
}
// draw a box depending on if all landmarks are in position
@@ -109,11 +114,81 @@ class Camera extends React.Component {
canvasContext.strokeStyle = '#00ffff';
canvasContext.lineWidth = 6;
canvasContext.strokeRect(start[0], start[1], size[0], size[1]);
// give positive feedback here if user is in correct position
this.giveFeedback(predictions.length, [], true);
} else {
canvasContext.fillStyle = 'rgba(255, 51, 0, 0.75)';
canvasContext.fillRect(start[0], start[1], size[0], size[1]);
this.giveFeedback(predictions.length, features[0], false);
}
});
if (predictions.length === 0) {
this.giveFeedback(predictions.length, [], false);
}
}
giveFeedback(numFaces, rightEye, isCorrect) {
if (this.state.shouldGiveFeedback) {
const currentFeedback = this.state.feedback;
let newFeedback = '';
if (numFaces === 1) {
// only give feedback if one face is detected otherwise
// it would be difficult to tell a user which face to move
if (isCorrect) {
newFeedback = this.props.intl.formatMessage(messages['id.verification.photo.feedback.correct']);
} else {
// give feedback based on where user is
newFeedback = this.props.intl.formatMessage(messages[this.getGridPosition(rightEye)]);
}
} else if (numFaces > 1) {
newFeedback = this.props.intl.formatMessage(messages['id.verification.photo.feedback.two.faces']);
} else {
newFeedback = this.props.intl.formatMessage(messages['id.verification.photo.feedback.no.faces']);
}
if (currentFeedback !== newFeedback) {
// only update status if it is different, so we don't overload the user with status updates
this.setState({ feedback: newFeedback });
}
// turn off feedback for one to ensure that instructions aren't disruptive/interrupting
this.setState({ shouldGiveFeedback: false });
setTimeout(() => {
this.setState({ shouldGiveFeedback: true });
}, 1000);
}
}
getGridPosition(coordinates) {
// Used to determine where a face is (i.e. top-left, center-right, bottom-center, etc.)
const x = coordinates[0];
const y = coordinates[1];
let messageBase = 'id.verification.photo.feedback';
const heightUpperLimit = 320;
const heightMiddleLimit = 160;
if (y < heightMiddleLimit) {
messageBase += '.top';
} else if (y < heightUpperLimit && y >= heightMiddleLimit) {
messageBase += '.center';
} else {
messageBase += '.bottom';
}
const widthRightLimit = 213;
const widthMiddleLimit = 427;
if (x < widthRightLimit) {
messageBase += '.right';
} else if (x >= widthRightLimit && x < widthMiddleLimit) {
messageBase += '.center';
} else {
messageBase += '.left';
}
return messageBase;
}
isInRangeForPortrait(x, y) {
@@ -207,16 +282,32 @@ class Camera extends React.Component {
autoPlay
className="camera-video"
onLoadedData={() => { this.setVideoHasLoaded(); }}
style={{ display: this.state.dataUri ? 'none' : 'block' }}
style={{
display: this.state.dataUri ? 'none' : 'block',
WebkitTransform: 'scaleX(-1)',
transform: 'scaleX(-1)',
}}
playsInline
/>
<canvas ref={this.canvasRef} data-testid="detection-canvas" className="canvas-video" style={{ display: !this.state.shouldDetect || this.state.dataUri ? 'none' : 'block' }} width="640" height="480" />
<canvas
ref={this.canvasRef}
data-testid="detection-canvas"
className="canvas-video"
style={{
display: !this.state.shouldDetect || this.state.dataUri ? 'none' : 'block',
WebkitTransform: 'scaleX(-1)',
transform: 'scaleX(-1)',
}}
width="640"
height="480"
/>
<img
alt="imgCamera"
src={this.state.dataUri}
className="camera-video"
style={{ display: this.state.dataUri ? 'block' : 'none' }}
/>
<div role="status" className="sr-only">{this.state.feedback}</div>
</div>
<button
className={`btn camera-btn ${

60
src/id-verification/IdVerification.messages.js
View File

@@ -96,6 +96,66 @@ const messages = defineMessages({
defaultMessage: 'If checked, a box will appear around the face on your ID card. The face can be seen clearly if the box around it is blue. If the face is not in a good position or undetectable, the box will be red.',
description: 'Help text that appears for enabling face detection on the portrait photo panel.',
},
'id.verification.photo.feedback.correct': {
id: 'id.verification.photo.feedback.correct',
defaultMessage: 'Face is in a good position.',
description: 'Text for screen reader when user\'s face is in a good position.',
},
'id.verification.photo.feedback.two.faces': {
id: 'id.verification.photo.feedback.two.faces',
defaultMessage: 'More than one face detected.',
description: 'Text for screen reader when more than one face detected.',
},
'id.verification.photo.feedback.no.faces': {
id: 'id.verification.photo.feedback.no.faces',
defaultMessage: 'No face detected.',
description: 'Text for screen reader when no face detected.',
},
'id.verification.photo.feedback.top.left': {
id: 'id.verification.photo.feedback.top.left',
defaultMessage: 'Incorrect position. Top left.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.top.center': {
id: 'id.verification.photo.feedback.top.center',
defaultMessage: 'Incorrect position. Top center.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.top.right': {
id: 'id.verification.photo.feedback.top.right',
defaultMessage: 'Incorrect position. Top right.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.center.left': {
id: 'id.verification.photo.feedback.center.left',
defaultMessage: 'Incorrect position. Center left.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.center.center': {
id: 'id.verification.photo.feedback.center.center',
defaultMessage: 'Incorrect position. Too close to camera.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.center.right': {
id: 'id.verification.photo.feedback.center.right',
defaultMessage: 'Incorrect position. Center right.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.bottom.left': {
id: 'id.verification.photo.feedback.bottom.left',
defaultMessage: 'Incorrect position. Bottom left.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.bottom.center': {
id: 'id.verification.photo.feedback.bottom.center',
defaultMessage: 'Incorrect position. Bottom center.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.photo.feedback.bottom.right': {
id: 'id.verification.photo.feedback.bottom.right',
defaultMessage: 'Incorrect position. Bottom right.',
description: 'Text for screen reader when face is in a bad position.',
},
'id.verification.camera.access.title': {
id: 'id.verification.camera.access.title',
defaultMessage: 'Camera Permissions',