Implementing face detection in the browser
First, here is the face detection demo.
To implement face detection in the browser, we first bring in OpenCV.
Here is a brief introduction to what it does:
- OpenCV is an open-source, cross-platform computer vision and machine learning library.
- It consists of C functions and a small number of C++ classes, with interfaces for languages including Python, Ruby, and MATLAB.
- OpenCV provides many image-processing functions, from displaying images and manipulating pixels to detecting objects, greatly simplifying the development of graphics and deep learning applications.
Because it is written in C, it must be compiled before it can run in a browser. We can get the precompiled version 4.5.1 directly from
the OpenCV.js documentation: OpenCV.js.
Basic Workflow
- Load an image.
- Store it as an OpenCV matrix.
- Use a face model to detect face locations.
- Draw the results.
What is the model in step 3?
A model is a set of classification rules (a neural network model) generated by a machine learning or deep learning framework. Common frameworks include TensorFlow, PyTorch, and Caffe.
These frameworks can produce models for faces, cats 🐱🐱, dogs 🐶🐶, and much more. Image classification is only the tip of the iceberg.
Malicious-text detection, speech recognition, semantic segmentation, human pose detection, and more: it is a vast field.
Demo Code
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Hello OpenCV.js</title>
<style>
.img_dectect, .video_dectect{
display: flex;
}
.caption {
text-align: center;
border: 2px solid rgb(6, 155, 193);
border-radius: 10px;
}
</style>
</head>
<body>
<h2>Hello OpenCV.js</h2>
<p id="status">Loading OpenCV.js...</p>
<div>
<h1>Image face detection</h1>
<div><input type="file" id="imgInput" name="file" accept="image/*"> Upload an image for detection</div>
<button id="imgDectectButton" disabled>Loading OpenCV...</button>
<div class="img_dectect">
<div class="caption"><p>Image input</p> <canvas id="imgCanvasInput" width="640" height="480"></canvas></div>
<div class="caption"><p>Detection output</p> <canvas id="imgCanvasOutput" width="640" height="480"></canvas></div>
</div>
<h1>Video face detection</h1>
<button id="videoDectectButton" disabled>Loading OpenCV...</button>
<div class="video_dectect">
<div class="caption">Video input<video id="videoInput" width="640" height="480"></video></div>
<div class="caption">Video output<canvas id="canvasOutput" width="640" height="480" ></canvas></div>
<div class="caption">Intermediate video output<canvas id="canvasOutput2"></canvas></div>
</div>
</div>
<script type="text/javascript">
// ----------------------- Image face detection ----------------------
let imgDectectButton = document.getElementById('imgDectectButton');
imgDectectButton.addEventListener('click', startImgDectect);
let inputElement = document.getElementById('imgInput');
let imgCanvasInput = document.getElementById('imgCanvasInput');
let imgCanvasOutput = document.getElementById('imgCanvasOutput');
inputElement.addEventListener('change', (e) => {
let files = e.target.files;
if (files.length > 0) {
let imgUrl = URL.createObjectURL(files[0]);
let ctx = imgCanvasInput.getContext('2d');
let img = new Image();
img.crossOrigin = 'anonymous';
img.onload = function() {
imgCanvasInput.width = img.width;
imgCanvasInput.height = img.height;
imgCanvasOutput.width = img.width;
imgCanvasOutput.height = img.height;
ctx.drawImage(img, 0, 0, img.width, img.height);
};
img.src = imgUrl;
}
}, false);
function startImgDectect () {
let src = new cv.imread('imgCanvasInput'); // Read the canvas image into a matrix
let gray = new cv.Mat();
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0);
let classifier = new cv.CascadeClassifier(); // Cascade classifier
classifier.load('face.xml'); // Load the trained model
let faces = new cv.RectVector();
classifier.detectMultiScale(gray, faces, 1.1, 3, 0); // Detect faces
for (let i = 0; i < faces.size(); ++i) {
let roiGray = gray.roi(faces.get(i));
let roiSrc = src.roi(faces.get(i));
let point1 = new cv.Point(faces.get(i).x, faces.get(i).y);
let point2 = new cv.Point(faces.get(i).x + faces.get(i).width,
faces.get(i).y + faces.get(i).height);
cv.rectangle(src, point1, point2, [255, 0, 0, 255]);
}
// Draw to the canvas
cv.imshow('imgCanvasOutput', src);
// Release allocated resources
src.delete();
gray.delete();
classifier.delete();
}
// ----------------------- Image face detection ----------------------
// ----------------------- Video face detection ----------------------
let videoDectectButton = document.getElementById('videoDectectButton');
videoDectectButton.addEventListener('click', startVideo);
let video = document.getElementById('videoInput');
function startVideo() {
let src = new cv.Mat(video.height, video.width, cv.CV_8UC4); // Image matrix: 8-bit, 4 channels
let gray = new cv.Mat(); // Grayscale matrix
let cap = new cv.VideoCapture(video); // Video capture
let classifier = new cv.CascadeClassifier(); // Cascade classifier
classifier.load('face.xml'); // Load the trained model
const FPS = 30;
function processVideo() {
let begin = Date.now();
// Capture one frame
cap.read(src);
// Convert to grayscale
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY);
let downSampled = new cv.Mat();
cv.pyrDown(gray, downSampled); // Downsample the image to reduce detection time
cv.pyrDown(downSampled, downSampled);
// Detect faces
let faces = new cv.RectVector();
classifier.detectMultiScale(downSampled, faces); // Detect faces
// Draw bounding boxes
let size = downSampled.size();
let xRatio = video.width / size.width;
let yRatio = video.height / size.height;
for (let i = 0; i < faces.size(); ++i) {
let face = faces.get(i);
let point1 = new cv.Point(face.x * xRatio, face.y * yRatio);
let point2 = new cv.Point((face.x + face.width) * xRatio, (face.y + face.height) * xRatio);
cv.rectangle(src, point1, point2, [255, 0, 0, 255])
}
// Draw the final results
cv.imshow('canvasOutput', src);
cv.imshow('canvasOutput2', downSampled);
// Free memory
downSampled.delete();
faces.delete();
let delay = 1000/FPS - (Date.now() - begin);
setTimeout(processVideo, delay);
};
setTimeout(processVideo, 0);
}
// ----------------------- Video face detection ----------------------
// Access the camera API and output to the canvas
let constraints = { audio: false, video: { width: video.width, height: video.height } };
navigator.mediaDevices.getUserMedia(constraints)
.then(function(mediaStream) {
let video = document.getElementById('videoInput');
video.srcObject = mediaStream;
video.onloadedmetadata = function(e) {
video.play();
};
})
.catch(function(err) { console.log(err.name + ": " + err.message); });
// ------------------------- Initialization ----------------------
// Load the trained model needed for face detection
function loadXmlData() {
let request = new XMLHttpRequest();
request.open('GET', 'face.xml');
request.responseType = 'arraybuffer';
request.onload = function(ev) {
if (request.readyState === 4) {
if (request.status === 200) {
let data = new Uint8Array(request.response);
cv.FS_createDataFile('/', 'face.xml', data, true, false, false);
imgDectectButton.removeAttribute('disabled');
videoDectectButton.removeAttribute('disabled');
document.getElementById('status').innerHTML = 'OpenCV.js is ready.';
imgDectectButton.innerHTML = 'Start detection';
videoDectectButton.innerHTML = 'Start detection';
}
}
};
request.send();
}
// Wait for OpenCV to load system information
async function onOpenCvReady() {
if (cv.getBuildInformation) {
console.log(cv.getBuildInformation());
loadXmlData();
} else {
if (cv instanceof Promise) {
cv = await cv;
console.log(cv.getBuildInformation());
loadXmlData();
} else {
cv.onRuntimeInitialized = () => {
console.log(cv.getBuildInformation());
loadXmlData();
}
}
}
}
// ------------------------- Initialization ----------------------
</script>
<script async src="opencv.js" onload="onOpenCvReady();" type="text/javascript"></script>
</body>
</html>
Pitfalls When Building the Demo
- Run the demo through a server; otherwise, fetching the model XML file causes cross-origin request issues. Use npm install http-server --save to install a simple local development server.
- When deploying to a server, enable HTTPS; otherwise, the browser will not grant the site access to the camera.
- During face detection and output, the canvas dimensions must match the matrix dimensions to avoid bugs.