A wrapper node for the epic face-api.js library. 

A wrapper node for the epic face-api.js library

thebigpotatoe

node-red-contrib-face-recognition

    let faceapi     = require('face-api.js');

    let formidable  = require('formidable');

    // Try load in Tfjs-node if it is installed

        require('@tensorflow/tfjs-node');

        if (e instanceof Error && e.code === "MODULE_NOT_FOUND")

            RED.log.info("[Face-api.js] - TensorFlow.js for Node.js was not found, running without it");

    // Monkey patch nodejs to faceapi with canvas

    const { Canvas, Image, ImageData } = canvas

    faceapi.env.monkeyPatch({ Canvas, Image, ImageData })

        RED.log.info("[Face-api.js] - Loading Models", "info")

            const modelPath = `${__dirname}/weights`;

            const ssdMobilenetv1Method          = faceapi.nets.ssdMobilenetv1.loadFromDisk(modelPath)

            const tinyFaceDetectorMethod        = faceapi.nets.tinyFaceDetector.loadFromDisk(modelPath)

            const faceLandmark68NetMethod       = faceapi.nets.faceLandmark68Net.loadFromDisk(modelPath)

            const faceLandmark68TinyNetMethod   = faceapi.nets.faceLandmark68TinyNet.loadFromDisk(modelPath)

            const faceExpressionNetMethod       = faceapi.nets.faceExpressionNet.loadFromDisk(modelPath)

            const ageGenderNetMethod            = faceapi.nets.ageGenderNet.loadFromDisk(modelPath)

            const faceRecognitionNetMethod      = faceapi.nets.faceRecognitionNet.loadFromDisk(modelPath)

            await faceLandmark68NetMethod

            await faceLandmark68TinyNetMethod

            await faceExpressionNetMethod

            await faceRecognitionNetMethod

            RED.log.info("[Face-api.js] - Models Loaded")

            RED.log.warn("[Face-api.js] - Models failed to load: \n" + error)

    // Create a descriptor file path if it does not exist

    async function checkDescriptorDir() {

        const saveDir = `${__dirname}/descriptors`;

            RED.log.info("[Face-a-pi.js] - Created descriptors directory at " + saveDir)

        RED.nodes.createNode(this, config);

        // Register all compute nodes into an array

        if (RED.nodes.getNode(config.computeNode1)) computeNodes.push(RED.nodes.getNode(config.computeNode1));

        if (RED.nodes.getNode(config.computeNode2)) computeNodes.push(RED.nodes.getNode(config.computeNode2));

        if (RED.nodes.getNode(config.computeNode3)) computeNodes.push(RED.nodes.getNode(config.computeNode3));

        if (RED.nodes.getNode(config.computeNode4)) computeNodes.push(RED.nodes.getNode(config.computeNode4));

        if (RED.nodes.getNode(config.computeNode5)) computeNodes.push(RED.nodes.getNode(config.computeNode5));

        if (RED.nodes.getNode(config.computeNode6)) computeNodes.push(RED.nodes.getNode(config.computeNode6));

        if (RED.nodes.getNode(config.computeNode7)) computeNodes.push(RED.nodes.getNode(config.computeNode7));

        if (RED.nodes.getNode(config.computeNode8)) computeNodes.push(RED.nodes.getNode(config.computeNode8));

        if (RED.nodes.getNode(config.computeNode9)) computeNodes.push(RED.nodes.getNode(config.computeNode9));

        if (RED.nodes.getNode(config.computeNode10)) computeNodes.push(RED.nodes.getNode(config.computeNode10));

        // Get the number of active nodes

        for (i = 0; i < computeNodes.length; i++) {

            if(computeNodes[i] !== null) numActiveNodes = numActiveNodes + 1

        // Set the initial status of the node

        this.status({fill:"green",shape:"dot",text:"ready"});

        // Get the facec API config node 

        this.computeNode = RED.nodes.getNode(config.computeNode);

        node.on('input', async function (msg, send, done) {

                // Set the status to computing

                                    let msg = {}

                                    msg["Matched Faces"] = []

                                    computeNodesOutput.forEach((output) => {

                                        msg[output.name] = output

                                        output.faces.forEach(face => {

                                            if (face.matchedLabel !== "unknown") {

                                                msg["Matched Faces"].push({

                                                    "name" : face.matchedLabel,

                                                    "confidence" : face.matchedDistance

                                                })

                                                // console.log(face)

                                            }

                                        })

                                    // Send the message

                                    send = send || function() { node.send.apply(node,arguments); };

                                    send({"payload": msg});

                                    // Set the busy boolean 

                                    node.busy = false;

                                    // Reset the array

                                    computeNodesOutput = []

    RED.nodes.registerType("face-api-input", faceApiInputNode);

    function faceApiComputeNode(config) {

        node.name                   = config.name                   || "face-api-compute";

        node.childHost              = config.childHost              || false;

        node.labelName              = (config.name === "face-api-compute") ? "known" : config.name;

        node.recognitionType        = config.recognitionType        || "SSD";

        node.multipleFaces          = config.multipleFaces          || "Multiple Faces";

        node.confidence             = config.confidence/100         || 0.5;

        node.inputSize              = parseInt(config.inputSize)    || 416;

        node.landmarks              = config.landmarks              || false;

        node.expressions            = config.expressions            || false;

        node.ageGender              = config.ageGender              || false;

        node.recognition            = config.recognition            || false;

        node.recognitionMetric      = config.recognitionMetric      || "Mean Squared Error";

        node.recognitionConfidence  = config.recognitionConfidence  || 0;       

        node.descriptors            = null;

        node.modelsLoaded           = false;

        node.labelledDescriptors    = null;

        node.msgCallback            = null;

        node.isComputing            = false;

        // Start the child node if required or kill it

            // Set up args and options for the child 

            const options = {stdio : "pipe"};

            // Start the forked child process

            node.childProcess = require('child_process').fork(`${__dirname}/face-api-cmd.js`, args, options)

            // Create the callback to handle a message event for info and warn messages

            node.childProcess.on('message', (msg) => {

                // Route the message appropriately 

                    RED.log.info("[Face-api.js : " + node.id + " : Child Node] - " + msg.info)

                else if ("warn" in msg) {

                    RED.log.warn("[Face-api.js : " + node.id + " : Child Node] - " + msg.warn)

                else if ("error" in msg) {

                    RED.log.error("[Face-api.js : " + node.id + " : Child Node] - " + msg.error)

                    msg.image = Buffer.from(msg.image)

                // Send messages to the callback if it has been set

                    node.msgCallback(msg);

                    // if (!("info" in msg)) node.msgCallback = null

                // Set the computing boolean to false

                node.isComputing = false;

            // Create a callback to handle a error events

            node.childProcess.on('error', (err, signal) => {

                RED.log.error("[Face-api.js : " + node.id + " : Child Node]:\n" + err);

            // Create callback to handle a exit events 

            node.childProcess.on('exit', (code, signal) => {

                const exitString = "[Face-api.js : " + node.id + " : Child Node] - child_process exited with " + `code ${code} and signal ${signal}`

                if (signal == "SIGINT") RED.log.info(exitString)

                else RED.log.error(exitString);

            // Create the stderr callback for errors that occur in the child node

            node.childProcess.stderr.on('data', (data) => {

                // Convert buffer to string 

                    errMsg = JSON.parse(data)

    "name": "node-red-contrib-face-recognition",

    "description": "A wrapper node for the epic face-api.js library",

        "test": "echo \"Error: no test specified\" && exit 1"

        "url": "git+https://github.com/thebigpotatoe/node-red-contrib-face-recognition.git"

        "url": "https://github.com/thebigpotatoe/node-red-contrib-face-recognition/issues"

    "homepage": "https://github.com/thebigpotatoe/node-red-contrib-face-recognition#readme",

This node aims to wrap the epic [Face-API.js library](https://github.com/justadudewhohacks/face-api.js) from [justadudewhohacks](https://github.com/justadudewhohacks) into a simple to import and use node in Node-Red. If you like anything in this repo be sure to also check out the original.

Usage of this node is designed to be very easy while allowing the user to choose any number of options exposed by the original face-api.js module. These are currently;

This module also utilizes the `child_process` module of Node.js to offload the complex calculations required to a separate thread. In turn, the offloaded task will not block the main event loop and allow Node-Red to continue other tasks. This is entirely optional and up to the user to decide to allow for better management of resources on a constrained device such as the Raspberry Pi.

From your .node-red directory, you can run;

`npm install node-red-contrib-face-recognition`

or you can go to the pallette manager in Node-Red and find `node-red-contrib-face-recognition` in the install tab.

> Linux users (including raspberry pi) should read [this issue](https://github.com/thebigpotatoe/node-red-contrib-face-recognition/issues/4#issuecomment-579821200) if having issues with installing canvas.

As an example on how to use the node, below is a flow that grabs an image from the internet and runs inference over it. Simply copy and paste it into Node-Red to use. 

As an example on how to use the node, below is a flow that grabs an image from the internet and runs inference over it. Copy and paste it into Node-Red to use, but make sure to install the following nodes from the pallet manager;

You can also optionally install TensorFlow for Node.js to make this package run faster. If you do not, the node will still run albeit much slower. To install TensorFlow navigate to your `.node-red` folder and run the following command. This will install TensorFlow in your Node-Red directory for use by the node.

> There are known issues with the working combinations version of Node.js, @tensorflow/tfjs-node and face-api.js. At the time of writing this, on a windows environment these were found to be;

> Please install these to gain the speed of the tf c++ backend and keep up to date on the face-api.js GitHib page for any errors relating to this.

This module comes with two nodes; the `face-api-input` node and `face-api-compute` node.

![Input Node](Images/face-api-input-node-menu.JPG)

The `face-api-input` node acts as a pipeline between a flow and the compute nodes. Multiple compute nodes can be created and at least one must be selected for the input node to work. By using a config node based approach, multiple input nodes can share the same resources on your device allowing for a smaller memory footprint of the module.

The input node is capable of utilizing 10 compute nodes to process an image. The output of the input node is an array of results from each individual compute nodes containing information about found faces, the over layed image and individual inference time. 

To add multiple compute nodes to the input simply click the __Add node__ button in the edit dialog. This will add an option to select another compute node from a list of already created nodes. If you have multiple nodes but do not fill in any of them, they will be ignored. Only Node 1 is required.

By design, if a node is computing and another image is sent to that node, it will be ignored until the compute node has finished. This allows users to use a stream of images as an input and not worry about queued images bogging down the event loop.

![Input Node](Images/face-api-compute-node-menu.JPG)

The `face-api-compute` node is where all the options are set and calculations done. There are numerous options, so as a brief outline these are;

- __Name__: The name of this specific node. Useful to change when you have multiple config nodes and need to know the difference between them. (defaults to "face-api-compute")

- __Detection Type__: The detection type to use. This is either `SSD` or `Yolo`, as `MTCNN` is not currently supported. (Defaults to SSD)

- __Faces to Detect__: The number of detections to calculate. Either all faces found as `"Multiple Faces"` or the highest confidence one as `"Single Face"`. (Defaults to Multiple)

- __Detection Confidence__: The minimum confidence score that each detected face much be above to be counted as a face. This option is available for both SSD and Yolo. (Defaults to 50%, Limited between 0 and 100)

- __Input Size__: The input size of the Yolo algorithm. This option is only available when using Yolo as the type. (Defaults to 416)

- __Child Process__: Select if you would like to run the algorithm in a `child_process`. This will offload from the main event loop but use more resources. Use carefully. (Defaults to true)

- __Facial Landmarks__: Select this if you would like to add facial landmarks of each face to the output. (Defaults to false)

- __Facial Expressions__: Select this if you would like to add a prediction of facial expressions of each face to the output. (Defaults to false)

- __Age and Gender__: Select this if you would like to add a prediction of the age and gender of each face to the output. (Defaults to false)

- __Recognise__: Select this if you would like to try recognise each face in the output. This will require adding a descriptor by uploading an image using the supplied add image button. (Defaults to false)

- __Recognition Metric__: Select the type of recognition metric to use when comparing faces with the recognition option. This is option is only shown when the recognise option is selected.

- __Matched Confidence__: This is the minimum cutoff value for recognition for each of the metrics. Keep in mind that the metrics will produce different ranges of values for recognition. This is option is only shown when the recognise option is selected.

- __Add Images__: Use this button to add an image to create a descriptors from. These descriptors will then be used in the compute node to predict against an input. This is option is only shown when the recognise option is selected.

- __Remove Descriptors__: Use this button to remove the currently stored descriptors. This is option is only shown when the recognise option is selected and is irreversible.

In order to use the facial recognition option, facial descriptors must be calculated first to allow a comparison between them and any input image. To do this, check `enable recognition`, then click `Add Images`. Once selected all images will be computed either on the next deploy if your node is new, or immediately if your node already exists.

Currently the node will compute the face descriptor in the main node-red thread. Due to this, the deploy action will be blocking until finished. This was by design so that the descriptors would be computed before using the compute node they are associated with. This may take some time if not using `tensorflow for nodejs`, so please be patient on deploys for large sets of images.

These descriptors are then saved to disk allowing it to survive restarts of Node-Red. A saved file will then be loaded on startup of Node-Red. Saving the descriptor is also safer than saving an image if your Node-Red instance is online as no data about the original image is stored.

As stated the compute node can offload the calculations to a `child_process` to allow the Node-Red event loop to continue. In doing this, another instance of Node.js is spawned which takes a certain amount of resources. Due to this it may not be desirable to run the calculations in this child node. With the option available, you can choose how to run this node for your specific application.

> On a windows environment, each child process take approx 85MB of ram. This should be similar for other platforms.

> It should be noted that the child node does not speed up the calculation. It only unblocks the main thread

The original Face-api.js library only supports the Euclidean distance method of comparison between descriptors when matching faces. To extend this, this node also supports 3 more type of metrics. These are; __Manhattan__, __Chebyshev__, and __Mean Squared Error__.

The outputs of these metrics from the node are all mapped roughly to the same output values of around 0 to 10000. This was to make it easier to compare the usefulness of each and allow the user to set a cutoff value within a similar range for each metric.

From testing, __Mean Squared Error__ gives the highest contrasting results from known to unknown faces. If you have another metric you feel may be useful, feel free to submit a pull request or create a discussion as I can now implement it along with the others.

If you like this node and want to contribute feel free to fork this repo and make a pull request. New features and suggestions are welcome, and there are several features I would like to implement, but lack the Javascript knowledge in Node.js. These are;

- [ ] Help on implementing tfjs-node on all platforms.

- [ ] General code improvements and clean ups

		{
		"name": "node-red-contrib-face-recognition",
		"version": "1.3.2",
		"version": "1.3.3",
		"description": "A wrapper node for the epic face-api.js library",
		@@ -5,0 +5,0 @@ "main": "index.js",

		@@ -25,6 +25,11 @@ # node-red-contrib-face-recognition

		> Linux users (including raspberry pi) should read [this issue](https://github.com/thebigpotatoe/node-red-contrib-face-recognition/issues/4#issuecomment-579821200) if having issues with installing canvas.

		## Example Flow

		As an example on how to use the node, below is a flow that grabs an image from the internet and runs inference over it. Simply copy and paste it into Node-Red to use.
		As an example on how to use the node, below is a flow that grabs an image from the internet and runs inference over it. Copy and paste it into Node-Red to use, but make sure to install the following nodes from the pallet manager;

		- node-red-node-base64
		- node-red-contrib-image-output

		![Example](Images/Example%20Flow.PNG)
		@@ -31,0 +36,0 @@

		@@ -146,4 +146,15 @@ module.exports = function (RED) {
		let msg = {}
		msg["Matched Faces"] = []

		computeNodesOutput.forEach((output) => {
		msg[output.name] = output
		output.faces.forEach(face => {
		if (face.matchedLabel !== "unknown") {
		msg["Matched Faces"].push({
		"name" : face.matchedLabel,
		"confidence" : face.matchedDistance
		})
		// console.log(face)
		}
		})
		})
		@@ -150,0 +161,0 @@

Improved metrics