Posenet

Playing with models released for tensorflow js.
Posenet is a neural network that allows the estimation of a human pose from an image.
For more technical information about how it actually works refer to this blog post https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5

import * as posenet from "@tensorflow-models/posenet";

var imageScaleFactor = 0.2;
var outputStride = 16;
var flipHorizontal = false;

var imageElement = document.getElementById("img");
imageElement.width = 400;
imageElement.height = 400;

const video = imageElement;
const canvas = document.querySelector("canvas");
canvas.width = imageElement.width;
canvas.height = imageElement.height;

const ctx = canvas.getContext("2d");

navigator.mediaDevices
  .getUserMedia({
    audio: false,
    video: {
      height: imageElement.height,
      width: imageElement.width,
      facingMode: "user"
    }
  })
  .then(function(stream) {
    video.srcObject = stream;
  })
  .catch(function(err0r) {
    console.log(err0r.message);
    console.log("Something went wrong!");
  });

const parts = {
  nose: {},
  leye: {},
  reye: {},
  lear: {},
  rear: {}
};

video.addEventListener("play", () => {
  posenet
    .load()
    .then(function(net) {
      console.log("Model loaded");
      const v = document.querySelector("video");
      setInterval(() => {
        net
          .estimateSinglePose(v, imageScaleFactor, flipHorizontal, outputStride)
          .then(function(pose) {
            ctx.fillStyle = "red";

            pose.keypoints.forEach(k => {
              if (k.part == "nose" && k.score > 0.8) parts.nose = k;
              if (k.part == "leftEye" && k.score > 0.8) parts.leye = k;
              if (k.part == "rightEye" && k.score > 0.8) parts.reye = k;
            });

            ctx.clearRect(0, 0, canvas.width, canvas.height);
            ctx.drawImage(
              imageElement,
              0,
              0,
              imageElement.width * imageScaleFactor,
              imageElement.height * imageScaleFactor,
              0,
              0,
              imageElement.width * imageScaleFactor,
              imageElement.height * imageScaleFactor
            );
            ctx.drawImage(imageElement, 0, 0);
            Object.keys(parts).forEach(k => {
              const part = parts[k];
              if (!part.position) return;
              const size = 10;
              ctx.fillRect(
                part.position.x - size / 2,
                part.position.y - size / 2,
                size,
                size
              );
            });
          });
      }, 100);
    })

    .catch(err => {
      console.log(err.message);
    });
});