From 97e4ca5a79da0c19fcbf4a73cd8e53ed21f1c696 Mon Sep 17 00:00:00 2001 From: SmahiHana <96351416+SmahiHana@users.noreply.github.com> Date: Tue, 11 Oct 2022 19:42:46 -0700 Subject: [PATCH] Adding in Pose Estimation html + js --- js/pose.js | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++++ pose.html | 56 +++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 js/pose.js create mode 100644 pose.html diff --git a/js/pose.js b/js/pose.js new file mode 100644 index 0000000..94f571d --- /dev/null +++ b/js/pose.js @@ -0,0 +1,128 @@ +//current questions: +/*How do you access POSE_LANDMARKS_LEFT/RIGHT/NEUTRAL +is there an x & y coordinate variable/field +is there a way to capture frame by frame to compare changes in x & y +should frame capturing be done in certain time frame (like every 1 sec or 0.5 etc) +How much shift should occur to be considered a great enough rep/change in movement +*/ + +//video5 = webcame input section +const video5 = document.getElementsByClassName('input_video5')[0]; +//out5 = mediapipe output to user + pose estimations +const out5 = document.getElementsByClassName('output5')[0]; +//controls which aspects of algorithm are on (ex: tracking upperbody Y/No) +const controlsElement5 = document.getElementsByClassName('control5')[0]; +const canvasCtx5 = out5.getContext('2d'); + +const fpsControl = new FPS(); + +const spinner = document.querySelector('.loading'); +spinner.ontransitionend = () => { + spinner.style.display = 'none'; +}; + +function zColor(data) { + const z = clamp(data.from.z + 0.5, 0, 1); + return `rgba(0, ${255 * z}, ${255 * (1 - z)}, 1)`; +} + +//gets called repeatdley/ frames +//creates global vairable for previous get previous and current frame landamrks and +//compare changes +function onResultsPose(results) { + document.body.classList.add('loaded'); + fpsControl.tick(); + + canvasCtx5.save(); + canvasCtx5.clearRect(0, 0, out5.width, out5.height); + canvasCtx5.drawImage( + results.image, 0, 0, out5.width, out5.height); + // console.log("results = ", results) + drawConnectors( + canvasCtx5, results.poseLandmarks, POSE_CONNECTIONS, { + color: (data) => { + const x0 = out5.width * data.from.x; + const y0 = out5.height * data.from.y; + const x1 = out5.width * data.to.x; + const y1 = out5.height * data.to.y; + + const z0 = clamp(data.from.z + 0.5, 0, 1); + const z1 = clamp(data.to.z + 0.5, 0, 1); + + const gradient = canvasCtx5.createLinearGradient(x0, y0, x1, y1); + gradient.addColorStop( + 0, `rgba(0, ${255 * z0}, ${255 * (1 - z0)}, 1)`); + gradient.addColorStop( + 1.0, `rgba(0, ${255 * z1}, ${255 * (1 - z1)}, 1)`); + return gradient; + } + }); + /*const noseLocation = -- const + let noseLocation --> changes/unstatic variable + */ + //GETTING NOSE LOCATION + const noseLocation = results.poseLandmarks[0]; + console.log("Nose: ", noseLocation.x); + drawLandmarks( + canvasCtx5, + Object.values(POSE_LANDMARKS_LEFT) + .map(index => results.poseLandmarks[index]), + {color: zColor, fillColor: '#FF0000'}); + drawLandmarks( + canvasCtx5, + Object.values(POSE_LANDMARKS_RIGHT) + .map(index => results.poseLandmarks[index]), + {color: zColor, fillColor: '#00FF00'}); + drawLandmarks( + canvasCtx5, + Object.values(POSE_LANDMARKS_NEUTRAL) + .map(index => results.poseLandmarks[index]), + {color: zColor, fillColor: '#AAAAAA'}); + canvasCtx5.restore(); +} + +const pose = new Pose({locateFile: (file) => { + return `https://cdn.jsdelivr.net/npm/@mediapipe/pose@0.2/${file}`; +}}); +pose.onResults(onResultsPose); + +const camera = new Camera(video5, { + onFrame: async () => { + await pose.send({image: video5}); + }, + width: 480, + height: 480 +}); +camera.start(); + +new ControlPanel(controlsElement5, { + selfieMode: true, + upperBodyOnly: false, + smoothLandmarks: true, + //I'm assuming the confidence for if the thing being detected is 50% sure? + minDetectionConfidence: 0.5, + minTrackingConfidence: 0.5 + }) + .add([ + new StaticText({title: 'MediaPipe Pose'}), + fpsControl, + new Toggle({title: 'Selfie Mode', field: 'selfieMode'}), + new Toggle({title: 'Upper-body Only', field: 'upperBodyOnly'}), + new Toggle({title: 'Smooth Landmarks', field: 'smoothLandmarks'}), + new Slider({ + title: 'Min Detection Confidence', + field: 'minDetectionConfidence', + range: [0, 1], + step: 0.01 + }), + new Slider({ + title: 'Min Tracking Confidence', + field: 'minTrackingConfidence', + range: [0, 1], + step: 0.01 + }), + ]) + .on(options => { + video5.classList.toggle('selfie', options.selfieMode); + pose.setOptions(options); + }); diff --git a/pose.html b/pose.html new file mode 100644 index 0000000..2fc2ce3 --- /dev/null +++ b/pose.html @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + +
+ +
+ + +
+
+

+ Webcam Input +

+
+ +
+
+
+ + +
+
+

+ Mediapipe Pose Detection +

+
+ +
+
+
+
+ +
+ +
+ + + + +