验证完毕

package.json

@@ -3,12 +3,20 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "@tensorflow-models/posenet": "^2.2.1",
+    "@tensorflow/tfjs": "^1.7.2",
+    "@tensorflow/tfjs-core": "^1.7.2",
     "@testing-library/jest-dom": "^4.2.4",
     "@testing-library/react": "^9.3.2",
     "@testing-library/user-event": "^7.1.2",
+    "canvas-sketch-util": "^1.10.0",
+    "matter-js": "^0.14.2",
+    "node-sass": "^4.13.1",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
-    "react-scripts": "3.4.1"
+    "react-scripts": "3.4.1",
+    "three": "^0.117.1",
+    "touches": "^1.2.2"
   },
   "scripts": {
     "start": "react-scripts start",

src/App.css

@@ -1,38 +0,0 @@
-.App {
-  text-align: center;
-}
-
-.App-logo {
-  height: 40vmin;
-  pointer-events: none;
-}
-
-@media (prefers-reduced-motion: no-preference) {
-  .App-logo {
-    animation: App-logo-spin infinite 20s linear;
-  }
-}
-
-.App-header {
-  background-color: #282c34;
-  min-height: 100vh;
-  display: flex;
-  flex-direction: column;
-  align-items: center;
-  justify-content: center;
-  font-size: calc(10px + 2vmin);
-  color: white;
-}
-
-.App-link {
-  color: #61dafb;
-}
-
-@keyframes App-logo-spin {
-  from {
-    transform: rotate(0deg);
-  }
-  to {
-    transform: rotate(360deg);
-  }
-}

src/App.js

@@ -1,26 +1,157 @@
-import React from 'react';
-import logo from './logo.svg';
-import './App.css';
+import React, { useState, useEffect } from "react";
+
+import * as posenet from "@tensorflow-models/posenet";
+import Wall from "./wall/Wall";
+import Sphere from "./wall/Sphere";
+import createTouches from "touches";
+import "./App.scss";
+import i1 from "./images/1.jpg";
+import i2 from "./images/2.jpg";
+import i3 from "./images/3.jpg";
+
+const videoWidth = 600;
+const videoHeight = 500;
+const width = 1080;
+const height = 1920;
+const radius = 412;
+const displacement = 412;
+const POSENET_URL =
+  "https://lg-cjdqwkbo-1256266248.cos.ap-shanghai.myqcloud.com/mobile-net/50/model-stride16.json";
+
+const Block = ({ style }) => <div style={style}></div>;
+
+const Mist = ({ style }) => (
+  <div className="mist" style={style}>
+    <div className="ball"></div>
+    <div className="text">
+      touch <br />
+      item
+    </div>
+  </div>
+);
+
+const App = () => {
+  const [posenetModel, setPosenetModel] = useState(undefined);
+  const [video, setVideo] = useState(undefined);
+  const [wall, setWall] = useState(undefined);
+  const [sphere, setSphere] = useState(undefined);
+  const [list, setList] = useState([]);
+
+  async function poseDetectionFrame() {
+    if (posenetModel) {
+      const poses = await posenetModel.estimatePoses(video, {
+        flipHorizontal: true,
+        decodingMethod: "single-person",
+      });
+      const minPoseConfidence = 0.3;
+      poses.forEach(({ score, keypoints }) => {
+        if (score >= minPoseConfidence) {
+          keypoints
+            .filter(({ part }) => part === "nose")
+            .forEach(({ position: { x } }) => {
+              if (sphere) sphere.rePosition(x / videoWidth);
+            });
+        }
+      });
+    }
+    requestAnimationFrame(poseDetectionFrame);
+  }
+
+  useEffect(() => {
+    if (!posenetModel) {
+      console.log("loading posenet model...");
+      posenet
+        .load({
+          architecture: "MobileNetV1",
+          outputStride: 16,
+          inputResolution: 500,
+          multiplier: 0.5,
+          modelUrl: POSENET_URL,
+        })
+        .then((model) => {
+          setPosenetModel(model);
+          console.log("model loaded.");
+        });
+    }
+  }, [posenetModel]);
+
+  useEffect(() => {
+    if (!video) {
+      var constraints = { audio: false, video: { width: 1080, height: 1920 } };
+      navigator.mediaDevices
+        .getUserMedia(constraints)
+        .then(function (mediaStream) {
+          var video = document.createElement("video");
+          video.width = videoWidth;
+          video.height = videoHeight;
+          video.srcObject = mediaStream;
+          video.onloadedmetadata = function (e) {
+            video.play();
+            setVideo(video);
+          };
+        })
+        .catch(function (err) {
+          console.log(err.name + ": " + err.message);
+        });
+    } else {
+      poseDetectionFrame();
+    }
+  }, [video]);
+
+  useEffect(() => {
+    if (!wall) {
+      const wall = new Wall({
+        items: [{ url: i1 }, { url: i2 }, { url: i3 }],
+        imgWidth: 90,
+        imgHeight: 100,
+        containerWidth: 1080,
+        containerHeight: 1920,
+        speed: 1,
+        onListChange: setList,
+      });
+      wall.init();
+      setWall(wall);
+      wall.getList();
+    } else {
+      let sphere = new Sphere({
+        x: width / 2,
+        y: height / 2,
+        radius,
+        displacement,
+        width,
+      });
+      if (wall) {
+        wall.attachSphere(sphere);
+      }
+      setSphere(sphere);
+    }
+    return () => {
+      if (wall) wall.dispose();
+    };
+  }, [wall]);
+
+  useEffect(() => {
+    if (sphere) {
+      const container = document.querySelector(".App");
+      const touchHandler = createTouches(container, {
+        target: container,
+        filtered: true,
+      });
+      touchHandler.on("move", (_, [x, y]) => {
+        sphere.x = x;
+        sphere.y = y;
+      });
+    }
+  }, [sphere]);
 
-function App() {
   return (
     <div className="App">
-      <header className="App-header">
-        <img src={logo} className="App-logo" alt="logo" />
-        <p>
-          Edit <code>src/App.js</code> and save to reload.
-        </p>
-        <a
-          className="App-link"
-          href="https://reactjs.org"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          Learn React
-        </a>
-      </header>
+      {list.map(([key, el]) => (
+        <Block key={key} style={el.style}></Block>
+      ))}
+      {sphere && <Mist style={sphere.style}></Mist>}
     </div>
   );
-}
+};
 
 export default App;

src/App.scss

@@ -0,0 +1,40 @@
+.App {
+  position: relative;
+  width: 100vw;
+  height: 100vh;
+  text-align: center;
+  background: rgb(218, 232, 255);
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+  canvas {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100vw;
+    height: 100vh;
+    z-index: 1;
+  }
+  .mist {
+    position: absolute;
+    z-index: 100;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    .ball {
+      position: absolute;
+      width: 100%;
+      height: 100%;
+      filter: blur(50px);
+      border-radius: 50%;
+      background: rgba(255, 255, 255, 0.8);
+    }
+    .text {
+      text-align: center;
+      color: #000;
+      font-size: 60px;
+      margin: auto;
+      z-index: 1;
+    }
+  }
+}

src/util.js

@@ -0,0 +1,205 @@
+import * as posenet from "@tensorflow-models/posenet";
+import * as tf from "@tensorflow/tfjs";
+
+const color = "aqua";
+const boundingBoxColor = "red";
+const lineWidth = 2;
+
+export const tryResNetButtonName = "tryResNetButton";
+export const tryResNetButtonText = "[New] Try ResNet50";
+const tryResNetButtonTextCss = "width:100%;text-decoration:underline;";
+const tryResNetButtonBackgroundCss = "background:#e61d5f;";
+
+function isAndroid() {
+  return /Android/i.test(navigator.userAgent);
+}
+
+function isiOS() {
+  return /iPhone|iPad|iPod/i.test(navigator.userAgent);
+}
+
+export function isMobile() {
+  return isAndroid() || isiOS();
+}
+
+function setDatGuiPropertyCss(propertyText, liCssString, spanCssString = "") {
+  var spans = document.getElementsByClassName("property-name");
+  for (var i = 0; i < spans.length; i++) {
+    var text = spans[i].textContent || spans[i].innerText;
+    if (text == propertyText) {
+      spans[i].parentNode.parentNode.style = liCssString;
+      if (spanCssString !== "") {
+        spans[i].style = spanCssString;
+      }
+    }
+  }
+}
+
+export function updateTryResNetButtonDatGuiCss() {
+  setDatGuiPropertyCss(
+    tryResNetButtonText,
+    tryResNetButtonBackgroundCss,
+    tryResNetButtonTextCss
+  );
+}
+
+/**
+ * Toggles between the loading UI and the main canvas UI.
+ */
+export function toggleLoadingUI(
+  showLoadingUI,
+  loadingDivId = "loading",
+  mainDivId = "main"
+) {
+  if (showLoadingUI) {
+    document.getElementById(loadingDivId).style.display = "block";
+    document.getElementById(mainDivId).style.display = "none";
+  } else {
+    document.getElementById(loadingDivId).style.display = "none";
+    document.getElementById(mainDivId).style.display = "block";
+  }
+}
+
+function toTuple({ y, x }) {
+  return [y, x];
+}
+
+export function drawPoint(ctx, y, x, r, color) {
+  ctx.beginPath();
+  ctx.arc(x, y, r, 0, 2 * Math.PI);
+  ctx.fillStyle = color;
+  ctx.fill();
+}
+
+/**
+ * Draws a line on a canvas, i.e. a joint
+ */
+export function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
+  ctx.beginPath();
+  ctx.moveTo(ax * scale, ay * scale);
+  ctx.lineTo(bx * scale, by * scale);
+  ctx.lineWidth = lineWidth;
+  ctx.strokeStyle = color;
+  ctx.stroke();
+}
+
+/**
+ * Draws a pose skeleton by looking up all adjacent keypoints/joints
+ */
+export function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
+  const adjacentKeyPoints = posenet.getAdjacentKeyPoints(
+    keypoints,
+    minConfidence
+  );
+
+  adjacentKeyPoints.forEach((keypoints) => {
+    drawSegment(
+      toTuple(keypoints[0].position),
+      toTuple(keypoints[1].position),
+      color,
+      scale,
+      ctx
+    );
+  });
+}
+
+/**
+ * Draw pose keypoints onto a canvas
+ */
+export function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
+  for (let i = 0; i < keypoints.length; i++) {
+    const keypoint = keypoints[i];
+
+    if (keypoint.score < minConfidence) {
+      continue;
+    }
+
+    const { y, x } = keypoint.position;
+    drawPoint(ctx, y * scale, x * scale, 3, color);
+  }
+}
+
+/**
+ * Draw the bounding box of a pose. For example, for a whole person standing
+ * in an image, the bounding box will begin at the nose and extend to one of
+ * ankles
+ */
+export function drawBoundingBox(keypoints, ctx) {
+  const boundingBox = posenet.getBoundingBox(keypoints);
+
+  ctx.rect(
+    boundingBox.minX,
+    boundingBox.minY,
+    boundingBox.maxX - boundingBox.minX,
+    boundingBox.maxY - boundingBox.minY
+  );
+
+  ctx.strokeStyle = boundingBoxColor;
+  ctx.stroke();
+}
+
+/**
+ * Converts an arary of pixel data into an ImageData object
+ */
+export async function renderToCanvas(a, ctx) {
+  const [height, width] = a.shape;
+  const imageData = new ImageData(width, height);
+
+  const data = await a.data();
+
+  for (let i = 0; i < height * width; ++i) {
+    const j = i * 4;
+    const k = i * 3;
+
+    imageData.data[j + 0] = data[k + 0];
+    imageData.data[j + 1] = data[k + 1];
+    imageData.data[j + 2] = data[k + 2];
+    imageData.data[j + 3] = 255;
+  }
+
+  ctx.putImageData(imageData, 0, 0);
+}
+
+/**
+ * Draw an image on a canvas
+ */
+export function renderImageToCanvas(image, size, canvas) {
+  canvas.width = size[0];
+  canvas.height = size[1];
+  const ctx = canvas.getContext("2d");
+
+  ctx.drawImage(image, 0, 0);
+}
+
+/**
+ * Draw heatmap values, one of the model outputs, on to the canvas
+ * Read our blog post for a description of PoseNet's heatmap outputs
+ * https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5
+ */
+export function drawHeatMapValues(heatMapValues, outputStride, canvas) {
+  const ctx = canvas.getContext("2d");
+  const radius = 5;
+  const scaledValues = heatMapValues.mul(tf.scalar(outputStride, "int32"));
+
+  drawPoints(ctx, scaledValues, radius, color);
+}
+
+/**
+ * Used by the drawHeatMapValues method to draw heatmap points on to
+ * the canvas
+ */
+function drawPoints(ctx, points, radius, color) {
+  const data = points.buffer().values;
+
+  for (let i = 0; i < data.length; i += 2) {
+    const pointY = data[i];
+    const pointX = data[i + 1];
+
+    if (pointX !== 0 && pointY !== 0) {
+      ctx.beginPath();
+      ctx.arc(pointX, pointY, radius, 0, 2 * Math.PI);
+      ctx.fillStyle = color;
+      ctx.fill();
+    }
+  }
+}

src/wall/Sphere.js

@@ -0,0 +1,22 @@
+import { Vector3 } from "three";
+export default class Sphere {
+  constructor({ radius, displacement, x, y, update, width }) {
+    Object.assign(this, { radius, displacement, x, y, update, width });
+  }
+  rePosition(ratio) {
+    const x = ratio * this.width;
+    Object.assign(this, { x });
+  }
+  get point() {
+    return new Vector3(this.x, this.y, 0);
+  }
+  get style() {
+    const { radius, x, y } = this;
+    return {
+      width: `${radius * 2}px`,
+      height: `${radius * 2}px`,
+      top: `${y - radius}px`,
+      left: `${x - radius}px`,
+    };
+  }
+}

src/wall/Wall.js

@@ -0,0 +1,183 @@
+import { Vector3 } from "three";
+export const fps = 60;
+export const mspf = 1000 / fps;
+
+class El {
+  constructor(seed) {
+    const { data, row, wall, key, index } = seed;
+    const { imgWidth, blockWidth, imgHeight, sphere } = wall;
+    const { ratio, y } = row;
+    let x = index * blockWidth + blockWidth / 2 - ratio * blockWidth;
+    const point = new Vector3(x, y, 0);
+    const targetPoint = new Vector3(x, y, 0);
+    const spherePoint = sphere ? sphere.point : null;
+    if (sphere && point.distanceTo(spherePoint) < sphere.displacement) {
+      const direction = point.clone().sub(spherePoint);
+      const displacementAmount = sphere.displacement - direction.length();
+      direction.setLength(displacementAmount);
+      direction.add(point);
+
+      point.lerp(direction, 1); // ✨ magic number
+    }
+
+    // and move them back to their original position
+    if (point.distanceTo(targetPoint) > 0.01) {
+      point.lerp(targetPoint, 0.27); // ✨ magic number
+    }
+    const top = point.y - imgHeight / 2;
+    const left = point.x - imgWidth / 2;
+    const { url } = data;
+
+    const style = {
+      position: "absolute",
+      width: `${imgWidth}px`,
+      height: `${imgHeight}px`,
+      top: `${top}px`,
+      left: `${left}px`,
+      backgroundSize: "cover",
+      backgroundImage: `url(${url})`,
+      opacity: 1,
+      zIndex: row.size - Math.floor(Math.abs(row.size / 2 - row.index)),
+    };
+    Object.assign(this, { seed, data, style, key, point });
+  }
+  update() {}
+}
+
+class Row {
+  constructor({ wall, index }) {
+    Object.assign(this, {
+      wall,
+      q: [],
+      minSize: wall.colNum + 1,
+      index,
+      ratio: Math.random(),
+      y: index * wall.blockHeight + wall.blockHeight / 2,
+    });
+  }
+  get size() {
+    return this.q.length;
+  }
+  init() {
+    while (this.size < this.minSize) {
+      this.push();
+    }
+  }
+  push() {
+    const { wall } = this;
+    const data = wall.getNext();
+    const key = wall.getKey();
+    const seed = { data, row: this, wall, key, index: this.q.length };
+    const el = new El(seed);
+    this.wall.elMap.set(key, el);
+    this.q.push(el);
+  }
+  shift() {
+    const el = this.q.shift();
+    this.wall.elMap.delete(el.key);
+  }
+  nextFrame(ratioSpeed) {
+    this.ratio += ratioSpeed;
+    if (this.ratio > 1) {
+      this.shift();
+      this.push();
+      this.q.forEach((el) => {
+        el.seed.index--;
+      });
+      this.ratio -= 1;
+    }
+    this.q.forEach((el) => {
+      this.wall.elMap.set(el.key, new El(el.seed));
+    });
+  }
+}
+
+export default class Wall {
+  #rafID;
+  #lastTime;
+  constructor({
+    items,
+    imgWidth,
+    imgHeight,
+    containerWidth,
+    containerHeight,
+    speed,
+    onListChange,
+  }) {
+    const colNum = Math.floor(containerWidth / imgWidth);
+    const rowNum = Math.floor(containerHeight / imgHeight);
+    const elNum = items.length;
+    const ratioSpeed = speed / (containerWidth / colNum);
+    const blockWidth = containerWidth / (colNum - 1);
+    const blockHeight = containerHeight / (rowNum - 1);
+    const elMap = new Map();
+    Object.assign(this, {
+      time: 0,
+      index: 0,
+      items,
+      colNum,
+      rowNum,
+      elNum,
+      imgWidth,
+      imgHeight,
+      ratioSpeed,
+      rows: [],
+      key: 0,
+      blockWidth,
+      blockHeight,
+      elMap,
+      onListChange,
+      maxDeltaTime: 1 / 30,
+    });
+  }
+  getNext() {
+    const { index, elNum, items } = this;
+    if (index === elNum) {
+      this.index = 1;
+      return items[0];
+    } else {
+      this.index++;
+      return items[index];
+    }
+  }
+  init() {
+    const { rowNum } = this;
+
+    for (let i = 0; i < rowNum; i++) {
+      const row = new Row({ wall: this, index: i });
+      row.init();
+      this.rows.push(row);
+    }
+    this.#rafID = window.requestAnimationFrame(this.animate);
+    this.isRunning = true;
+    return this;
+  }
+  animate = () => {
+    if (!this.isRunning) return;
+    window.requestAnimationFrame(this.animate);
+
+    const now = performance.now();
+    const dt = Math.min(this.maxDeltaTime, (now - this.#lastTime) / 1000);
+    this.rows.forEach((row) => row.nextFrame(this.ratioSpeed));
+    this.getList();
+    this.time += dt;
+    this.#lastTime = now;
+  };
+  dispose() {
+    if (this.#rafID === null) return;
+    window.cancelAnimationFrame(this.#rafID);
+    this.#rafID = null;
+    this.isRunning = false;
+    return this;
+  }
+  getKey() {
+    this.key++;
+    return this.key;
+  }
+  getList() {
+    if (this.onListChange) this.onListChange(Array.from(this.elMap.entries()));
+  }
+  attachSphere(sphere) {
+    this.sphere = sphere;
+  }
+}