init commit

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

README.md

@@ -0,0 +1 @@
+# 🐱Cat 🔫Cannon

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "catcannon",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@tensorflow-models/coco-ssd": "^2.2.1",
+    "@tensorflow/tfjs": "^2.8.2"
+  },
+  "devDependencies": {
+    "ts-loader": "^8.0.13",
+    "typescript": "^4.1.3",
+    "webpack": "^5.11.1"
+  }
+}

src/Controllers/PredictedObjectCollectionController.ts

@@ -0,0 +1,32 @@
+import PredictedObject from "../Models/PredictedObject"
+import PredictedObjectCollection from "../Models/PredictedObjectCollection"
+import PredictedObjectView from "../Views/PredictedObjectView"
+
+class PredictedObjectCollectionController {
+  private model: PredictedObjectCollection
+
+  constructor () {
+    this.model = new PredictedObjectCollection()
+    this.renderView()
+  }
+
+  set predictedObjects (objects: PredictedObject[]) {
+    this.model.objects = objects
+    this.renderView()
+  }
+
+  public renderView = () => {
+    const existingPredictedObjectViews = document.querySelectorAll('.PredictedObject')
+    existingPredictedObjectViews.forEach(v => {
+      v.outerHTML = ''
+    })
+
+    const body = document.querySelector('body')!
+    this.model.objects.forEach((object: PredictedObject) => {
+      const predictedObjectView = new PredictedObjectView(object)
+      body.appendChild(predictedObjectView.element)
+    })
+  }
+}
+
+export default PredictedObjectCollectionController

src/Controllers/VideoController.ts

@@ -0,0 +1,48 @@
+import Video from "../Models/Video"
+import VideoView from '../Views/VideoView'
+
+class VideoController {
+  private defaultWidth: number = 640
+  private defaultHeight: number = 480
+  private userMediaConstraints = { video: true }
+  public model: Video
+  private view: VideoView
+
+  constructor (props: { width?: number, height?: number } = {}) {
+    this.model = new Video({
+      width: props.width || this.defaultWidth,
+      height: props.height || this.defaultHeight
+    })
+
+    this.view = new VideoView(this.model)
+    this.renderView()
+    this.enableCamera()
+  }
+
+  get imageData () {
+    if (!this.view.element.srcObject) return null
+
+    const canvas: HTMLCanvasElement = document.createElement('canvas')
+    canvas.width = this.model.width
+    canvas.height = this.model.height
+    const context = canvas.getContext('2d')!
+    context.drawImage(this.view.element, 0, 0, this.model.width, this.model.height)
+
+    return context.getImageData(0, 0, this.model.width, this.model.height)
+  }
+
+  private enableCamera = async () => {
+    const stream = await navigator.mediaDevices.getUserMedia(this.userMediaConstraints)
+    this.view.srcObject = stream
+  }
+
+  private renderView () {
+    const existingVideoView = document.querySelector('#videoView')
+    if (existingVideoView) existingVideoView.outerHTML = ''
+
+    const body = document.querySelector('body')!
+    body.appendChild(this.view.element)
+  }
+}
+
+export default VideoController

src/Interfaces/IPredictedObject.ts

@@ -0,0 +1,9 @@
+interface IPredictedObject {
+  xOrigin: number,
+  yOrigin: number,
+  width: number,
+  height: number,
+  class: string
+}
+
+export default IPredictedObject

src/Interfaces/IVideo.ts

@@ -0,0 +1,7 @@
+interface IVideo {
+  height: number,
+  width: number,
+  autoplay?: boolean
+}
+
+export default IVideo

src/Models/ObjectDetector.ts

@@ -0,0 +1,43 @@
+import * as tf from '@tensorflow/tfjs'
+import * as cocossd from '@tensorflow-models/coco-ssd'
+
+let instance: ObjectDetector | null = null
+
+class ObjectDetector {
+  private mlModel: cocossd.ObjectDetection | null = null
+  private filterPredicates: Function[] = []
+
+  constructor (props?: { filterPredicates?: Function[] }) {
+    if (!instance) instance = this
+    
+    if (props?.filterPredicates) this.filterPredicates = props.filterPredicates
+    tf.getBackend()
+
+    return instance
+  }
+
+  private doesPredictionPassFilterPredicates (prediction: cocossd.DetectedObject): boolean {
+    let failedPredictions = []
+    this.filterPredicates.forEach(filter => {
+      if (!filter(prediction)) failedPredictions.push(filter)
+    })
+
+    if (failedPredictions.length > 0) return false
+    else return true
+  }
+
+  public predictImageStream = async (videoImage: ImageData) => {
+    const mlModel = await this.loadMlModel()
+    const predictions = await mlModel.detect(videoImage)
+    const filteredPredictions = predictions.filter(p => this.doesPredictionPassFilterPredicates(p))
+
+    return filteredPredictions
+  }
+
+  public async loadMlModel (): Promise<cocossd.ObjectDetection> {
+    if (!this.mlModel) this.mlModel = await cocossd.load()
+    return this.mlModel
+  }
+}
+
+export default ObjectDetector

src/Models/PredictedObject.ts

@@ -0,0 +1,19 @@
+import IPredictedObject from "../Interfaces/IPredictedObject"
+
+class PredictedObject {
+  public xOrigin: number
+  public yOrigin: number
+  public width: number
+  public height: number
+  public class: string
+
+  constructor (props: IPredictedObject) {
+    this.xOrigin = props.xOrigin
+    this.yOrigin = props.yOrigin
+    this.width = props.width
+    this.height = props.height
+    this.class = props.class
+  }
+}
+
+export default PredictedObject

src/Models/PredictedObjectCollection.ts

@@ -0,0 +1,26 @@
+import { DetectedObject } from '@tensorflow-models/coco-ssd'
+import PredictedObject from "./PredictedObject"
+
+let instance: PredictedObjectCollection | null = null
+
+class PredictedObjectCollection {
+  public objects: PredictedObject[] = []
+
+  constructor () {
+    if (!instance) instance = this
+    return instance
+  }
+
+  public addDetectedObjectObject(object: DetectedObject) {
+    const newPredictedObject = new PredictedObject({
+      xOrigin: object.bbox[0],
+      yOrigin: object.bbox[1],
+      width: object.bbox[2],
+      height: object.bbox[3],
+      class: object.class
+    })
+    this.objects.push(newPredictedObject)
+  }
+}
+
+export default PredictedObjectCollection

src/Models/Video.ts

@@ -0,0 +1,35 @@
+import IVideo from '../Interfaces/IVideo'
+
+let instance: Video | null = null
+
+class Video {
+  public height: number = 0
+  public width: number = 0
+  public autoplay: boolean = true
+
+  constructor (props?: IVideo) {
+    if (!instance) instance = this
+
+    if (props) {
+      this.height = props.height
+      this.width = props.width
+      if (typeof props.autoplay === 'boolean') this.autoplay = props.autoplay
+    }
+
+    return instance
+  }
+
+  get centerCoordinates (): { x: number, y: number } {
+    return { x: this.width / 2, y: this.height / 2 }
+  }
+
+  get props () {
+    return {
+      height: this.height,
+      width: this.width,
+      autoplay: this.autoplay
+    }
+  }
+}
+
+export default Video

src/Views/PredictedObjectView.ts

@@ -0,0 +1,45 @@
+import IPredictedObject from "../Interfaces/IPredictedObject"
+
+class PredictedObjectView {
+
+  private xOrigin: number
+  private yOrigin: number
+  private width: number
+  private height: number
+  private class: string
+  public element: HTMLElement
+
+  constructor (props: IPredictedObject) {
+    this.xOrigin = props.xOrigin
+    this.yOrigin = props.yOrigin
+    this.width = props.width
+    this.height = props.height
+    this.class = props.class
+    this.element = this.createElement()
+  }
+
+  createElement (): HTMLElement {
+    let predictedObjectElement: HTMLElement = document.createElement('div')
+    predictedObjectElement.style.position = 'absolute'
+    predictedObjectElement.style.left = `${this.xOrigin}px`
+    predictedObjectElement.style.top = `${this.yOrigin}px`
+    predictedObjectElement.style.width = `${this.width}px`
+    predictedObjectElement.style.height = `${this.height}px`
+    predictedObjectElement.setAttribute('class', `PredictedObject ${this.class}`)
+
+    const predictedObjectLabel = this.createLabel()
+
+    predictedObjectElement.appendChild(predictedObjectLabel)
+
+    return predictedObjectElement
+  }
+
+  createLabel (): HTMLLabelElement {
+    const labelElement = document.createElement('label')
+    labelElement.setAttribute('class', 'predictedObjectLabel')
+    labelElement.innerText = this.class
+    return labelElement
+  }
+}
+
+export default PredictedObjectView

src/Views/VideoView.ts

@@ -0,0 +1,28 @@
+import IVideo from '../Interfaces/IVideo'
+
+class VideoView {
+  public width: number
+  public height: number
+  public autoplay: boolean
+  public element: HTMLVideoElement
+
+  constructor (props: IVideo) {
+    this.width = props.width
+    this.height = props.height
+    this.autoplay = props.autoplay!
+    this.element = this.createElement()
+  }
+
+  createElement (): HTMLVideoElement {
+    let videoElement: HTMLVideoElement = document.createElement('video')
+    videoElement.setAttribute('id', 'VideoView')
+    videoElement.setAttribute('width', this.width.toString())
+    videoElement.setAttribute('height', this.height.toString())
+    videoElement.autoplay = this.autoplay
+    return videoElement
+  }
+
+  set srcObject (media: MediaStream | null) { this.element.srcObject = media }
+}
+
+export default VideoView

src/app.ts

@@ -0,0 +1,52 @@
+import { DetectedObject } from "@tensorflow-models/coco-ssd"
+import PredictedObjectCollectionController from "./Controllers/PredictedObjectCollectionController"
+import VideoController from './Controllers/VideoController'
+import ObjectDetector from './Models/ObjectDetector'
+import PredictedObject from "./Models/PredictedObject"
+
+class App {
+  private predictedObjectCollectionController: PredictedObjectCollectionController
+  private videoController: VideoController
+  private objectDetector: ObjectDetector
+
+  constructor () {
+    this.objectDetector = new ObjectDetector({
+      filterPredicates: [
+        (prediction: DetectedObject) => prediction.score > 0.6,
+        (prediction: DetectedObject) => prediction.class === 'cat',
+      ]
+    })
+
+    this.predictedObjectCollectionController = new PredictedObjectCollectionController()
+
+    this.videoController = new VideoController({
+      width: 640,
+      height: 480
+    })
+    this.predictImage()
+  }
+
+  predictImage = async () => {
+    const imageData = this.videoController.imageData
+
+    if (!imageData) {
+      window.requestAnimationFrame(this.predictImage)
+      return
+    }
+
+    const predictions: DetectedObject[] = await this.objectDetector.predictImageStream(imageData)
+    const predictedObjects = predictions.map(p => new PredictedObject({
+      xOrigin: p.bbox[0],
+      yOrigin: p.bbox[1],
+      width: p.bbox[2],
+      height: p.bbox[3],
+      class: p.class
+    }))
+
+    this.predictedObjectCollectionController.predictedObjects = predictedObjects
+
+    window.requestAnimationFrame(this.predictImage)
+  }
+}
+
+new App()

tsconfig.json

@@ -0,0 +1,69 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Basic Options */
+    // "incremental": true,                   /* Enable incremental compilation */
+    "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./app.js",                       /* Concatenate and emit output to single file. */
+    "outDir": "./build",                        /* Redirect output structure to the directory. */
+    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true,                  /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "skipLibCheck": true,                     /* Skip type checking of declaration files. */
+    "forceConsistentCasingInFileNames": true  /* Disallow inconsistently-cased references to the same file. */
+  }
+}

webpack.config.js

@@ -0,0 +1,22 @@
+const path = require('path');
+
+module.exports = {
+  entry: './src/app.ts',
+  devtool: 'inline-source-map',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  resolve: {
+    extensions: [ '.tsx', '.ts', '.js' ],
+  },
+  output: {
+    filename: 'app.js',
+    path: path.resolve(__dirname, 'dist'),
+  },
+};