From fc16465fa50feb026713d179ed616424faa585c1 Mon Sep 17 00:00:00 2001 From: Joshua Shoemaker Date: Sun, 17 Jan 2021 02:34:10 -0600 Subject: [PATCH] refact: Dependency Inversion Interfaces for source code dependencies --- src/Interfaces/IObjectDetector.ts | 7 +++ src/Interfaces/IObjectLocator.ts | 8 +++ src/Interfaces/IOffset.ts | 7 +++ src/Interfaces/IUiRenderer.ts | 8 +++ src/Interfaces/IVideoCapturer.ts | 5 ++ src/UseCases/Factories/makeObjectDetector.ts | 14 ++++++ src/UseCases/Factories/makeObjectLocator.ts | 14 ++++++ src/UseCases/Factories/makePredictedObject.ts | 15 ++++++ src/UseCases/Factories/makeUiRenderer.ts | 7 +++ src/UseCases/Factories/makeVideoCatpurer.ts | 14 ++++++ src/UseCases/ObjectDetector.ts | 18 +++---- src/UseCases/ObjectLocator.ts | 25 +++++----- src/UseCases/UiRenderer.ts | 44 +++++++++++++++++ src/UseCases/VideoCapturer.ts | 49 +++++++++++++++++++ src/app.ts | 49 +++++++++---------- 15 files changed, 235 insertions(+), 49 deletions(-) create mode 100644 src/Interfaces/IObjectDetector.ts create mode 100644 src/Interfaces/IObjectLocator.ts create mode 100644 src/Interfaces/IOffset.ts create mode 100644 src/Interfaces/IUiRenderer.ts create mode 100644 src/Interfaces/IVideoCapturer.ts create mode 100644 src/UseCases/Factories/makeObjectDetector.ts create mode 100644 src/UseCases/Factories/makeObjectLocator.ts create mode 100644 src/UseCases/Factories/makePredictedObject.ts create mode 100644 src/UseCases/Factories/makeUiRenderer.ts create mode 100644 src/UseCases/Factories/makeVideoCatpurer.ts create mode 100644 src/UseCases/UiRenderer.ts create mode 100644 src/UseCases/VideoCapturer.ts diff --git a/src/Interfaces/IObjectDetector.ts b/src/Interfaces/IObjectDetector.ts new file mode 100644 index 0000000..4bf51bc --- /dev/null +++ b/src/Interfaces/IObjectDetector.ts @@ -0,0 +1,7 @@ +import PredictedObject from '../Models/PredictedObject' + +interface IObjectDetector { + getPredictionsFromImageData(videoImage: ImageData): Promise +} + +export default IObjectDetector diff --git a/src/Interfaces/IObjectLocator.ts b/src/Interfaces/IObjectLocator.ts new file mode 100644 index 0000000..2c44c2a --- /dev/null +++ b/src/Interfaces/IObjectLocator.ts @@ -0,0 +1,8 @@ +import IOffset from "./IOffset" +import IPredictedObject from "./IPredictedObject" + +interface IObjectLocator { + getOffsetsFromPredictions(predictedObject: IPredictedObject): IOffset +} + +export default IObjectLocator \ No newline at end of file diff --git a/src/Interfaces/IOffset.ts b/src/Interfaces/IOffset.ts new file mode 100644 index 0000000..be726e7 --- /dev/null +++ b/src/Interfaces/IOffset.ts @@ -0,0 +1,7 @@ +interface IOffset { + x: number, + y: number, + hypotenuse: number +} + +export default IOffset \ No newline at end of file diff --git a/src/Interfaces/IUiRenderer.ts b/src/Interfaces/IUiRenderer.ts new file mode 100644 index 0000000..3f2e596 --- /dev/null +++ b/src/Interfaces/IUiRenderer.ts @@ -0,0 +1,8 @@ +import IOffset from "./IOffset" +import IPredictedObject from "./IPredictedObject" + +interface IUiRenderer { + render(props: { imageData: ImageData, predictedObjects: IPredictedObject[], offsets: IOffset[] }): void +} + +export default IUiRenderer diff --git a/src/Interfaces/IVideoCapturer.ts b/src/Interfaces/IVideoCapturer.ts new file mode 100644 index 0000000..2575503 --- /dev/null +++ b/src/Interfaces/IVideoCapturer.ts @@ -0,0 +1,5 @@ +interface IVideoCapturer { + imageData: ImageData | null +} + +export default IVideoCapturer diff --git a/src/UseCases/Factories/makeObjectDetector.ts b/src/UseCases/Factories/makeObjectDetector.ts new file mode 100644 index 0000000..bf82047 --- /dev/null +++ b/src/UseCases/Factories/makeObjectDetector.ts @@ -0,0 +1,14 @@ +import { DetectedObject } from "@tensorflow-models/coco-ssd" +import ObjectDetector from "../ObjectDetector" + +const defaultPredictions = [ + (prediction: DetectedObject) => prediction.score > 0.6, + (prediction: DetectedObject) => prediction.class === 'cat', +] + +function makeObjectDetector (filterPredicates?: Function[]): ObjectDetector { + if (!filterPredicates) filterPredicates = defaultPredictions + return new ObjectDetector({ filterPredicates }) +} + +export default makeObjectDetector \ No newline at end of file diff --git a/src/UseCases/Factories/makeObjectLocator.ts b/src/UseCases/Factories/makeObjectLocator.ts new file mode 100644 index 0000000..8366d71 --- /dev/null +++ b/src/UseCases/Factories/makeObjectLocator.ts @@ -0,0 +1,14 @@ +import IVideo from "../../Interfaces/IVideo" +import ObjectLocator from "../ObjectLocator" + +const defaultProps = { + width: 640, + height: 480 +} + +function makeObjectLocator (props?: IVideo): ObjectLocator { + const videoProps = props || defaultProps + return new ObjectLocator(videoProps) +} + +export default makeObjectLocator \ No newline at end of file diff --git a/src/UseCases/Factories/makePredictedObject.ts b/src/UseCases/Factories/makePredictedObject.ts new file mode 100644 index 0000000..4350c7a --- /dev/null +++ b/src/UseCases/Factories/makePredictedObject.ts @@ -0,0 +1,15 @@ +import { DetectedObject } from "@tensorflow-models/coco-ssd" +import IPredictedObject from "../../Interfaces/IPredictedObject" +import PredictedObject from "../../Models/PredictedObject" + +function makePredictedObject (p: IPredictedObject) { + return new PredictedObject({ + xOrigin: p.xOrigin, + yOrigin: p.yOrigin, + width: p.width, + height: p.height, + class: p.class + }) +} + +export default makePredictedObject \ No newline at end of file diff --git a/src/UseCases/Factories/makeUiRenderer.ts b/src/UseCases/Factories/makeUiRenderer.ts new file mode 100644 index 0000000..301f9ca --- /dev/null +++ b/src/UseCases/Factories/makeUiRenderer.ts @@ -0,0 +1,7 @@ +import UiRenderer from "../UiRenderer"; + +function makeUiRenderer (): UiRenderer { + return new UiRenderer() +} + +export default makeUiRenderer diff --git a/src/UseCases/Factories/makeVideoCatpurer.ts b/src/UseCases/Factories/makeVideoCatpurer.ts new file mode 100644 index 0000000..d3394f7 --- /dev/null +++ b/src/UseCases/Factories/makeVideoCatpurer.ts @@ -0,0 +1,14 @@ +import IVideo from "../../Interfaces/IVideo" +import VideoCapturer from "../VideoCapturer" + +const defaultProps = { + width: 640, + height: 480 +} + +function makeVideoCapturer (props?: IVideo): VideoCapturer { + const videoProps = props || defaultProps + return new VideoCapturer(videoProps) +} + +export default makeVideoCapturer diff --git a/src/UseCases/ObjectDetector.ts b/src/UseCases/ObjectDetector.ts index f16b989..3fe68da 100644 --- a/src/UseCases/ObjectDetector.ts +++ b/src/UseCases/ObjectDetector.ts @@ -1,24 +1,20 @@ import * as tf from '@tensorflow/tfjs' import * as cocossd from '@tensorflow-models/coco-ssd' -import PredictedObject from '../Models/PredictedObject' +import IObjectDetector from '../Interfaces/IObjectDetector' +import IPredictedObject from '../Interfaces/IPredictedObject' +import makePredictedObject from './Factories/makePredictedObject' -let instance: ObjectDetector | null = null - -class ObjectDetector { +class ObjectDetector implements IObjectDetector { private mlModel: cocossd.ObjectDetection | null = null private filterPredicates: Function[] = [] constructor (props?: { filterPredicates?: Function[] }) { - if (!instance) instance = this - if (props?.filterPredicates) this.filterPredicates = props.filterPredicates tf.getBackend() - - return instance } - private convertDetectedToPredictedObjects = (detectedObjects: cocossd.DetectedObject[]) => { - const predictedObjects: PredictedObject[] = detectedObjects.map(p => new PredictedObject({ + private convertDetectedToPredictedObjects (detectedObjects: cocossd.DetectedObject[]) { + const predictedObjects: IPredictedObject[] = detectedObjects.map(p => makePredictedObject({ xOrigin: p.bbox[0], yOrigin: p.bbox[1], width: p.bbox[2], @@ -39,7 +35,7 @@ class ObjectDetector { else return true } - public predictImageStream = async (videoImage: ImageData) => { + public async getPredictionsFromImageData (videoImage: ImageData): Promise { const mlModel = await this.loadMlModel() const detectedObjects = await mlModel.detect(videoImage) const filteredDetections = detectedObjects.filter(p => this.doesDetectionPassFilterPredicates(p)) diff --git a/src/UseCases/ObjectLocator.ts b/src/UseCases/ObjectLocator.ts index 8aa5565..2f7e05e 100644 --- a/src/UseCases/ObjectLocator.ts +++ b/src/UseCases/ObjectLocator.ts @@ -1,20 +1,19 @@ -import PredictedObject from "../Models/PredictedObject" -import Video from "../Models/Video" +import IObjectLocator from "../Interfaces/IObjectLocator" +import IOffset from "../Interfaces/IOffset" +import IPredictedObject from "../Interfaces/IPredictedObject" +import IVideo from "../Interfaces/IVideo" -interface Offset { - x: number, - y: number, - hypotenuse: number -} +class ObjectLocator implements IObjectLocator { + private videoWidth: number + private videoHeight: number -class ObjectLocator { - private video: Video - constructor (video: Video) { - this.video = video + constructor (props: IVideo) { + this.videoWidth = props.width + this.videoHeight = props.height } - detectPredictedObjectLocationFromVideo = (predictedObject: PredictedObject): Offset => { - const videoCenter = { x: this.video.width / 2, y: this.video.height / 2 } + getOffsetsFromPredictions = (predictedObject: IPredictedObject): IOffset => { + const videoCenter = { x: this.videoWidth / 2, y: this.videoHeight / 2 } const objectCenter = { x: predictedObject.xOrigin + (predictedObject.width / 2), y: predictedObject.yOrigin + (predictedObject.height / 2) diff --git a/src/UseCases/UiRenderer.ts b/src/UseCases/UiRenderer.ts new file mode 100644 index 0000000..9918672 --- /dev/null +++ b/src/UseCases/UiRenderer.ts @@ -0,0 +1,44 @@ +import IOffset from "../Interfaces/IOffset" +import IPredictedObject from "../Interfaces/IPredictedObject" +import IUiRenderer from "../Interfaces/IUiRenderer" + +class UiRenderer implements IUiRenderer { + render (props: { imageData: ImageData, predictedObjects: IPredictedObject[], offsets: IOffset[] }) { + + const body: HTMLBodyElement = document.querySelector('body')! + + let canvasElement: HTMLCanvasElement = document.querySelector('#videoOutput') as HTMLCanvasElement + if (!canvasElement) { + canvasElement = document.createElement('canvas') + canvasElement.id = 'videoOutput' + canvasElement.width = props.imageData.width + canvasElement.height = props.imageData.height + body.append(canvasElement) + } + + const canvasContext = canvasElement.getContext('2d')! + canvasContext.clearRect(0, 0, canvasElement.width, canvasElement.height) + canvasContext.putImageData(props.imageData, 0, 0) + + props.predictedObjects.forEach(obj => { + canvasContext.strokeStyle = 'rgb(0, 255, 0)' + canvasContext.strokeRect(obj.xOrigin, obj.yOrigin, obj.width, obj.height) + }) + + const startPoint = { + x: props.imageData.width / 2, + y: props.imageData.height / 2 + } + + props.offsets.forEach(offset => { + canvasContext.strokeStyle = 'rgb(255, 0, 0)' + canvasContext.beginPath() + canvasContext.moveTo(startPoint.x, startPoint.y) + canvasContext.lineTo(startPoint.x - offset.x, startPoint.y - offset.y) + canvasContext.closePath() + canvasContext.stroke() + }) + } +} + +export default UiRenderer diff --git a/src/UseCases/VideoCapturer.ts b/src/UseCases/VideoCapturer.ts new file mode 100644 index 0000000..7455875 --- /dev/null +++ b/src/UseCases/VideoCapturer.ts @@ -0,0 +1,49 @@ +import IVideo from '../Interfaces/IVideo' +import IVideoCapturer from '../Interfaces/IVideoCapturer' + +class VideoCapturer implements IVideoCapturer { + private videoWidth: number + private videoHeight: number + private videoStream: MediaStream | null = null + + constructor (props: IVideo) { + this.videoWidth = props.width + this.videoHeight = props.height + this.enableCamera() + } + + private enableCamera = async () => { + const webCameraStream = await navigator.mediaDevices.getUserMedia({ video: true }) + this.videoStream = webCameraStream + } + + get imageData () { + if (!this.videoStream) return null + + let videoElement: HTMLVideoElement = document.querySelector('#videoView') as HTMLVideoElement + if (!videoElement) { + videoElement = document.createElement('video') + videoElement.width = this.videoWidth + videoElement.height = this.videoHeight + videoElement.autoplay = true + videoElement.srcObject = this.videoStream + videoElement.id = 'videoView' + videoElement.style.display = 'none' + + const body = document.querySelector('body')! + body.appendChild(videoElement) + } + + const canvasElement: HTMLCanvasElement = document.createElement('canvas') + canvasElement.width = this.videoWidth + canvasElement.height = this.videoHeight + + const canvasContext = canvasElement.getContext('2d')! + canvasContext.drawImage(videoElement, 0, 0, this.videoWidth, this.videoHeight) + + return canvasContext.getImageData(0, 0, this.videoWidth, this.videoHeight) + } + +} + +export default VideoCapturer diff --git a/src/app.ts b/src/app.ts index 00f2e11..4890411 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,43 +1,42 @@ -import { DetectedObject } from "@tensorflow-models/coco-ssd" -import PredictedObjectCollectionController from "./Controllers/PredictedObjectCollectionController" -import VideoController from './Controllers/VideoController' -import ObjectDetector from './UseCases/ObjectDetector' -import ObjectLocator from "./UseCases/ObjectLocator" +import IObjectDetector from './Interfaces/IObjectDetector' +import IObjectLocator from './Interfaces/IObjectLocator' +import IOffset from './Interfaces/IOffset' +import IUiRenderer from './Interfaces/IUiRenderer' +import IVideoCapturer from "./Interfaces/IVideoCapturer" -const defaultPredictions = [ - (prediction: DetectedObject) => prediction.score > 0.6, - (prediction: DetectedObject) => prediction.class === 'person', // TODO: change to cat -] +import makeObjectDetector from './UseCases/Factories/makeObjectDetector' +import makeObjectLocator from './UseCases/Factories/makeObjectLocator' +import makeUiRenderer from './UseCases/Factories/makeUiRenderer' +import makeVideoCapturer from './UseCases/Factories/makeVideoCatpurer' class App { - private predictedObjectCollectionController: PredictedObjectCollectionController - private videoController: VideoController - private objectDetector: ObjectDetector - private objectLocator: ObjectLocator + private objectDetector: IObjectDetector + private objectLocator: IObjectLocator + private videoCapturer: IVideoCapturer + private uiRenderer: IUiRenderer constructor () { - this.objectDetector = new ObjectDetector({ filterPredicates: defaultPredictions }) - this.predictedObjectCollectionController = new PredictedObjectCollectionController() - this.videoController = new VideoController({ width: 640, height: 480 }) - this.objectLocator = new ObjectLocator(this.videoController.model) + this.videoCapturer = makeVideoCapturer() + this.objectDetector = makeObjectDetector() + this.objectLocator = makeObjectLocator() + this.uiRenderer = makeUiRenderer() + + const eventTarget = new EventTarget() + eventTarget.addEventListener('onMediaStreamReady', this.predictImage) this.predictImage() } predictImage = async () => { - const imageData = this.videoController.imageData + const imageData = this.videoCapturer.imageData if (!imageData) { window.requestAnimationFrame(this.predictImage) return } - const predictedObjects = await this.objectDetector.predictImageStream(imageData) - this.predictedObjectCollectionController.predictedObjects = predictedObjects - const offsets = predictedObjects.map(obj => { - return this.objectLocator.detectPredictedObjectLocationFromVideo(obj) - }) - - console.log(offsets) + const predictedObjects = await this.objectDetector.getPredictionsFromImageData(imageData) + const offsets: IOffset[] = predictedObjects.map(obj => this.objectLocator.getOffsetsFromPredictions(obj)) + this.uiRenderer.render({ imageData, predictedObjects, offsets }) window.requestAnimationFrame(this.predictImage) } -- 2.47.2