Feat/v3: This request adds a new plugin to execute custom object detectors on ios and it also fixes a bug related to automatic zooming on ios

.gitignore

@@ -43,4 +43,5 @@ Thumbs.db
 *.tgz
 packages/**/angular/dist
 
-.nx/cache
+.nx/cache
+.nx/workspace-data

README.md

@@ -1,5 +1,6 @@
 - [@nativescript/mlkit-barcode-scanning](packages/mlkit-barcode-scanning/README.md)
 - [@nativescript/mlkit-core](packages/mlkit-core/README.md)
+- [@nativescript/mlkit-custom-object-detection](packages/mlkit-custom-object-detection/README.md)
 - [@nativescript/mlkit-digital-ink-recognition](packages/mlkit-digital-ink-recognition/README.md)
 - [@nativescript/mlkit-face-detection](packages/mlkit-face-detection/README.md)
 - [@nativescript/mlkit-image-labeling](packages/mlkit-image-labeling/README.md)

apps/demo-angular/package.json

@@ -10,7 +10,8 @@
     "@nativescript/mlkit-object-detection": "file:../../dist/packages/mlkit-object-detection",
     "@nativescript/mlkit-digital-ink-recognition": "file:../../dist/packages/mlkit-digital-ink-recognition",
     "@nativescript/mlkit-pose-detection": "file:../../dist/packages/mlkit-pose-detection",
-    "@nativescript/mlkit-selfie-segmentation": "file:../../dist/packages/mlkit-selfie-segmentation"
+    "@nativescript/mlkit-selfie-segmentation": "file:../../dist/packages/mlkit-selfie-segmentation",
+    "@nativescript/mlkit-custom-object-detection": "file:../../dist/packages/mlkit-custom-object-detection"
   },
   "devDependencies": {
     "@nativescript/android": "8.8.0-alpha.7",

apps/demo-angular/src/app-routing.module.ts

@@ -5,21 +5,22 @@ import { NativeScriptRouterModule } from '@nativescript/angular';
 import { HomeComponent } from './home.component';
 
 const routes: Routes = [
-   { path: '', redirectTo: '/home', pathMatch: 'full' },
-   { path: 'home', component: HomeComponent },
-	{ path: 'mlkit-barcode-scanning', loadChildren: () => import('./plugin-demos/mlkit-barcode-scanning.module').then(m => m.MlkitBarcodeScanningModule) },
-	{ path: 'mlkit-core', loadChildren: () => import('./plugin-demos/mlkit-core.module').then(m => m.MlkitCoreModule) },
-	{ path: 'mlkit-digital-ink-recognition', loadChildren: () => import('./plugin-demos/mlkit-digital-ink-recognition.module').then(m => m.MlkitDigitalInkRecognitionModule) },
-	{ path: 'mlkit-face-detection', loadChildren: () => import('./plugin-demos/mlkit-face-detection.module').then(m => m.MlkitFaceDetectionModule) },
-	{ path: 'mlkit-image-labeling', loadChildren: () => import('./plugin-demos/mlkit-image-labeling.module').then(m => m.MlkitImageLabelingModule) },
-	{ path: 'mlkit-object-detection', loadChildren: () => import('./plugin-demos/mlkit-object-detection.module').then(m => m.MlkitObjectDetectionModule) },
-	{ path: 'mlkit-pose-detection', loadChildren: () => import('./plugin-demos/mlkit-pose-detection.module').then(m => m.MlkitPoseDetectionModule) },
-	{ path: 'mlkit-selfie-segmentation', loadChildren: () => import('./plugin-demos/mlkit-selfie-segmentation.module').then(m => m.MlkitSelfieSegmentationModule) },
-	{ path: 'mlkit-text-recognition', loadChildren: () => import('./plugin-demos/mlkit-text-recognition.module').then(m => m.MlkitTextRecognitionModule) }
+  { path: '', redirectTo: '/home', pathMatch: 'full' },
+  { path: 'home', component: HomeComponent },
+  { path: 'mlkit-barcode-scanning', loadChildren: () => import('./plugin-demos/mlkit-barcode-scanning.module').then((m) => m.MlkitBarcodeScanningModule) },
+  { path: 'mlkit-core', loadChildren: () => import('./plugin-demos/mlkit-core.module').then((m) => m.MlkitCoreModule) },
+  { path: 'mlkit-custom-object-detection', loadChildren: () => import('./plugin-demos/mlkit-custom-object-detection.module').then((m) => m.MlkitCustomObjectDetectionModule) },
+  { path: 'mlkit-digital-ink-recognition', loadChildren: () => import('./plugin-demos/mlkit-digital-ink-recognition.module').then((m) => m.MlkitDigitalInkRecognitionModule) },
+  { path: 'mlkit-face-detection', loadChildren: () => import('./plugin-demos/mlkit-face-detection.module').then((m) => m.MlkitFaceDetectionModule) },
+  { path: 'mlkit-image-labeling', loadChildren: () => import('./plugin-demos/mlkit-image-labeling.module').then((m) => m.MlkitImageLabelingModule) },
+  { path: 'mlkit-object-detection', loadChildren: () => import('./plugin-demos/mlkit-object-detection.module').then((m) => m.MlkitObjectDetectionModule) },
+  { path: 'mlkit-pose-detection', loadChildren: () => import('./plugin-demos/mlkit-pose-detection.module').then((m) => m.MlkitPoseDetectionModule) },
+  { path: 'mlkit-selfie-segmentation', loadChildren: () => import('./plugin-demos/mlkit-selfie-segmentation.module').then((m) => m.MlkitSelfieSegmentationModule) },
+  { path: 'mlkit-text-recognition', loadChildren: () => import('./plugin-demos/mlkit-text-recognition.module').then((m) => m.MlkitTextRecognitionModule) },
 ];
 
 @NgModule({
-	imports: [NativeScriptRouterModule.forRoot(routes)],
-	exports: [NativeScriptRouterModule],
+  imports: [NativeScriptRouterModule.forRoot(routes)],
+  exports: [NativeScriptRouterModule],
 })
 export class AppRoutingModule {}

apps/demo-angular/src/home.component.ts

@@ -2,44 +2,40 @@ import { Component } from '@angular/core';
 import { RouterExtensions } from '@nativescript/angular';
 
 @Component({
-	selector: 'demo-home',
-	templateUrl: 'home.component.html',
+  selector: 'demo-home',
+  templateUrl: 'home.component.html',
 })
 export class HomeComponent {
-	demos = [
-		{
-			name: 'mlkit-barcode-scanning'
-		},
-		{
-			name: 'mlkit-core'
-		},
-		{
-			name: 'mlkit-digital-ink-recognition'
-		},
-		{
-			name: 'mlkit-face-detection'
-		},
-		{
-			name: 'mlkit-image-labeling'
-		},
-		{
-			name: 'mlkit-object-detection'
-		},
-		{
-			name: 'mlkit-pose-detection'
-		},
-		{
-			name: 'mlkit-selfie-segmentation'
-		},
-		{
-			name: 'mlkit-text-recognition'
-		}
-	];
-
-	constructor(private router: RouterExtensions) { }
-
-	onTap(event) {
-		const item = this.demos[event.index];
-		this.router.navigate(['/' + item.name]);
-	}
-}
+  demos = [
+    {
+      name: 'mlkit-barcode-scanning',
+    },
+    {
+      name: 'mlkit-core',
+    },
+    {
+      name: 'mlkit-custom-object-detection',
+    },
+    {
+      name: 'mlkit-digital-ink-recognition',
+    },
+    {
+      name: 'mlkit-face-detection',
+    },
+    {
+      name: 'mlkit-image-labeling',
+    },
+    {
+      name: 'mlkit-object-detection',
+    },
+    {
+      name: 'mlkit-pose-detection',
+    },
+    {
+      name: 'mlkit-selfie-segmentation',
+    },
+    {
+      name: 'mlkit-text-recognition',
+    },
+  ];
+}

apps/demo-angular/src/plugin-demos/mlkit-core.component.html

@@ -1,16 +1,15 @@
 <ActionBar title="mlkit-core" class="action-bar"> </ActionBar>
 <StackLayout class="p-20">
-	<GridLayout rows="*,auto,auto,auto,auto,auto,auto,auto,auto,auto" height="100%">
-		<MLKitView [torchOn]="torchOn" [pause]="isPaused" height="100%" rowSpan="3" (loaded)="onLoaded($event)" cameraPosition="back"
-			[detectionType]="detectorType" (detection)="onDetection($event)"></MLKitView>
-		<Button row="1" height="40" text="Toggle Camera" (tap)="toggleCamera($event)"></Button>
-		<Button row="2" height="40" text="Request Camera Permission" (tap)="requestPermission($event)"></Button>
-		<Label row="3" [text]="'Detecting ' + detectorType"></Label>
-		<Button row="4" height="40" text="Change Detector Type" (tap)="changeType($event)"></Button>
-		<Label row="5" [text]="'isPaused : ' + isPaused"></Label>
-		<Button row="6" height="40" text="Toggle Pause" (tap)="togglePause($event)"></Button>
-		<Label row="7" [text]="'Torch On : ' + torchOn"></Label>
-		<Button row="8" height="40" text="Toggle Torch" (tap)="toggleTorch($event)"></Button>
-		<Button row="9" height="40" text="Process Still Image" (tap)="processStill($event)"></Button>
-	</GridLayout>
-</StackLayout>
+  <GridLayout rows="*,auto,auto,auto,auto,auto,auto,auto,auto,auto" height="100%">
+    <MLKitView [torchOn]="torchOn" [pause]="isPaused" height="100%" rowSpan="3" (loaded)="onLoaded($event)" cameraPosition="back" [detectionType]="detectorType" (detection)="onDetection($event)"></MLKitView>
+    <Button row="1" height="40" text="Toggle Camera" (tap)="toggleCamera($event)"></Button>
+    <Button row="2" height="40" text="Request Camera Permission" (tap)="requestPermission($event)"></Button>
+    <Label row="3" [text]="'Detecting ' + detectorType"></Label>
+    <Button row="4" height="40" text="Change Detector Type" (tap)="changeType($event)"></Button>
+    <Label row="5" [text]="'isPaused : ' + isPaused"></Label>
+    <Button row="6" height="40" text="Toggle Pause" (tap)="togglePause($event)"></Button>
+    <Label row="7" [text]="'Torch On : ' + torchOn"></Label>
+    <Button row="8" height="40" text="Toggle Torch" (tap)="toggleTorch($event)"></Button>
+    <Button row="9" height="40" text="Process Still Image" (tap)="processStill($event)"></Button>
+  </GridLayout>
+</StackLayout>

apps/demo-angular/src/plugin-demos/mlkit-custom-object-detection.component.html

@@ -0,0 +1,8 @@
+<ActionBar title="mlkit-custom-object-detection" class="action-bar"> </ActionBar>
+<StackLayout class="p-20">
+  <ScrollView class="h-full">
+    <StackLayout>
+      <Button text="Test mlkit-custom-object-detection" (tap)="demoShared.testIt()" class="btn btn-primary"></Button>
+    </StackLayout>
+  </ScrollView>
+</StackLayout>

apps/demo-angular/src/plugin-demos/mlkit-custom-object-detection.component.ts

@@ -0,0 +1,17 @@
+import { Component, NgZone } from '@angular/core';
+import { DemoSharedMlkitCustomObjectDetection } from '@demo/shared';
+import {} from '@nativescript/mlkit-custom-object-detection';
+
+@Component({
+  selector: 'demo-mlkit-custom-object-detection',
+  templateUrl: 'mlkit-custom-object-detection.component.html',
+})
+export class MlkitCustomObjectDetectionComponent {
+  demoShared: DemoSharedMlkitCustomObjectDetection;
+
+  constructor(private _ngZone: NgZone) {}
+
+  ngOnInit() {
+    this.demoShared = new DemoSharedMlkitCustomObjectDetection();
+  }
+}

apps/demo-angular/src/plugin-demos/mlkit-custom-object-detection.module.ts

@@ -0,0 +1,10 @@
+import { NgModule, NO_ERRORS_SCHEMA } from '@angular/core';
+import { NativeScriptCommonModule, NativeScriptRouterModule } from '@nativescript/angular';
+import { MlkitCustomObjectDetectionComponent } from './mlkit-custom-object-detection.component';
+
+@NgModule({
+  imports: [NativeScriptCommonModule, NativeScriptRouterModule.forChild([{ path: '', component: MlkitCustomObjectDetectionComponent }])],
+  declarations: [MlkitCustomObjectDetectionComponent],
+  schemas: [NO_ERRORS_SCHEMA],
+})
+export class MlkitCustomObjectDetectionModule {}

apps/demo-angular/tsconfig.json

@@ -7,6 +7,7 @@
       "@nativescript/mlkit-text-recognition": ["packages/mlkit-text-recognition/index.ts"],
       "@nativescript/mlkit-core": ["packages/mlkit-core/index.d.ts"],
       "@nativescript/mlkit-core/angular": ["packages/mlkit-core/angular/index.ts"],
+      "@nativescript/mlkit-custom-object-detection": ["../../packages/mlkit-custom-object-detection/index.d.ts"],
       "@nativescript/mlkit-barcode-scanning": ["packages/mlkit-barcode-scanning/index.ts"],
       "@nativescript/mlkit-face-detection": ["packages/mlkit-face-detection/index.ts"],
       "@nativescript/mlkit-image-labeling": ["packages/mlkit-image-labeling/index.ts"],

apps/demo/package.json

@@ -13,7 +13,8 @@
     "@nativescript/mlkit-object-detection": "file:../../packages/mlkit-object-detection",
     "@nativescript/mlkit-pose-detection": "file:../../packages/mlkit-pose-detection",
     "@nativescript/mlkit-selfie-segmentation": "file:../../packages/mlkit-selfie-segmentation",
-    "@nativescript/mlkit-text-recognition": "file:../../packages/mlkit-text-recognition"
+    "@nativescript/mlkit-text-recognition": "file:../../packages/mlkit-text-recognition",
+    "@nativescript/mlkit-custom-object-detection": "file:../../packages/mlkit-custom-object-detection"
   },
   "devDependencies": {
     "@nativescript/android": "8.8.0-alpha.8",

apps/demo/references.d.ts

@@ -1 +1,2 @@
 /// <reference path="../../references.d.ts" />
+/// <reference path="../../node_modules/@nativescript/types-ios/lib/ios/objc-x86_64/objc!CoreImage.d.ts" />

apps/demo/src/main-page.xml

@@ -15,6 +15,7 @@
 <Button text="mlkit-digital-ink-recognition" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
 <Button text="mlkit-pose-detection" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
 <Button text="mlkit-selfie-segmentation" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
+<Button text="mlkit-custom-object-detection" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
       </StackLayout>
     </ScrollView>
   </StackLayout>

apps/demo/src/plugin-demos/mlkit-core.ts

@@ -1,4 +1,4 @@
-import { Observable, EventData, Page, Dialogs, ImageSource } from '@nativescript/core';
+import { Observable, EventData, Page, Dialogs, ImageSource, Frame, Image } from '@nativescript/core';
 import { DemoSharedMlkitCore } from '@demo/shared';
 import { DetectionType, MLKitView, DetectionEvent, detectWithStillImage } from '@nativescript/mlkit-core';
 import { BarcodeResult } from '@nativescript/mlkit-barcode-scanning';
@@ -7,6 +7,7 @@ import { ImageLabelingResult } from '@nativescript/mlkit-image-labeling';
 import { ObjectResult } from '@nativescript/mlkit-object-detection';
 import { PoseResult } from '@nativescript/mlkit-pose-detection';
 import { TextResult } from '@nativescript/mlkit-text-recognition';
+import { BoundingBoxSettings, TNSObjectDetectionResult } from '@nativescript/mlkit-core/common';
 
 export function navigatingTo(args: EventData) {
   const page = <Page>args.object;
@@ -18,6 +19,23 @@ export class DemoModel extends DemoSharedMlkitCore {
   detectorType = DetectionType.Barcode;
   isPaused = false;
   torchOn = false;
+  _bbox: { x: number; y: number; width: number; height: number };
+  bboxSettings: BoundingBoxSettings = {
+    drawBBoxes: true,
+    drawEdgeMarks: true,
+    drawLabels: true,
+    bBoxLineColor: '#FF0000FF',
+    bBoxLineWidth: 1,
+    bBoxCornerness: 7,
+    edgeMarkLengthFactor: 0.2,
+    edgeMarkLineWidth: 5,
+    showConfidence: true,
+    labelTextColor: '#FFFFFFFF',
+    labelBackgroundColor: '#00000090',
+    labelCornerness: 5,
+    labelAlignment: 'center',
+    labelMappings: { laptop: 'Laptop' },
+  };
 
   onLoaded(args) {
     this.camera = args.object;
@@ -31,6 +49,48 @@ export class DemoModel extends DemoSharedMlkitCore {
       const first = event.data[0] as BarcodeResult;
       console.log('bounds', first.bounds);
     }
+    if (event.type === DetectionType.CustomObject) {
+      const first = event.data[0] as TNSObjectDetectionResult;
+      if (first?.bounds) {
+        this._bbox = first.bounds;
+      }
+    }
+  }
+
+  public drawBBox(args: EventData) {
+    const page = Frame.topmost()?.currentPage;
+    const imgView = page.getViewById('imageView') as Image;
+    const mlkitview = page.getViewById('mlKitView') as MLKitView;
+    imgView.visibility = 'visible';
+    if (this._bbox && mlkitview.latestImage) {
+      imgView.src = this.extractRectFromImageBuffer(mlkitview.latestImage, this._bbox);
+    }
+  }
+
+  private extractRectFromImageBuffer(imageSource: ImageSource, rect: { x: number; y: number; width: number; height: number }): ImageSource | null {
+    if (global.isIOS) {
+      const uiImage = imageSource.ios; // Get the UIImage
+      const buffer = uiImage.CIImage?.extent || uiImage.CGImage;
+      if (!buffer) {
+        console.error("UIImage doesn't have a valid buffer representation");
+        return null;
+      }
+      const context = CIContext.context();
+      const scale = uiImage.scale || 1;
+      // CGImage origin is not at top right but bottom left, we therefore have to adopt y axis and bbox ref. point
+      const y_inCGImageCoords = buffer.size.height - rect.y - rect.height;
+      const scaledRect = CGRectMake(rect.x * scale, y_inCGImageCoords * scale, rect.width * scale, rect.height * scale);
+      const croppedCGImage = context.createCGImageFromRect(uiImage.CIImage, scaledRect);
+
+      if (!croppedCGImage) {
+        console.error('Failed to crop CGImage from buffer');
+        return null;
+      }
+      const croppedUIImage = UIImage.alloc().initWithCGImageScaleOrientation(croppedCGImage, scale, uiImage.imageOrientation);
+      const croppedImageSource = new ImageSource();
+      croppedImageSource.setNativeSource(croppedUIImage);
+      return croppedImageSource;
+    }
   }
 
   toggleCamera() {
@@ -47,7 +107,8 @@ export class DemoModel extends DemoSharedMlkitCore {
   }
 
   changeType(args) {
-    Dialogs.action('Change Detector Type', 'Cancel', ['all', 'barcode', 'digitalInk (unsupport atm)', 'face', 'image', 'object', 'pose', 'text', 'none']).then((value) => {
+    (Frame.topmost()?.currentPage.getViewById('imageView') as Image).visibility = 'hidden';
+    Dialogs.action('Change Detector Type', 'Cancel', ['all', 'barcode', 'digitalInk (unsupport atm)', 'face', 'image', 'object', 'customObject', 'pose', 'text', 'none']).then((value) => {
       if (value === 'Cancel') {
         return;
       }
@@ -61,6 +122,7 @@ export class DemoModel extends DemoSharedMlkitCore {
   }
 
   togglePause(args) {
+    (Frame.topmost()?.currentPage.getViewById('imageView') as Image).visibility = 'hidden';
     this.camera.pause = !this.camera.pause;
     this.set('isPaused', this.camera.pause);
   }