nativescript-plugin-firebase - npm Package Compare versions

Comparing version 7.6.1 to 7.7.0

admob/admob.d.ts

@@ -151,4 +151,4 @@ /**
   onLeftApplication?: () => void;
-  onLoaded: () => void,
-  onFailedToLoad: (err) => void,
+  onLoaded?: () => void,
+  onFailedToLoad?: (err) => void,
   onRewarded?: (reward: RewardedVideoAdReward) => void;
@@ -155,0 +155,0 @@ }

app/database/index.d.ts

@@ -0,1 +1,2 @@
+import * as firebase from "../../firebase";
 export declare module database {
@@ -30,2 +31,3 @@ interface DataSnapshot {
         remove(onComplete?: (a: Error | null) => any): Promise<any>;
+        onDisconnect(): firebase.OnDisconnect;
     }
@@ -32,0 +34,0 @@ interface ThenableReference extends Reference {

app/database/index.js

@@ -15,3 +15,3 @@ "use strict";
                 if (result.error) {
-                    callback(result.error);
+                    callback(result);
                 }
@@ -178,2 +178,5 @@ else {
         };
+        Reference.prototype.onDisconnect = function () {
+            return firebase.onDisconnect(this.path);
+        };
         return Reference;
@@ -180,0 +183,0 @@ }(Query));

firebase.d.ts

@@ -520,2 +520,13 @@ /**
 // Database
+export interface OnDisconnect {
+  cancel(): Promise<any>;
+  remove(): Promise<any>;
+  set(value: any): Promise<any>;
+  setWithPriority(
+      value: any,
+      priority: number | string
+  ): Promise<any>;
+  update(values: Object): Promise<any>;
+}
+
 export function push(path: string, value: any): Promise<PushResult>;
@@ -539,2 +550,4 @@
 
+export function onDisconnect(path: string): OnDisconnect;
+
 /**
@@ -541,0 +554,0 @@ * Tells the client to keep its local cache in sync with the server automatically.

mlkit/custommodel/custommodel-common.d.ts

@@ -0,4 +1,21 @@
+import { Property } from "tns-core-modules/ui/core/properties";
 import { MLKitCameraView } from "../mlkit-cameraview";
+import { MLKitCustomModelType } from "./index";
+export declare const localModelFileProperty: any;
+export declare const labelsFileProperty: Property<MLKitCustomModel, string>;
+export declare const modelInputShapeProperty: Property<MLKitCustomModel, string>;
+export declare const modelInputTypeProperty: Property<MLKitCustomModel, string>;
+export declare const maxResultsProperty: Property<MLKitCustomModel, number>;
 export declare abstract class MLKitCustomModel extends MLKitCameraView {
     static scanResultEvent: string;
+    protected localModelFile: string;
+    protected labelsFile: string;
+    protected maxResults: number;
+    protected modelInputShape: Array<number>;
+    protected modelInputType: MLKitCustomModelType;
+    protected onSuccessListener: any;
+    protected detectorBusy: boolean;
+    protected labels: Array<string>;
 }
+export declare function getLabelsFromAppFolder(labelsFile: string): Array<string>;
+export declare function getLabelsFromFile(labelsFile: string): Array<string>;

mlkit/custommodel/custommodel-common.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+var fs = require("tns-core-modules/file-system");
+var properties_1 = require("tns-core-modules/ui/core/properties");
 var mlkit_cameraview_1 = require("../mlkit-cameraview");
+exports.localModelFileProperty = new properties_1.Property({
+    name: "localModelFile",
+    defaultValue: null,
+});
+exports.labelsFileProperty = new properties_1.Property({
+    name: "labelsFile",
+    defaultValue: null,
+});
+exports.modelInputShapeProperty = new properties_1.Property({
+    name: "modelInputShape",
+    defaultValue: null,
+});
+exports.modelInputTypeProperty = new properties_1.Property({
+    name: "modelInputType",
+    defaultValue: null,
+});
+exports.maxResultsProperty = new properties_1.Property({
+    name: "maxResults",
+    defaultValue: 5
+});
 var MLKitCustomModel = (function (_super) {
@@ -9,2 +31,26 @@ __extends(MLKitCustomModel, _super);
     }
+    MLKitCustomModel.prototype[exports.localModelFileProperty.setNative] = function (value) {
+        this.localModelFile = value;
+    };
+    MLKitCustomModel.prototype[exports.labelsFileProperty.setNative] = function (value) {
+        this.labelsFile = value;
+        if (value.indexOf("~/") === 0) {
+            this.labels = getLabelsFromAppFolder(value);
+        }
+        else {
+            console.log("For the 'labelsFile' property, use the ~/ prefix for now..");
+            return;
+        }
+    };
+    MLKitCustomModel.prototype[exports.maxResultsProperty.setNative] = function (value) {
+        this.maxResults = parseInt(value);
+    };
+    MLKitCustomModel.prototype[exports.modelInputShapeProperty.setNative] = function (value) {
+        if ((typeof value) === "string") {
+            this.modelInputShape = value.split(",").map(function (v) { return parseInt(v.trim()); });
+        }
+    };
+    MLKitCustomModel.prototype[exports.modelInputTypeProperty.setNative] = function (value) {
+        this.modelInputType = value;
+    };
     MLKitCustomModel.scanResultEvent = "scanResult";
@@ -14,1 +60,20 @@ return MLKitCustomModel;
 exports.MLKitCustomModel = MLKitCustomModel;
+exports.localModelFileProperty.register(MLKitCustomModel);
+exports.labelsFileProperty.register(MLKitCustomModel);
+exports.maxResultsProperty.register(MLKitCustomModel);
+exports.modelInputShapeProperty.register(MLKitCustomModel);
+exports.modelInputTypeProperty.register(MLKitCustomModel);
+function getLabelsFromAppFolder(labelsFile) {
+    var labelsPath = fs.knownFolders.currentApp().path + labelsFile.substring(1);
+    return getLabelsFromFile(labelsPath);
+}
+exports.getLabelsFromAppFolder = getLabelsFromAppFolder;
+function getLabelsFromFile(labelsFile) {
+    var fileContents = fs.File.fromPath(labelsFile).readTextSync();
+    var lines = fileContents.split("\n");
+    while (lines[lines.length - 1].trim() === "") {
+        lines.pop();
+    }
+    return lines;
+}
+exports.getLabelsFromFile = getLabelsFromFile;

mlkit/custommodel/index.android.d.ts

 import { MLKitCustomModelOptions, MLKitCustomModelResult } from "./";
 import { MLKitCustomModel as MLKitCustomModelBase } from "./custommodel-common";
 export declare class MLKitCustomModel extends MLKitCustomModelBase {
+    private detector;
+    private onFailureListener;
+    private inputOutputOptions;
     protected createDetector(): any;
+    protected runDetector(imageByteBuffer: any, previewWidth: any, previewHeight: any): void;
     protected createSuccessListener(): any;
 }
 export declare function useCustomModel(options: MLKitCustomModelOptions): Promise<MLKitCustomModelResult>;

mlkit/custommodel/index.android.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+var fs = require("tns-core-modules/file-system");
 var image_source_1 = require("tns-core-modules/image-source");
@@ -11,20 +12,58 @@ var custommodel_common_1 = require("./custommodel-common");
     MLKitCustomModel.prototype.createDetector = function () {
-        return getInterpreter();
+        this.detector = getInterpreter(this.localModelFile);
+        return this.detector;
     };
+    MLKitCustomModel.prototype.runDetector = function (imageByteBuffer, previewWidth, previewHeight) {
+        var _this = this;
+        if (this.detectorBusy) {
+            return;
+        }
+        this.detectorBusy = true;
+        if (!this.onFailureListener) {
+            this.onFailureListener = new com.google.android.gms.tasks.OnFailureListener({
+                onFailure: function (exception) {
+                    console.log(exception.getMessage());
+                    _this.detectorBusy = false;
+                }
+            });
+        }
+        var modelExpectsWidth = this.modelInputShape[1];
+        var modelExpectsHeight = this.modelInputShape[2];
+        var isQuantized = this.modelInputType !== "FLOAT32";
+        if (!this.inputOutputOptions) {
+            var intArrayIn = Array.create("int", 4);
+            intArrayIn[0] = this.modelInputShape[0];
+            intArrayIn[1] = modelExpectsWidth;
+            intArrayIn[2] = modelExpectsHeight;
+            intArrayIn[3] = this.modelInputShape[3];
+            var inputType = isQuantized ? com.google.firebase.ml.custom.FirebaseModelDataType.BYTE : com.google.firebase.ml.custom.FirebaseModelDataType.FLOAT32;
+            var intArrayOut = Array.create("int", 2);
+            intArrayOut[0] = 1;
+            intArrayOut[1] = this.labels.length;
+            this.inputOutputOptions = new com.google.firebase.ml.custom.FirebaseModelInputOutputOptions.Builder()
+                .setInputFormat(0, inputType, intArrayIn)
+                .setOutputFormat(0, inputType, intArrayOut)
+                .build();
+        }
+        var input = org.nativescript.plugins.firebase.mlkit.BitmapUtil.byteBufferToByteBuffer(imageByteBuffer, previewWidth, previewHeight, modelExpectsWidth, modelExpectsHeight, isQuantized);
+        var inputs = new com.google.firebase.ml.custom.FirebaseModelInputs.Builder()
+            .add(input)
+            .build();
+        this.detector
+            .run(inputs, this.inputOutputOptions)
+            .addOnSuccessListener(this.onSuccessListener)
+            .addOnFailureListener(this.onFailureListener);
+    };
     MLKitCustomModel.prototype.createSuccessListener = function () {
         var _this = this;
-        return new com.google.android.gms.tasks.OnSuccessListener({
-            onSuccess: function (labels) {
-                if (labels.size() === 0)
+        this.onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
+            onSuccess: function (output) {
+                var probabilities = output.getOutput(0)[0];
+                if (_this.labels.length !== probabilities.length) {
+                    console.log("The number of labels (" + _this.labels.length + ") is not equal to the interpretation result (" + probabilities.length + ")!");
                     return;
+                }
                 var result = {
-                    result: []
+                    result: getSortedResult(_this.labels, probabilities, _this.maxResults)
                 };
-                for (var i = 0; i < labels.size(); i++) {
-                    var label = labels.get(i);
-                    result.result.push({
-                        text: label.getLabel(),
-                        confidence: label.getConfidence()
-                    });
-                }
                 _this.notify({
@@ -35,4 +74,6 @@ eventName: MLKitCustomModel.scanResultEvent,
                 });
+                _this.detectorBusy = false;
             }
         });
+        return this.onSuccessListener;
     };
@@ -42,11 +83,33 @@ return MLKitCustomModel;
 exports.MLKitCustomModel = MLKitCustomModel;
-function getInterpreter() {
-    var localSource = new com.google.firebase.ml.custom.model.FirebaseLocalModelSource.Builder("my_local_model")
-        .setAssetFilePath("mobilenet_quant_v1_224.tflite")
-        .build();
-    com.google.firebase.ml.custom.FirebaseModelManager.getInstance().registerLocalModelSource(localSource);
-    var options = new com.google.firebase.ml.custom.FirebaseModelOptions.Builder()
-        .setLocalModelName("my_local_model")
-        .build();
-    return com.google.firebase.ml.custom.FirebaseModelInterpreter.getInstance(options);
+function getInterpreter(localModelFile) {
+    var firModelOptionsBuilder = new com.google.firebase.ml.custom.FirebaseModelOptions.Builder();
+    var localModelRegistrationSuccess = false;
+    var cloudModelRegistrationSuccess = false;
+    var localModelName;
+    if (localModelFile) {
+        localModelName = localModelFile.lastIndexOf("/") === -1 ? localModelFile : localModelFile.substring(localModelFile.lastIndexOf("/") + 1);
+        if (com.google.firebase.ml.custom.FirebaseModelManager.getInstance().getLocalModelSource(localModelName)) {
+            localModelRegistrationSuccess = true;
+            firModelOptionsBuilder.setLocalModelName(localModelName);
+        }
+        else {
+            console.log("model not yet loaded: " + localModelFile);
+            var firModelLocalSourceBuilder = new com.google.firebase.ml.custom.model.FirebaseLocalModelSource.Builder(localModelName);
+            if (localModelFile.indexOf("~/") === 0) {
+                firModelLocalSourceBuilder.setFilePath(fs.knownFolders.currentApp().path + localModelFile.substring(1));
+            }
+            else {
+                firModelLocalSourceBuilder.setAssetFilePath(localModelFile);
+            }
+            localModelRegistrationSuccess = com.google.firebase.ml.custom.FirebaseModelManager.getInstance().registerLocalModelSource(firModelLocalSourceBuilder.build());
+            if (localModelRegistrationSuccess) {
+                firModelOptionsBuilder.setLocalModelName(localModelName);
+            }
+        }
+    }
+    if (!localModelRegistrationSuccess && !cloudModelRegistrationSuccess) {
+        console.log("No (cloud or local) model was successfully loaded.");
+        return null;
+    }
+    return com.google.firebase.ml.custom.FirebaseModelInterpreter.getInstance(firModelOptionsBuilder.build());
 }
@@ -56,7 +119,20 @@ function useCustomModel(options) {
         try {
-            var interpreter_1 = getInterpreter();
+            var interpreter_1 = getInterpreter(options.localModelFile);
+            var labels_1;
+            if (options.labelsFile.indexOf("~/") === 0) {
+                labels_1 = custommodel_common_1.getLabelsFromAppFolder(options.labelsFile);
+            }
+            else {
+                reject("Use the ~/ prefix for now..");
+                return;
+            }
             var onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
-                onSuccess: function (labels) {
+                onSuccess: function (output) {
+                    var probabilities = output.getOutput(0)[0];
+                    if (labels_1.length !== probabilities.length) {
+                        console.log("The number of labels in " + options.labelsFile + " (" + labels_1.length + ") is not equal to the interpretation result (" + probabilities.length + ")!");
+                        return;
+                    }
                     var result = {
-                        result: []
+                        result: getSortedResult(labels_1, probabilities, options.maxResults)
                     };
@@ -70,15 +146,18 @@ resolve(result);
             });
-            var intArrayIn = Array.create('int', 4);
-            intArrayIn[0] = 1;
-            intArrayIn[1] = 640;
-            intArrayIn[2] = 480;
-            intArrayIn[3] = 3;
-            var intArrayOut = Array.create('int', 2);
+            var intArrayIn = Array.create("int", 4);
+            intArrayIn[0] = options.modelInput[0].shape[0];
+            intArrayIn[1] = options.modelInput[0].shape[1];
+            intArrayIn[2] = options.modelInput[0].shape[2];
+            intArrayIn[3] = options.modelInput[0].shape[3];
+            var isQuantized = options.modelInput[0].type !== "FLOAT32";
+            var inputType = isQuantized ? com.google.firebase.ml.custom.FirebaseModelDataType.BYTE : com.google.firebase.ml.custom.FirebaseModelDataType.FLOAT32;
+            var intArrayOut = Array.create("int", 2);
             intArrayOut[0] = 1;
-            intArrayOut[1] = 1000;
+            intArrayOut[1] = labels_1.length;
             var inputOutputOptions = new com.google.firebase.ml.custom.FirebaseModelInputOutputOptions.Builder()
-                .setInputFormat(0, com.google.firebase.ml.custom.FirebaseModelDataType.BYTE, intArrayIn)
-                .setOutputFormat(0, com.google.firebase.ml.custom.FirebaseModelDataType.FLOAT32, intArrayOut)
+                .setInputFormat(0, inputType, intArrayIn)
+                .setOutputFormat(0, inputType, intArrayOut)
                 .build();
-            var input = null;
+            var image = options.image instanceof image_source_1.ImageSource ? options.image.android : options.image.imageSource.android;
+            var input = org.nativescript.plugins.firebase.mlkit.BitmapUtil.bitmapToByteBuffer(image, options.modelInput[0].shape[1], options.modelInput[0].shape[2], isQuantized);
             var inputs = new com.google.firebase.ml.custom.FirebaseModelInputs.Builder()
@@ -99,5 +178,12 @@ .add(input)
 exports.useCustomModel = useCustomModel;
-function getImage(options) {
-    var image = options.image instanceof image_source_1.ImageSource ? options.image.android : options.image.imageSource.android;
-    return com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
+function getSortedResult(labels, probabilities, maxResults) {
+    if (maxResults === void 0) { maxResults = 5; }
+    var result = [];
+    labels.forEach(function (text, i) { return result.push({ text: text, confidence: probabilities[i] }); });
+    result.sort(function (a, b) { return a.confidence < b.confidence ? 1 : (a.confidence === b.confidence ? 0 : -1); });
+    if (result.length > maxResults) {
+        result.splice(maxResults);
+    }
+    result.map(function (r) { return r.confidence = (r.confidence & 0xff) / 255.0; });
+    return result;
 }

mlkit/custommodel/index.d.ts

 import { MLKitCameraView, MLKitOptions, MLKitResult } from "../index";
 
+export interface MLKitCustomModelResultValue {
+  text: string;
+  confidence: number;
+}
+
 export interface MLKitCustomModelResult extends MLKitResult {
-  result: any;
+  result: Array<MLKitCustomModelResultValue>;
 }
 
+export type MLKitCustomModelType = "FLOAT32" | "QUANT";
+
+export interface TNSCustomModelInput {
+  shape: Array<number>,
+  type: MLKitCustomModelType
+}
+
+// see https://firebase.google.com/docs/ml-kit/ios/use-custom-models
 export interface MLKitCustomModelOptions extends MLKitOptions {
+  localModelFile?: string;
+  labelsFile: string;
+  /**
+   * Default 5
+   */
+  maxResults?: number;
+  modelInput: Array<TNSCustomModelInput>
+  /**
+   * Ignoring this for now as we deduct it from the model spec.
+   */
+  // modelOutput?: Array<{
+  //   shape: Array<number>,
+  //   type: MLKitCustomModelType
+  // }>
+  /**
+   * Never got this working, so not supporting it for now.
+   */
+  // cloudModelName?: string;
+  /**
+   * Default false
+   */
+  // requireWifiForCloudModelDownload?: boolean;
 }
@@ -12,2 +47,3 @@
 
-export declare class MLKitCustomModel extends MLKitCameraView {}
+export declare class MLKitCustomModel extends MLKitCameraView {
+}

mlkit/custommodel/index.ios.d.ts

 import { MLKitCustomModelOptions, MLKitCustomModelResult } from "./";
 import { MLKitCustomModel as MLKitCustomModelBase } from "./custommodel-common";
 export declare class MLKitCustomModel extends MLKitCustomModelBase {
+    private modelInterpreter;
+    private inputOutputOptions;
     protected createDetector(): any;
+    runDetector(image: UIImage): void;
     protected createSuccessListener(): any;
@@ -6,0 +9,0 @@ protected rotateRecording(): boolean;

mlkit/custommodel/index.ios.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+var fs = require("tns-core-modules/file-system");
 var image_source_1 = require("tns-core-modules/image-source");
@@ -11,4 +12,57 @@ var custommodel_common_1 = require("./custommodel-common");
     MLKitCustomModel.prototype.createDetector = function () {
-        return getInterpreter();
+        this.modelInterpreter = getInterpreter(this.localModelFile);
+        return this.modelInterpreter;
     };
+    MLKitCustomModel.prototype.runDetector = function (image) {
+        var _this = this;
+        if (this.detectorBusy) {
+            return;
+        }
+        this.detectorBusy = true;
+        var modelExpectsWidth = this.modelInputShape[1];
+        var modelExpectsHeight = this.modelInputShape[2];
+        var isQuantized = this.modelInputType !== "FLOAT32";
+        if (!this.inputOutputOptions) {
+            this.inputOutputOptions = FIRModelInputOutputOptions.new();
+            var inputType = void 0;
+            var arrIn_1 = NSMutableArray.new();
+            this.modelInputShape.forEach(function (dim) { return arrIn_1.addObject(dim); });
+            inputType = isQuantized ? 3 : 1;
+            this.inputOutputOptions.setInputFormatForIndexTypeDimensionsError(0, inputType, arrIn_1);
+            var arrOut = NSMutableArray.new();
+            arrOut.addObject(1);
+            arrOut.addObject(this.labels.length);
+            this.inputOutputOptions.setOutputFormatForIndexTypeDimensionsError(0, inputType, arrOut);
+        }
+        var inputData;
+        if (isQuantized) {
+            inputData = TNSMLKitCameraView.scaledDataWithSizeByteCountIsQuantized(image, CGSizeMake(modelExpectsWidth, modelExpectsHeight), modelExpectsWidth * modelExpectsHeight * this.modelInputShape[3] * this.modelInputShape[0], isQuantized);
+        }
+        else {
+            inputData = TNSMLKitCameraView.getInputDataWithRowsAndColumnsAndType(image, modelExpectsWidth, modelExpectsHeight, "Float32");
+        }
+        var inputs = FIRModelInputs.new();
+        inputs.addInputError(inputData);
+        this.modelInterpreter.runWithInputsOptionsCompletion(inputs, this.inputOutputOptions, function (outputs, error) {
+            if (error !== null) {
+                console.log(error.localizedDescription);
+            }
+            else if (outputs !== null) {
+                var probabilities = outputs.outputAtIndexError(0)[0];
+                if (_this.labels.length !== probabilities.count) {
+                    console.log("The number of labels (" + _this.labels.length + ") is not equal to the interpretation result (" + probabilities.count + ")!");
+                    return;
+                }
+                var result = {
+                    result: getSortedResult(_this.labels, probabilities, _this.maxResults)
+                };
+                _this.notify({
+                    eventName: MLKitCustomModel.scanResultEvent,
+                    object: _this,
+                    value: result
+                });
+            }
+            _this.detectorBusy = false;
+        });
+    };
     MLKitCustomModel.prototype.createSuccessListener = function () {
@@ -39,39 +93,66 @@ var _this = this;
 exports.MLKitCustomModel = MLKitCustomModel;
-function getInterpreter() {
-    var fIRModelDownloadConditions = FIRModelDownloadConditions.alloc().initWithIsWiFiRequiredCanDownloadInBackground(false, true);
-    var fIRCloudModelSource = FIRCloudModelSource.alloc().initWithModelNameEnableModelUpdatesInitialConditionsUpdateConditions("my-custom-model", true, fIRModelDownloadConditions, fIRModelDownloadConditions);
-    loadLocalModel();
-    var fIRModelOptions = FIRModelOptions.alloc().initWithCloudModelNameLocalModelName(null, "mobilenet");
+function getInterpreter(localModelFile) {
+    var localModelRegistrationSuccess = false;
+    var cloudModelRegistrationSuccess = false;
+    var localModelName;
+    if (localModelFile) {
+        localModelName = localModelFile.lastIndexOf("/") === -1 ? localModelFile : localModelFile.substring(localModelFile.lastIndexOf("/") + 1);
+        if (FIRModelManager.modelManager().localModelSourceForModelName(localModelName)) {
+            localModelRegistrationSuccess = true;
+        }
+        else {
+            var localModelFilePath = void 0;
+            if (localModelFile.indexOf("~/") === 0) {
+                localModelFilePath = fs.knownFolders.currentApp().path + localModelFile.substring(1);
+            }
+            else {
+                localModelFilePath = NSBundle.mainBundle.pathForResourceOfType(localModelFile.substring(0, localModelFile.lastIndexOf(".")), localModelFile.substring(localModelFile.lastIndexOf(".") + 1));
+            }
+            var localModelSource = FIRLocalModelSource.alloc().initWithModelNamePath(localModelName, localModelFilePath);
+            localModelRegistrationSuccess = FIRModelManager.modelManager().registerLocalModelSource(localModelSource);
+        }
+    }
+    if (!localModelRegistrationSuccess && !cloudModelRegistrationSuccess) {
+        console.log("No (cloud or local) model was successfully loaded.");
+        return null;
+    }
+    var fIRModelOptions = FIRModelOptions.alloc().initWithCloudModelNameLocalModelName(null, localModelRegistrationSuccess ? localModelName : null);
     return FIRModelInterpreter.modelInterpreterWithOptions(fIRModelOptions);
 }
-function loadLocalModel() {
-    var localModelFilePath = NSBundle.mainBundle.pathForResourceOfType("mobilenet_quant_v1_224", "tflite");
-    console.log(">>> localModelFilePath: " + localModelFilePath);
-    var localModelSource = FIRLocalModelSource.alloc().initWithModelNamePath("mobilenet", localModelFilePath);
-    console.log(">>> localModelSource: " + localModelSource);
-    var localModelRegistrationSuccess = FIRModelManager.modelManager().registerLocalModelSource(localModelSource);
-    console.log("localModelRegistrationSuccess: " + localModelRegistrationSuccess);
-}
 function useCustomModel(options) {
     return new Promise(function (resolve, reject) {
         try {
-            var modelInterpreter = getInterpreter();
+            var image = options.image instanceof image_source_1.ImageSource ? options.image.ios : options.image.imageSource.ios;
+            var isQuant = options.modelInput[0].type !== "FLOAT32";
+            var inputData = void 0;
+            if (isQuant) {
+                inputData = TNSMLKitCameraView.scaledDataWithSizeByteCountIsQuantized(image, CGSizeMake(options.modelInput[0].shape[1], options.modelInput[0].shape[2]), options.modelInput[0].shape[1] * options.modelInput[0].shape[2] * options.modelInput[0].shape[3] * options.modelInput[0].shape[0], options.modelInput[0].type !== "FLOAT32");
+            }
+            else {
+                inputData = TNSMLKitCameraView.getInputDataWithRowsAndColumnsAndType(image, options.modelInput[0].shape[1], options.modelInput[0].shape[2], "Float32");
+            }
             var inputs = FIRModelInputs.new();
-            var image = options.image instanceof image_source_1.ImageSource ? options.image.ios : options.image.imageSource.ios;
-            var resizedImg = TNSMLKitCameraView.resizeImage(image);
-            var successAddingInput = inputs.addInputError(resizedImg);
-            var inputOptions = FIRModelInputOutputOptions.new();
-            var arrIn = NSMutableArray.new();
-            arrIn.addObject(1);
-            arrIn.addObject(image.size.width);
-            arrIn.addObject(image.size.height);
-            arrIn.addObject(3);
+            inputs.addInputError(inputData);
+            var inputOptions_1 = FIRModelInputOutputOptions.new();
+            var inputType_1;
+            options.modelInput.forEach(function (dimensionAndType, i) {
+                var arrIn = NSMutableArray.new();
+                dimensionAndType.shape.forEach(function (dim) { return arrIn.addObject(dim); });
+                inputType_1 = dimensionAndType.type === "FLOAT32" ? 1 : 3;
+                inputOptions_1.setInputFormatForIndexTypeDimensionsError(i, inputType_1, arrIn);
+            });
+            var labels_1;
+            if (options.labelsFile.indexOf("~/") === 0) {
+                labels_1 = custommodel_common_1.getLabelsFromAppFolder(options.labelsFile);
+            }
+            else {
+                var labelsFile = NSBundle.mainBundle.pathForResourceOfType(options.labelsFile.substring(0, options.labelsFile.lastIndexOf(".")), options.labelsFile.substring(options.labelsFile.lastIndexOf(".") + 1));
+                labels_1 = custommodel_common_1.getLabelsFromFile(labelsFile);
+            }
             var arrOut = NSMutableArray.new();
             arrOut.addObject(1);
-            arrOut.addObject(1001);
-            inputOptions.setInputFormatForIndexTypeDimensionsError(0, 3, arrIn);
-            inputOptions.setOutputFormatForIndexTypeDimensionsError(0, 3, arrOut);
-            modelInterpreter.runWithInputsOptionsCompletion(inputs, inputOptions, function (outputs, error) {
-                console.log(">>> error: " + error);
-                console.log(">>> outputs: " + outputs);
+            arrOut.addObject(labels_1.length);
+            inputOptions_1.setOutputFormatForIndexTypeDimensionsError(0, inputType_1, arrOut);
+            var modelInterpreter = getInterpreter(options.localModelFile);
+            modelInterpreter.runWithInputsOptionsCompletion(inputs, inputOptions_1, function (outputs, error) {
                 if (error !== null) {
@@ -81,7 +162,10 @@ reject(error.localizedDescription);
                 else if (outputs !== null) {
-                    console.log(">>> outputs.count: " + outputs.outputAtIndexError(0));
+                    var probabilities = outputs.outputAtIndexError(0)[0];
+                    if (labels_1.length !== probabilities.count) {
+                        console.log("The number of labels in " + options.labelsFile + " (" + labels_1.length + ") is not equal to the interpretation result (" + probabilities.count + ")!");
+                        return;
+                    }
                     var result = {
-                        result: []
+                        result: getSortedResult(labels_1, probabilities, options.maxResults)
                     };
-                    console.log(">>> outputs: " + outputs);
                     resolve(result);
@@ -98,1 +182,13 @@ }
 exports.useCustomModel = useCustomModel;
+function getSortedResult(labels, probabilities, maxResults) {
+    if (maxResults === void 0) { maxResults = 5; }
+    var result = [];
+    labels.forEach(function (text, i) { return result.push({ text: text, confidence: probabilities.objectAtIndex(i) }); });
+    result.sort(function (a, b) { return a.confidence < b.confidence ? 1 : (a.confidence === b.confidence ? 0 : -1); });
+    if (result.length > maxResults) {
+        result.splice(maxResults);
+    }
+    var softmaxScale = 1.0 / 256.0;
+    result.map(function (r) { return r.confidence = NSNumber.numberWithFloat(softmaxScale * r.confidence); });
+    return result;
+}

mlkit/index.d.ts

@@ -6,2 +6,3 @@ import * as textrecognition from "./textrecognition";
 import * as landmarkrecognition from "./landmarkrecognition";
+import * as custommodel from "./custommodel";
 import { ImageSource } from "tns-core-modules/image-source";
@@ -21,2 +22,2 @@ import { Image } from "tns-core-modules/ui/image";
 }
-export { textrecognition, barcodescanning, facedetection, imagelabeling, landmarkrecognition };
+export { textrecognition, barcodescanning, facedetection, imagelabeling, landmarkrecognition, custommodel };

mlkit/index.js

@@ -13,1 +13,3 @@ "use strict";
 exports.landmarkrecognition = landmarkrecognition;
+var custommodel = require("./custommodel");
+exports.custommodel = custommodel;

mlkit/mlkit-cameraview.android.d.ts

@@ -21,2 +21,3 @@ import { MLKitCameraView as MLKitCameraViewBase } from "./mlkit-cameraview-common";
     protected abstract createSuccessListener(): any;
+    protected runDetector(imageByteBuffer: any, width: any, height: any): void;
     private createFailureListener;
@@ -23,0 +24,0 @@ private generateValidPreviewSizeList;

mlkit/mlkit-cameraview.android.js

@@ -113,2 +113,4 @@ "use strict";
             var previewSize = sizePair.previewSize;
+            console.log("sizePair.pictureSize: " + pictureSize.width + "x" + pictureSize.height);
+            console.log("sizePair.previewSize: " + previewSize.width + "x" + previewSize.height);
             var parameters = _this.camera.getParameters();
@@ -155,4 +157,4 @@ if (pictureSize) {
                     var data = _this.pendingFrameData;
-                    _this.lastVisionImage = com.google.firebase.ml.vision.common.FirebaseVisionImage.fromByteBuffer(data, metadata);
                     if (_this.detector.processImage) {
+                        _this.lastVisionImage = com.google.firebase.ml.vision.common.FirebaseVisionImage.fromByteBuffer(data, metadata);
                         _this.detector
@@ -163,3 +165,4 @@ .processImage(_this.lastVisionImage)
                     }
-                    else {
+                    else if (_this.detector.detectInImage) {
+                        _this.lastVisionImage = com.google.firebase.ml.vision.common.FirebaseVisionImage.fromByteBuffer(data, metadata);
                         _this.detector
@@ -170,2 +173,5 @@ .detectInImage(_this.lastVisionImage)
                     }
+                    else {
+                        _this.runDetector(data, previewSize.width, previewSize.height);
+                    }
                 }
@@ -199,2 +205,5 @@ }));
     };
+    MLKitCameraView.prototype.runDetector = function (imageByteBuffer, width, height) {
+        throw new Error("No custom detector implemented for detector " + this.detector + ", so 'runDetector' can't do its thing");
+    };
     MLKitCameraView.prototype.createFailureListener = function () {
@@ -201,0 +210,0 @@ return new com.google.android.gms.tasks.OnFailureListener({

mlkit/mlkit-cameraview.ios.d.ts

@@ -20,2 +20,3 @@ import { MLKitCameraView as MLKitCameraViewBase } from "./mlkit-cameraview-common";
     abstract createSuccessListener(): any;
+    runDetector(image: UIImage): void;
 }

mlkit/mlkit-cameraview.ios.js

@@ -149,2 +149,5 @@ "use strict";
     };
+    MLKitCameraView.prototype.runDetector = function (image) {
+        throw new Error("No custom detector implemented, so 'runDetector' can't do its thing");
+    };
     return MLKitCameraView;
@@ -172,16 +175,22 @@ }(mlkit_cameraview_common_1.MLKitCameraView));
         if (image) {
-            var fIRVisionImage = FIRVisionImage.alloc().initWithImage(image);
-            var fIRVisionImageMetadata = FIRVisionImageMetadata.new();
-            fIRVisionImageMetadata.orientation = this.owner.get().getVisionOrientation(image.imageOrientation);
-            fIRVisionImage.metadata = fIRVisionImageMetadata;
             if (this.detector.detectInImageCompletion) {
-                this.detector.detectInImageCompletion(fIRVisionImage, this.onSuccessListener);
+                this.detector.detectInImageCompletion(this.uiImageToFIRVisionImage(image), this.onSuccessListener);
             }
+            else if (this.detector.processImageCompletion) {
+                this.detector.processImageCompletion(this.uiImageToFIRVisionImage(image), this.onSuccessListener);
+            }
             else {
-                this.detector.processImageCompletion(fIRVisionImage, this.onSuccessListener);
+                this.owner.get().runDetector(image);
             }
         }
     };
+    TNSMLKitCameraViewDelegateImpl.prototype.uiImageToFIRVisionImage = function (image) {
+        var fIRVisionImage = FIRVisionImage.alloc().initWithImage(image);
+        var fIRVisionImageMetadata = FIRVisionImageMetadata.new();
+        fIRVisionImageMetadata.orientation = this.owner.get().getVisionOrientation(image.imageOrientation);
+        fIRVisionImage.metadata = fIRVisionImageMetadata;
+        return fIRVisionImage;
+    };
     TNSMLKitCameraViewDelegateImpl.ObjCProtocols = [];
     return TNSMLKitCameraViewDelegateImpl;
 }(NSObject));

package.json

 {
   "name": "nativescript-plugin-firebase",
-  "version": "7.6.1",
+  "version": "7.7.0",
   "description": "Fire. Base. Firebase!",
@@ -5,0 +5,0 @@ "main": "firebase",

No alert changes

Improved metrics

Total package byte prevSize

921005

increased by5.26%

Lines of code

15268

increased by3.76%

No dependency changes