capacitor-plugin-multi-camera - npm Package Compare versions

Comparing version 0.0.2 to 0.0.3

android/src/main/assets/aip-ocr.license

+�k���6�P?jw?Ƞ���D0��;WnG}���H�<���E�{P��s֙X�Vy�ˆ��~��KD-
�}
+�*�UE�90m�������	�瞢�8�LºEH����E��
+��

android/src/main/j...idcardquality/IDcardQualityProcess.java

+/*
+ * Copyright (C) 2017 Baidu, Inc. All Rights Reserved.
+ */
+package com.baidu.idcardquality;
+
+import android.content.res.AssetManager;
+import android.graphics.Bitmap;
+
+import com.baidu.idl.authority.AlgorithmOnMainThreadException;
+import com.baidu.idl.authority.IDLAuthorityException;
+import com.baidu.idl.license.License;
+import com.baidu.idl.util.UIThread;
+
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class IDcardQualityProcess {
+    final ReentrantReadWriteLock nativeModelLock = new ReentrantReadWriteLock();
+    private static IDcardQualityProcess mInstance;
+    private static String tokenString;
+    private static int mAuthorityStatus;
+    private static Throwable loadNativeException = null;
+    private static volatile boolean hasReleased;
+
+    public IDcardQualityProcess() {
+    }
+
+    public static synchronized IDcardQualityProcess getInstance() {
+        if (null == mInstance) {
+            mInstance = new IDcardQualityProcess();
+        }
+
+        return mInstance;
+    }
+
+    public native byte[] convertRGBImage(int[] colors, int width, int height);
+
+    public native int idcardQualityModelInit(AssetManager var1, String var2);
+
+    public native int idcardQualityCaptchaRelease();
+
+    public native int idcardQualityProcess(byte[] var1, int var2, int var3, boolean var4, int var5);
+
+    public static synchronized int init(String token) throws AlgorithmOnMainThreadException, IDLAuthorityException {
+        if (UIThread.isUITread()) {
+            throw new AlgorithmOnMainThreadException();
+        } else {
+            tokenString = token;
+
+            try {
+                mAuthorityStatus = License.getInstance().init(tokenString);
+            } catch (Exception var2) {
+                var2.printStackTrace();
+            }
+
+            return mAuthorityStatus;
+        }
+    }
+
+    public int idcardQualityInit(AssetManager assetManager, String modelPath) {
+        if (mAuthorityStatus == 0) {
+            hasReleased = false;
+            nativeModelLock.writeLock().lock();
+            int status = this.idcardQualityModelInit(assetManager, modelPath);
+            nativeModelLock.writeLock().unlock();
+            return status;
+        } else {
+            return mAuthorityStatus;
+        }
+    }
+
+    public int idcardQualityRelease() {
+        if (mAuthorityStatus == 0) {
+            hasReleased = true;
+            nativeModelLock.writeLock().lock();
+            this.idcardQualityCaptchaRelease();
+            nativeModelLock.writeLock().unlock();
+            return 0;
+        } else {
+            return mAuthorityStatus;
+        }
+    }
+
+    public int idcardQualityDetectionImg(Bitmap img, boolean isFont) {
+        int status;
+        if (mAuthorityStatus == 0) {
+            if (hasReleased) {
+                return -1;
+            }
+            int imgHeight = img.getHeight();
+            int imgWidth = img.getWidth();
+            byte[] imageData = this.getRGBImageData(img);
+            nativeModelLock.readLock().lock();
+            status = this.idcardQualityProcess(imageData, imgHeight, imgWidth, isFont, 3);
+            nativeModelLock.readLock().unlock();
+            return status;
+        } else {
+            return mAuthorityStatus;
+        }
+    }
+
+    public static Throwable getLoadSoException() {
+        return loadNativeException;
+    }
+
+    public byte[] getRGBImageData(Bitmap img) {
+        int imgWidth = img.getWidth();
+        int imgHeight = img.getHeight();
+        int[] pixels = new int[imgWidth * imgHeight];
+        img.getPixels(pixels, 0, imgWidth, 0, 0, imgWidth, imgHeight);
+        byte[] imageData = convertRGBImage(pixels, imgWidth, imgHeight);
+        return imageData;
+    }
+
+    public void releaseModel() {
+        this.idcardQualityRelease();
+    }
+
+    static {
+        try {
+            System.loadLibrary("idl_license");
+            System.loadLibrary("idcard_quality.1.1.1");
+        } catch (Throwable e) {
+            loadNativeException = e;
+        }
+        mInstance = null;
+        mAuthorityStatus = 256;
+    }
+}

android/src/main/j...fu/multi/camera/CameraNativeHelper.java

+/*
+ * Copyright (C) 2018 Baidu, Inc. All Rights Reserved.
+ */
+package com.yufu.multi.camera;
+
+import android.content.Context;
+
+import com.baidu.idcardquality.IDcardQualityProcess;
+
+
+/**
+ * Created by ruanshimin on 2018/1/23.
+ */
+
+public class CameraNativeHelper {
+    /**
+     * 本地模型授权，加载成功
+     */
+    public static final int NATIVE_AUTH_INIT_SUCCESS = 0;
+
+    /**
+     * 本地模型授权，缺少SO
+     */
+    public static final int NATIVE_SO_LOAD_FAIL = 10;
+
+    /**
+     * 本地模型授权，授权失败，token异常
+     */
+    public static final int NATIVE_AUTH_FAIL = 11;
+
+    /**
+     * 本地模型授权，模型加载失败
+     */
+    public static final int NATIVE_INIT_FAIL = 12;
+
+    /**
+     * 是否已经通过本地质量控制扫描
+     */
+    public static final int SCAN_SUCCESS = 0;
+
+
+    public interface CameraNativeInitCallback {
+        /**
+         * 加载本地库异常回调
+         *
+         * @param errorCode 错误代码
+         * @param e         如果加载so异常则会有异常对象传入
+         */
+        void onError(int errorCode, Throwable e);
+    }
+
+    public static void init(final Context ctx, final String token, final CameraNativeInitCallback cb) {
+        CameraThreadPool.execute(new Runnable() {
+            @Override
+            public void run() {
+                int status;
+                // 加载本地so失败, 异常返回getloadSoException
+                if (IDcardQualityProcess.getLoadSoException() != null) {
+                    // 本地模型授权，缺少SO
+                    status = NATIVE_SO_LOAD_FAIL;
+                    System.out.println("本地模型授权，缺少SO");
+                    cb.onError(status, IDcardQualityProcess.getLoadSoException());
+                    return;
+                }
+                // 授权状态
+                int authStatus = IDcardQualityProcess.init(token);
+                if (authStatus != 0) {
+                    // 本地模型授权，授权失败，token异常
+                    System.out.println("本地模型授权，授权失败，token异常");
+                    System.out.println(token);
+                    cb.onError(NATIVE_AUTH_FAIL, null);
+                    return;
+                }
+
+                // 加载模型状态
+                int initModelStatus = IDcardQualityProcess.getInstance().idcardQualityInit(ctx.getAssets(), "models");
+                if (initModelStatus != 0) {
+                    // 本地模型授权，模型加载失败
+                    cb.onError(NATIVE_INIT_FAIL, null);
+                }
+            }
+        });
+    }
+
+    public static void release() {
+        IDcardQualityProcess.getInstance().releaseModel();
+    }
+}

android/src/main/j...fu/multi/camera/util/DimensionUtil.java

+/*
+ * Copyright (C) 2017 Baidu, Inc. All Rights Reserved.
+ */
+package com.yufu.multi.camera.util;
+
+import android.content.res.Resources;
+
+public class DimensionUtil {
+
+    public static int dpToPx(int dp) {
+        return (int) (dp * Resources.getSystem().getDisplayMetrics().density);
+    }
+
+}

android/src/main/j...m/yufu/multi/camera/util/ImageUtil.java

+/*
+ * Copyright (C) 2017 Baidu, Inc. All Rights Reserved.
+ */
+package com.yufu.multi.camera.util;
+
+import android.graphics.BitmapFactory;
+import android.media.ExifInterface;
+import android.util.Log;
+
+public class ImageUtil {
+  private static final String TAG = "CameraExif";
+
+  public static int exifToDegrees(int exifOrientation) {
+    if (exifOrientation == ExifInterface.ORIENTATION_ROTATE_90) {
+      return 90;
+    } else if (exifOrientation == ExifInterface.ORIENTATION_ROTATE_180) {
+      return 180;
+    } else if (exifOrientation == ExifInterface.ORIENTATION_ROTATE_270) {
+      return 270;
+    }
+    return 0;
+  }
+
+  // Returns the degrees in clockwise. Values are 0, 90, 180, or 270.
+  public static int getOrientation(byte[] jpeg) {
+    if (jpeg == null) {
+      return 0;
+    }
+
+    int offset = 0;
+    int length = 0;
+
+    // ISO/IEC 10918-1:1993(E)
+    while (offset + 3 < jpeg.length && (jpeg[offset++] & 0xFF) == 0xFF) {
+      int marker = jpeg[offset] & 0xFF;
+
+      // Check if the marker is a padding.
+      if (marker == 0xFF) {
+        continue;
+      }
+      offset++;
+
+      // Check if the marker is SOI or TEM.
+      if (marker == 0xD8 || marker == 0x01) {
+        continue;
+      }
+      // Check if the marker is EOI or SOS.
+      if (marker == 0xD9 || marker == 0xDA) {
+        break;
+      }
+
+      // Get the length and check if it is reasonable.
+      length = pack(jpeg, offset, 2, false);
+      if (length < 2 || offset + length > jpeg.length) {
+        Log.e(TAG, "Invalid length");
+        return 0;
+      }
+
+      // Break if the marker is EXIF in APP1.
+      if (marker == 0xE1 && length >= 8
+          && pack(jpeg, offset + 2, 4, false) == 0x45786966
+          && pack(jpeg, offset + 6, 2, false) == 0) {
+        offset += 8;
+        length -= 8;
+        break;
+      }
+
+      // Skip other markers.
+      offset += length;
+      length = 0;
+    }
+
+    // JEITA CP-3451 Exif Version 2.2
+    if (length > 8) {
+      // Identify the byte order.
+      int tag = pack(jpeg, offset, 4, false);
+      if (tag != 0x49492A00 && tag != 0x4D4D002A) {
+        Log.e(TAG, "Invalid byte order");
+        return 0;
+      }
+      boolean littleEndian = (tag == 0x49492A00);
+
+      // Get the offset and check if it is reasonable.
+      int count = pack(jpeg, offset + 4, 4, littleEndian) + 2;
+      if (count < 10 || count > length) {
+        Log.e(TAG, "Invalid offset");
+        return 0;
+      }
+      offset += count;
+      length -= count;
+
+      // Get the count and go through all the elements.
+      count = pack(jpeg, offset - 2, 2, littleEndian);
+      while (count-- > 0 && length >= 12) {
+        // Get the tag and check if it is orientation.
+        tag = pack(jpeg, offset, 2, littleEndian);
+        if (tag == 0x0112) {
+          // We do not really care about type and count, do we?
+          int orientation = pack(jpeg, offset + 8, 2, littleEndian);
+          switch (orientation) {
+            case 1:
+              return 0;
+            case 3:
+              return 180;
+            case 6:
+              return 90;
+            case 8:
+              return 270;
+            default:
+              return 0;
+          }
+        }
+        offset += 12;
+        length -= 12;
+      }
+    }
+
+    Log.i(TAG, "Orientation not found");
+    return 0;
+  }
+
+  private static int pack(byte[] bytes, int offset, int length,
+      boolean littleEndian) {
+    int step = 1;
+    if (littleEndian) {
+      offset += length - 1;
+      step = -1;
+    }
+
+    int value = 0;
+    while (length-- > 0) {
+      value = (value << 8) | (bytes[offset] & 0xFF);
+      offset += step;
+    }
+    return value;
+  }
+
+  public static int calculateInSampleSize(
+      BitmapFactory.Options options, int reqWidth, int reqHeight) {
+    // Raw height and width of image
+    final int height = options.outHeight;
+    final int width = options.outWidth;
+    int inSampleSize = 1;
+
+    if (height > reqHeight || width > reqWidth) {
+
+      final int halfHeight = height / 2;
+      final int halfWidth = width / 2;
+      // Calculate the largest inSampleSize value that is a power of 2 and keeps both
+      // height and width larger than the requested height and width.
+      while ((halfHeight / inSampleSize) >= reqHeight
+          && (halfWidth / inSampleSize) >= reqWidth) {
+        inSampleSize *= 2;
+      }
+    }
+
+    return inSampleSize;
+  }
+}

android/build.gradle

@@ -52,3 +52,3 @@ ext {
 dependencies {
-    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation fileTree(dir: 'libs', include: ['*.jar','*.aar'])
     implementation project(':capacitor-android')
@@ -55,0 +55,0 @@ implementation "androidx.appcompat:appcompat:$androidxAppCompatVersion"

android/src/main/j...m/yufu/multi/camera/Camera1Control.java

@@ -7,4 +7,7 @@ package com.yufu.multi.camera;
 import android.graphics.ImageFormat;
+import android.graphics.Matrix;
 import android.graphics.Rect;
+import android.graphics.RectF;
 import android.graphics.SurfaceTexture;
+import android.graphics.YuvImage;
 import android.hardware.Camera;
@@ -20,2 +23,3 @@ import android.view.SurfaceHolder;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -54,2 +58,22 @@ import java.util.ArrayList;
 
+    private Camera.FaceDetectionListener faceDetectionListener;
+    private OnDetectPictureCallback detectCallback;
+    private int previewFrameCount = 0;
+
+    /*
+     * 非扫描模式
+     */
+    private final int MODEL_NOSCAN = 0;
+
+    /*
+     * 本地质量控制扫描模式
+     */
+    private final int MODEL_SCAN = 1;
+
+    private int detectType = MODEL_NOSCAN;
+
+    public int getCameraRotation() {
+        return rotation;
+    }
+
     private void initCamera() {
@@ -95,12 +119,77 @@ try {
 
+
     private void setPreviewCallbackImpl() {
-//        if (buffer == null) {
-//            buffer = new byte[displayView.getWidth() * displayView.getHeight() * ImageFormat.getBitsPerPixel(ImageFormat.NV21) / 8];
-//        }
-//        if (camera != null && detectType == MODEL_SCAN) {
-//            camera.addCallbackBuffer(buffer);
-//            camera.setPreviewCallback(previewCallback);
-//        }
+        if (buffer == null) {
+            buffer = new byte[displayView.getWidth() * displayView.getHeight() * ImageFormat.getBitsPerPixel(ImageFormat.NV21) / 8];
+        }
+        if (camera != null && detectType == MODEL_SCAN) {
+            camera.addCallbackBuffer(buffer);
+            camera.setPreviewCallback(previewCallback);
+        }
     }
 
+    private void clearPreviewCallback() {
+        if (camera != null && detectType == MODEL_SCAN) {
+            camera.setPreviewCallback(null);
+            stopPreview();
+        }
+    }
+
+    Camera.PreviewCallback previewCallback = new Camera.PreviewCallback() {
+        @Override
+        public void onPreviewFrame(final byte[] data, Camera camera) {
+            // 扫描成功阻止打开新线程处理
+            if (abortingScan.get()) {
+                return;
+            }
+            // 节流
+            if (previewFrameCount++ % 5 != 0) {
+                return;
+            }
+
+            // 在某些机型和某项项目中，某些帧的data的数据不符合nv21的格式，需要过滤，否则后续处理会导致crash
+            if (data.length != parameters.getPreviewSize().width * parameters.getPreviewSize().height * 1.5) {
+                return;
+            }
+
+            camera.addCallbackBuffer(buffer);
+
+            CameraThreadPool.execute(new Runnable() {
+                @Override
+                public void run() {
+                    Camera1Control.this.onRequestDetect(data);
+                }
+            });
+        }
+    };
+
+    private void onRequestDetect(byte[] data) {
+        // 相机已经关闭
+        if (camera == null || data == null || optSize == null) {
+            return;
+        }
+
+        YuvImage img = new YuvImage(data, ImageFormat.NV21, optSize.width, optSize.height, null);
+        ByteArrayOutputStream os = null;
+        try {
+            os = new ByteArrayOutputStream(data.length);
+            img.compressToJpeg(new Rect(0, 0, optSize.width, optSize.height), 80, os);
+            byte[] jpeg = os.toByteArray();
+            int status = detectCallback.onDetect(jpeg, getCameraRotation());
+
+            if (status == 0) {
+                clearPreviewCallback();
+            }
+        } catch (OutOfMemoryError e) {
+            // 内存溢出则取消当次操作
+        } finally {
+            try {
+                os.close();
+            } catch (Throwable e) {
+                e.printStackTrace();
+            }
+        }
+
+    }
+
     // 开启预览
@@ -118,2 +207,8 @@ private void startPreview(boolean checkPermission) {
             camera.startPreview();
+            System.out.println("startPreview 开启预览");
+            System.out.println(faceDetectionListener);
+            if (faceDetectionListener != null) {
+                camera.startFaceDetection();
+                camera.setFaceDetectionListener(faceDetectionListener);
+            }
             startAutoFocus();
@@ -236,2 +331,5 @@ }
             camera.stopPreview();
+            if (faceDetectionListener != null) {
+                camera.stopFaceDetection();
+            }
         }
@@ -248,3 +346,4 @@ }
     public void setDetectCallback(OnDetectPictureCallback callback) {
-
+        detectType = MODEL_SCAN;
+        detectCallback = callback;
     }
@@ -254,5 +353,37 @@
     public void setFaceDetectCallback(FaceDetectionListener faceDetectionListener) {
+        this.faceDetectionListener = (faces, camera) -> {
+            Matrix matrix = new Matrix();
+            boolean mirror = (this.facing == Camera.CameraInfo.CAMERA_FACING_FRONT);
+            matrix.setScale(mirror ? -1 : 1, 1);
+            matrix.postRotate(rotation);
+            matrix.postScale(previewView.getWidth() / 2000f, previewView.getHeight() / 2000f);
+            matrix.postTranslate(previewView.getWidth() / 2f, previewView.getHeight() / 2f);
 
+            ArrayList<Face> faceList = new ArrayList<>();
+            for (Camera.Face value : faces) {
+                // 过滤掉置信度小于50的Face
+                if (value.score < 50) {
+                    continue;
+                }
+
+                RectF srcRect = new RectF(value.rect);
+                RectF dstRect = new RectF(0f, 0f, 0f, 0f);
+                matrix.mapRect(dstRect, srcRect);
+
+                Face face = new Face();
+                face.setId(value.id);
+                face.setLeftEye(value.leftEye);
+                face.setRightEye(value.rightEye);
+                face.setMouth(value.mouth);
+                face.setScore(value.score);
+                face.setRect(dstRect);
+                faceList.add(face);
+            }
+
+            Face[] faceResult = faceList.toArray(new Face[0]);
+            faceDetectionListener.onFaceDetection(faceResult);
+        };
     }
 
+
     @Override
@@ -331,3 +462,3 @@ public void start() {
     public AtomicBoolean getAbortingScan() {
-        return null;
+        return abortingScan;
     }
@@ -334,0 +465,0 @@

android/src/main/j...m/yufu/multi/camera/ICameraControl.java

@@ -6,4 +6,6 @@ /*
 
+import android.graphics.Matrix;
 import android.graphics.Point;
 import android.graphics.Rect;
+import android.graphics.RectF;
 import android.view.Surface;
@@ -15,2 +17,4 @@ import android.view.View;
 
+import com.getcapacitor.JSObject;
+
 import org.json.JSONException;
@@ -68,2 +72,22 @@ import org.json.JSONObject;
 
+    /**
+     * 人脸识别
+     */
+    int DETECT_ABILITY_FACE = 1;
+
+    /**
+     * 身份证正面识别
+     */
+    int DETECT_ABILITY_ID_CARD_FRONT = 2;
+
+    /**
+     * 身份证背面识别
+     */
+    int DETECT_ABILITY_ID_CARD_BACK = 3;
+
+    @IntDef({DETECT_ABILITY_FACE, DETECT_ABILITY_ID_CARD_FRONT, DETECT_ABILITY_ID_CARD_BACK})
+    @interface DetectAbility {
+    }
+
+
     @StringDef({CAMERA_FACING_FRONT, CAMERA_FACING_BACK})
@@ -74,3 +98,3 @@ @interface CameraFacing {
     @IntDef({ORIENTATION_PORTRAIT, ORIENTATION_HORIZONTAL, ORIENTATION_INVERT})
-    public @interface Orientation {
+    @interface Orientation {
 
@@ -84,3 +108,3 @@ }
 
-    @IntDef( value = {
+    @IntDef(value = {
             Surface.ROTATION_0,
@@ -92,3 +116,4 @@ Surface.ROTATION_90,
     @Retention(RetentionPolicy.SOURCE)
-    public @interface Rotation {}
+    public @interface Rotation {
+    }
 
@@ -229,9 +254,9 @@ /**
     class Face {
-        public Rect rect = new Rect();
+        private RectF rect = new RectF();
         // 人脸检测的置信度。
-        public int score = 0;
-        public int id = -1;
-        public Point leftEye = null;
-        public Point rightEye = null;
-        public Point mouth = null;
+        private int score = 0;
+        private int id = -1;
+        private Point leftEye = null;
+        private Point rightEye = null;
+        private Point mouth = null;
 
@@ -270,7 +295,7 @@ public void setId(int id) {
 
-        public void setRect(Rect rect) {
+        public void setRect(RectF rect) {
             this.rect = rect;
         }
 
-        public Rect getRect() {
+        public RectF getRect() {
             return rect;
@@ -287,30 +312,45 @@ }
 
-        public JSONObject toJSONObject() {
-            JSONObject data = new JSONObject();
-            try {
-                data.put("id", getId());
-            } catch (JSONException ignored) {
-            }
-            try {
-                data.put("mouth", getMouth());
-            } catch (JSONException ignored) {
-            }
-            try {
-                data.put("leftEye", getLeftEye());
-            } catch (JSONException ignored) {
-            }
-            try {
-                data.put("rightEye", getRightEye());
-            } catch (JSONException ignored) {
-            }
-            try {
-                data.put("rect", getRect());
-            } catch (JSONException ignored) {
-            }
-            try {
-                data.put("score", getScore());
-            } catch (JSONException ignored) {
-            }
+        public JSObject getJSObject() {
+            JSObject data = new JSObject();
+            data.put("id", getId());
+            data.put("mouth", getMouth());
+            data.put("leftEye", getLeftEye());
+            data.put("rightEye", getRightEye());
+            data.put("score", getScore());
+            JSObject rect = new JSObject();
+            rect.put("width", this.rect.width());
+            rect.put("height", this.rect.height());
+            rect.put("top", this.rect.top);
+            rect.put("right", this.rect.right);
+            rect.put("bottom", this.rect.bottom);
+            rect.put("left", this.rect.left);
+            rect.put("centerX", this.rect.centerX());
+            rect.put("centerY", this.rect.centerY());
+            data.put("rect", rect);
             return data;
         }
+
+        public JSObject getJSObject(float density){
+            JSObject data = new JSObject();
+            data.put("id", getId());
+            data.put("mouth", getMouth());
+            data.put("leftEye", getLeftEye());
+            data.put("rightEye", getRightEye());
+            data.put("score", getScore());
+            JSObject rect = new JSObject();
+            rect.put("width", dpToPx(this.rect.width(), density));
+            rect.put("height", dpToPx(this.rect.height(), density));
+            rect.put("top", dpToPx(this.rect.top, density));
+            rect.put("right", dpToPx(this.rect.right, density));
+            rect.put("bottom", dpToPx(this.rect.bottom, density));
+            rect.put("left", dpToPx(this.rect.left, density));
+            rect.put("centerX", dpToPx(this.rect.centerX(), density));
+            rect.put("centerY", dpToPx(this.rect.centerY(), density));
+            data.put("rect", rect);
+            return data;
+        }
+
+        private int dpToPx(float dp, float density) {
+            return (int) (dp / density + 0.5f);
+        }
     }
@@ -317,0 +357,0 @@ }

android/src/main/java/com/yufu/multi/camera/MultiCamera.java

@@ -24,2 +24,4 @@ package com.yufu.multi.camera;
 
+
+
     public void setOverlay() {
@@ -26,0 +28,0 @@ isOverlay = true;

android/src/main/j...u/multi/camera/MultiCameraActivity.java

 package com.yufu.multi.camera;
 
 import android.app.Fragment;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.BitmapRegionDecoder;
+import android.graphics.Matrix;
+import android.graphics.Rect;
 import android.os.Bundle;
@@ -12,4 +17,13 @@ import android.view.LayoutInflater;
 
+import com.baidu.idcardquality.IDcardQualityProcess;
+import com.baidu.ocr.sdk.OCR;
+import com.baidu.ocr.sdk.OnResultListener;
+import com.baidu.ocr.sdk.exception.OCRError;
+import com.baidu.ocr.sdk.model.AccessToken;
+import com.yufu.multi.camera.util.ImageUtil;
+
 import androidx.annotation.Nullable;
 
+import java.io.IOException;
+
 @SuppressWarnings("deprecation")
@@ -21,2 +35,11 @@ public class MultiCameraActivity extends Fragment {
         void onRequestPermission();
+
+        // 人脸识别
+        void onFaceDetection(ICameraControl.Face[] faces);
+
+        // 身份证本地质量控制状态回调
+        void onIDCardMessage(int status, String message);
+
+        // 拍照
+        void onPictureTaken(Bitmap bitmap);
     }
@@ -32,2 +55,4 @@
     private View view;
+    private boolean hasGotToken = false;
+    private String baiduAccessToken;
 
@@ -39,3 +64,20 @@ private int width;
     private String cameraFacing = ICameraControl.CAMERA_FACING_BACK;
+    private Rect maskRect = new Rect();
 
+    public void setInitNativeStatus(int initNativeStatus) {
+        this.initNativeStatus = initNativeStatus;
+    }
+
+    public void setMaskRect(Rect maskRect) {
+        this.maskRect = maskRect;
+    }
+
+    /**
+     * 本地检测初始化，模型加载标识
+     */
+    private int initNativeStatus = CameraNativeHelper.NATIVE_AUTH_INIT_SUCCESS;
+
+    @ICameraControl.DetectAbility
+    private int detectAbility;
+
     @Nullable
@@ -69,2 +111,21 @@ @Override
         cameraControl.setPermissionCallback(permissionCallback);
+        switch (detectAbility) {
+            case ICameraControl.DETECT_ABILITY_ID_CARD_BACK:
+            case ICameraControl.DETECT_ABILITY_ID_CARD_FRONT:
+                if (baiduAccessToken == null) {
+                    initBaiduAccessToken();
+                } else {
+                    initNative();
+                }
+                cameraControl.setDetectCallback(new ICameraControl.OnDetectPictureCallback() {
+                    @Override
+                    public int onDetect(byte[] data, int rotation) {
+                        return detect(data, rotation);
+                    }
+                });
+                break;
+            case ICameraControl.DETECT_ABILITY_FACE:
+                cameraControl.setFaceDetectCallback(faces -> eventListener.onFaceDetection(faces));
+                break;
+        }
         relativeLayout.addView(cameraControl.getDisplayView());
@@ -84,3 +145,216 @@
 
+    private String getScanMessage(int status) {
+        String message;
+        switch (status) {
+            case 0:
+                message = "";
+                break;
+            case 2:
+                message = "身份证模糊，请重新尝试";
+                break;
+            case 3:
+                message = "身份证反光，请重新尝试";
+                break;
+            case 4:
+                message = "请将身份证前后反转再进行识别";
+                break;
+            case 5:
+                message = "请拿稳镜头和身份证";
+                break;
+            case 6:
+                message = "请将镜头靠近身份证";
+                break;
+            case 7:
+                message = "请将身份证完整置于取景框内";
+                break;
+            case CameraNativeHelper.NATIVE_AUTH_FAIL:
+                message = "本地质量控制授权失败";
+                break;
+            case CameraNativeHelper.NATIVE_INIT_FAIL:
+                message = "本地模型加载失败";
+                break;
+            case CameraNativeHelper.NATIVE_SO_LOAD_FAIL:
+                message = "本地SO库加载失败";
+                break;
+            case 1:
+            default:
+                message = "请将身份证置于取景框内";
+        }
+        return message;
+    }
 
+    private int detect(byte[] data, final int rotation) {
+        if (initNativeStatus != CameraNativeHelper.NATIVE_AUTH_INIT_SUCCESS) {
+            eventListener.onIDCardMessage(initNativeStatus, getScanMessage(initNativeStatus));
+            return 1;
+        }
+        // 扫描成功阻止多余的操作
+        if (cameraControl.getAbortingScan().get()) {
+            return 0;
+        }
+
+        Rect previewFrame = cameraControl.getPreviewFrame();
+
+        if (previewFrame.width() == 0 || previewFrame.height() == 0) {
+            return 0;
+        }
+
+        // BitmapRegionDecoder不会将整个图片加载到内存。
+        BitmapRegionDecoder decoder = null;
+        try {
+            decoder = BitmapRegionDecoder.newInstance(data, 0, data.length, true);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        int width;
+        if (rotation % 180 == 0) {
+            assert decoder != null;
+            width = decoder.getWidth();
+        } else {
+            assert decoder != null;
+            width = decoder.getHeight();
+        }
+        int height = rotation % 180 == 0 ? decoder.getHeight() : decoder.getWidth();
+
+
+        int left =  maskRect.left;
+        int top = maskRect.top;
+        int right = maskRect.right;
+        int bottom = maskRect.bottom;
+
+//        // 高度大于图片
+//        if (previewFrame.top < 0) {
+//            // 宽度对齐。
+//            int adjustedPreviewHeight = previewFrame.height() * getWidth() / previewFrame.width();
+//            int topInFrame = ((adjustedPreviewHeight - frameRect.height()) / 2)
+//                    * getWidth() / previewFrame.width();
+//            int bottomInFrame = ((adjustedPreviewHeight + frameRect.height()) / 2) * getWidth()
+//                    / previewFrame.width();
+//
+//            // 等比例投射到照片当中。
+//            top = topInFrame * height / previewFrame.height();
+//            bottom = bottomInFrame * height / previewFrame.height();
+//        } else {
+//            // 宽度大于图片
+//            if (previewFrame.left < 0) {
+//                // 高度对齐
+//                int adjustedPreviewWidth = previewFrame.width() * getHeight() / previewFrame.height();
+//                int leftInFrame = ((adjustedPreviewWidth - maskView.getFrameRect().width()) / 2) * getHeight()
+//                        / previewFrame.height();
+//                int rightInFrame = ((adjustedPreviewWidth + maskView.getFrameRect().width()) / 2) * getHeight()
+//                        / previewFrame.height();
+//
+//                // 等比例投射到照片当中。
+//                left = leftInFrame * width / previewFrame.width();
+//                right = rightInFrame * width / previewFrame.width();
+//            }
+//        }
+//
+        Rect region = new Rect();
+        region.left = left;
+        region.top = top;
+        region.right = right;
+        region.bottom = bottom;
+
+        // 90度或者270度旋转
+        if (rotation % 180 == 90) {
+            int x = decoder.getWidth() / 2;
+            int y = decoder.getHeight() / 2;
+
+            int rotatedWidth = region.height();
+            int rotated = region.width();
+
+            // 计算，裁剪框旋转后的坐标
+            region.left = x - rotatedWidth / 2;
+            region.top = y - rotated / 2;
+            region.right = x + rotatedWidth / 2;
+            region.bottom = y + rotated / 2;
+            region.sort();
+        }
+
+        BitmapFactory.Options options = new BitmapFactory.Options();
+
+        // 最大图片大小。
+        int maxPreviewImageSize = 2560;
+        int size = Math.min(decoder.getWidth(), decoder.getHeight());
+        size = Math.min(size, maxPreviewImageSize);
+
+        options.inSampleSize = ImageUtil.calculateInSampleSize(options, size, size);
+        options.inScaled = true;
+        options.inDensity = Math.max(options.outWidth, options.outHeight);
+        options.inTargetDensity = size;
+        options.inPreferredConfig = Bitmap.Config.RGB_565;
+        Bitmap bitmap = decoder.decodeRegion(region, options);
+        if (rotation != 0) {
+            // 只能是裁剪完之后再旋转了。有没有别的更好的方案呢？
+            Matrix matrix = new Matrix();
+            matrix.postRotate(rotation);
+            Bitmap rotatedBitmap = Bitmap.createBitmap(
+                    bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, false);
+            if (bitmap != rotatedBitmap) {
+                // 有时候 createBitmap会复用对象
+                bitmap.recycle();
+            }
+            bitmap = rotatedBitmap;
+        }
+
+        final int status;
+
+        // 调用本地质量控制请求
+        switch (detectAbility) {
+            case ICameraControl.DETECT_ABILITY_ID_CARD_FRONT:
+                status = IDcardQualityProcess.getInstance().idcardQualityDetectionImg(bitmap, true);
+                break;
+            case ICameraControl.DETECT_ABILITY_ID_CARD_BACK:
+                status = IDcardQualityProcess.getInstance().idcardQualityDetectionImg(bitmap, false);
+                break;
+            case ICameraControl.DETECT_ABILITY_FACE:
+            default:
+                status = 1;
+                break;
+        }
+
+        // 当有某个扫描处理线程调用成功后，阻止其他线程继续调用本地控制代码
+        if (status == CameraNativeHelper.SCAN_SUCCESS) {
+            // 扫描成功阻止多线程同时回调
+            if (!cameraControl.getAbortingScan().compareAndSet(false, true)) {
+                bitmap.recycle();
+                return 0;
+            }
+            eventListener.onPictureTaken(bitmap);
+        }
+
+        eventListener.onIDCardMessage(status, getScanMessage(status));
+        return 4;
+    }
+
+    /**
+     * 以license文件方式初始化
+     */
+    private void initBaiduAccessToken() {
+        OCR.getInstance(getActivity().getApplicationContext()).initAccessToken(new OnResultListener<AccessToken>() {
+            @Override
+            public void onResult(AccessToken accessToken) {
+                baiduAccessToken = accessToken.getAccessToken();
+                initNative();
+                hasGotToken = true;
+            }
+
+            @Override
+            public void onError(OCRError error) {
+                error.printStackTrace();
+                System.out.println("licence方式获取token失败:" + error.getMessage());
+            }
+        }, getActivity().getApplicationContext());
+    }
+
+    private void initNative() {
+        CameraNativeHelper.init(getActivity(), OCR.getInstance(getActivity().getApplicationContext()).getLicense(), (errorCode, e) -> setInitNativeStatus(errorCode));
+    }
+
+    public void setDetectAbility(@ICameraControl.DetectAbility int detectAbility) {
+        this.detectAbility = detectAbility;
+    }
+
     public void setEventListener(MultiCameraListener listener) {
@@ -116,2 +390,3 @@ eventListener = listener;
 
+
     @Override
@@ -118,0 +393,0 @@ public void onResume() {

android/src/main/j...ufu/multi/camera/MultiCameraPlugin.java

 package com.yufu.multi.camera;
 
 import android.Manifest;
-import android.annotation.SuppressLint;
-import android.app.FragmentManager;
 import android.app.FragmentTransaction;
+import android.graphics.Bitmap;
 import android.graphics.Color;
 import android.graphics.Point;
+import android.graphics.Rect;
 import android.util.DisplayMetrics;
@@ -13,7 +13,10 @@ import android.util.TypedValue;
 import android.view.ViewGroup;
+import android.view.View;
 import android.widget.FrameLayout;
+import android.widget.RelativeLayout;
+import android.widget.LinearLayout;
 
 import androidx.appcompat.app.AppCompatActivity;
 
-import com.getcapacitor.Bridge;
+import com.getcapacitor.JSArray;
 import com.getcapacitor.JSObject;
@@ -69,2 +72,3 @@ import com.getcapacitor.PermissionState;
     public void startPreview(PluginCall call) {
+
         String camera = call.getString("camera");
@@ -79,4 +83,5 @@ if (!Objects.equals(camera, ICameraControl.CAMERA_FACING_FRONT)) {
         final Boolean toBack = call.getBoolean("toBack", false);
-        final Boolean overlay = call.getBoolean("overlay", false);
-        final Boolean landscape = call.getBoolean("landscape", false);
+        final String ability = call.getString("ability", "");
+        final JSObject mask = call.getObject("mask");
+
         assert x != null;
@@ -86,8 +91,19 @@ assert y != null;
         assert height != null;
+        assert ability != null;
 
-        if (Boolean.TRUE.equals(landscape)) {
-            multiCamera.lockLandscape();
+        cameraFragment = new MultiCameraActivity();
+
+        switch (ability) {
+            case "face":
+                cameraFragment.setDetectAbility(ICameraControl.DETECT_ABILITY_FACE);
+                break;
+            case "idCardFront":
+                cameraFragment.setDetectAbility(ICameraControl.DETECT_ABILITY_ID_CARD_FRONT);
+                break;
+            case "idCardBack":
+                cameraFragment.setDetectAbility(ICameraControl.DETECT_ABILITY_ID_CARD_BACK);
+                break;
+            default:
+                break;
         }
-
-        cameraFragment = new MultiCameraActivity();
         cameraFragment.setCameraFacing(camera);
@@ -129,8 +145,36 @@ cameraFragment.setEventListener(this);
 
-                if (Boolean.TRUE.equals(overlay)) {
-                    multiCamera.setOverlay();
+
+
+
+                if(mask != null){
+                    Integer iLeft = mask.getInteger("left", 0);
+                    Integer iTop = mask.getInteger("top", 0);
+                    Integer iWidth = mask.getInteger("width", 0);
+                    Integer iHeight = mask.getInteger("height", 0);
+                    assert iLeft != null;
+                    assert iTop != null;
+                    assert iWidth != null;
+                    assert iHeight != null;
+                    int maskLeft = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, iLeft, metrics);
+                    int maskTop = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, iTop, metrics);
+                    int maskWidth = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, iWidth, metrics);
+                    int maskHeight = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, iHeight, metrics);
+                    Rect maskRect = new Rect(maskLeft, maskTop, maskLeft + maskWidth, maskTop + maskHeight);
+                    cameraFragment.setMaskRect(maskRect);
+                    int maskViewId = 99;
+                    View maskView = new View(getActivity());
+                    maskView.setAlpha(0.2f);
+                    maskView.setId(maskViewId);
+                    maskView.setBackgroundColor(Color.YELLOW);
+                    LinearLayout.LayoutParams maskLayout = new LinearLayout.LayoutParams(maskWidth,maskHeight);
+                    maskView.setLayoutParams(maskLayout);
+                    ViewGroup.MarginLayoutParams margin = new ViewGroup.MarginLayoutParams(maskView.getLayoutParams());
+                    margin.setMargins(maskLeft, maskTop, 0, 0);
+                    LinearLayout.LayoutParams maskMarginLayout = new LinearLayout.LayoutParams(margin);
+                    maskView.setLayoutParams(maskMarginLayout);
+                    ((ViewGroup) getBridge().getWebView().getParent()).addView(maskView);
                 }
 
+
                 FragmentTransaction fragmentTransaction = compatActivity.getFragmentManager().beginTransaction();
-                System.out.println(cameraFragment);
                 cameraFragment.setRect(computedX, computedY, computedWidth, computedHeight);
@@ -163,4 +207,2 @@ fragmentTransaction.add(containerView.getId(), cameraFragment);
                 cameraFragment = null;
-                multiCamera.unlockLandscape();
-                multiCamera.unsetOverlay();
                 call.resolve();
@@ -200,13 +242,2 @@ } else {
     /**
-     * 开始人脸检测
-     */
-    @PluginMethod
-    public void startFaceDetection(PluginCall call) {
-        String value = call.getString("value");
-
-        JSObject ret = new JSObject();
-        call.resolve(ret);
-    }
-
-    /**
      * 停止人脸检测
@@ -242,2 +273,30 @@ */
 
+    @Override
+    public void onFaceDetection(ICameraControl.Face[] faces) {
+        if (hasListeners("FaceDetectionEvent")) {
+            final float density = getActivity().getResources().getDisplayMetrics().density;
+            JSArray faceArray = new JSArray();
+            for (ICameraControl.Face face : faces) {
+                faceArray.put(face.getJSObject(density));
+            }
+            JSObject ret = new JSObject();
+            ret.put("faces", faceArray);
+            notifyListeners("FaceDetectionEvent", ret);
+        }
+    }
+
+    @Override
+    public void onIDCardMessage(int status, String message) {
+        JSObject ret = new JSObject();
+        ret.put("code", status);
+        ret.put("message", message);
+        notifyListeners("IDCardDetectionEvent", ret);
+    }
+
+    @Override
+    public void onPictureTaken(Bitmap bitmap){
+        System.out.println("图片 bitmap");
+        System.out.println(bitmap);
+    }
+
     @PermissionCallback
@@ -244,0 +303,0 @@ public void onPermissionCameraResult(PluginCall call) {

dist/docs.json

@@ -167,16 +167,2 @@ {
           "type": "number | undefined"
-        },
-        {
-          "name": "overlay",
-          "tags": [],
-          "docs": "",
-          "complexTypes": [],
-          "type": "boolean | undefined"
-        },
-        {
-          "name": "landscape",
-          "tags": [],
-          "docs": "",
-          "complexTypes": [],
-          "type": "boolean | undefined"
         }
@@ -183,0 +169,0 @@ ]

dist/esm/definitions.d.ts

@@ -13,4 +13,2 @@ /**
     y?: number;
-    overlay?: boolean;
-    landscape?: boolean;
 }
@@ -17,0 +15,0 @@ export interface CaptureOption {

dist/esm/definitions.js.map

@@ -1,1 +0,1 @@
-{"version":3,"file":"definitions.js","sourceRoot":"","sources":["../../src/definitions.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * 相机选项\n */\nexport type CameraType = 'back' | 'front';\n\nexport declare type IDCardType = 'back' | 'front';\n\nexport interface StartOption {\n  // 默认相机\n  camera?: CameraType;\n  // 将 html 放在你预览前\n  toBack?: boolean;\n  // 以像素为单位的预览宽度，默认window.screen.width\n  width?: number;\n  // 以像素为单位的预览高度，默认window.screen.height\n  height?: number;\n  // x 原点，默认为 0\n  x?: number;\n  // y 原点，默认为 0\n  y?: number;\n  // 沉浸式\n  overlay?: boolean;\n  // 横屏\n  landscape?: boolean;\n}\n\nexport interface CaptureOption {\n  // 质量\n  quality: number;\n  // 保存到文件\n  storeToFile?: boolean;\n}\nexport interface CaptureResult {\n  data?: string;\n  path?: string;\n}\nexport interface IDCardDetectionOption {\n  type: IDCardType;\n}\n\nexport interface MultiCameraPlugin {\n  // 开始预览\n  startPreview(options: StartOption): Promise<void>;\n  // 停止预览\n  stopPreview(): Promise<void>;\n  // 前后相机翻转\n  flip(): Promise<void>;\n  // 拍照\n  capture(options: CaptureOption): Promise<CaptureResult>;\n  // 开始人脸检测\n  startFaceDetection(): Promise<void>;\n  // 停止人脸检测\n  stopFaceDetection(): Promise<void>;\n  // 开始身份证识别\n  startIDCardDetection(options: IDCardDetectionOption): Promise<void>;\n  // 停止身份证识别\n  stopIDCardDetection(): Promise<void>;\n}\n"]}
+{"version":3,"file":"definitions.js","sourceRoot":"","sources":["../../src/definitions.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * 相机选项\n */\nexport type CameraType = 'back' | 'front';\n\nexport declare type IDCardType = 'back' | 'front';\n\nexport interface StartOption {\n  // 默认相机\n  camera?: CameraType;\n  // 将 html 放在你预览前\n  toBack?: boolean;\n  // 以像素为单位的预览宽度，默认window.screen.width\n  width?: number;\n  // 以像素为单位的预览高度，默认window.screen.height\n  height?: number;\n  // x 原点，默认为 0\n  x?: number;\n  // y 原点，默认为 0\n  y?: number;\n}\n\nexport interface CaptureOption {\n  // 质量\n  quality: number;\n  // 保存到文件\n  storeToFile?: boolean;\n}\nexport interface CaptureResult {\n  data?: string;\n  path?: string;\n}\nexport interface IDCardDetectionOption {\n  type: IDCardType;\n}\n\nexport interface MultiCameraPlugin {\n  // 开始预览\n  startPreview(options: StartOption): Promise<void>;\n  // 停止预览\n  stopPreview(): Promise<void>;\n  // 前后相机翻转\n  flip(): Promise<void>;\n  // 拍照\n  capture(options: CaptureOption): Promise<CaptureResult>;\n  // 开始人脸检测\n  startFaceDetection(): Promise<void>;\n  // 停止人脸检测\n  stopFaceDetection(): Promise<void>;\n  // 开始身份证识别\n  startIDCardDetection(options: IDCardDetectionOption): Promise<void>;\n  // 停止身份证识别\n  stopIDCardDetection(): Promise<void>;\n}\n"]}

package.json

 {
   "name": "capacitor-plugin-multi-camera",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "多功能相机，预览、百度OCR、人脸检测等功能",
@@ -10,2 +10,3 @@ "main": "dist/plugin.cjs.js",
   "files": [
+    "android/libs/",
     "android/src/main/",
@@ -12,0 +13,0 @@ "android/build.gradle",

README.md

@@ -123,12 +123,10 @@ # capacitor-plugin-multi-camera
 
-| Prop            | Type                                              |
-| --------------- | ------------------------------------------------- |
-| **`camera`**    | <code><a href="#cameratype">CameraType</a></code> |
-| **`toBack`**    | <code>boolean</code>                              |
-| **`width`**     | <code>number</code>                               |
-| **`height`**    | <code>number</code>                               |
-| **`x`**         | <code>number</code>                               |
-| **`y`**         | <code>number</code>                               |
-| **`overlay`**   | <code>boolean</code>                              |
-| **`landscape`** | <code>boolean</code>                              |
+| Prop         | Type                                              |
+| ------------ | ------------------------------------------------- |
+| **`camera`** | <code><a href="#cameratype">CameraType</a></code> |
+| **`toBack`** | <code>boolean</code>                              |
+| **`width`**  | <code>number</code>                               |
+| **`height`** | <code>number</code>                               |
+| **`x`**      | <code>number</code>                               |
+| **`y`**      | <code>number</code>                               |
 
@@ -135,0 +133,0 @@

New alerts

Native code

Supply chain risk

Contains native code (e.g., compiled binaries or shared libraries). Including native code can obscure malicious behavior.

Found 1 instance in 1 package

Misc. License Issues

License

A package's licensing information has fine-grained problems.

Found 4 instances in 1 package

Unidentified License

License

Something that seems like a license was found, but its contents could not be matched with a known license.

Found 5 instances in 1 package

Improved metrics

Total package byte prevSize

24597718

30680.24%

Number of package files

53

60.61%

Lines of code

2393

45.91%

Worsened metrics

Number of low license alerts

5

Infinity%

Number of medium license alerts

4

Infinity%

Number of lines in readme file

171

-1.16%

Number of medium supply chain risk alerts

8

Infinity%

No dependency changes