From 67efa6ee152e05e0b92950412218ea60ded6508c Mon Sep 17 00:00:00 2001 From: Ricardo Souza Date: Tue, 3 Jan 2023 11:44:23 -0300 Subject: [PATCH 1/3] Fixed an error when users not grant the permission so app crashes because barcodeScanner is not initialized. Improved code with a new mlkit so now the camera can track QRcode and Texts. Removed `QrCodeDrawable.kt` and created `BuildRect.kt` instead. Improved the request permission to new resultLauncher API. --- CameraX-MLKit/README.md | 14 ++- CameraX-MLKit/app/build.gradle | 1 + .../com/example/camerax_mlkit/MainActivity.kt | 102 ++++++++++-------- .../example/camerax_mlkit/QrCodeDrawable.kt | 86 --------------- .../example/camerax_mlkit/TextViewModel.kt | 19 ++++ .../example/camerax_mlkit/utils/BuildRect.kt | 65 +++++++++++ .../app/src/main/res/layout/activity_main.xml | 1 + README.md | 2 +- 8 files changed, 157 insertions(+), 133 deletions(-) delete mode 100644 CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/QrCodeDrawable.kt create mode 100644 CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/TextViewModel.kt create mode 100644 CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/utils/BuildRect.kt diff --git a/CameraX-MLKit/README.md b/CameraX-MLKit/README.md index 4df2e048..409a36bc 100644 --- a/CameraX-MLKit/README.md +++ b/CameraX-MLKit/README.md @@ -1,8 +1,14 @@ # CameraX-MLKit -This example uses CameraX's MlKitAnalyzer to perform QR Code scanning. For QR Codes that encode Urls, this app will prompt the user to open the Url in a broswer. This app can be adapted to handle other types of QR Code data. +This example uses CameraX's MlKitAnalyzer to perform QR Code scanning. For QR Codes that encode Urls, this app will prompt the user to open the Url in +a broswer. This app can be adapted to handle other types of QR Code data. This example also uses CameraX's MlKitAnalyzer to perform Text Recognition. -The interesting part of the code is in `MainActivity.kt` in the `startCamera()` function. There, we set up BarcodeScannerOptions to match on QR Codes. Then we call `cameraController.setImageAnalysisAnalyzer` with an `MlKitAnalyzer` (available as of CameraX 1.2). We also pass in `COORDINATE_SYSTEM_VIEW_REFERENCED` so that CameraX will handle the cordinates coming off of the camera sensor, making it easy to draw a box around the QR Code. Finally, we create a QrCodeDrawable, which is a class defined in this sample, extending View, for displaying an overlay on the QR Code and handling tap events on the QR Code. +On `onCreate()` we set up BarcodeScannerOptions to match on QR Codes and TextRecognizerOptions to match on Text on image. + +The interesting part of the code is in `MainActivity.kt` in the `startCamera()` function. Then we call `cameraController.setImageAnalysisAnalyzer` +with an `MlKitAnalyzer` (available as of CameraX 1.2). We also pass in `COORDINATE_SYSTEM_VIEW_REFERENCED` so that CameraX will handle the cordinates +coming off of the camera sensor, making it easy to draw a box around the QR Code. Finally, we create a QrCodeDrawable, which is a class defined in +this sample, extending View, for displaying an overlay on the QR Code and handling tap events on the QR Code. You can open this project in Android Studio to explore the code further, and to build and run the application on a test device. @@ -10,11 +16,11 @@ You can open this project in Android Studio to explore the code further, and to Screenshot of QR-code reader app scanning a QR code for the website google.com -## Command line options +## Command line options ### Build -To build the app directly from the command line, run: +To build the app directly from the command line, run: '' ```sh ./gradlew assembleDebug ``` diff --git a/CameraX-MLKit/app/build.gradle b/CameraX-MLKit/app/build.gradle index ff5a4f5c..29614370 100644 --- a/CameraX-MLKit/app/build.gradle +++ b/CameraX-MLKit/app/build.gradle @@ -69,4 +69,5 @@ dependencies { implementation "androidx.camera:camera-view:${camerax_version}" implementation 'com.google.mlkit:barcode-scanning:17.0.2' + implementation 'com.google.mlkit:text-recognition:16.0.0-beta6' } \ No newline at end of file diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt index e47a5e7b..2c37a4b8 100644 --- a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt +++ b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt @@ -18,21 +18,26 @@ package com.example.camerax_mlkit import android.Manifest import android.content.pm.PackageManager +import android.os.Build import android.os.Bundle -import android.view.View import android.widget.Toast +import androidx.activity.result.contract.ActivityResultContracts import androidx.appcompat.app.AppCompatActivity import androidx.camera.mlkit.vision.MlKitAnalyzer import androidx.camera.view.CameraController.COORDINATE_SYSTEM_VIEW_REFERENCED import androidx.camera.view.LifecycleCameraController import androidx.camera.view.PreviewView -import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import com.example.camerax_mlkit.databinding.ActivityMainBinding +import com.example.camerax_mlkit.utils.BuildRect +import com.google.android.material.snackbar.Snackbar import com.google.mlkit.vision.barcode.BarcodeScanner import com.google.mlkit.vision.barcode.BarcodeScannerOptions import com.google.mlkit.vision.barcode.BarcodeScanning import com.google.mlkit.vision.barcode.common.Barcode +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.TextRecognizer +import com.google.mlkit.vision.text.latin.TextRecognizerOptions import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -40,6 +45,14 @@ class MainActivity : AppCompatActivity() { private lateinit var viewBinding: ActivityMainBinding private lateinit var cameraExecutor: ExecutorService private lateinit var barcodeScanner: BarcodeScanner + private lateinit var textRecognizer: TextRecognizer + + companion object { + private const val TAG = "CameraX-MLKit" + private const val REQUEST_CODE_PERMISSIONS = 10 + private val REQUIRED_PERMISSIONS = + mutableListOf(Manifest.permission.CAMERA).toTypedArray() + } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -50,46 +63,52 @@ class MainActivity : AppCompatActivity() { if (allPermissionsGranted()) { startCamera() } else { - ActivityCompat.requestPermissions( - this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS - ) + requestCameraPermission() } cameraExecutor = Executors.newSingleThreadExecutor() - } - - private fun startCamera() { - var cameraController = LifecycleCameraController(baseContext) - val previewView: PreviewView = viewBinding.viewFinder - val options = BarcodeScannerOptions.Builder() .setBarcodeFormats(Barcode.FORMAT_QR_CODE) .build() + + val textOptions = TextRecognizerOptions.Builder().build() barcodeScanner = BarcodeScanning.getClient(options) + textRecognizer = TextRecognition.getClient(textOptions) + } + + private fun startCamera() { + val cameraController = LifecycleCameraController(baseContext) + val previewView: PreviewView = viewBinding.viewFinder cameraController.setImageAnalysisAnalyzer( ContextCompat.getMainExecutor(this), MlKitAnalyzer( - listOf(barcodeScanner), + listOf(barcodeScanner, textRecognizer), COORDINATE_SYSTEM_VIEW_REFERENCED, ContextCompat.getMainExecutor(this) ) { result: MlKitAnalyzer.Result? -> + val textResults = result?.getValue(textRecognizer) val barcodeResults = result?.getValue(barcodeScanner) - if ((barcodeResults == null) || - (barcodeResults.size == 0) || - (barcodeResults.first() == null) - ) { + + previewView.overlay.clear() + + barcodeResults?.getOrNull(0)?.let { + val qrCodeViewModel = QrCodeViewModel(it) + val qrCodeDrawable = BuildRect(qrCodeViewModel.boundingRect, qrCodeViewModel.qrContent) + previewView.setOnTouchListener(qrCodeViewModel.qrCodeTouchCallback) + previewView.overlay.add(qrCodeDrawable) + } ?: kotlin.run { previewView.overlay.clear() previewView.setOnTouchListener { _, _ -> false } //no-op - return@MlKitAnalyzer } - val qrCodeViewModel = QrCodeViewModel(barcodeResults[0]) - val qrCodeDrawable = QrCodeDrawable(qrCodeViewModel) - - previewView.setOnTouchListener(qrCodeViewModel.qrCodeTouchCallback) - previewView.overlay.clear() - previewView.overlay.add(qrCodeDrawable) + textResults?.textBlocks?.flatMap { it.lines }?.forEach { + val textViewModel = TextViewModel(it) + val textDrawable = BuildRect(textViewModel.boundingRect, textViewModel.lineContent) + previewView.overlay.add(textDrawable) + } ?: kotlin.run { + previewView.overlay.clear() + } } ) @@ -98,8 +117,7 @@ class MainActivity : AppCompatActivity() { } private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all { - ContextCompat.checkSelfPermission( - baseContext, it) == PackageManager.PERMISSION_GRANTED + ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED } override fun onDestroy() { @@ -108,28 +126,28 @@ class MainActivity : AppCompatActivity() { barcodeScanner.close() } - companion object { - private const val TAG = "CameraX-MLKit" - private const val REQUEST_CODE_PERMISSIONS = 10 - private val REQUIRED_PERMISSIONS = - mutableListOf ( - Manifest.permission.CAMERA - ).toTypedArray() + private val requestMultiplePermissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> + if (REQUIRED_PERMISSIONS.all { result[it] == true }) { + startCamera() + } else { + requestCameraPermission() + } } - override fun onRequestPermissionsResult( - requestCode: Int, permissions: Array, grantResults: - IntArray) { - super.onRequestPermissionsResult(requestCode, permissions, grantResults) - if (requestCode == REQUEST_CODE_PERMISSIONS) { - if (allPermissionsGranted()) { - startCamera() + private fun requestCameraPermission() { + if (Build.VERSION.SDK_INT > Build.VERSION_CODES.M) { + if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) { + Snackbar.make(viewBinding.myConstraintLayout, "You need to provide camera permission to use this.", Snackbar.LENGTH_INDEFINITE) + .setAction("Request Camera Permission") { + requestMultiplePermissionLauncher.launch(REQUIRED_PERMISSIONS) + }.show() } else { - Toast.makeText(this, - "Permissions not granted by the user.", - Toast.LENGTH_SHORT).show() + Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show() finish() } + } else { + Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show() + finish() } } } \ No newline at end of file diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/QrCodeDrawable.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/QrCodeDrawable.kt deleted file mode 100644 index d2c5a4d6..00000000 --- a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/QrCodeDrawable.kt +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.camerax_mlkit - -import android.content.Intent -import android.graphics.* -import android.graphics.drawable.Drawable -import android.net.Uri -import android.view.MotionEvent -import android.view.View -import com.google.mlkit.vision.barcode.common.Barcode - -/** - * A Drawable that handles displaying a QR Code's data and a bounding box around the QR code. - */ -class QrCodeDrawable(qrCodeViewModel: QrCodeViewModel) : Drawable() { - private val boundingRectPaint = Paint().apply { - style = Paint.Style.STROKE - color = Color.YELLOW - strokeWidth = 5F - alpha = 200 - } - - private val contentRectPaint = Paint().apply { - style = Paint.Style.FILL - color = Color.YELLOW - alpha = 255 - } - - private val contentTextPaint = Paint().apply { - color = Color.DKGRAY - alpha = 255 - textSize = 36F - } - - private val qrCodeViewModel = qrCodeViewModel - private val contentPadding = 25 - private var textWidth = contentTextPaint.measureText(qrCodeViewModel.qrContent).toInt() - - override fun draw(canvas: Canvas) { - canvas.drawRect(qrCodeViewModel.boundingRect, boundingRectPaint) - canvas.drawRect( - Rect( - qrCodeViewModel.boundingRect.left, - qrCodeViewModel.boundingRect.bottom + contentPadding/2, - qrCodeViewModel.boundingRect.left + textWidth + contentPadding*2, - qrCodeViewModel.boundingRect.bottom + contentTextPaint.textSize.toInt() + contentPadding), - contentRectPaint - ) - canvas.drawText( - qrCodeViewModel.qrContent, - (qrCodeViewModel.boundingRect.left + contentPadding).toFloat(), - (qrCodeViewModel.boundingRect.bottom + contentPadding*2).toFloat(), - contentTextPaint - ) - } - - override fun setAlpha(alpha: Int) { - boundingRectPaint.alpha = alpha - contentRectPaint.alpha = alpha - contentTextPaint.alpha = alpha - } - - override fun setColorFilter(colorFiter: ColorFilter?) { - boundingRectPaint.colorFilter = colorFilter - contentRectPaint.colorFilter = colorFilter - contentTextPaint.colorFilter = colorFilter - } - - @Deprecated("Deprecated in Java") - override fun getOpacity(): Int = PixelFormat.TRANSLUCENT -} \ No newline at end of file diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/TextViewModel.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/TextViewModel.kt new file mode 100644 index 00000000..91fc2a00 --- /dev/null +++ b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/TextViewModel.kt @@ -0,0 +1,19 @@ +package com.example.camerax_mlkit + +import android.graphics.Rect +import android.view.MotionEvent +import android.view.View +import com.google.mlkit.vision.text.Text + +class TextViewModel(line: Text.Line) { + var boundingRect: Rect? = line.boundingBox + var lineContent: String = "" + var lineTouchCallback = { v: View, e: MotionEvent -> false } + + init { + lineContent = line.text + lineTouchCallback = { v: View, e: MotionEvent -> + true // return true from the callback to signify the event was handled + } + } +} \ No newline at end of file diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/utils/BuildRect.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/utils/BuildRect.kt new file mode 100644 index 00000000..23b310ff --- /dev/null +++ b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/utils/BuildRect.kt @@ -0,0 +1,65 @@ +package com.example.camerax_mlkit.utils + +import android.graphics.* +import android.graphics.drawable.Drawable + +class BuildRect(private val boundingRect: Rect?, private val content: String) : Drawable() { + + private val boundingRectPaint = Paint().apply { + style = Paint.Style.STROKE + color = Color.YELLOW + strokeWidth = 5F + alpha = 200 + } + + private val contentRectPaint = Paint().apply { + style = Paint.Style.FILL + color = Color.YELLOW + alpha = 255 + } + + private val contentTextPaint = Paint().apply { + color = Color.DKGRAY + alpha = 255 + textSize = 36F + } + + private val contentPadding = 25 + private var textWidth = contentTextPaint.measureText(content).toInt() + + override fun draw(canvas: Canvas) { + boundingRect?.let { rect -> + canvas.drawRect(rect, boundingRectPaint) + canvas.drawRect( + Rect( + rect.left, + rect.bottom + contentPadding / 2, + rect.left + textWidth + contentPadding * 2, + rect.bottom + contentTextPaint.textSize.toInt() + contentPadding + ), + contentRectPaint + ) + canvas.drawText( + content, + (rect.left + contentPadding).toFloat(), + (rect.bottom + contentPadding * 2).toFloat(), + contentTextPaint + ) + } + } + + override fun setAlpha(alpha: Int) { + boundingRectPaint.alpha = alpha + contentRectPaint.alpha = alpha + contentTextPaint.alpha = alpha + } + + override fun setColorFilter(colorFilter: ColorFilter?) { + boundingRectPaint.colorFilter = colorFilter + contentRectPaint.colorFilter = colorFilter + contentTextPaint.colorFilter = colorFilter + } + + @Deprecated("Deprecated in Java", ReplaceWith("PixelFormat.TRANSLUCENT", "android.graphics.PixelFormat")) + override fun getOpacity(): Int = PixelFormat.TRANSLUCENT +} \ No newline at end of file diff --git a/CameraX-MLKit/app/src/main/res/layout/activity_main.xml b/CameraX-MLKit/app/src/main/res/layout/activity_main.xml index 90d64126..94236507 100644 --- a/CameraX-MLKit/app/src/main/res/layout/activity_main.xml +++ b/CameraX-MLKit/app/src/main/res/layout/activity_main.xml @@ -20,6 +20,7 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" + android:id="@+id/myConstraintLayout" tools:context=".MainActivity"> Date: Tue, 3 Jan 2023 12:31:59 -0300 Subject: [PATCH 2/3] Closing textRecognizer when onDestroy --- .../app/src/main/java/com/example/camerax_mlkit/MainActivity.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt index 2c37a4b8..02ff9476 100644 --- a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt +++ b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt @@ -124,6 +124,7 @@ class MainActivity : AppCompatActivity() { super.onDestroy() cameraExecutor.shutdown() barcodeScanner.close() + textRecognizer.close() } private val requestMultiplePermissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> From 4b3adba3da398870af8d4f18bb59b9e0e3359b66 Mon Sep 17 00:00:00 2001 From: Ricardo Souza Date: Thu, 5 Jan 2023 18:09:27 -0300 Subject: [PATCH 3/3] Fixes crash on application and fix permission call --- .../com/example/camerax_mlkit/MainActivity.kt | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt index 02ff9476..ccaa9d2c 100644 --- a/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt +++ b/CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt @@ -59,13 +59,6 @@ class MainActivity : AppCompatActivity() { viewBinding = ActivityMainBinding.inflate(layoutInflater) setContentView(viewBinding.root) - // Request camera permissions - if (allPermissionsGranted()) { - startCamera() - } else { - requestCameraPermission() - } - cameraExecutor = Executors.newSingleThreadExecutor() val options = BarcodeScannerOptions.Builder() .setBarcodeFormats(Barcode.FORMAT_QR_CODE) @@ -76,6 +69,16 @@ class MainActivity : AppCompatActivity() { textRecognizer = TextRecognition.getClient(textOptions) } + override fun onStart() { + super.onStart() + // Request camera permissions + if (allPermissionsGranted()) { + startCamera() + } else { + requestCameraPermission() + } + } + private fun startCamera() { val cameraController = LifecycleCameraController(baseContext) val previewView: PreviewView = viewBinding.viewFinder @@ -143,8 +146,7 @@ class MainActivity : AppCompatActivity() { requestMultiplePermissionLauncher.launch(REQUIRED_PERMISSIONS) }.show() } else { - Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show() - finish() + requestMultiplePermissionLauncher.launch(REQUIRED_PERMISSIONS) } } else { Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show()