Improved the CameraX-MLKit sample to read QRCode and Text.

CameraX-MLKit/README.md

@@ -1,20 +1,26 @@
 # CameraX-MLKit
 
-This example uses CameraX's MlKitAnalyzer to perform QR Code scanning. For QR Codes that encode Urls, this app will prompt the user to open the Url in a broswer. This app can be adapted to handle other types of QR Code data.
+This example uses CameraX's MlKitAnalyzer to perform QR Code scanning. For QR Codes that encode Urls, this app will prompt the user to open the Url in
+a broswer. This app can be adapted to handle other types of QR Code data. This example also uses CameraX's MlKitAnalyzer to perform Text Recognition.
 
-The interesting part of the code is in `MainActivity.kt` in the `startCamera()` function. There, we set up BarcodeScannerOptions to match on QR Codes. Then we call `cameraController.setImageAnalysisAnalyzer` with an `MlKitAnalyzer` (available as of CameraX 1.2). We also pass in `COORDINATE_SYSTEM_VIEW_REFERENCED` so that CameraX will handle the cordinates coming off of the camera sensor, making it easy to draw a box around the QR Code. Finally, we create a QrCodeDrawable, which is a class defined in this sample, extending View, for displaying an overlay on the QR Code and handling tap events on the QR Code.
+On `onCreate()` we set up BarcodeScannerOptions to match on QR Codes and TextRecognizerOptions to match on Text on image.
+
+The interesting part of the code is in `MainActivity.kt` in the `startCamera()` function. Then we call `cameraController.setImageAnalysisAnalyzer`
+with an `MlKitAnalyzer` (available as of CameraX 1.2). We also pass in `COORDINATE_SYSTEM_VIEW_REFERENCED` so that CameraX will handle the cordinates
+coming off of the camera sensor, making it easy to draw a box around the QR Code. Finally, we create a QrCodeDrawable, which is a class defined in
+this sample, extending View, for displaying an overlay on the QR Code and handling tap events on the QR Code.
 
 You can open this project in Android Studio to explore the code further, and to build and run the application on a test device.
 
 ## Screenshots
 
 <img src="screenshots/camerax-mlkit.png" height="700" alt="Screenshot of QR-code reader app scanning a QR code for the website google.com"/>
 
-## Command line options 
+## Command line options
 
 ### Build
 
-To build the app directly from the command line, run:
+To build the app directly from the command line, run:   ''
 ```sh
 ./gradlew assembleDebug
 ```

CameraX-MLKit/app/build.gradle

@@ -69,4 +69,5 @@ dependencies {
     implementation "androidx.camera:camera-view:${camerax_version}"
 
     implementation 'com.google.mlkit:barcode-scanning:17.0.2'
+    implementation 'com.google.mlkit:text-recognition:16.0.0-beta6'
 }

CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/MainActivity.kt

@@ -18,78 +18,100 @@ package com.example.camerax_mlkit
 
 import android.Manifest
 import android.content.pm.PackageManager
+import android.os.Build
 import android.os.Bundle
-import android.view.View
 import android.widget.Toast
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AppCompatActivity
 import androidx.camera.mlkit.vision.MlKitAnalyzer
 import androidx.camera.view.CameraController.COORDINATE_SYSTEM_VIEW_REFERENCED
 import androidx.camera.view.LifecycleCameraController
 import androidx.camera.view.PreviewView
-import androidx.core.app.ActivityCompat
 import androidx.core.content.ContextCompat
 import com.example.camerax_mlkit.databinding.ActivityMainBinding
+import com.example.camerax_mlkit.utils.BuildRect
+import com.google.android.material.snackbar.Snackbar
 import com.google.mlkit.vision.barcode.BarcodeScanner
 import com.google.mlkit.vision.barcode.BarcodeScannerOptions
 import com.google.mlkit.vision.barcode.BarcodeScanning
 import com.google.mlkit.vision.barcode.common.Barcode
+import com.google.mlkit.vision.text.TextRecognition
+import com.google.mlkit.vision.text.TextRecognizer
+import com.google.mlkit.vision.text.latin.TextRecognizerOptions
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 
 class MainActivity : AppCompatActivity() {
     private lateinit var viewBinding: ActivityMainBinding
     private lateinit var cameraExecutor: ExecutorService
     private lateinit var barcodeScanner: BarcodeScanner
+    private lateinit var textRecognizer: TextRecognizer
+
+    companion object {
+        private const val TAG = "CameraX-MLKit"
+        private const val REQUEST_CODE_PERMISSIONS = 10
+        private val REQUIRED_PERMISSIONS =
+            mutableListOf(Manifest.permission.CAMERA).toTypedArray()
+    }
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         viewBinding = ActivityMainBinding.inflate(layoutInflater)
         setContentView(viewBinding.root)
 
+        cameraExecutor = Executors.newSingleThreadExecutor()
+        val options = BarcodeScannerOptions.Builder()
+            .setBarcodeFormats(Barcode.FORMAT_QR_CODE)
+            .build()
+
+        val textOptions = TextRecognizerOptions.Builder().build()
+        barcodeScanner = BarcodeScanning.getClient(options)
+        textRecognizer = TextRecognition.getClient(textOptions)
+    }
+
+    override fun onStart() {
+        super.onStart()
         // Request camera permissions
         if (allPermissionsGranted()) {
             startCamera()
         } else {
-            ActivityCompat.requestPermissions(
-                this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS
-            )
+            requestCameraPermission()
         }
-
-        cameraExecutor = Executors.newSingleThreadExecutor()
     }
 
     private fun startCamera() {
-        var cameraController = LifecycleCameraController(baseContext)
+        val cameraController = LifecycleCameraController(baseContext)
         val previewView: PreviewView = viewBinding.viewFinder
 
-        val options = BarcodeScannerOptions.Builder()
-            .setBarcodeFormats(Barcode.FORMAT_QR_CODE)
-            .build()
-        barcodeScanner = BarcodeScanning.getClient(options)
-
         cameraController.setImageAnalysisAnalyzer(
             ContextCompat.getMainExecutor(this),
             MlKitAnalyzer(
-                listOf(barcodeScanner),
+                listOf(barcodeScanner, textRecognizer),
                 COORDINATE_SYSTEM_VIEW_REFERENCED,
                 ContextCompat.getMainExecutor(this)
             ) { result: MlKitAnalyzer.Result? ->
+                val textResults = result?.getValue(textRecognizer)
                 val barcodeResults = result?.getValue(barcodeScanner)
-                if ((barcodeResults == null) ||
-                    (barcodeResults.size == 0) ||
-                    (barcodeResults.first() == null)
-                ) {
+
+                previewView.overlay.clear()
+
+                barcodeResults?.getOrNull(0)?.let {
+                    val qrCodeViewModel = QrCodeViewModel(it)
+                    val qrCodeDrawable = BuildRect(qrCodeViewModel.boundingRect, qrCodeViewModel.qrContent)
+                    previewView.setOnTouchListener(qrCodeViewModel.qrCodeTouchCallback)
+                    previewView.overlay.add(qrCodeDrawable)
+                } ?: kotlin.run {
                     previewView.overlay.clear()
                     previewView.setOnTouchListener { _, _ -> false } //no-op
-                    return@MlKitAnalyzer
                 }
 
-                val qrCodeViewModel = QrCodeViewModel(barcodeResults[0])
-                val qrCodeDrawable = QrCodeDrawable(qrCodeViewModel)
-
-                previewView.setOnTouchListener(qrCodeViewModel.qrCodeTouchCallback)
-                previewView.overlay.clear()
-                previewView.overlay.add(qrCodeDrawable)
+                textResults?.textBlocks?.flatMap { it.lines }?.forEach {
+                    val textViewModel = TextViewModel(it)
+                    val textDrawable = BuildRect(textViewModel.boundingRect, textViewModel.lineContent)
+                    previewView.overlay.add(textDrawable)
+                } ?: kotlin.run {
+                    previewView.overlay.clear()
+                }
             }
         )
 
@@ -98,38 +120,37 @@ class MainActivity : AppCompatActivity() {
     }
 
     private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all {
-        ContextCompat.checkSelfPermission(
-            baseContext, it) == PackageManager.PERMISSION_GRANTED
+        ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED
     }
 
     override fun onDestroy() {
         super.onDestroy()
         cameraExecutor.shutdown()
         barcodeScanner.close()
+        textRecognizer.close()
     }
 
-    companion object {
-        private const val TAG = "CameraX-MLKit"
-        private const val REQUEST_CODE_PERMISSIONS = 10
-        private val REQUIRED_PERMISSIONS =
-            mutableListOf (
-                Manifest.permission.CAMERA
-            ).toTypedArray()
+    private val requestMultiplePermissionLauncher = registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
+        if (REQUIRED_PERMISSIONS.all { result[it] == true }) {
+            startCamera()
+        } else {
+            requestCameraPermission()
+        }
     }
 
-    override fun onRequestPermissionsResult(
-        requestCode: Int, permissions: Array<String>, grantResults:
-        IntArray) {
-        super.onRequestPermissionsResult(requestCode, permissions, grantResults)
-        if (requestCode == REQUEST_CODE_PERMISSIONS) {
-            if (allPermissionsGranted()) {
-                startCamera()
+    private fun requestCameraPermission() {
+        if (Build.VERSION.SDK_INT > Build.VERSION_CODES.M) {
+            if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) {
+                Snackbar.make(viewBinding.myConstraintLayout, "You need to provide camera permission to use this.", Snackbar.LENGTH_INDEFINITE)
+                    .setAction("Request Camera Permission") {
+                        requestMultiplePermissionLauncher.launch(REQUIRED_PERMISSIONS)
+                    }.show()
             } else {
-                Toast.makeText(this,
-                    "Permissions not granted by the user.",
-                    Toast.LENGTH_SHORT).show()
-                finish()
+                requestMultiplePermissionLauncher.launch(REQUIRED_PERMISSIONS)
             }
+        } else {
+            Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show()
+            finish()
         }
     }
 }

CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/TextViewModel.kt

@@ -0,0 +1,19 @@
+package com.example.camerax_mlkit
+
+import android.graphics.Rect
+import android.view.MotionEvent
+import android.view.View
+import com.google.mlkit.vision.text.Text
+
+class TextViewModel(line: Text.Line) {
+    var boundingRect: Rect? = line.boundingBox
+    var lineContent: String = ""
+    var lineTouchCallback = { v: View, e: MotionEvent -> false }
+
+    init {
+        lineContent = line.text
+        lineTouchCallback = { v: View, e: MotionEvent ->
+            true // return true from the callback to signify the event was handled
+        }
+    }
+}

CameraX-MLKit/app/src/main/java/com/example/camerax_mlkit/utils/BuildRect.kt

@@ -0,0 +1,65 @@
+package com.example.camerax_mlkit.utils
+
+import android.graphics.*
+import android.graphics.drawable.Drawable
+
+class BuildRect(private val boundingRect: Rect?, private val content: String) : Drawable() {
+
+    private val boundingRectPaint = Paint().apply {
+        style = Paint.Style.STROKE
+        color = Color.YELLOW
+        strokeWidth = 5F
+        alpha = 200
+    }
+
+    private val contentRectPaint = Paint().apply {
+        style = Paint.Style.FILL
+        color = Color.YELLOW
+        alpha = 255
+    }
+
+    private val contentTextPaint = Paint().apply {
+        color = Color.DKGRAY
+        alpha = 255
+        textSize = 36F
+    }
+
+    private val contentPadding = 25
+    private var textWidth = contentTextPaint.measureText(content).toInt()
+
+    override fun draw(canvas: Canvas) {
+        boundingRect?.let { rect ->
+            canvas.drawRect(rect, boundingRectPaint)
+            canvas.drawRect(
+                Rect(
+                    rect.left,
+                    rect.bottom + contentPadding / 2,
+                    rect.left + textWidth + contentPadding * 2,
+                    rect.bottom + contentTextPaint.textSize.toInt() + contentPadding
+                ),
+                contentRectPaint
+            )
+            canvas.drawText(
+                content,
+                (rect.left + contentPadding).toFloat(),
+                (rect.bottom + contentPadding * 2).toFloat(),
+                contentTextPaint
+            )
+        }
+    }
+
+    override fun setAlpha(alpha: Int) {
+        boundingRectPaint.alpha = alpha
+        contentRectPaint.alpha = alpha
+        contentTextPaint.alpha = alpha
+    }
+
+    override fun setColorFilter(colorFilter: ColorFilter?) {
+        boundingRectPaint.colorFilter = colorFilter
+        contentRectPaint.colorFilter = colorFilter
+        contentTextPaint.colorFilter = colorFilter
+    }
+
+    @Deprecated("Deprecated in Java", ReplaceWith("PixelFormat.TRANSLUCENT", "android.graphics.PixelFormat"))
+    override fun getOpacity(): Int = PixelFormat.TRANSLUCENT
+}