patch of Add functionality to request camera data · round #0 · pull #6 · nateholland.bsky.social/PoseDetection

+1

gradle/wrapper/gradle-wrapper.properties

···

       5
       5
        
       validateDistributionUrl=true

     

       6
       6
        
       zipStoreBase=GRADLE_USER_HOME

     

       7
       7
        
       zipStorePath=wrapper/dists

     

       8
       8
       +
       org.gradle.jvmargs=-Xmx4G

+15 -1

posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt

···

       44
       44
        
       import androidx.camera.core.Preview

     

       45
       45
        
       import androidx.camera.core.ImageAnalysis

     

       46
       46
        
       import androidx.camera.core.ImageProxy

     

       47
       47
       +
       import androidx.compose.runtime.DisposableEffect

     

       47
       48
        
       import androidx.compose.runtime.rememberCoroutineScope

     

       48
       49
        
       import androidx.compose.ui.geometry.Rect

     

       49
       50
        
       import com.google.mlkit.vision.pose.PoseDetector

     
···

       75
       76
        
           frontCamera: Boolean,

     

       76
       77
        
           recordingId: String?,

     

       77
       78
        
           focusArea: Rect?,

     

       79
       79
       +
           controller: CameraViewController?,

     

       78
       80
        
           onRecordToggled: (Boolean) -> Unit,

     

       79
       81
        
           onVideoSaved: (String, String) -> Unit,

     

       80
       82
        
       ) {

     
···

       214
       216
        
                           bitmap =

     

       215
       217
        
                               imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat())

     

       216
       218
        
                                   .asImageBitmap().let { inbmp ->

     

       219
       219
       +
                                       controller?.setRequestDataProvider {

     

       220
       220
       +
                                           CameraViewData(

     

       221
       221
       +
                                               width = inbmp.width.toFloat(),

     

       222
       222
       +
                                               height = inbmp.height.toFloat(),

     

       223
       223
       +
                                               rotation = when (imageProxy.imageInfo.rotationDegrees) {

     

       224
       224
       +
                                                   0 -> SensorRotation.ROTATION_0

     

       225
       225
       +
                                                   90 -> SensorRotation.ROTATION_90

     

       226
       226
       +
                                                   180 -> SensorRotation.ROTATION_180

     

       227
       227
       +
                                                   270 -> SensorRotation.ROTATION_270

     

       228
       228
       +
                                                   else -> SensorRotation.ROTATION_0

     

       229
       229
       +
                                               }

     

       230
       230
       +
                                           )

     

       231
       231
       +
                                       }

     

       217
       232
        
                                       addFrameToActiveRecordings(inbmp, timestamp)

     

       218
       233
        
                                       inbmp.drawResults(if(drawSkeleton) it.skeleton else null, drawObjects?.invoke(it.objects)?: emptyList())

     

       219
       234
        
                                   }

     
···

       229
       244
        
               )

     

       230
       245
        
               previewView.scaleType = PreviewView.ScaleType.FIT_CENTER

     

       231
       246
        
           }

     

       232
       232
       -
       

     

       233
       247
        
           Box(

     

       234
       248
        
               modifier = modifier

     

       235
       249
        
           ) {

+27

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt

···

       33
       33
        
           frontCamera: Boolean = true,

     

       34
       34
        
           recordingId: String? = null,

     

       35
       35
        
           focusArea: Rect? = null,

     

       36
       36
       +
           controller: CameraViewController? = null,

     

       36
       37
        
           onRecordToggled: (Boolean) -> Unit = {},

     

       37
       38
        
           onVideoSaved: (String, String) -> Unit

     

       38
       39
        
       )

     
···

       47
       48
        
       enum class DrawableShape {

     

       48
       49
        
       OVAL,RECTANGLE

     

       49
       50
        
       }

     

       51
       51
       +
       

     

       52
       52
       +
       data class CameraViewData(

     

       53
       53
       +
           val width: Float,

     

       54
       54
       +
           val height: Float,

     

       55
       55
       +
           val rotation: SensorRotation,

     

       56
       56
       +
       )

     

       57
       57
       +
       

     

       58
       58
       +
       enum class SensorRotation {

     

       59
       59
       +
           ROTATION_0, ROTATION_90, ROTATION_180, ROTATION_270

     

       60
       60
       +
       }

     

       61
       61
       +
       

     

       62
       62
       +
       

     

       63
       63
       +
       interface CameraViewController {

     

       64
       64
       +
           fun requestData(onResult: (CameraViewData) -> Unit)

     

       65
       65
       +
           fun setRequestDataProvider(provider: (() -> CameraViewData)?)

     

       66
       66
       +
       }

     

       67
       67
       +
       

     

       68
       68
       +
       class CameraViewControllerImpl : CameraViewController {

     

       69
       69
       +
           private var dataProvider: (() -> CameraViewData)? = null

     

       70
       70
       +
           override fun requestData(onResult: (CameraViewData) -> Unit) {

     

       71
       71
       +
               dataProvider?.let { onResult(it()) }

     

       72
       72
       +
           }

     

       73
       73
       +
           override fun setRequestDataProvider(provider: (() -> CameraViewData)?) {

     

       74
       74
       +
               dataProvider = provider

     

       75
       75
       +
           }

     

       76
       76
       +
       }

+2

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt

···

       36
       36
        
           frontCamera: Boolean,

     

       37
       37
        
           recordingId: String?,

     

       38
       38
        
           focusArea: Rect?,

     

       39
       39
       +
           controller: CameraViewController?,

     

       39
       40
        
           onRecordToggled: (Boolean) -> Unit,

     

       40
       41
        
           onVideoSaved: (String, String) -> Unit,

     

       41
       42
        
           ) {

     
···

       58
       59
        
               cameraEngine.value?.apply {

     

       59
       60
        
                   addSkeletonRepository(skeletonRepository)

     

       60
       61
        
                   addCustomObjectRepository(customObjectRepository)

     

       62
       62
       +
                   addCameraViewController(controller)

     

       61
       63
        
                   addFrameListener(frameListener)

     

       62
       64
        
                   setOnVideoSavedCallback(recordingDone)

     

       63
       65
        
                   setDrawOptions(

+10 -4

sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt

···

       35
       35
        
       import chaintech.videoplayer.model.VideoPlayerConfig

     

       36
       36
        
       import chaintech.videoplayer.ui.video.VideoPlayerComposable

     

       37
       37
        
       import chaintech.videoplayer.util.RetrieveMediaDuration

     

       38
       38
       +
       import co.touchlab.kermit.Logger

     

       38
       39
        
       import com.nate.posedetection.theme.AppTheme

     

       39
       40
        
       import com.performancecoachlab.posedetection.camera.CameraView

     

       41
       41
       +
       import com.performancecoachlab.posedetection.camera.CameraViewControllerImpl

     

       40
       42
        
       import com.performancecoachlab.posedetection.camera.DetectMode

     

       41
       43
        
       import com.performancecoachlab.posedetection.camera.DrawableObject

     

       42
       44
        
       import com.performancecoachlab.posedetection.camera.DrawableShape

     
···

       72
       74
        
       

     

       73
       75
        
       @Composable

     

       74
       76
        
       internal fun App() = AppTheme {

     

       75
       75
       -
           var selectedTabIndex by remember { mutableStateOf(1) }

     

       77
       77
       +
           var selectedTabIndex by remember { mutableStateOf(0) }

     

       76
       78
        
           val tabs = listOf("Camera Feed", "Recorded Video")

     

       77
       79
        
           Column {

     

       78
       80
        
               TabRow(selectedTabIndex = selectedTabIndex) {

     
···

       307
       309
        
                   "YOLOv3FP16"

     

       308
       310
        
               )

     

       309
       311
        
           )

     

       312
       312
       +
           val controller = remember { CameraViewControllerImpl() }

     

       310
       313
        
           PermissionProvider().apply {

     

       311
       314
        
               if (!hasCameraPermission()) RequestCameraPermission(onGranted = {

     

       312
       315
        
                   permissionGranted = true

     
···

       344
       347
        
                       CameraView(

     

       345
       348
        
                           skeletonRepository = skeletonRepository,

     

       346
       349
        
                           customObjectRepository = customObjectRespository,

     

       347
       347
       -
                           detectMode = DetectMode.BOTH,

     

       350
       350
       +
                           detectMode = DetectMode.NONE,

     

       348
       351
        
                           drawSkeleton = true,

     

       349
       352
        
                           drawObjects = { obj ->

     

       350
       350
       -
       

     

       351
       353
        
                               obj.map {

     

       352
       354
        
                                   DrawableObject(

     

       353
       355
        
                                       obj = it,

     
···

       361
       363
        
                           modifier = Modifier.weight(1f),

     

       362
       364
        
                           frontCamera = false,

     

       363
       365
        
                           recordingId = recordingId,

     

       366
       366
       +
                           controller = controller,

     

       364
       367
        
                           onVideoSaved = {id,url -> path = url },

     

       365
       368
        
                       )

     

       366
       369
        
                   }

     

       367
       370
        
                   Button(

     

       368
       371
        
                       onClick = {

     

       369
       369
       -
                           recordingId = "${Clock.System.now().epochSeconds}"

     

       372
       372
       +
                           //recordingId = "${Clock.System.now().epochSeconds}"

     

       373
       373
       +
                           controller.requestData { data ->

     

       374
       374
       +
                               Logger.d("CameraViewData: $data")

     

       375
       375
       +
                           }

     

       370
       376
        
                       },

     

       371
       377
        
                       modifier = Modifier.imePadding().padding(16.dp).align(Alignment.TopStart)

     

       372
       378
        
                   ) {

+1

sample/gradle.properties

···

       10
       10
        
       #Android

     

       11
       11
        
       android.useAndroidX=true

     

       12
       12
        
       android.nonTransitiveRClass=true

     

       13
       13
       +
       android.enableJetifier=true

+3

settings.gradle.kts

···

       14
       14
        
               mavenCentral()

     

       15
       15
        
           }

     

       16
       16
        
       }

     

       17
       17
       +
       plugins {

     

       18
       18
       +
           id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0"

     

       19
       19
       +
       }

     

       17
       20
        
       

     

       18
       21
        
       dependencyResolutionManagement {

     

       19
       22
        
           repositories {

+4

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt

···

       19
       19
        
       import kotlinx.cinterop.value

     

       20
       20
        
       import platform.AVFoundation.AVCaptureVideoOrientationLandscapeRight

     

       21
       21
        
       import platform.AVFoundation.AVCaptureVideoPreviewLayer

     

       22
       22
       +
       import platform.CoreFoundation.CFRelease

     

       23
       23
       +
       import platform.CoreFoundation.CFRetain

     

       22
       24
        
       import platform.CoreGraphics.CGImageGetHeight

     

       23
       25
        
       import platform.CoreGraphics.CGImageGetWidth

     

       24
       26
        
       import platform.CoreGraphics.CGImageRef

     
···

       372
       374
        
                       onSkeletonProcessed(null)

     

       373
       375
        
                       return

     

       374
       376
        
                   }

     

       377
       377
       +
                   val retainedBuffer = CFRetain(buffer)

     

       375
       378
        
                   val width = 480uL // You may want to get actual width from buffer if needed

     

       376
       379
        
                   val height = 360uL // You may want to get actual height from buffer if needed

     

       377
       380
        
                   memScoped {

     
···

       553
       556
        
                           handler.performRequests(

     

       554
       557
        
                               listOfNotNull(requestForObjects, requestForSkeleton), errorPtr.ptr

     

       555
       558
        
                           )

     

       559
       559
       +
                           CFRelease(retainedBuffer)

     

       556
       560
        
                           if (errorPtr.value != null) {

     

       557
       561
        
                               //println("Error performing object detection request: ${errorPtr.value}")

     

       558
       562
        
                               onObjectsProcessed(emptyList())