From a9fac6ec2e7c13df4725753dee34160a19dc56e0 Mon Sep 17 00:00:00 2001 From: nate Date: Fri, 26 Sep 2025 22:45:58 +0300 Subject: [PATCH] chore: checking --- gradle/wrapper/gradle-wrapper.properties | 1 + .../camera/CameraView.android.kt | 16 ++++++++++- .../posedetection/camera/CameraView.kt | 27 +++++++++++++++++++ .../posedetection/camera/CameraEngine.kt | 19 ++++++++++++- .../posedetection/camera/CameraView.ios.kt | 2 ++ .../kotlin/com/nate/posedetection/App.kt | 14 +++++++--- sample/gradle.properties | 1 + settings.gradle.kts | 3 +++ 8 files changed, 77 insertions(+), 6 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37f853b..92bcbed 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -5,3 +5,4 @@ networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +org.gradle.jvmargs=-Xmx4G \ No newline at end of file diff --git a/posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt b/posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt index 9fcf432..17e4ceb 100644 --- a/posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt +++ b/posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt @@ -44,6 +44,7 @@ import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.core.Preview import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageProxy +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.ui.geometry.Rect import com.google.mlkit.vision.pose.PoseDetector @@ -75,6 +76,7 @@ actual fun CameraView( frontCamera: Boolean, recordingId: String?, focusArea: Rect?, + controller: CameraViewController?, onRecordToggled: (Boolean) -> Unit, onVideoSaved: (String, String) -> Unit, ) { @@ -214,6 +216,19 @@ actual fun CameraView( bitmap = imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat()) .asImageBitmap().let { inbmp -> + controller?.setRequestDataProvider { + CameraViewData( + width = inbmp.width.toFloat(), + height = inbmp.height.toFloat(), + rotation = when (imageProxy.imageInfo.rotationDegrees) { + 0 -> SensorRotation.ROTATION_0 + 90 -> SensorRotation.ROTATION_90 + 180 -> SensorRotation.ROTATION_180 + 270 -> SensorRotation.ROTATION_270 + else -> SensorRotation.ROTATION_0 + } + ) + } addFrameToActiveRecordings(inbmp, timestamp) inbmp.drawResults(if(drawSkeleton) it.skeleton else null, drawObjects?.invoke(it.objects)?: emptyList()) } @@ -229,7 +244,6 @@ actual fun CameraView( ) previewView.scaleType = PreviewView.ScaleType.FIT_CENTER } - Box( modifier = modifier ) { diff --git a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt index 91d1f68..b97e994 100644 --- a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt +++ b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt @@ -33,6 +33,7 @@ expect fun CameraView( frontCamera: Boolean = true, recordingId: String? = null, focusArea: Rect? = null, + controller: CameraViewController? = null, onRecordToggled: (Boolean) -> Unit = {}, onVideoSaved: (String, String) -> Unit ) @@ -47,3 +48,29 @@ data class DrawableObject( enum class DrawableShape { OVAL,RECTANGLE } + +data class CameraViewData( + val width: Float, + val height: Float, + val rotation: SensorRotation, +) + +enum class SensorRotation { + ROTATION_0, ROTATION_90, ROTATION_180, ROTATION_270 +} + + +interface CameraViewController { + fun requestData(onResult: (CameraViewData) -> Unit) + fun setRequestDataProvider(provider: (() -> CameraViewData)?) +} + +class CameraViewControllerImpl : CameraViewController { + private var dataProvider: (() -> CameraViewData)? = null + override fun requestData(onResult: (CameraViewData) -> Unit) { + dataProvider?.let { onResult(it()) } + } + override fun setRequestDataProvider(provider: (() -> CameraViewData)?) { + dataProvider = provider + } +} \ No newline at end of file diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt index f0c7c9a..31e0004 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt @@ -201,6 +201,10 @@ class CameraEngine : UIViewController(null, null) { cameraController.customObjectRepository = repository } + fun addCameraViewController(controller: CameraViewController?) { + cameraController.cameraViewController = controller + } + fun addFrameListener(listener: FrameRepository) { cameraController.frameListener = listener } @@ -246,6 +250,7 @@ class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegat var isUsingFrontCamera = true var skeletonRepository: SkeletonRepository? = null var customObjectRepository: CustomObjectRespository? = null + var cameraViewController: CameraViewController? = null var frameListener: FrameRepository? = null var onError: ((CameraException) -> Unit)? = null var startTime: Long? = null @@ -532,7 +537,6 @@ class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegat previewObjects?.also { objects -> customObjectRepository?.updateCustomObject(objects) } - preview.bounds.useContents { Pair( size.width.toInt(), size.height.toInt() @@ -544,6 +548,19 @@ class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegat if (drawSkeleton) previewSkeleton else null, drawObjects?.invoke(previewObjects) ?: emptyList() ).also { drawn -> + cameraViewController?.setRequestDataProvider { + CameraViewData( + width = bo.first.toFloat(), + height = bo.second.toFloat(), + rotation = when (currentVideoOrientation()) { + AVCaptureVideoOrientationPortrait -> SensorRotation.ROTATION_90 + AVCaptureVideoOrientationLandscapeRight -> SensorRotation.ROTATION_180 + AVCaptureVideoOrientationPortraitUpsideDown -> SensorRotation.ROTATION_270 + AVCaptureVideoOrientationLandscapeLeft -> SensorRotation.ROTATION_0 + else -> SensorRotation.ROTATION_0 + } + ) + } frameListener?.updateFrame( drawn ) diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt index 45a2e25..8d6155b 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt @@ -36,6 +36,7 @@ actual fun CameraView( frontCamera: Boolean, recordingId: String?, focusArea: Rect?, + controller: CameraViewController?, onRecordToggled: (Boolean) -> Unit, onVideoSaved: (String, String) -> Unit, ) { @@ -58,6 +59,7 @@ actual fun CameraView( cameraEngine.value?.apply { addSkeletonRepository(skeletonRepository) addCustomObjectRepository(customObjectRepository) + addCameraViewController(controller) addFrameListener(frameListener) setOnVideoSavedCallback(recordingDone) setDrawOptions( diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 7b095cc..cb7ce19 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -35,8 +35,10 @@ import chaintech.videoplayer.model.ScreenResize import chaintech.videoplayer.model.VideoPlayerConfig import chaintech.videoplayer.ui.video.VideoPlayerComposable import chaintech.videoplayer.util.RetrieveMediaDuration +import co.touchlab.kermit.Logger import com.nate.posedetection.theme.AppTheme import com.performancecoachlab.posedetection.camera.CameraView +import com.performancecoachlab.posedetection.camera.CameraViewControllerImpl import com.performancecoachlab.posedetection.camera.DetectMode import com.performancecoachlab.posedetection.camera.DrawableObject import com.performancecoachlab.posedetection.camera.DrawableShape @@ -72,7 +74,7 @@ import kotlin.time.ExperimentalTime @Composable internal fun App() = AppTheme { - var selectedTabIndex by remember { mutableStateOf(1) } + var selectedTabIndex by remember { mutableStateOf(0) } val tabs = listOf("Camera Feed", "Recorded Video") Column { TabRow(selectedTabIndex = selectedTabIndex) { @@ -307,6 +309,7 @@ fun CameraSample() { "YOLOv3FP16" ) ) + val controller = remember { CameraViewControllerImpl() } PermissionProvider().apply { if (!hasCameraPermission()) RequestCameraPermission(onGranted = { permissionGranted = true @@ -344,10 +347,9 @@ fun CameraSample() { CameraView( skeletonRepository = skeletonRepository, customObjectRepository = customObjectRespository, - detectMode = DetectMode.BOTH, + detectMode = DetectMode.NONE, drawSkeleton = true, drawObjects = { obj -> - obj.map { DrawableObject( obj = it, @@ -361,12 +363,16 @@ fun CameraSample() { modifier = Modifier.weight(1f), frontCamera = false, recordingId = recordingId, + controller = controller, onVideoSaved = {id,url -> path = url }, ) } Button( onClick = { - recordingId = "${Clock.System.now().epochSeconds}" + //recordingId = "${Clock.System.now().epochSeconds}" + controller.requestData { data -> + Logger.d("CameraViewData: $data") + } }, modifier = Modifier.imePadding().padding(16.dp).align(Alignment.TopStart) ) { diff --git a/sample/gradle.properties b/sample/gradle.properties index 0996e58..aed498c 100644 --- a/sample/gradle.properties +++ b/sample/gradle.properties @@ -10,3 +10,4 @@ kotlin.daemon.jvmargs=-Xmx4G #Android android.useAndroidX=true android.nonTransitiveRClass=true +android.enableJetifier=true \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index b0a87c0..d9c637b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,6 +14,9 @@ pluginManagement { mavenCentral() } } +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0" +} dependencyResolutionManagement { repositories { -- 2.43.0 From 78dfdfa0bf7ecff83b79db337edd34910a021238 Mon Sep 17 00:00:00 2001 From: nate Date: Fri, 26 Sep 2025 23:37:44 +0300 Subject: [PATCH] fix: retain buffer --- .../posedetection/camera/FrameProcessor.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt index a940535..84a8892 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt @@ -19,6 +19,8 @@ import kotlinx.cinterop.useContents import kotlinx.cinterop.value import platform.AVFoundation.AVCaptureVideoOrientationLandscapeRight import platform.AVFoundation.AVCaptureVideoPreviewLayer +import platform.CoreFoundation.CFRelease +import platform.CoreFoundation.CFRetain import platform.CoreGraphics.CGImageGetHeight import platform.CoreGraphics.CGImageGetWidth import platform.CoreGraphics.CGImageRef @@ -372,6 +374,7 @@ class FrameProcessor(var modelObj: VNCoreMLModel?) { onSkeletonProcessed(null) return } + val retainedBuffer = CFRetain(buffer) val width = 480uL // You may want to get actual width from buffer if needed val height = 360uL // You may want to get actual height from buffer if needed memScoped { @@ -553,6 +556,7 @@ class FrameProcessor(var modelObj: VNCoreMLModel?) { handler.performRequests( listOfNotNull(requestForObjects, requestForSkeleton), errorPtr.ptr ) + CFRelease(retainedBuffer) if (errorPtr.value != null) { //println("Error performing object detection request: ${errorPtr.value}") onObjectsProcessed(emptyList()) -- 2.43.0 From fafc5a534c8c7fb8909a00b37da77c1b23f10af6 Mon Sep 17 00:00:00 2001 From: nate Date: Fri, 26 Sep 2025 23:43:39 +0300 Subject: [PATCH] feat: function for requesting camera data --- .../performancecoachlab/posedetection/camera/CameraEngine.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt index 31e0004..8d48db3 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt @@ -554,9 +554,9 @@ class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegat height = bo.second.toFloat(), rotation = when (currentVideoOrientation()) { AVCaptureVideoOrientationPortrait -> SensorRotation.ROTATION_90 - AVCaptureVideoOrientationLandscapeRight -> SensorRotation.ROTATION_180 + AVCaptureVideoOrientationLandscapeRight -> SensorRotation.ROTATION_0 AVCaptureVideoOrientationPortraitUpsideDown -> SensorRotation.ROTATION_270 - AVCaptureVideoOrientationLandscapeLeft -> SensorRotation.ROTATION_0 + AVCaptureVideoOrientationLandscapeLeft -> SensorRotation.ROTATION_180 else -> SensorRotation.ROTATION_0 } ) -- 2.43.0