From 544031aeae868f1d7884b4398a91d1070d4a1819 Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 17:49:11 +0300 Subject: [PATCH] feat: retrieve timestamps from video --- .../recording/VideoUtils.android.kt | 46 ++++++ .../posedetection/recording/VideoUtils.kt | 4 + .../posedetection/recording/VideoUtils.ios.kt | 142 +++++++++++++++++- .../kotlin/com/nate/posedetection/App.kt | 8 +- 4 files changed, 198 insertions(+), 2 deletions(-) diff --git a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt index 94f8ed9..1f2e7ea 100644 --- a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt +++ b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt @@ -2,6 +2,8 @@ package com.performancecoachlab.posedetection.recording import android.app.Application import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat import android.media.MediaMetadataRetriever import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -39,6 +41,50 @@ actual suspend fun extractFrame( } } +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val ctx = VideoExtractionContext.get() + val uri = videoPath.toUri() + val extractor = MediaExtractor() + + val out = ArrayList() + try { + extractor.setDataSource(ctx, uri, null) + + // Select video track + var videoTrack = -1 + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith("video/")) { + videoTrack = i + break + } + } + if (videoTrack == -1) return@withContext out + + extractor.selectTrack(videoTrack) + + var idx = 0 + while (true) { + val sampleTimeUs = extractor.sampleTime + if (sampleTimeUs < 0) break + + out.add(sampleTimeUs / 1000L) // ms + if (out.size >= Int.MAX_VALUE) break + idx++ + + if (!extractor.advance()) break + } + } catch (_: Throwable) { + // swallow; return what we have + } finally { + extractor.release() + } + out +} + actual object VideoExtractionContext { private lateinit var application: Application diff --git a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt index 009314a..f6566de 100644 --- a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt +++ b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt @@ -4,5 +4,9 @@ expect suspend fun extractFrame( videoPath: String, frameTimestamp: Long ): InputFrame? +expect suspend fun listVideoFrameTimestamps( + videoPath: String, +): List + @Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") expect object VideoExtractionContext \ No newline at end of file diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt index fcdb320..7898ac7 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt @@ -1,17 +1,33 @@ package com.performancecoachlab.posedetection.recording +import kotlinx.cinterop.BetaInteropApi import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.ObjCObjectVar +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.value import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.withContext import platform.AVFoundation.AVAsset import platform.AVFoundation.AVAssetImageGenerator +import platform.AVFoundation.AVAssetReader +import platform.AVFoundation.AVAssetReaderTrackOutput +import platform.AVFoundation.AVAssetTrack +import platform.AVFoundation.AVMediaTypeVideo import platform.AVFoundation.CMTimeValue +import platform.AVFoundation.tracksWithMediaType import platform.AVFoundation.valueWithCMTime import platform.CoreMedia.CMTimeMake import platform.CoreMedia.CMTimeMakeWithSeconds +import platform.CoreMedia.CMTimeGetSeconds +import platform.CoreMedia.CMSampleBufferGetPresentationTimeStamp +import platform.CoreFoundation.CFRelease +import platform.Foundation.NSError import platform.Foundation.NSURL import platform.Foundation.NSValue +import platform.Foundation.NSFileManager @OptIn(ExperimentalForeignApi::class) actual suspend fun extractFrame( @@ -39,8 +55,132 @@ actual suspend fun extractFrame( } } +@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class) +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val out = mutableListOf() + + try { + val resolvedUrl = createUrl(url = videoPath) + println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}") + + // AVAssetReader requires a local file URL. Bail out early for remote URLs. + if (!resolvedUrl.isFileURL()) { + println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...") + return@withContext out + } + + // Verify the file actually exists at path + val path = resolvedUrl.path + if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) { + println("File does not exist at path: ${'$'}path") + return@withContext out + } + + val asset = AVAsset.assetWithURL(resolvedUrl) + + // Use synchronous approach - get tracks directly using tracksWithMediaType + val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo) + + if (videoTracks.isEmpty()) { + println("No video tracks found in video: ${'$'}videoPath") + return@withContext out + } + + val videoTrack = videoTracks.first() as AVAssetTrack + println("Found video track: ${'$'}{videoTrack}") + + memScoped { + val errorPtr = alloc>() + val reader = AVAssetReader.assetReaderWithAsset(asset, error = errorPtr.ptr) + + if (reader == null) { + val err = errorPtr.value + println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}") + return@withContext out + } + + val trackOutput = AVAssetReaderTrackOutput.assetReaderTrackOutputWithTrack( + track = videoTrack, + outputSettings = null + ) + + if (!reader.canAddOutput(trackOutput)) { + println("Cannot add track output to reader") + return@withContext out + } + + reader.addOutput(trackOutput) + + if (!reader.startReading()) { + println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}") + return@withContext out + } + + println("Started reading video frames...") + var frameCount = 0 + while (true) { + val sampleBuffer = trackOutput.copyNextSampleBuffer() + if (sampleBuffer == null) { + break + } + + val presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + val timeInSeconds = CMTimeGetSeconds(presentationTime) + val timeInMillis = (timeInSeconds * 1000.0).toLong() + + out.add(timeInMillis) + frameCount++ + + // Release the sample buffer + CFRelease(sampleBuffer) + + if (out.size >= Int.MAX_VALUE) break + } + + println("Extracted ${'$'}frameCount frame timestamps") + } + + } catch (e: Exception) { + println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}") + e.printStackTrace() + } + + println("Returning ${'$'}{out.size} timestamps") + out +} + fun createUrl(url: String): NSURL { - return NSURL.fileURLWithPath(url) + val trimmed = url.trim() + + // Remote URLs: return as-is + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return requireNotNull(NSURL.URLWithString(trimmed)) { "Invalid http(s) URL: ${'$'}trimmed" } + } + + // Well-formed file URLs + if (trimmed.startsWith("file://")) { + // Normalize to a proper file URL via its path component to avoid odd host parts + NSURL.URLWithString(trimmed)?.let { fileUrl -> + fileUrl.path?.let { path -> + return NSURL.fileURLWithPath(path) + } + return fileUrl + } + } + + // Malformed file: prefix (e.g., "file:/var/..."), coerce to path + if (trimmed.startsWith("file:")) { + var path = trimmed.removePrefix("file:") + // Remove duplicate leading slashes + while (path.startsWith("//")) path = path.removePrefix("//") + if (!path.startsWith("/")) path = "/${'$'}path" + return NSURL.fileURLWithPath(path) + } + + // Otherwise treat as a filesystem path + return NSURL.fileURLWithPath(trimmed) } actual object VideoExtractionContext \ No newline at end of file diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 217bc14..13953f8 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -49,6 +49,7 @@ import com.performancecoachlab.posedetection.recording.AnalysisObject import com.performancecoachlab.posedetection.recording.FrameAnalyser import com.performancecoachlab.posedetection.recording.InputFrame import com.performancecoachlab.posedetection.recording.extractFrame +import com.performancecoachlab.posedetection.recording.listVideoFrameTimestamps import com.performancecoachlab.posedetection.skeleton.Pose import com.performancecoachlab.posedetection.skeleton.SkeletonRepository import io.github.vinceglb.filekit.FileKit @@ -68,7 +69,7 @@ import kotlin.time.ExperimentalTime @Composable internal fun App() = AppTheme { - var selectedTabIndex by remember { mutableStateOf(0) } + var selectedTabIndex by remember { mutableStateOf(1) } val tabs = listOf("Camera Feed", "Recorded Video") Column { TabRow(selectedTabIndex = selectedTabIndex) { @@ -186,6 +187,11 @@ fun FrameAnalysis( } } + LaunchedEffect(url){ + val timestamps = listVideoFrameTimestamps(url) + println("Timestamps: $timestamps") + } + LaunchedEffect(url, frame) { currentJob?.cancel() currentJob = coroutineScope.launch { -- 2.43.0 From 939fc7fe7a010273514eb7d706ea669100504a5d Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 18:08:36 +0300 Subject: [PATCH] feat: use explicit timestamps for better accuracy --- .../src/commonMain/kotlin/com/nate/posedetection/App.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 13953f8..60efc68 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -187,9 +187,11 @@ fun FrameAnalysis( } } + val frameMap: MutableMap> = mutableMapOf() + LaunchedEffect(url){ val timestamps = listVideoFrameTimestamps(url) - println("Timestamps: $timestamps") + frameMap[url] = timestamps } LaunchedEffect(url, frame) { @@ -224,7 +226,10 @@ fun FrameAnalysis( // Handle any error that may occur e.printStackTrace() } finally { - (frame + 20L).also { + val nextFrame = frameMap[url]?.let { frameTimeStamps -> + frameTimeStamps.firstOrNull { it > frame} + }?:(frame + 20L) + nextFrame.also { if (it < timeRange.second) { frame = it } else { -- 2.43.0 From fda161167d02b581716fe7fc1dee4a4b4010fe3c Mon Sep 17 00:00:00 2001 From: florian-kima Date: Wed, 17 Sep 2025 09:13:06 +0300 Subject: [PATCH] chore: add Kermit for Logging --- posedetection/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/posedetection/build.gradle.kts b/posedetection/build.gradle.kts index bd714d5..2d855da 100644 --- a/posedetection/build.gradle.kts +++ b/posedetection/build.gradle.kts @@ -68,6 +68,7 @@ kotlin { implementation(compose.material3) implementation(compose.components.resources) implementation(compose.components.uiToolingPreview) + api("co.touchlab:kermit:2.0.4") } commonTest.dependencies { -- 2.43.0 From 8fecd3aa08ef03a5784e95639c797e548421bbaf Mon Sep 17 00:00:00 2001 From: florian-kima Date: Wed, 17 Sep 2025 13:04:38 +0300 Subject: [PATCH] feat: handle Exceptions correctly and proper Logging --- .../recording/VideoUtils.android.kt | 13 +++++-- .../posedetection/recording/VideoUtils.ios.kt | 38 +++++++++++++------ .../kotlin/com/nate/posedetection/App.kt | 9 ++++- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt index 1f2e7ea..39be3bc 100644 --- a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt +++ b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt @@ -8,6 +8,7 @@ import android.media.MediaMetadataRetriever import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import androidx.core.net.toUri +import co.touchlab.kermit.Logger actual suspend fun extractFrame( videoPath: String, frameTimestamp: Long @@ -62,7 +63,11 @@ actual suspend fun listVideoFrameTimestamps( break } } - if (videoTrack == -1) return@withContext out + + if (videoTrack == -1) { + Logger.i { "No video track found in file: $videoPath" } + return@withContext out + } extractor.selectTrack(videoTrack) @@ -77,8 +82,10 @@ actual suspend fun listVideoFrameTimestamps( if (!extractor.advance()) break } - } catch (_: Throwable) { - // swallow; return what we have + } catch (e: Exception) { + Logger.i { "Exception extracting frames from $videoPath: ${e::class.simpleName} - ${e.message}" } + } catch (t: Throwable) { + Logger.i { "CRASH prevented during frame extraction for $videoPath: ${t::class.simpleName}" } } finally { extractor.release() } diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt index 7898ac7..6cc85ee 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt @@ -28,6 +28,7 @@ import platform.Foundation.NSError import platform.Foundation.NSURL import platform.Foundation.NSValue import platform.Foundation.NSFileManager +import co.touchlab.kermit.Logger @OptIn(ExperimentalForeignApi::class) actual suspend fun extractFrame( @@ -55,6 +56,16 @@ actual suspend fun extractFrame( } } +private fun NSURL.safeDescription(): String { + val scheme = this.scheme ?: "unknown" + val fileName = this.lastPathComponent ?: "unknown" + return "$scheme://.../$fileName" +} + +private fun safeFileName(path: String?): String { + return path?.substringAfterLast("/") ?: "unknown" +} + @OptIn(ExperimentalForeignApi::class, BetaInteropApi::class) actual suspend fun listVideoFrameTimestamps( videoPath: String, @@ -63,18 +74,18 @@ actual suspend fun listVideoFrameTimestamps( try { val resolvedUrl = createUrl(url = videoPath) - println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}") + Logger.i { "[iOS] listVideoFrameTimestamps using URL: ${resolvedUrl.safeDescription()}" } // AVAssetReader requires a local file URL. Bail out early for remote URLs. if (!resolvedUrl.isFileURL()) { - println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...") + Logger.i { "AVAssetReader requires a local file URL; got scheme: ${resolvedUrl.scheme}" } return@withContext out } // Verify the file actually exists at path val path = resolvedUrl.path if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) { - println("File does not exist at path: ${'$'}path") + Logger.i { "File does not exist: ${safeFileName(path)}" } return@withContext out } @@ -84,12 +95,12 @@ actual suspend fun listVideoFrameTimestamps( val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo) if (videoTracks.isEmpty()) { - println("No video tracks found in video: ${'$'}videoPath") + Logger.i { "No video tracks found in video: ${safeFileName(videoPath)}" } return@withContext out } val videoTrack = videoTracks.first() as AVAssetTrack - println("Found video track: ${'$'}{videoTrack}") + Logger.i { "Found video track with ID: ${videoTrack.trackID}" } memScoped { val errorPtr = alloc>() @@ -97,7 +108,7 @@ actual suspend fun listVideoFrameTimestamps( if (reader == null) { val err = errorPtr.value - println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}") + Logger.i { "Failed to create AVAssetReader: ${err?.localizedDescription}" } return@withContext out } @@ -107,18 +118,18 @@ actual suspend fun listVideoFrameTimestamps( ) if (!reader.canAddOutput(trackOutput)) { - println("Cannot add track output to reader") + Logger.i { "Cannot add track output to reader" } return@withContext out } reader.addOutput(trackOutput) if (!reader.startReading()) { - println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}") + Logger.i { "Failed to start reading: ${reader.error?.localizedDescription}" } return@withContext out } - println("Started reading video frames...") + Logger.i { "Started reading video frames..." } var frameCount = 0 while (true) { val sampleBuffer = trackOutput.copyNextSampleBuffer() @@ -139,15 +150,18 @@ actual suspend fun listVideoFrameTimestamps( if (out.size >= Int.MAX_VALUE) break } - println("Extracted ${'$'}frameCount frame timestamps") + Logger.i { "Extracted $frameCount frame timestamps" } } } catch (e: Exception) { - println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}") + Logger.i { + val safeName = safeFileName(videoPath) + "Exception extracting frames from $safeName: ${e::class.simpleName} - ${e.message ?: "No message"}" + } e.printStackTrace() } - println("Returning ${'$'}{out.size} timestamps") + Logger.i { "Returning ${out.size} timestamps" } out } diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 60efc68..9fc4308 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -66,6 +66,7 @@ import kotlinx.coroutines.launch import kotlin.math.roundToLong import kotlin.time.Clock import kotlin.time.ExperimentalTime +import co.touchlab.kermit.Logger @Composable internal fun App() = AppTheme { @@ -228,7 +229,13 @@ fun FrameAnalysis( } finally { val nextFrame = frameMap[url]?.let { frameTimeStamps -> frameTimeStamps.firstOrNull { it > frame} - }?:(frame + 20L) + }?:( + // Fallback: + // If no timestamp is found (e.g., end of list or metadata missing), + // assume a 50 FPS video and increment by 20ms. + // This keeps playback and processing moving forward smoothly. + frame + 20L + ) nextFrame.also { if (it < timeRange.second) { frame = it -- 2.43.0 From a49c31c2042cbb34e8aa2f909c6c7f90bbfba5ef Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 17:49:11 +0300 Subject: [PATCH] feat: retrieve timestamps from video --- .../recording/VideoUtils.android.kt | 46 ++++++ .../posedetection/recording/VideoUtils.kt | 4 + .../posedetection/recording/VideoUtils.ios.kt | 142 +++++++++++++++++- .../kotlin/com/nate/posedetection/App.kt | 8 +- 4 files changed, 198 insertions(+), 2 deletions(-) diff --git a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt index 94f8ed9..1f2e7ea 100644 --- a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt +++ b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt @@ -2,6 +2,8 @@ package com.performancecoachlab.posedetection.recording import android.app.Application import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat import android.media.MediaMetadataRetriever import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -39,6 +41,50 @@ actual suspend fun extractFrame( } } +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val ctx = VideoExtractionContext.get() + val uri = videoPath.toUri() + val extractor = MediaExtractor() + + val out = ArrayList() + try { + extractor.setDataSource(ctx, uri, null) + + // Select video track + var videoTrack = -1 + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith("video/")) { + videoTrack = i + break + } + } + if (videoTrack == -1) return@withContext out + + extractor.selectTrack(videoTrack) + + var idx = 0 + while (true) { + val sampleTimeUs = extractor.sampleTime + if (sampleTimeUs < 0) break + + out.add(sampleTimeUs / 1000L) // ms + if (out.size >= Int.MAX_VALUE) break + idx++ + + if (!extractor.advance()) break + } + } catch (_: Throwable) { + // swallow; return what we have + } finally { + extractor.release() + } + out +} + actual object VideoExtractionContext { private lateinit var application: Application diff --git a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt index 009314a..f6566de 100644 --- a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt +++ b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt @@ -4,5 +4,9 @@ expect suspend fun extractFrame( videoPath: String, frameTimestamp: Long ): InputFrame? +expect suspend fun listVideoFrameTimestamps( + videoPath: String, +): List + @Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") expect object VideoExtractionContext \ No newline at end of file diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt index fcdb320..7898ac7 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt @@ -1,17 +1,33 @@ package com.performancecoachlab.posedetection.recording +import kotlinx.cinterop.BetaInteropApi import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.ObjCObjectVar +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.value import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.withContext import platform.AVFoundation.AVAsset import platform.AVFoundation.AVAssetImageGenerator +import platform.AVFoundation.AVAssetReader +import platform.AVFoundation.AVAssetReaderTrackOutput +import platform.AVFoundation.AVAssetTrack +import platform.AVFoundation.AVMediaTypeVideo import platform.AVFoundation.CMTimeValue +import platform.AVFoundation.tracksWithMediaType import platform.AVFoundation.valueWithCMTime import platform.CoreMedia.CMTimeMake import platform.CoreMedia.CMTimeMakeWithSeconds +import platform.CoreMedia.CMTimeGetSeconds +import platform.CoreMedia.CMSampleBufferGetPresentationTimeStamp +import platform.CoreFoundation.CFRelease +import platform.Foundation.NSError import platform.Foundation.NSURL import platform.Foundation.NSValue +import platform.Foundation.NSFileManager @OptIn(ExperimentalForeignApi::class) actual suspend fun extractFrame( @@ -39,8 +55,132 @@ actual suspend fun extractFrame( } } +@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class) +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val out = mutableListOf() + + try { + val resolvedUrl = createUrl(url = videoPath) + println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}") + + // AVAssetReader requires a local file URL. Bail out early for remote URLs. + if (!resolvedUrl.isFileURL()) { + println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...") + return@withContext out + } + + // Verify the file actually exists at path + val path = resolvedUrl.path + if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) { + println("File does not exist at path: ${'$'}path") + return@withContext out + } + + val asset = AVAsset.assetWithURL(resolvedUrl) + + // Use synchronous approach - get tracks directly using tracksWithMediaType + val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo) + + if (videoTracks.isEmpty()) { + println("No video tracks found in video: ${'$'}videoPath") + return@withContext out + } + + val videoTrack = videoTracks.first() as AVAssetTrack + println("Found video track: ${'$'}{videoTrack}") + + memScoped { + val errorPtr = alloc>() + val reader = AVAssetReader.assetReaderWithAsset(asset, error = errorPtr.ptr) + + if (reader == null) { + val err = errorPtr.value + println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}") + return@withContext out + } + + val trackOutput = AVAssetReaderTrackOutput.assetReaderTrackOutputWithTrack( + track = videoTrack, + outputSettings = null + ) + + if (!reader.canAddOutput(trackOutput)) { + println("Cannot add track output to reader") + return@withContext out + } + + reader.addOutput(trackOutput) + + if (!reader.startReading()) { + println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}") + return@withContext out + } + + println("Started reading video frames...") + var frameCount = 0 + while (true) { + val sampleBuffer = trackOutput.copyNextSampleBuffer() + if (sampleBuffer == null) { + break + } + + val presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + val timeInSeconds = CMTimeGetSeconds(presentationTime) + val timeInMillis = (timeInSeconds * 1000.0).toLong() + + out.add(timeInMillis) + frameCount++ + + // Release the sample buffer + CFRelease(sampleBuffer) + + if (out.size >= Int.MAX_VALUE) break + } + + println("Extracted ${'$'}frameCount frame timestamps") + } + + } catch (e: Exception) { + println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}") + e.printStackTrace() + } + + println("Returning ${'$'}{out.size} timestamps") + out +} + fun createUrl(url: String): NSURL { - return NSURL.fileURLWithPath(url) + val trimmed = url.trim() + + // Remote URLs: return as-is + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return requireNotNull(NSURL.URLWithString(trimmed)) { "Invalid http(s) URL: ${'$'}trimmed" } + } + + // Well-formed file URLs + if (trimmed.startsWith("file://")) { + // Normalize to a proper file URL via its path component to avoid odd host parts + NSURL.URLWithString(trimmed)?.let { fileUrl -> + fileUrl.path?.let { path -> + return NSURL.fileURLWithPath(path) + } + return fileUrl + } + } + + // Malformed file: prefix (e.g., "file:/var/..."), coerce to path + if (trimmed.startsWith("file:")) { + var path = trimmed.removePrefix("file:") + // Remove duplicate leading slashes + while (path.startsWith("//")) path = path.removePrefix("//") + if (!path.startsWith("/")) path = "/${'$'}path" + return NSURL.fileURLWithPath(path) + } + + // Otherwise treat as a filesystem path + return NSURL.fileURLWithPath(trimmed) } actual object VideoExtractionContext \ No newline at end of file diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 99fe2f9..2aef605 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -52,6 +52,7 @@ import com.performancecoachlab.posedetection.recording.FrameSize import com.performancecoachlab.posedetection.recording.InputFrame import com.performancecoachlab.posedetection.recording.Label import com.performancecoachlab.posedetection.recording.extractFrame +import com.performancecoachlab.posedetection.recording.listVideoFrameTimestamps import com.performancecoachlab.posedetection.skeleton.Pose import com.performancecoachlab.posedetection.skeleton.SkeletonRepository import io.github.vinceglb.filekit.FileKit @@ -71,7 +72,7 @@ import kotlin.time.ExperimentalTime @Composable internal fun App() = AppTheme { - var selectedTabIndex by remember { mutableStateOf(0) } + var selectedTabIndex by remember { mutableStateOf(1) } val tabs = listOf("Camera Feed", "Recorded Video") Column { TabRow(selectedTabIndex = selectedTabIndex) { @@ -189,6 +190,11 @@ fun FrameAnalysis( } } + LaunchedEffect(url){ + val timestamps = listVideoFrameTimestamps(url) + println("Timestamps: $timestamps") + } + LaunchedEffect(url, frame) { currentJob?.cancel() currentJob = coroutineScope.launch { -- 2.43.0 From 2a8592b1d64d60b1187788ddcd66d8cfe4d07dbd Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 18:08:36 +0300 Subject: [PATCH] feat: use explicit timestamps for better accuracy --- .../src/commonMain/kotlin/com/nate/posedetection/App.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 2aef605..694ee8f 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -190,9 +190,11 @@ fun FrameAnalysis( } } + val frameMap: MutableMap> = mutableMapOf() + LaunchedEffect(url){ val timestamps = listVideoFrameTimestamps(url) - println("Timestamps: $timestamps") + frameMap[url] = timestamps } LaunchedEffect(url, frame) { @@ -220,7 +222,10 @@ fun FrameAnalysis( // Handle any error that may occur e.printStackTrace() } finally { - (frame + 20L).also { + val nextFrame = frameMap[url]?.let { frameTimeStamps -> + frameTimeStamps.firstOrNull { it > frame} + }?:(frame + 20L) + nextFrame.also { if (it < timeRange.second) { frame = it } else { -- 2.43.0 From dc7afe18c59200b5bd6ef969ec0c913904559f4f Mon Sep 17 00:00:00 2001 From: florian-kima Date: Wed, 17 Sep 2025 09:13:06 +0300 Subject: [PATCH] chore: add Kermit for Logging --- posedetection/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/posedetection/build.gradle.kts b/posedetection/build.gradle.kts index be2fdea..7eb730b 100644 --- a/posedetection/build.gradle.kts +++ b/posedetection/build.gradle.kts @@ -68,6 +68,7 @@ kotlin { implementation(compose.material3) implementation(compose.components.resources) implementation(compose.components.uiToolingPreview) + api("co.touchlab:kermit:2.0.4") } commonTest.dependencies { -- 2.43.0 From bcbbf77a7cb1cb98a73ff6dd415d36ccd822745c Mon Sep 17 00:00:00 2001 From: florian-kima Date: Wed, 17 Sep 2025 13:04:38 +0300 Subject: [PATCH] feat: handle Exceptions correctly and proper Logging --- .../recording/VideoUtils.android.kt | 13 +++++-- .../posedetection/recording/VideoUtils.ios.kt | 38 +++++++++++++------ .../kotlin/com/nate/posedetection/App.kt | 9 ++++- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt index 1f2e7ea..39be3bc 100644 --- a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt +++ b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt @@ -8,6 +8,7 @@ import android.media.MediaMetadataRetriever import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import androidx.core.net.toUri +import co.touchlab.kermit.Logger actual suspend fun extractFrame( videoPath: String, frameTimestamp: Long @@ -62,7 +63,11 @@ actual suspend fun listVideoFrameTimestamps( break } } - if (videoTrack == -1) return@withContext out + + if (videoTrack == -1) { + Logger.i { "No video track found in file: $videoPath" } + return@withContext out + } extractor.selectTrack(videoTrack) @@ -77,8 +82,10 @@ actual suspend fun listVideoFrameTimestamps( if (!extractor.advance()) break } - } catch (_: Throwable) { - // swallow; return what we have + } catch (e: Exception) { + Logger.i { "Exception extracting frames from $videoPath: ${e::class.simpleName} - ${e.message}" } + } catch (t: Throwable) { + Logger.i { "CRASH prevented during frame extraction for $videoPath: ${t::class.simpleName}" } } finally { extractor.release() } diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt index 7898ac7..6cc85ee 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt @@ -28,6 +28,7 @@ import platform.Foundation.NSError import platform.Foundation.NSURL import platform.Foundation.NSValue import platform.Foundation.NSFileManager +import co.touchlab.kermit.Logger @OptIn(ExperimentalForeignApi::class) actual suspend fun extractFrame( @@ -55,6 +56,16 @@ actual suspend fun extractFrame( } } +private fun NSURL.safeDescription(): String { + val scheme = this.scheme ?: "unknown" + val fileName = this.lastPathComponent ?: "unknown" + return "$scheme://.../$fileName" +} + +private fun safeFileName(path: String?): String { + return path?.substringAfterLast("/") ?: "unknown" +} + @OptIn(ExperimentalForeignApi::class, BetaInteropApi::class) actual suspend fun listVideoFrameTimestamps( videoPath: String, @@ -63,18 +74,18 @@ actual suspend fun listVideoFrameTimestamps( try { val resolvedUrl = createUrl(url = videoPath) - println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}") + Logger.i { "[iOS] listVideoFrameTimestamps using URL: ${resolvedUrl.safeDescription()}" } // AVAssetReader requires a local file URL. Bail out early for remote URLs. if (!resolvedUrl.isFileURL()) { - println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...") + Logger.i { "AVAssetReader requires a local file URL; got scheme: ${resolvedUrl.scheme}" } return@withContext out } // Verify the file actually exists at path val path = resolvedUrl.path if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) { - println("File does not exist at path: ${'$'}path") + Logger.i { "File does not exist: ${safeFileName(path)}" } return@withContext out } @@ -84,12 +95,12 @@ actual suspend fun listVideoFrameTimestamps( val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo) if (videoTracks.isEmpty()) { - println("No video tracks found in video: ${'$'}videoPath") + Logger.i { "No video tracks found in video: ${safeFileName(videoPath)}" } return@withContext out } val videoTrack = videoTracks.first() as AVAssetTrack - println("Found video track: ${'$'}{videoTrack}") + Logger.i { "Found video track with ID: ${videoTrack.trackID}" } memScoped { val errorPtr = alloc>() @@ -97,7 +108,7 @@ actual suspend fun listVideoFrameTimestamps( if (reader == null) { val err = errorPtr.value - println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}") + Logger.i { "Failed to create AVAssetReader: ${err?.localizedDescription}" } return@withContext out } @@ -107,18 +118,18 @@ actual suspend fun listVideoFrameTimestamps( ) if (!reader.canAddOutput(trackOutput)) { - println("Cannot add track output to reader") + Logger.i { "Cannot add track output to reader" } return@withContext out } reader.addOutput(trackOutput) if (!reader.startReading()) { - println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}") + Logger.i { "Failed to start reading: ${reader.error?.localizedDescription}" } return@withContext out } - println("Started reading video frames...") + Logger.i { "Started reading video frames..." } var frameCount = 0 while (true) { val sampleBuffer = trackOutput.copyNextSampleBuffer() @@ -139,15 +150,18 @@ actual suspend fun listVideoFrameTimestamps( if (out.size >= Int.MAX_VALUE) break } - println("Extracted ${'$'}frameCount frame timestamps") + Logger.i { "Extracted $frameCount frame timestamps" } } } catch (e: Exception) { - println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}") + Logger.i { + val safeName = safeFileName(videoPath) + "Exception extracting frames from $safeName: ${e::class.simpleName} - ${e.message ?: "No message"}" + } e.printStackTrace() } - println("Returning ${'$'}{out.size} timestamps") + Logger.i { "Returning ${out.size} timestamps" } out } diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 694ee8f..c8d4cb1 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -69,6 +69,7 @@ import kotlinx.coroutines.launch import kotlin.math.roundToLong import kotlin.time.Clock import kotlin.time.ExperimentalTime +import co.touchlab.kermit.Logger @Composable internal fun App() = AppTheme { @@ -224,7 +225,13 @@ fun FrameAnalysis( } finally { val nextFrame = frameMap[url]?.let { frameTimeStamps -> frameTimeStamps.firstOrNull { it > frame} - }?:(frame + 20L) + }?:( + // Fallback: + // If no timestamp is found (e.g., end of list or metadata missing), + // assume a 50 FPS video and increment by 20ms. + // This keeps playback and processing moving forward smoothly. + frame + 20L + ) nextFrame.also { if (it < timeRange.second) { frame = it -- 2.43.0