From 544031aeae868f1d7884b4398a91d1070d4a1819 Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 17:49:11 +0300 Subject: [PATCH] feat: retrieve timestamps from video --- .../recording/VideoUtils.android.kt | 46 ++++++ .../posedetection/recording/VideoUtils.kt | 4 + .../posedetection/recording/VideoUtils.ios.kt | 142 +++++++++++++++++- .../kotlin/com/nate/posedetection/App.kt | 8 +- 4 files changed, 198 insertions(+), 2 deletions(-) diff --git a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt index 94f8ed9..1f2e7ea 100644 --- a/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt +++ b/posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt @@ -2,6 +2,8 @@ package com.performancecoachlab.posedetection.recording import android.app.Application import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat import android.media.MediaMetadataRetriever import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -39,6 +41,50 @@ actual suspend fun extractFrame( } } +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val ctx = VideoExtractionContext.get() + val uri = videoPath.toUri() + val extractor = MediaExtractor() + + val out = ArrayList() + try { + extractor.setDataSource(ctx, uri, null) + + // Select video track + var videoTrack = -1 + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith("video/")) { + videoTrack = i + break + } + } + if (videoTrack == -1) return@withContext out + + extractor.selectTrack(videoTrack) + + var idx = 0 + while (true) { + val sampleTimeUs = extractor.sampleTime + if (sampleTimeUs < 0) break + + out.add(sampleTimeUs / 1000L) // ms + if (out.size >= Int.MAX_VALUE) break + idx++ + + if (!extractor.advance()) break + } + } catch (_: Throwable) { + // swallow; return what we have + } finally { + extractor.release() + } + out +} + actual object VideoExtractionContext { private lateinit var application: Application diff --git a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt index 009314a..f6566de 100644 --- a/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt +++ b/posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt @@ -4,5 +4,9 @@ expect suspend fun extractFrame( videoPath: String, frameTimestamp: Long ): InputFrame? +expect suspend fun listVideoFrameTimestamps( + videoPath: String, +): List + @Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") expect object VideoExtractionContext \ No newline at end of file diff --git a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt index fcdb320..7898ac7 100644 --- a/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt +++ b/posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt @@ -1,17 +1,33 @@ package com.performancecoachlab.posedetection.recording +import kotlinx.cinterop.BetaInteropApi import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.ObjCObjectVar +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.value import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.withContext import platform.AVFoundation.AVAsset import platform.AVFoundation.AVAssetImageGenerator +import platform.AVFoundation.AVAssetReader +import platform.AVFoundation.AVAssetReaderTrackOutput +import platform.AVFoundation.AVAssetTrack +import platform.AVFoundation.AVMediaTypeVideo import platform.AVFoundation.CMTimeValue +import platform.AVFoundation.tracksWithMediaType import platform.AVFoundation.valueWithCMTime import platform.CoreMedia.CMTimeMake import platform.CoreMedia.CMTimeMakeWithSeconds +import platform.CoreMedia.CMTimeGetSeconds +import platform.CoreMedia.CMSampleBufferGetPresentationTimeStamp +import platform.CoreFoundation.CFRelease +import platform.Foundation.NSError import platform.Foundation.NSURL import platform.Foundation.NSValue +import platform.Foundation.NSFileManager @OptIn(ExperimentalForeignApi::class) actual suspend fun extractFrame( @@ -39,8 +55,132 @@ actual suspend fun extractFrame( } } +@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class) +actual suspend fun listVideoFrameTimestamps( + videoPath: String, +): List = withContext(Dispatchers.IO) { + val out = mutableListOf() + + try { + val resolvedUrl = createUrl(url = videoPath) + println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}") + + // AVAssetReader requires a local file URL. Bail out early for remote URLs. + if (!resolvedUrl.isFileURL()) { + println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...") + return@withContext out + } + + // Verify the file actually exists at path + val path = resolvedUrl.path + if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) { + println("File does not exist at path: ${'$'}path") + return@withContext out + } + + val asset = AVAsset.assetWithURL(resolvedUrl) + + // Use synchronous approach - get tracks directly using tracksWithMediaType + val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo) + + if (videoTracks.isEmpty()) { + println("No video tracks found in video: ${'$'}videoPath") + return@withContext out + } + + val videoTrack = videoTracks.first() as AVAssetTrack + println("Found video track: ${'$'}{videoTrack}") + + memScoped { + val errorPtr = alloc>() + val reader = AVAssetReader.assetReaderWithAsset(asset, error = errorPtr.ptr) + + if (reader == null) { + val err = errorPtr.value + println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}") + return@withContext out + } + + val trackOutput = AVAssetReaderTrackOutput.assetReaderTrackOutputWithTrack( + track = videoTrack, + outputSettings = null + ) + + if (!reader.canAddOutput(trackOutput)) { + println("Cannot add track output to reader") + return@withContext out + } + + reader.addOutput(trackOutput) + + if (!reader.startReading()) { + println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}") + return@withContext out + } + + println("Started reading video frames...") + var frameCount = 0 + while (true) { + val sampleBuffer = trackOutput.copyNextSampleBuffer() + if (sampleBuffer == null) { + break + } + + val presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + val timeInSeconds = CMTimeGetSeconds(presentationTime) + val timeInMillis = (timeInSeconds * 1000.0).toLong() + + out.add(timeInMillis) + frameCount++ + + // Release the sample buffer + CFRelease(sampleBuffer) + + if (out.size >= Int.MAX_VALUE) break + } + + println("Extracted ${'$'}frameCount frame timestamps") + } + + } catch (e: Exception) { + println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}") + e.printStackTrace() + } + + println("Returning ${'$'}{out.size} timestamps") + out +} + fun createUrl(url: String): NSURL { - return NSURL.fileURLWithPath(url) + val trimmed = url.trim() + + // Remote URLs: return as-is + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return requireNotNull(NSURL.URLWithString(trimmed)) { "Invalid http(s) URL: ${'$'}trimmed" } + } + + // Well-formed file URLs + if (trimmed.startsWith("file://")) { + // Normalize to a proper file URL via its path component to avoid odd host parts + NSURL.URLWithString(trimmed)?.let { fileUrl -> + fileUrl.path?.let { path -> + return NSURL.fileURLWithPath(path) + } + return fileUrl + } + } + + // Malformed file: prefix (e.g., "file:/var/..."), coerce to path + if (trimmed.startsWith("file:")) { + var path = trimmed.removePrefix("file:") + // Remove duplicate leading slashes + while (path.startsWith("//")) path = path.removePrefix("//") + if (!path.startsWith("/")) path = "/${'$'}path" + return NSURL.fileURLWithPath(path) + } + + // Otherwise treat as a filesystem path + return NSURL.fileURLWithPath(trimmed) } actual object VideoExtractionContext \ No newline at end of file diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 217bc14..13953f8 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -49,6 +49,7 @@ import com.performancecoachlab.posedetection.recording.AnalysisObject import com.performancecoachlab.posedetection.recording.FrameAnalyser import com.performancecoachlab.posedetection.recording.InputFrame import com.performancecoachlab.posedetection.recording.extractFrame +import com.performancecoachlab.posedetection.recording.listVideoFrameTimestamps import com.performancecoachlab.posedetection.skeleton.Pose import com.performancecoachlab.posedetection.skeleton.SkeletonRepository import io.github.vinceglb.filekit.FileKit @@ -68,7 +69,7 @@ import kotlin.time.ExperimentalTime @Composable internal fun App() = AppTheme { - var selectedTabIndex by remember { mutableStateOf(0) } + var selectedTabIndex by remember { mutableStateOf(1) } val tabs = listOf("Camera Feed", "Recorded Video") Column { TabRow(selectedTabIndex = selectedTabIndex) { @@ -186,6 +187,11 @@ fun FrameAnalysis( } } + LaunchedEffect(url){ + val timestamps = listVideoFrameTimestamps(url) + println("Timestamps: $timestamps") + } + LaunchedEffect(url, frame) { currentJob?.cancel() currentJob = coroutineScope.launch { -- 2.43.0 From 939fc7fe7a010273514eb7d706ea669100504a5d Mon Sep 17 00:00:00 2001 From: nathan holland Date: Mon, 25 Aug 2025 18:08:36 +0300 Subject: [PATCH] feat: use explicit timestamps for better accuracy --- .../src/commonMain/kotlin/com/nate/posedetection/App.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt index 13953f8..60efc68 100644 --- a/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt +++ b/sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt @@ -187,9 +187,11 @@ fun FrameAnalysis( } } + val frameMap: MutableMap> = mutableMapOf() + LaunchedEffect(url){ val timestamps = listVideoFrameTimestamps(url) - println("Timestamps: $timestamps") + frameMap[url] = timestamps } LaunchedEffect(url, frame) { @@ -224,7 +226,10 @@ fun FrameAnalysis( // Handle any error that may occur e.printStackTrace() } finally { - (frame + 20L).also { + val nextFrame = frameMap[url]?.let { frameTimeStamps -> + frameTimeStamps.firstOrNull { it > frame} + }?:(frame + 20L) + nextFrame.also { if (it < timeRange.second) { frame = it } else { -- 2.43.0