creates video voice memos from audio clips; with bluesky integration.
trill.ptr.pet
ptr.pet
1import {
2 Output as MediaOutput,
3 Mp4OutputFormat,
4 BufferTarget,
5 CanvasSource,
6 QUALITY_MEDIUM,
7 getFirstEncodableVideoCodec,
8 Input as MediaInput,
9 BlobSource,
10 ALL_FORMATS,
11 Conversion,
12} from "mediabunny";
13
14const renderCanvas = new OffscreenCanvas(1280, 720);
15
16// claude generated visualizer code cuz im lazy and it works okay ig
17
18/**
19 * Extracts frequency data from audio file for visualization
20 * Efficient FFT-like approach with adaptive sensitivity based on track volume
21 */
22const extractFrequencyData = async (file: File, fps: number = 30) => {
23 if (fps <= 0)
24 throw new Error("invalid frame rate: must be greater than zero");
25
26 const audioContext = new AudioContext();
27 const arrayBuffer = await file.arrayBuffer();
28 const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
29
30 const sampleRate = audioBuffer.sampleRate;
31 const channelData = audioBuffer.getChannelData(0);
32 const fftSize = 2048;
33 const frequencyBinCount = fftSize / 2;
34 const hopSize = Math.floor(sampleRate / fps);
35 const totalFrames = Math.floor(channelData.length / hopSize);
36
37 // Pre-calculate volume levels for adaptive scaling
38 const volumeWindowSize = Math.floor(sampleRate * 2); // 2 second windows
39 const volumeLevels: number[] = [];
40
41 for (let i = 0; i < channelData.length; i += volumeWindowSize) {
42 let sum = 0;
43 const end = Math.min(i + volumeWindowSize, channelData.length);
44 for (let j = i; j < end; j++) {
45 sum += Math.abs(channelData[j]);
46 }
47 volumeLevels.push(sum / (end - i));
48 }
49
50 // Calculate overall average and max volume for normalization
51 const avgVolume =
52 volumeLevels.reduce((a, b) => a + b, 0) / volumeLevels.length;
53 const maxVolume = Math.max(...volumeLevels);
54
55 const allFrequencyData: Uint8Array[] = [];
56 let previousFrame: Float32Array | null = null;
57 const smoothingFactor = 0.45;
58
59 // Pre-compute sine/cosine tables for efficiency
60 const numBands = 128;
61 const cosTable: number[][] = [];
62 const sinTable: number[][] = [];
63
64 for (let i = 0; i < numBands; i++) {
65 cosTable[i] = [];
66 sinTable[i] = [];
67 const freqIndex = Math.pow(i / numBands, 1.5) * frequencyBinCount;
68
69 for (let j = 0; j < fftSize; j++) {
70 const angle = (2 * Math.PI * freqIndex * j) / fftSize;
71 cosTable[i][j] = Math.cos(angle);
72 sinTable[i][j] = Math.sin(angle);
73 }
74 }
75
76 for (let frame = 0; frame < totalFrames; frame++) {
77 const offset = frame * hopSize;
78
79 // Determine which volume window this frame is in
80 const volumeIndex = Math.floor(offset / volumeWindowSize);
81 const currentVolume =
82 volumeLevels[Math.min(volumeIndex, volumeLevels.length - 1)];
83
84 // Calculate adaptive boost: quieter sections get more boost
85 // Range from 1.5x to 8x boost depending on how quiet the section is
86 const volumeRatio = currentVolume / (avgVolume + 0.0001);
87 const adaptiveBoost = Math.pow(1 / (volumeRatio + 0.3), 0.6) * 4;
88
89 const slice = new Float32Array(fftSize);
90
91 // Apply Hann window
92 for (let i = 0; i < fftSize; i++) {
93 const idx = offset + i;
94 const sample = idx < channelData.length ? channelData[idx] : 0;
95 const window = 0.5 * (1 - Math.cos((2 * Math.PI * i) / fftSize));
96 slice[i] = sample * window;
97 }
98
99 // Calculate magnitudes for reduced frequency bands
100 const frequencyData = new Float32Array(numBands);
101
102 for (let i = 0; i < numBands; i++) {
103 let real = 0;
104 let imag = 0;
105
106 for (let j = 0; j < fftSize; j += 4) {
107 real += slice[j] * cosTable[i][j];
108 imag += slice[j] * sinTable[i][j];
109 }
110
111 const magnitude = Math.sqrt(real * real + imag * imag);
112
113 // Apply logarithmic scaling with adaptive boost
114 const boosted = Math.log(magnitude * 30 * adaptiveBoost + 1) * 20;
115 frequencyData[i] = boosted;
116 }
117
118 // Temporal smoothing
119 if (previousFrame) {
120 for (let i = 0; i < numBands; i++) {
121 frequencyData[i] =
122 smoothingFactor * previousFrame[i] +
123 (1 - smoothingFactor) * frequencyData[i];
124 }
125 }
126
127 // Spatial smoothing
128 const smoothed = new Float32Array(numBands);
129 for (let i = 0; i < numBands; i++) {
130 const prev = i > 0 ? frequencyData[i - 1] : frequencyData[i];
131 const next = i < numBands - 1 ? frequencyData[i + 1] : frequencyData[i];
132 smoothed[i] = (prev + frequencyData[i] * 2 + next) / 4;
133 }
134
135 // Expand back to full frequencyBinCount for circular visualization
136 const uint8Data = new Uint8Array(frequencyBinCount);
137 for (let i = 0; i < frequencyBinCount; i++) {
138 const bandIndex = (i / frequencyBinCount) * numBands;
139 const lower = Math.floor(bandIndex);
140 const upper = Math.min(numBands - 1, Math.ceil(bandIndex));
141 const t = bandIndex - lower;
142
143 const value = smoothed[lower] * (1 - t) + smoothed[upper] * t;
144 uint8Data[i] = Math.min(255, Math.max(0, Math.floor(value)));
145 }
146
147 allFrequencyData.push(uint8Data);
148 previousFrame = smoothed;
149 }
150
151 return allFrequencyData;
152};
153
154const drawPfp = (
155 ctx: OffscreenCanvasRenderingContext2D,
156 pfpImg: HTMLImageElement,
157 centerX: number,
158 centerY: number,
159 baseRadius: number,
160) => {
161 const pfpSize = baseRadius * 1.9;
162 const pfpX = centerX - pfpSize / 2;
163 const pfpY = centerY - pfpSize / 2;
164
165 ctx.save();
166 ctx.beginPath();
167 ctx.arc(centerX, centerY, pfpSize / 2, 0, Math.PI * 2);
168 ctx.closePath();
169 ctx.clip();
170 ctx.drawImage(pfpImg, pfpX, pfpY, pfpSize, pfpSize);
171 ctx.restore();
172};
173
174/**
175 * Draws circular audio visualizer around center
176 * Inspired by circular spectrum visualizers: https://codepen.io/nfj525/pen/rVBaab
177 */
178const drawCircularVisualizer = (
179 ctx: OffscreenCanvasRenderingContext2D,
180 canvas: OffscreenCanvas,
181 frequencyData: Uint8Array,
182 pfpImg: HTMLImageElement | null,
183) => {
184 const centerX = canvas.width / 2;
185 const centerY = canvas.height / 2;
186 const baseRadius = Math.min(canvas.width, canvas.height) * 0.15;
187 const maxBarHeight = baseRadius * 1.5;
188
189 // Draw profile picture in center if provided
190 if (pfpImg) drawPfp(ctx, pfpImg, centerX, centerY, baseRadius);
191
192 // Draw circular bars
193 const barCount = 128;
194 const angleStep = (Math.PI * 2) / barCount;
195
196 for (let i = 0; i < barCount; i++) {
197 const dataIndex = Math.floor((i / barCount) * frequencyData.length);
198 const value = frequencyData[dataIndex] / 255;
199 const barHeight = value * maxBarHeight;
200
201 const angle = i * angleStep - Math.PI / 2;
202 const innerX = centerX + Math.cos(angle) * baseRadius;
203 const innerY = centerY + Math.sin(angle) * baseRadius;
204 const outerX = centerX + Math.cos(angle) * (baseRadius + barHeight);
205 const outerY = centerY + Math.sin(angle) * (baseRadius + barHeight);
206
207 const hue = (i / barCount) * 360;
208 const brightness = 50 + value * 50;
209 ctx.strokeStyle = `hsl(${hue}, 100%, ${brightness}%)`;
210 ctx.lineWidth = 3;
211
212 ctx.beginPath();
213 ctx.moveTo(innerX, innerY);
214 ctx.lineTo(outerX, outerY);
215 ctx.stroke();
216 }
217};
218
219/**
220 * Draws flat horizontal audio visualizer bars
221 */
222const drawFlatVisualizer = (
223 ctx: OffscreenCanvasRenderingContext2D,
224 canvas: OffscreenCanvas,
225 frequencyData: Uint8Array,
226) => {
227 const barCount = 64;
228 const barWidth = canvas.width / barCount;
229 const maxBarHeight = canvas.height * 0.8;
230 const baseY = canvas.height / 2;
231
232 for (let i = 0; i < barCount; i++) {
233 const dataIndex = Math.floor((i / barCount) * frequencyData.length);
234 const value = frequencyData[dataIndex] / 255;
235 const barHeight = (value * maxBarHeight) / 2;
236
237 const hue = (i / barCount) * 360;
238 const brightness = 50 + value * 50;
239 ctx.fillStyle = `hsl(${hue}, 100%, ${brightness}%)`;
240
241 // Draw mirrored bars (top and bottom)
242 ctx.fillRect(i * barWidth, baseY - barHeight, barWidth - 2, barHeight);
243 ctx.fillRect(i * barWidth, baseY, barWidth - 2, barHeight);
244 }
245};
246
247type RenderOptions = {
248 pfpUrl: string | undefined;
249 visualizer: boolean;
250 frameRate: number;
251 bgColor: string;
252 duration?: number;
253};
254
255export const render = async (file: File, opts: RenderOptions) => {
256 // load pfp picture
257 let pfpImg: HTMLImageElement | null = null;
258 if (opts.pfpUrl) {
259 pfpImg = new Image();
260 pfpImg.crossOrigin = "anonymous";
261 await new Promise((resolve, reject) => {
262 pfpImg!.onload = resolve;
263 pfpImg!.onerror = reject;
264 pfpImg!.src = opts.pfpUrl!;
265 });
266 }
267
268 const input = new MediaInput({
269 source: new BlobSource(file),
270 formats: ALL_FORMATS,
271 });
272
273 const audioTrack = await input.getPrimaryAudioTrack();
274 if (!audioTrack) throw "no audio track found.";
275
276 if (!(await audioTrack.canDecode()))
277 throw "audio track cannot be decoded by browser.";
278
279 const duration = opts.duration ?? (await audioTrack.computeDuration());
280 if (!duration) throw "couldn't get audio duration.";
281
282 const videoCodec = await getFirstEncodableVideoCodec(
283 new Mp4OutputFormat().getSupportedVideoCodecs(),
284 {
285 width: renderCanvas.width,
286 height: renderCanvas.height,
287 },
288 );
289 if (!videoCodec) throw "your browser doesn't support video encoding.";
290
291 const ctx = renderCanvas.getContext("2d");
292 if (!ctx) throw "couldn't get canvas context.";
293
294 const output = new MediaOutput({
295 format: new Mp4OutputFormat({
296 fastStart: "in-memory",
297 }),
298 target: new BufferTarget(),
299 });
300 const conversion = await Conversion.init({
301 input,
302 output,
303 });
304 const videoSource = new CanvasSource(renderCanvas, {
305 codec: "avc",
306 bitrate: QUALITY_MEDIUM,
307 });
308 output.addVideoTrack(videoSource);
309 await output.start();
310
311 const bgColor = opts.bgColor;
312 const drawBackground = () => {
313 ctx.fillStyle = bgColor;
314 ctx.fillRect(0, 0, renderCanvas.width, renderCanvas.height);
315 };
316
317 if (opts.visualizer) {
318 const freqData = await extractFrequencyData(file, opts.frameRate);
319 const frameDuration = 1 / opts.frameRate;
320
321 // Render animated frames
322 for (let i = 0; i < freqData.length; i++) {
323 const timestamp = i * frameDuration;
324 if (timestamp >= duration) break;
325
326 // bg
327 drawBackground();
328
329 if (pfpImg)
330 drawCircularVisualizer(ctx, renderCanvas, freqData[i], pfpImg);
331 else drawFlatVisualizer(ctx, renderCanvas, freqData[i]);
332
333 await videoSource.add(timestamp);
334 }
335 } else {
336 drawBackground();
337
338 if (pfpImg) {
339 const centerX = renderCanvas.width / 2;
340 const centerY = renderCanvas.height / 2;
341 const baseRadius =
342 Math.min(renderCanvas.width, renderCanvas.height) * 0.15;
343
344 drawPfp(ctx, pfpImg, centerX, centerY, baseRadius);
345 }
346
347 await videoSource.add(0, duration);
348 }
349
350 videoSource.close();
351 await conversion.execute();
352
353 return new Blob([output.target.buffer!], { type: "video/mp4" });
354};