a scrappy gimbal that insults you in shakespearean english
dunkirk.sh
1import "./global.css";
2import * as faceapi from "face-api.js";
3
4let port: SerialPort;
5let videoElement: HTMLVideoElement;
6let lastError = 0;
7let integral = 0;
8let isDragging = false;
9let joystickX = 0;
10let joystickY = 0;
11
12// PID constants
13const Kp = 0.5;
14const Ki = 0.1;
15const Kd = 0.2;
16
17// Check if Serial API is supported
18if (!("serial" in navigator)) {
19 alert(
20 "Web Serial API is not supported in this browser. Please use Chrome or Edge.",
21 );
22}
23
24const createTemplate = () => `
25 <div style="display: flex; flex-direction: row; height: 100vh;">
26 <div style="display: flex; flex-direction: column; align-items: center; justify-content: center; flex: 1; gap: 20px;">
27 <button id="connect">Connect Serial Port</button>
28 <button id="startTracking">Start Face Tracking</button>
29 <video id="webcam" width="640" height="480" autoplay muted></video>
30 <canvas id="overlay" style="position: absolute;"></canvas>
31
32 <div id="joystick" style="width: 200px; height: 200px; border: 2px solid black; border-radius: 50%; position: relative; margin: 20px;">
33 <div id="joystickHandle" style="width: 20px; height: 20px; background: red; border-radius: 50%; position: absolute; top: 90px; left: 90px; cursor: pointer;"></div>
34 </div>
35 <div>
36 <span>Motor Values - X: </span><span id="motor1Value">0</span>
37 <span>, Y: </span><span id="motor2Value">0</span>
38 <button id="sendJoystick">Send</button>
39 </div>
40 </div>
41 <div style="width: 300px; padding: 20px; border-left: 1px solid #ccc; overflow-y: auto;">
42 <h3>Serial Log</h3>
43 <pre id="serialLog" style="white-space: pre-wrap; margin: 0;"></pre>
44 </div>
45 </div>
46`;
47
48async function loadFaceDetectionModels() {
49 await faceapi.nets.tinyFaceDetector.loadFromUri("/models");
50 await faceapi.nets.faceLandmark68Net.loadFromUri("/models");
51}
52
53async function startWebcam() {
54 try {
55 const stream = await navigator.mediaDevices.getUserMedia({ video: true });
56 videoElement.srcObject = stream;
57 } catch (err) {
58 console.error("Error accessing webcam:", err);
59 alert("Failed to access webcam");
60 }
61}
62
63function calculatePID(error: number) {
64 integral += error;
65 const derivative = error - lastError;
66 lastError = error;
67
68 return Kp * error + Ki * integral + Kd * derivative;
69}
70
71async function trackFaces() {
72 const canvas = document.getElementById("overlay") as HTMLCanvasElement;
73 canvas.width = videoElement.width;
74 canvas.height = videoElement.height;
75 const displaySize = {
76 width: videoElement.width,
77 height: videoElement.height,
78 };
79
80 setInterval(async () => {
81 const detections = await faceapi.detectAllFaces(
82 videoElement,
83 new faceapi.TinyFaceDetectorOptions(),
84 );
85
86 if (detections.length > 0) {
87 const face = detections[0];
88 const centerX = face.box.x + face.box.width / 2;
89 const targetX = videoElement.width / 2;
90 const error = (centerX - targetX) / videoElement.width;
91
92 const adjustment = calculatePID(error);
93 await sendMotorCommand(1, adjustment);
94 await sendMotorCommand(2, -adjustment);
95
96 // Draw face detection
97 const context = canvas.getContext("2d");
98 if (context) {
99 context.clearRect(0, 0, canvas.width, canvas.height);
100 faceapi.draw.drawDetections(canvas, detections);
101 }
102 }
103 }, 100);
104}
105
106async function connectSerial() {
107 try {
108 port = await navigator.serial.requestPort();
109 await port.open({ baudRate: 115200 });
110
111 if (port.writable == null) {
112 throw new Error("Failed to open serial port - port is not writable");
113 }
114
115 console.log("Connected to serial port");
116 appendToLog("Connected to serial port");
117
118 while (port.readable) {
119 const reader = port.readable.getReader();
120 try {
121 while (true) {
122 const { value, done } = await reader.read();
123 if (done) break;
124 const decoded = new TextDecoder().decode(value);
125 appendToLog(decoded);
126 }
127 } catch (error) {
128 console.error(error);
129 } finally {
130 reader.releaseLock();
131 }
132 }
133 } catch (err) {
134 console.error("Serial port error:", err);
135 alert(
136 "Failed to open serial port. Please check your connection and permissions.",
137 );
138 }
139}
140
141function appendToLog(message: string) {
142 const log = document.getElementById("serialLog");
143 if (log) {
144 log.textContent += message + "\n";
145 log.scrollTop = log.scrollHeight;
146 }
147}
148
149async function sendMotorCommand(motorNum: number, rotation: number) {
150 if (!port) {
151 alert("Please connect serial port first");
152 return;
153 }
154
155 if (!port.writable) {
156 alert("Serial port is not writable");
157 return;
158 }
159
160 const writer = port.writable.getWriter();
161 const encoder = new TextEncoder();
162 const data = `${motorNum} ${rotation}\r`;
163
164 try {
165 await writer.write(encoder.encode(data));
166 appendToLog(`Sent to motor ${motorNum}: ${rotation} rotations`);
167 } catch (err) {
168 console.error("Write error:", err);
169 appendToLog(`Error sending to motor ${motorNum}: ${err}`);
170 } finally {
171 writer.releaseLock();
172 }
173}
174
175function updateJoystickPosition(x: number, y: number) {
176 const joystick = document.getElementById("joystick");
177 const handle = document.getElementById("joystickHandle");
178 if (!joystick || !handle) return;
179
180 const bounds = joystick.getBoundingClientRect();
181 const radius = bounds.width / 2;
182
183 // Calculate relative position from center
184 const relX = x - bounds.left - radius;
185 const relY = y - bounds.top - radius;
186
187 // Calculate distance from center
188 const distance = Math.sqrt(relX * relX + relY * relY);
189
190 // Normalize to radius if outside circle
191 const normalizedX = distance > radius ? (relX / distance) * radius : relX;
192 const normalizedY = distance > radius ? (relY / distance) * radius : relY;
193
194 // Update handle position
195 handle.style.left = normalizedX + radius - 10 + "px";
196 handle.style.top = normalizedY + radius - 10 + "px";
197
198 // Update values (-0.5 to 0.5 range)
199 joystickX = normalizedX / (radius * 2);
200 joystickY = normalizedY / (radius * 2);
201
202 document.getElementById("motor1Value")!.textContent = joystickX.toFixed(2);
203 document.getElementById("motor2Value")!.textContent = joystickY.toFixed(2);
204}
205
206function defaultPageRender() {
207 const app = document.querySelector<HTMLDivElement>("#app");
208 if (!app) throw new Error("App element not found");
209 app.innerHTML = createTemplate();
210
211 videoElement = document.getElementById("webcam") as HTMLVideoElement;
212
213 const joystick = document.getElementById("joystick");
214 const handle = document.getElementById("joystickHandle");
215
216 if (joystick && handle) {
217 handle.addEventListener("mousedown", () => {
218 isDragging = true;
219 });
220
221 document.addEventListener("mousemove", (e) => {
222 if (isDragging) {
223 updateJoystickPosition(e.clientX, e.clientY);
224 }
225 });
226
227 document.addEventListener("mouseup", () => {
228 if (isDragging) {
229 isDragging = false;
230 }
231 });
232 }
233
234 document.getElementById("connect")?.addEventListener("click", connectSerial);
235 document
236 .getElementById("startTracking")
237 ?.addEventListener("click", async () => {
238 await loadFaceDetectionModels();
239 await startWebcam();
240 trackFaces();
241 });
242 document.getElementById("sendJoystick")?.addEventListener("click", () => {
243 sendMotorCommand(1, joystickX);
244 sendMotorCommand(2, joystickY);
245 });
246
247 document.addEventListener("keydown", (e) => {
248 if (e.key === "Enter") {
249 sendMotorCommand(1, joystickX);
250 sendMotorCommand(2, joystickY);
251 }
252 });
253}
254
255function handleRoute() {
256 defaultPageRender();
257}
258
259handleRoute();