Skip to content

Commit 34af945

Browse files
committed
fix: preserve paired uploads when backup cleanup stalls
1 parent c7c5680 commit 34af945

7 files changed

Lines changed: 195 additions & 7 deletions

File tree

‎apps/chrome-extension/src/offscreen/recorder.ts‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
InstantRecordingUploader,
1111
initialLocalRecordingState,
1212
initiateMultipartUpload,
13+
isRecordingSpoolUploaded,
1314
isUserCancellationError,
1415
type LocalRecordingState,
1516
listRecordingSpoolSessions,
@@ -758,6 +759,12 @@ const sweepOrphanedRecordingSpools = async () => {
758759
remainingSessions.add(orphan.sessionId);
759760
continue;
760761
}
762+
if (isRecordingSpoolUploaded(orphan.sessionId)) {
763+
void deleteRecoveredRecordingSpool(orphan.sessionId).catch(
764+
() => undefined,
765+
);
766+
continue;
767+
}
761768
const manifest = manifestsBySession.get(orphan.sessionId);
762769
// A surviving manifest belongs to a previous offscreen document. This
763770
// document is the extension's only spool writer, so the shared live
@@ -2037,14 +2044,30 @@ const finalizeRecording = async (recording: ActiveRecording) => {
20372044
fps: recording.fps,
20382045
subpath: recording.subpath,
20392046
});
2040-
await recording.spool.dispose();
2041-
await recording.cameraSpool?.dispose();
2042-
await recording.inputSidecar?.dispose().catch(() => undefined);
2043-
await Promise.all(
2044-
recording.audioSidecars.map((sidecar) =>
2045-
sidecar.disposeBackup().catch(() => undefined),
2047+
recording.spool.markUploaded();
2048+
recording.cameraSpool?.markUploaded();
2049+
if (recording.inputSidecar?.isUploadCompleted) {
2050+
recording.inputSidecar.spool.markUploaded();
2051+
}
2052+
for (const sidecar of recording.audioSidecars) {
2053+
sidecar.markUploadedBackup();
2054+
}
2055+
void Promise.all([
2056+
recording.spool.dispose().catch((error) => {
2057+
console.error("Failed to remove uploaded screen backup", error);
2058+
}),
2059+
recording.cameraSpool?.dispose().catch((error) => {
2060+
console.error("Failed to remove uploaded camera backup", error);
2061+
}),
2062+
recording.inputSidecar?.dispose().catch((error) => {
2063+
console.error("Failed to remove uploaded input backup", error);
2064+
}),
2065+
...recording.audioSidecars.map((sidecar) =>
2066+
sidecar.disposeBackup().catch((error) => {
2067+
console.error("Failed to remove uploaded audio backup", error);
2068+
}),
20462069
),
2047-
);
2070+
]);
20482071
await removeFailedRecording(recording.spool.sessionId).catch(
20492072
() => undefined,
20502073
);

‎apps/web/__tests__/web-recorder-browser-harness.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,22 @@
1+
import {
2+
RecordingSpool,
3+
recoverOrphanedRecordingSpools,
4+
} from "@cap/recorder-core/recording-spool";
15
import { useEffect } from "react";
26
import { createRoot } from "react-dom/client";
37
import { useWebRecorder } from "../app/(org)/dashboard/caps/components/web-recorder-dialog/useWebRecorder";
48

59
declare global {
610
interface Window {
711
capRecorderHarness?: ReturnType<typeof useWebRecorder>;
12+
capRecorderSpool?: typeof RecordingSpool;
13+
capRecorderRecoverOrphans?: typeof recoverOrphanedRecordingSpools;
814
}
915
}
1016

17+
window.capRecorderSpool = RecordingSpool;
18+
window.capRecorderRecoverOrphans = recoverOrphanedRecordingSpools;
19+
1120
function RecorderHarness() {
1221
const captureMic = new URLSearchParams(location.search).get("mic") === "1";
1322
const recorder = useWebRecorder({

‎apps/web/__tests__/web-recorder-paired-browser-replay.mjs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ async function replayPairedCapture(
118118
failCameraSpool = false,
119119
failCameraCompletion = false,
120120
failAudioResume = false,
121+
failBackupDeletion = false,
121122
) {
122123
const requests = [];
123124
const parts = [];
@@ -413,6 +414,13 @@ async function replayPairedCapture(
413414
);
414415
}
415416
await page.waitForTimeout(engine.name === "WebKit" ? 5000 : 1700);
417+
if (failBackupDeletion) {
418+
await page.evaluate(() => {
419+
window.capRecorderSpool.prototype.dispose = async () => {
420+
throw new Error("Simulated backup deletion failure");
421+
};
422+
});
423+
}
416424
await page.evaluate(() => window.capRecorderHarness.stopRecording());
417425
try {
418426
await page.waitForFunction(
@@ -482,6 +490,12 @@ async function replayPairedCapture(
482490
if (failAudioResume) {
483491
assert.equal(browserErrors.length, 1);
484492
assert.ok(browserErrors[0].includes("Failed to resume recording"));
493+
} else if (failBackupDeletion) {
494+
assert.ok(
495+
browserErrors.some((error) =>
496+
error.includes("Failed to dispose camera recording spool"),
497+
),
498+
);
485499
} else {
486500
assert.deepEqual(browserErrors, []);
487501
}
@@ -567,10 +581,33 @@ async function replayPairedCapture(
567581
[screenComplete.body.width, screenComplete.body.height],
568582
[640, 360],
569583
);
584+
if (failBackupDeletion) {
585+
const markers = await page.evaluate(() =>
586+
Object.keys(localStorage).filter((key) =>
587+
key.startsWith("cap-recording-spool-uploaded:"),
588+
),
589+
);
590+
assert.equal(markers.length, engine.extension === "webm" ? 2 : 1);
591+
if (engine.extension === "webm") {
592+
assert.deepEqual(
593+
await page.evaluate(() => window.capRecorderRecoverOrphans()),
594+
[],
595+
);
596+
await page.waitForFunction(
597+
() =>
598+
Object.keys(localStorage).every(
599+
(key) => !key.startsWith("cap-recording-spool-uploaded:"),
600+
),
601+
null,
602+
{ timeout: 5000 },
603+
);
604+
}
605+
}
570606
return {
571607
engine: engine.name,
572608
pauseResume,
573609
failAudioResume,
610+
failBackupDeletion,
574611
failCameraSpool,
575612
cameraBytes,
576613
screenBytes,
@@ -630,6 +667,18 @@ try {
630667
),
631668
);
632669
}
670+
results.push(
671+
await replayPairedCapture(
672+
browser,
673+
bundle,
674+
false,
675+
engine,
676+
false,
677+
false,
678+
false,
679+
true,
680+
),
681+
);
633682
} finally {
634683
await browser.close();
635684
}

‎apps/web/app/(org)/dashboard/caps/components/web-recorder-dialog/useWebRecorder.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ export const useWebRecorder = ({
297297
const audioErrorDownloadUrlsRef = useRef<string[]>([]);
298298
const stopInFlightRef = useRef(false);
299299
const recordingSpoolRef = useRef<RecordingSpool | null>(null);
300+
const degradedRecordingSpoolRef = useRef<RecordingSpool | null>(null);
300301
const recordingSpoolDegradingRef = useRef(false);
301302
const recordingSpoolWarningShownRef = useRef(false);
302303
const recordingSpoolHeartbeatRef = useRef<number | null>(null);
@@ -580,6 +581,7 @@ export const useWebRecorder = ({
580581
const disposeRecordingSpool = useCallback(async () => {
581582
const spool = recordingSpoolRef.current;
582583
recordingSpoolRef.current = null;
584+
degradedRecordingSpoolRef.current = null;
583585
recordingSpoolDegradingRef.current = false;
584586
recordingSpoolWarningShownRef.current = false;
585587
stopRecordingSpoolHeartbeat();
@@ -743,6 +745,7 @@ export const useWebRecorder = ({
743745

744746
recordingSpoolDegradingRef.current = true;
745747
recordingSpoolRef.current = null;
748+
degradedRecordingSpoolRef.current = spool;
746749
stopRecordingSpoolHeartbeat();
747750
const backupOverflowed = await moveRecordingSpoolToInMemoryBackup({
748751
spool,
@@ -760,6 +763,9 @@ export const useWebRecorder = ({
760763

761764
try {
762765
await spool.dispose();
766+
if (degradedRecordingSpoolRef.current === spool) {
767+
degradedRecordingSpoolRef.current = null;
768+
}
763769
} catch (disposeError) {
764770
console.error(
765771
"Failed to dispose degraded recording spool",
@@ -2102,6 +2108,11 @@ export const useWebRecorder = ({
21022108
cameraSettingsRef.current = undefined;
21032109
recordingPipelineRef.current = null;
21042110
pendingInstantVideoIdRef.current = null;
2111+
recordingSpoolRef.current?.markUploaded();
2112+
degradedRecordingSpoolRef.current?.markUploaded();
2113+
degradedRecordingSpoolRef.current = null;
2114+
cameraSpoolRef.current?.markUploaded();
2115+
for (const sidecar of audioSidecars) sidecar.markUploadedBackup();
21052116
void Promise.all([
21062117
disposeRecordingSpool(),
21072118
disposeCameraSpool(),

‎packages/recorder-core/__tests__/recording-spool.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,51 @@ describe("RecordingSpool", () => {
242242
expect(backend.getChunkCount("session-2")).toBe(0);
243243
});
244244

245+
it("hides uploaded backups until a stalled deletion can finish", async () => {
246+
const values = new Map<string, string>();
247+
vi.stubGlobal("localStorage", {
248+
getItem: (key: string) => values.get(key) ?? null,
249+
setItem: (key: string, value: string) => values.set(key, value),
250+
removeItem: (key: string) => values.delete(key),
251+
});
252+
const consoleError = vi
253+
.spyOn(console, "error")
254+
.mockImplementation(() => {});
255+
try {
256+
const backend = new MemoryRecordingSpoolBackend();
257+
const spool = await RecordingSpool.create(
258+
{ mimeType: "video/webm", sessionId: "uploaded-backup" },
259+
backend,
260+
);
261+
await spool.appendChunk(new Blob(["uploaded"], { type: "video/webm" }));
262+
spool.markUploaded();
263+
const key = "cap-recording-spool-uploaded:uploaded-backup";
264+
expect(values.get(key)).toBe("1");
265+
266+
const originalDeleteSession = backend.deleteSession.bind(backend);
267+
backend.deleteSession = async () => {
268+
throw new Error("Backup deletion stalled");
269+
};
270+
await expect(spool.dispose()).rejects.toThrow("Backup deletion stalled");
271+
const readChunks = vi.spyOn(backend, "readChunks");
272+
expect(await recoverOrphanedRecordingSpools(backend)).toEqual([]);
273+
expect(readChunks).not.toHaveBeenCalled();
274+
expect(backend.getSessionCount()).toBe(1);
275+
expect(values.get(key)).toBe("1");
276+
await vi.waitFor(() => expect(consoleError).toHaveBeenCalled());
277+
278+
backend.deleteSession = originalDeleteSession;
279+
expect(await recoverOrphanedRecordingSpools(backend)).toEqual([]);
280+
await vi.waitFor(() => {
281+
expect(backend.getSessionCount()).toBe(0);
282+
expect(values.has(key)).toBe(false);
283+
});
284+
} finally {
285+
consoleError.mockRestore();
286+
vi.unstubAllGlobals();
287+
}
288+
});
289+
245290
it("fails fast when the pending spool backlog grows beyond its limit", async () => {
246291
const backend = new DelayedRecordingSpoolBackend();
247292
const spool = await RecordingSpool.create(

‎packages/recorder-core/src/audio-sidecar.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,10 @@ export class AudioRecordingSidecar {
306306
this.memoryBackup = initialLocalRecordingState();
307307
}
308308

309+
markUploadedBackup() {
310+
this.spool?.markUploaded();
311+
}
312+
309313
async prepareRetryMetadata(): Promise<AudioSidecarMetadata | null> {
310314
await this.stop().catch(() => undefined);
311315
if (this.bytes === 0) return null;

‎packages/recorder-core/src/recording-spool.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,34 @@ const SESSIONS_STORE = "sessions";
3838
const CHUNKS_STORE = "chunks";
3939
const DEFAULT_MAX_PENDING_CHUNK_BYTES = 32 * 1024 * 1024;
4040
const TRANSACTION_TIMEOUT_MS = 15_000;
41+
const UPLOADED_SESSION_KEY_PREFIX = "cap-recording-spool-uploaded:";
42+
43+
const uploadedSessionKey = (sessionId: string) =>
44+
`${UPLOADED_SESSION_KEY_PREFIX}${sessionId}`;
45+
46+
const isUploadedSession = (sessionId: string) => {
47+
try {
48+
return (
49+
globalThis.localStorage?.getItem(uploadedSessionKey(sessionId)) === "1"
50+
);
51+
} catch {
52+
return false;
53+
}
54+
};
55+
56+
export const isRecordingSpoolUploaded = isUploadedSession;
57+
58+
const markUploadedSession = (sessionId: string) => {
59+
try {
60+
globalThis.localStorage?.setItem(uploadedSessionKey(sessionId), "1");
61+
} catch {}
62+
};
63+
64+
const clearUploadedSession = (sessionId: string) => {
65+
try {
66+
globalThis.localStorage?.removeItem(uploadedSessionKey(sessionId));
67+
} catch {}
68+
};
4169

4270
// Liveness contract between live recorders and recovery sweeps: a spool whose
4371
// session was updated within this window must be treated as live and never
@@ -296,6 +324,10 @@ export class RecordingSpool {
296324
return [...this.pendingChunks];
297325
}
298326

327+
markUploaded() {
328+
markUploadedSession(this.session.sessionId);
329+
}
330+
299331
appendChunk(chunk: Blob) {
300332
if (this.disposed) {
301333
return Promise.reject(new Error("Recording spool has been disposed"));
@@ -414,6 +446,7 @@ export class RecordingSpool {
414446
await this.pendingWrite;
415447
} catch {}
416448
await this.backend.deleteSession(this.session.sessionId);
449+
clearUploadedSession(this.session.sessionId);
417450
}
418451

419452
private enqueue(task: () => Promise<void>) {
@@ -457,6 +490,19 @@ export const recoverOrphanedRecordingSpools = async (
457490
if (minIdleMs > 0 && now - session.updatedAt < minIdleMs) {
458491
continue;
459492
}
493+
if (isUploadedSession(session.sessionId)) {
494+
void backend
495+
.deleteSession(session.sessionId)
496+
.then(() => clearUploadedSession(session.sessionId))
497+
.catch((error) => {
498+
console.error(
499+
"Failed to remove uploaded recording backup",
500+
session.sessionId,
501+
error,
502+
);
503+
});
504+
continue;
505+
}
460506

461507
try {
462508
const chunks = await backend.readChunks(session.sessionId);
@@ -527,4 +573,5 @@ export const deleteRecoveredRecordingSpool = async (
527573
) => {
528574
await backend.initialize();
529575
await backend.deleteSession(sessionId);
576+
clearUploadedSession(sessionId);
530577
};

0 commit comments

Comments
 (0)