diff --git a/.gitignore b/.gitignore index 3784c0cf8..284656c38 100644 --- a/.gitignore +++ b/.gitignore @@ -34,14 +34,21 @@ vite.config.js vite.config.d.ts # Native capture build artifacts -electron/native/wgc-capture/build/ -electron/native/cursor-monitor/build/ -electron/native/gpu-export-probe/build/ -electron/native/nvidia-cuda-compositor/build/ -electron/native/bin/*/whisper-* -electron/native/bin/*/whisper-runtime.json - -# Local debug helpers -tmp-*.ps1 -.tmp-*.ps1 -gpu-export-probe.mp4 +electron/native/wgc-capture/build/ +electron/native/cursor-monitor/build/ +electron/native/gpu-export-probe/build/ +electron/native/nvidia-cuda-compositor/build/ +electron/native/bin/*/whisper-* +electron/native/bin/*/whisper-runtime.json + +# Bundled caption models (downloaded at postinstall) +/models/ + +# Local debug helpers +tmp-*.ps1 +.tmp-*.ps1 +gpu-export-probe.mp4 + +# 开发日志内部版（不公开） +dev_docs/articles/internal/ +dev_docs/.drafts/ diff --git a/dev_docs/INDEX.md b/dev_docs/INDEX.md new file mode 100644 index 000000000..46d6e8bfa --- /dev/null +++ b/dev_docs/INDEX.md @@ -0,0 +1,14 @@ +# cc-Recordly-cn — 开发日志 +> Build in public 系列 + +## 文章 +| 日期 | 标题 | 类型 | +|------|------|------| + +## 命令 +```bash +journal push # 生成草稿 +journal status # 查看状态 +journal sync # 同步到全局 +journal search # 全局搜索 +``` diff --git a/dev_docs/journal.db b/dev_docs/journal.db new file mode 100644 index 000000000..cd89f7a11 Binary files /dev/null and b/dev_docs/journal.db differ diff --git a/electron-builder.json5 b/electron-builder.json5 index a333a8723..d8d0a35b4 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -36,6 +36,10 @@ { "from": "public/wallpapers", "to": "assets/wallpapers" + }, + { + "from": "models", + "to": "assets/models" } ], "publish": [ @@ -53,15 +57,23 @@ "entitlements": "build/entitlements.mac.plist", "entitlementsInherit": "build/entitlements.mac.inherit.plist", "target": [ - { - "target": "dmg", - "arch": ["x64", "arm64"] - }, - { - "target": "zip", - "arch": ["x64", "arm64"] - } + { + "target": "dmg", + "arch": ["x64", "arm64"] + }, + { + "target": "zip", + "arch": ["x64", "arm64"] + } ], + "entitlements": "build/entitlements.mac.plist", + "entitlementsInherit": "build/entitlements.mac.inherit.plist", + "target": [ + { + "target": "dir", + "arch": ["arm64"] + } + ], "icon": "icons/icons/mac/icon.icns", "artifactName": "${productName}-${arch}.${ext}", "extendInfo": { diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..06b959888 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -673,21 +673,32 @@ interface Window { canceled?: boolean; error?: string; }>; - getWhisperSmallModelStatus: () => Promise<{ + // ── Multi-model API ───────────────────────────────────────────── + getAvailableModels: () => Promise< + Array<{ + id: string; + name: string; + engine: string; + sizeLabel?: string; + languages: string[]; + description: string; + }> + >; + getModelStatus: (modelId: string) => Promise<{ success: boolean; exists: boolean; path?: string | null; error?: string; }>; - downloadWhisperSmallModel: () => Promise<{ + downloadModel: (modelId: string) => Promise<{ success: boolean; path?: string; - alreadyDownloaded?: boolean; error?: string; }>; - deleteWhisperSmallModel: () => Promise<{ success: boolean; error?: string }>; - onWhisperSmallModelDownloadProgress: ( + deleteModel: (modelId: string) => Promise<{ success: boolean; error?: string }>; + onModelDownloadProgress: ( callback: (state: { + modelId: string; status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; @@ -698,6 +709,7 @@ interface Window { videoPath: string; whisperExecutablePath?: string; whisperModelPath: string; + modelId?: string; language?: string; }) => Promise<{ success: boolean; diff --git a/electron/ipc/captions/engine.ts b/electron/ipc/captions/engine.ts new file mode 100644 index 000000000..6234fd3b0 --- /dev/null +++ b/electron/ipc/captions/engine.ts @@ -0,0 +1,55 @@ +import type { CaptionModel } from "./models"; + +/** + * Result from caption generation. + */ +export interface CaptionWordPayload { + text: string; + startMs: number; + endMs: number; +} + +export interface CaptionCuePayload { + id: string; + text: string; + startMs: number; + endMs: number; + words?: CaptionWordPayload[]; +} + +export interface GenerateCaptionResult { + success: boolean; + cues: CaptionCuePayload[]; + message?: string; + error?: string; +} + +/** + * Options passed to an engine for caption generation. + */ +export interface GenerateCaptionOptions { + /** Path to the pre-extracted 16kHz mono WAV file */ + audioPath: string; + /** Absolute path to the downloaded model file */ + modelPath: string; + /** The model metadata */ + model: CaptionModel; + /** Language hint (BCP-47 code or "auto") */ + language?: string; + /** Temp directory for intermediate files */ + tempDir: string; +} + +/** + * Abstract engine interface. + * Each engine (whisper, sensevoice) implements this. + */ +export interface CaptionEngine { + readonly engineType: string; + + /** + * Run caption generation. + * Receives fully resolved paths — no model resolution needed. + */ + generate(options: GenerateCaptionOptions): Promise; +} diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 2a3f49cd7..c64693cd3 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -96,6 +96,13 @@ export async function resolveCaptionAudioCandidates(videoPath: string) { candidates.push({ path: normalizedCandidatePath, label }); }; + // Companion audio (mic/system) should be tried FIRST - they have the best audio quality + const videoDir = path.dirname(videoPath); + const videoExt = path.extname(videoPath); + const videoBase = path.basename(videoPath, videoExt); + pushCandidate(path.join(videoDir, `${videoBase}.mic.wav`), "microphone"); + pushCandidate(path.join(videoDir, `${videoBase}.system.wav`), "system audio"); + pushCandidate(videoPath, "recording"); const requestedRecordingSession = await resolveRecordingSession(videoPath); @@ -190,8 +197,47 @@ export async function generateAutoCaptionsFromVideo(options: { videoPath: string; whisperExecutablePath?: string; whisperModelPath: string; + modelId?: string; language?: string; }) { + // ── Engine dispatch: SenseVoice ────────────────────────────────────── + if (options.modelId) { + const { getModelById } = await import("./models"); + const model = getModelById(options.modelId); + if (model?.engine === "sensevoice") { + const { SenseVoiceEngine } = await import("./sensevoice"); + const engine = new SenseVoiceEngine(); + const ffmpegPath = getFfmpegBinaryPath(); + const tempDir = app.getPath("temp"); + const wavPath = path.join(tempDir, `sensevoice-${Date.now()}.wav`); + + // Use the same audio extraction as whisper path + const audioSource = await extractCaptionAudioSource({ + videoPath: options.videoPath, + ffmpegPath, + wavPath, + }); + + const result = await engine.generate({ + audioPath: wavPath, + modelPath: options.whisperModelPath, + model, + language: options.language, + tempDir, + }); + + if (!result.success) { + await fs.rm(wavPath, { force: true }).catch(() => undefined); + throw new Error(result.error || "SenseVoice caption generation failed."); + } + + await fs.rm(wavPath, { force: true }).catch(() => undefined); + + return { success: true, cues: result.cues, audioSourceLabel: audioSource.label }; + } + } + + // ── Engine: whisper.cpp (default) ──────────────────────────────────── const ffmpegPath = getFfmpegBinaryPath(); const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath); if (!normalizedVideoPath) { @@ -292,6 +338,7 @@ export async function generateAutoCaptionsFromVideo(options: { } return { + success: true, cues: cuesToReturn, audioSourceLabel: audioSource.label, }; diff --git a/electron/ipc/captions/models.ts b/electron/ipc/captions/models.ts new file mode 100644 index 000000000..2c7e85516 --- /dev/null +++ b/electron/ipc/captions/models.ts @@ -0,0 +1,217 @@ +import path from "node:path"; +import { app } from "electron"; +import { existsSync } from "node:fs"; + +/** + * Model engine type. + * - "whisper": whisper.cpp subprocess (GGML models) + * - "sensevoice": sherpa-onnx (ONNX models) + */ +export type ModelEngine = "whisper" | "sensevoice"; + +export interface CaptionModel { + /** Unique model identifier, e.g. "whisper-large-v3" */ + id: string; + /** Display name shown in the UI */ + name: string; + /** Engine that runs this model */ + engine: ModelEngine; + /** Primary download URL (HuggingFace / ModelScope mirror) */ + downloadUrl: string; + /** File name stored under the model directory */ + fileName: string; + /** For multi-file models (SenseVoice ONNX), additional files to download */ + auxiliaryFiles?: Array<{ url: string; fileName: string }>; + /** Approximate download size in bytes (for UI display) */ + sizeBytes?: number; + /** Human-readable size label, e.g. "~3 GB" */ + sizeLabel?: string; + /** Languages this model excels at */ + languages: string[]; + /** Short description shown in the model picker */ + description: string; + /** If true, pre-selected as the default model */ + isDefault?: boolean; +} + +const HF_BASE = "https://hf-mirror.com"; +const WHISPER_REPO = "ggerganov/whisper.cpp"; + +function hfUrl(repo: string, file: string): string { + return `${HF_BASE}/${repo}/resolve/main/${file}`; +} + +/** + * All available caption models. + * Models are grouped by engine and sorted by quality (best first). + */ +export const CAPTION_MODELS: CaptionModel[] = [ + // ── SenseVoice (通义 FunASR via sherpa-onnx) — best for Chinese ── + { + id: "sensevoice-small", + name: "SenseVoice Small (通义·int8)", + engine: "sensevoice", + downloadUrl: + "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/model.int8.onnx", + auxiliaryFiles: [ + { + url: "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/tokens.txt", + fileName: "tokens.txt", + }, + ], + fileName: "model.int8.onnx", + sizeBytes: 239_233_841, + sizeLabel: "~239 MB", + languages: ["zh", "en", "ja", "ko", "yue", "auto"], + description: "阿里通义 SenseVoice (int8)，中文识别最佳，速度快", + isDefault: true, + }, + { + id: "sensevoice-small-fp32", + name: "SenseVoice Small (通义·完整版)", + engine: "sensevoice", + downloadUrl: + "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/model.onnx", + auxiliaryFiles: [ + { + url: "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/tokens.txt", + fileName: "tokens.txt", + }, + ], + fileName: "model.onnx", + sizeBytes: 937_617_178, + sizeLabel: "~937 MB", + languages: ["zh", "en", "ja", "ko", "yue", "auto"], + description: "阿里通义 SenseVoice (FP32)，最高精度", + }, + + // ── Whisper (whisper.cpp) ── + { + id: "whisper-large-v3", + name: "Whisper Large V3", + engine: "whisper", + downloadUrl: hfUrl(WHISPER_REPO, "ggml-large-v3.bin"), + fileName: "ggml-large-v3.bin", + sizeBytes: 3_095_000_000, + sizeLabel: "~3 GB", + languages: ["auto", "zh", "en", "ja", "ko"], + description: "OpenAI Whisper，多语言效果最好", + }, + { + id: "whisper-large-v3-turbo", + name: "Whisper Large V3 Turbo", + engine: "whisper", + downloadUrl: hfUrl(WHISPER_REPO, "ggml-large-v3-turbo.bin"), + fileName: "ggml-large-v3-turbo.bin", + sizeBytes: 1_543_000_000, + sizeLabel: "~1.5 GB", + languages: ["auto", "zh", "en", "ja", "ko"], + description: "Whisper 加速版，速度快质量高", + }, + { + id: "whisper-medium", + name: "Whisper Medium", + engine: "whisper", + downloadUrl: hfUrl(WHISPER_REPO, "ggml-medium.bin"), + fileName: "ggml-medium.bin", + sizeBytes: 1_533_000_000, + sizeLabel: "~1.5 GB", + languages: ["auto", "zh", "en", "ja", "ko"], + description: "Whisper 中等模型，平衡速度与质量", + }, + { + id: "whisper-small", + name: "Whisper Small", + engine: "whisper", + downloadUrl: hfUrl(WHISPER_REPO, "ggml-small.bin"), + fileName: "ggml-small.bin", + sizeBytes: 466_000_000, + sizeLabel: "~466 MB", + languages: ["auto", "zh", "en"], + description: "Whisper 小模型，下载快", + }, + { + id: "whisper-base", + name: "Whisper Base", + engine: "whisper", + downloadUrl: hfUrl(WHISPER_REPO, "ggml-base.bin"), + fileName: "ggml-base.bin", + sizeBytes: 147_000_000, + sizeLabel: "~147 MB", + languages: ["auto", "en"], + description: "Whisper 基础模型，体积最小", + }, +]; + +/** Default model ID — SenseVoice for best Chinese support */ +export const DEFAULT_MODEL_ID = "sensevoice-small"; + +/** + * Resolve the bundled (pre-installed) storage directory for a given model. + * In development: /models// + * In production: /assets/models// + */ +export function getBundledModelDir(model: CaptionModel): string { + if (app.isPackaged) { + return path.join(process.resourcesPath, "assets", "models", model.id); + } + // In dev, models/ sits at the project root + return path.join(__dirname, "..", "models", model.id); +} + +/** + * Check if the model exists in the bundled (pre-installed) location. + */ +export function bundledModelExists(model: CaptionModel): boolean { + const dir = getBundledModelDir(model); + const filePath = path.join(dir, model.fileName); + return existsSync(filePath); +} + +/** + * Resolve the local storage directory for a given model. + * All models live under /models// + */ +export function getModelStorageDir(model: CaptionModel, userDataPath: string): string { + return path.join(userDataPath, "models", model.id); +} + +/** + * Resolve the primary model file path on disk. + * Checks bundled path first, then user-data download path. + */ +export function getModelFilePath(model: CaptionModel, userDataPath: string): string { + // Prefer bundled model + const bundledDir = getBundledModelDir(model); + const bundledPath = path.join(bundledDir, model.fileName); + if (existsSync(bundledPath)) { + return bundledPath; + } + // Fall back to user-data download + return path.join(getModelStorageDir(model, userDataPath), model.fileName); +} + +/** + * Resolve the storage directory — returns bundled dir if it exists, else user-data dir. + */ +export function getModelDir(model: CaptionModel, userDataPath: string): string { + const bundledDir = getBundledModelDir(model); + if (existsSync(bundledDir)) { + return bundledDir; + } + return getModelStorageDir(model, userDataPath); +} + +/** + * Look up a model by its ID. + */ +export function getModelById(id: string): CaptionModel | undefined { + return CAPTION_MODELS.find((m) => m.id === id); +} + +/** + * Get the default model. + */ +export function getDefaultModel(): CaptionModel { + return CAPTION_MODELS.find((m) => m.id === DEFAULT_MODEL_ID) ?? CAPTION_MODELS[0]; +} diff --git a/electron/ipc/captions/sensevoice.ts b/electron/ipc/captions/sensevoice.ts new file mode 100644 index 000000000..346382791 --- /dev/null +++ b/electron/ipc/captions/sensevoice.ts @@ -0,0 +1,195 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import * as sherpa from "sherpa-onnx"; +import type { CaptionEngine, CaptionCuePayload, GenerateCaptionOptions, GenerateCaptionResult } from "./engine"; + +function mapLanguage(language: string): string { + switch (language) { + case "zh": case "yue": case "ja": case "ko": case "en": + return language; + default: + return "auto"; + } +} + +/** + * Merge BPE subword tokens into readable words with timing spans. + * Returns [{text, startMs, endMs}, ...] for use by the existing phrase segmenter. + */ +function tokensToWords(tokens: string[], timestamps: number[]): Array<{ text: string; startMs: number; endMs: number }> { + if (tokens.length === 0 || timestamps.length === 0) return []; + + const words: Array<{ text: string; startMs: number; endMs: number }> = []; + let buffer = ""; + let startMs = Math.round(timestamps[0] * 1000); + + for (let i = 0; i < tokens.length; i++) { + const tok = tokens[i]; + const tsMs = Math.round(timestamps[i] * 1000); + + // BPE subword: "hel" + "lo" → "hello" + const isSubword = + buffer.length > 0 && + !tok.startsWith(" ") && + !/[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(tok) && + !/[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(buffer.slice(-1)); + + if (isSubword) { + buffer += tok; + continue; + } + + // Compute endMs as midpoint between this word's start and the next token's start. + // This creates real gaps the segmenter can use. + const nextTsMs = + i + 1 < tokens.length ? Math.round(timestamps[i + 1] * 1000) : tsMs + 200; + const endMs = Math.round((tsMs + nextTsMs) / 2); + + // Finalize previous word + if (buffer) { + words.push({ text: buffer, startMs, endMs }); + } + + // Start new word + if (tok.startsWith(" ")) { + buffer = tok.slice(1); + } else { + buffer = tok; + } + startMs = endMs; + } + + // Finalize last word + if (buffer) { + const lastTs = Math.round(timestamps[tokens.length - 1] * 1000); + words.push({ text: buffer, startMs, endMs: lastTs + 200 }); + } + + return words; +} + +export class SenseVoiceEngine implements CaptionEngine { + readonly engineType = "sensevoice"; + + async generate(options: GenerateCaptionOptions): Promise { + const { audioPath, modelPath, language } = options; + const modelDir = path.dirname(modelPath); + const tokensPath = path.join(modelDir, "tokens.txt"); + + try { + await fs.access(audioPath); + } catch { + return { success: false, cues: [], error: `Audio file not found: ${audioPath}` }; + } + try { + await fs.access(tokensPath); + } catch { + return { success: false, cues: [], error: `Tokenizer file not found: ${tokensPath}` }; + } + + const svLanguage = mapLanguage(language || "auto"); + + const recognizer = sherpa.createOfflineRecognizer({ + featConfig: { sampleRate: 16000, featureDim: 80 }, + modelConfig: { + senseVoice: { + model: modelPath, + language: svLanguage, + useInverseTextNormalization: 1, + }, + tokens: tokensPath, + }, + decodingMethod: "greedy_search", + }); + + const wave = sherpa.readWave(audioPath); + + let resultText = ""; + let resultTokens: string[] = []; + let resultTimestamps: number[] = []; + let recognizerError: string | null = null; + + const stream = recognizer.createStream(); + try { + stream.acceptWaveform(wave.sampleRate, wave.samples); + recognizer.decode(stream); + const raw = recognizer.getResult(stream) as Record; + resultText = (raw.text as string) ?? ""; + resultTokens = (raw.tokens as string[]) ?? []; + resultTimestamps = (raw.timestamps as number[]) ?? []; + } catch (error) { + recognizerError = error instanceof Error ? error.message : String(error); + } finally { + stream.free(); + } + recognizer.free(); + + if (recognizerError) { + return { success: false, cues: [], error: `SenseVoice: ${recognizerError}` }; + } + + const text = resultText.trim(); + if (!text) { + return { + success: false, + cues: [], + error: "SenseVoice produced no recognizable text. Try a different language setting.", + }; + } + + // Build cues directly from word timestamps. + // Split at pauses (≥1s) or at ~12 char cap for standard subtitle length. + const words = tokensToWords(resultTokens, resultTimestamps); + const cues: CaptionCuePayload[] = []; + + if (words.length > 0) { + let segStartMs = words[0].startMs; + let segText = ""; + + for (let i = 0; i < words.length; i++) { + const w = words[i]; + const gapToPrev = i > 0 ? w.startMs - words[i - 1].endMs : 0; + + if (segText.length > 0 && (gapToPrev >= 1000 || segText.length + w.text.length > 15)) { + let t = segText.trim(); + cues.push({ + id: `cue-${cues.length}`, + text: t, + startMs: segStartMs, + endMs: words[i - 1].endMs, + }); + segStartMs = w.startMs; + segText = ""; + } + segText += w.text; + } + + // Last segment + if (segText.length > 0) { + let t = segText.trim(); + cues.push({ + id: `cue-${cues.length}`, + text: t, + startMs: segStartMs, + endMs: words[words.length - 1].endMs, + }); + } + return { success: true, cues, message: "SenseVoice transcription complete." }; + } + + // Fallback: no word timings, single cue + return { + success: true, + cues: [ + { + id: "cue-0", + text, + startMs: 0, + endMs: text.length * 80, + words: [{ text, startMs: 0, endMs: text.length * 80 }], + }, + ], + message: "SenseVoice transcription complete (no timestamps).", + }; + } +} diff --git a/electron/ipc/captions/sherpa-onnx.d.ts b/electron/ipc/captions/sherpa-onnx.d.ts new file mode 100644 index 000000000..64b488ad9 --- /dev/null +++ b/electron/ipc/captions/sherpa-onnx.d.ts @@ -0,0 +1,61 @@ +declare module "sherpa-onnx" { + interface FeatConfig { + sampleRate: number; + featureDim: number; + } + + interface SenseVoiceModelConfig { + model: string; + language: string; + useInverseTextNormalization?: number; + } + + interface OfflineModelConfig { + senseVoice: SenseVoiceModelConfig; + tokens: string; + } + + interface OfflineRecognizerConfig { + featConfig: FeatConfig; + modelConfig: OfflineModelConfig; + lmConfig?: { model?: string; scale?: number }; + decodingMethod?: string; + maxActivePaths?: number; + hotwordsFile?: string; + hotwordsScore?: number; + } + + interface WaveData { + sampleRate: number; + samples: Float32Array; + } + + interface OfflineSegment { + text: string; + start: number; + end: number; + } + + interface OfflineResult { + text?: string; + segments?: OfflineSegment[]; + } + + interface OfflineStream { + acceptWaveform(sampleRate: number, samples: Float32Array): void; + free(): void; + } + + interface OfflineRecognizer { + createStream(): OfflineStream; + decode(stream: OfflineStream): void; + getResult(stream: OfflineStream): OfflineResult; + free(): void; + } + + export function createOfflineRecognizer(config: OfflineRecognizerConfig): OfflineRecognizer; + export function readWave(path: string): WaveData; + export function readWaveFromBinaryData(data: ArrayBuffer): WaveData; + + export const version: string; +} diff --git a/electron/ipc/captions/whisper.ts b/electron/ipc/captions/whisper.ts index c8e774c62..3bfa185ee 100644 --- a/electron/ipc/captions/whisper.ts +++ b/electron/ipc/captions/whisper.ts @@ -2,148 +2,210 @@ import { createWriteStream } from "node:fs"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import { get as httpsGet } from "node:https"; -import type Electron from "electron"; -import { WHISPER_MODEL_DIR, WHISPER_MODEL_DOWNLOAD_URL, WHISPER_SMALL_MODEL_PATH } from "../constants"; - -export function sendWhisperModelDownloadProgress( - webContents: Electron.WebContents, - payload: { - status: "idle" | "downloading" | "downloaded" | "error"; - progress: number; - path?: string | null; - error?: string; - }, +import path from "node:path"; +import { app, type WebContents } from "electron"; +import { getModelById, getModelFilePath, getModelStorageDir } from "./models"; + +// ─── IPC Event Helpers ────────────────────────────────────────────────── + +export type ModelDownloadStatus = "idle" | "downloading" | "downloaded" | "error"; + +export interface ModelDownloadProgressPayload { + modelId: string; + status: ModelDownloadStatus; + progress: number; + path?: string | null; + error?: string; +} + +/** + * Send model download progress to the renderer. + * Event name is per-model so the UI can track multiple models independently. + */ +export function sendModelDownloadProgress( + webContents: WebContents, + payload: ModelDownloadProgressPayload, ) { - webContents.send("whisper-small-model-download-progress", payload); + webContents.send("model-download-progress", payload); } -export async function getWhisperSmallModelStatus() { +// ─── Model Status ─────────────────────────────────────────────────────── + +export async function getModelStatus(modelId: string): Promise<{ + success: boolean; + exists: boolean; + path?: string | null; +}> { + const model = getModelById(modelId); + if (!model) return { success: false, exists: false }; + + // getModelFilePath checks bundled path first, then user-data download + const filePath = getModelFilePath(model, app.getPath("userData")); try { - await fs.access(WHISPER_SMALL_MODEL_PATH, fsConstants.R_OK); - return { - success: true, - exists: true, - path: WHISPER_SMALL_MODEL_PATH, - }; + await fs.access(filePath, fsConstants.R_OK); + return { success: true, exists: true, path: filePath }; } catch { - return { - success: true, - exists: false, - path: null, - }; + return { success: true, exists: false, path: null }; } } + + +// ─── File Download ────────────────────────────────────────────────────── + export function downloadFileWithProgress( url: string, destinationPath: string, onProgress: (progress: number) => void, ): Promise { - const request = (currentUrl: string, redirectCount = 0): Promise => { - return new Promise((resolve, reject) => { - const req = httpsGet(currentUrl, { timeout: 30_000 }, (response) => { + const request = (currentUrl: string, redirectCount = 0): Promise => + new Promise((resolve, reject) => { + if (redirectCount >= 5) { + reject(new Error("Too many redirects while downloading model.")); + return; + } + + const req = httpsGet(currentUrl, (response) => { const statusCode = response.statusCode ?? 0; - const location = response.headers.location; - if (statusCode >= 300 && statusCode < 400 && location) { + if (statusCode >= 300 && statusCode < 400 && response.headers.location) { response.resume(); - if (redirectCount >= 5) { - reject(new Error("Too many redirects while downloading Whisper model.")); - return; - } - - const nextUrl = new URL(location, currentUrl).toString(); - void request(nextUrl, redirectCount + 1) - .then(resolve) - .catch(reject); - return; + return request(response.headers.location, redirectCount + 1).then(resolve, reject); } - if (statusCode < 200 || statusCode >= 300) { + if (statusCode !== 200) { response.resume(); - reject(new Error(`Whisper model download failed with status ${statusCode}.`)); + reject(new Error(`Model download failed with status ${statusCode}.`)); return; } - const totalBytes = Number.parseInt( - String(response.headers["content-length"] ?? "0"), - 10, - ); + const totalBytes = Number.parseInt(response.headers["content-length"] ?? "0", 10); let downloadedBytes = 0; + const fileStream = createWriteStream(destinationPath); response.on("data", (chunk: Buffer) => { downloadedBytes += chunk.length; - if (Number.isFinite(totalBytes) && totalBytes > 0) { - onProgress(Math.min(100, Math.round((downloadedBytes / totalBytes) * 100))); + if (totalBytes > 0) { + onProgress((downloadedBytes / totalBytes) * 100); } }); - response.on("error", (error) => { - fileStream.destroy(error); + response.pipe(fileStream); + + fileStream.on("finish", () => { + fileStream.close(); + onProgress(100); + resolve(); }); fileStream.on("error", (error) => { - response.destroy(error); + fileStream.close(); reject(error); }); - fileStream.on("finish", () => { - onProgress(100); - resolve(); + response.on("error", (error) => { + fileStream.close(); + reject(error); }); - - response.pipe(fileStream); }); req.on("error", reject); req.on("timeout", () => { - req.destroy(new Error("Whisper model download timed out.")); + req.destroy(new Error("Model download timed out.")); }); + req.setTimeout(30_000); }); - }; return request(url); } -export async function downloadWhisperSmallModel(webContents: Electron.WebContents): Promise { - await fs.mkdir(WHISPER_MODEL_DIR, { recursive: true }); - const tempPath = `${WHISPER_SMALL_MODEL_PATH}.download`; +// ─── Model Download ───────────────────────────────────────────────────── + +/** + * Download a model (and its auxiliary files) by model ID. + * Reports progress via IPC to the renderer. + */ +export async function downloadModel( + webContents: WebContents, + modelId: string, +): Promise { + const model = getModelById(modelId); + if (!model) throw new Error(`Unknown model: ${modelId}`); - sendWhisperModelDownloadProgress(webContents, { + const storageDir = getModelStorageDir(model, app.getPath("userData")); + await fs.mkdir(storageDir, { recursive: true }); + + const primaryPath = getModelFilePath(model, app.getPath("userData")); + const tempPath = `${primaryPath}.download`; + + sendModelDownloadProgress(webContents, { + modelId, status: "downloading", progress: 0, path: null, }); try { + // Clean up any stale temp file await fs.rm(tempPath, { force: true }); - await downloadFileWithProgress(WHISPER_MODEL_DOWNLOAD_URL, tempPath, (progress) => { - sendWhisperModelDownloadProgress(webContents, { + + // Download primary model file + await downloadFileWithProgress(model.downloadUrl, tempPath, (progress) => { + sendModelDownloadProgress(webContents, { + modelId, status: "downloading", - progress, + progress: progress * 0.9, // Reserve 10% for auxiliary files path: null, }); }); - await fs.rename(tempPath, WHISPER_SMALL_MODEL_PATH); - sendWhisperModelDownloadProgress(webContents, { + await fs.rename(tempPath, primaryPath); + + // Download auxiliary files (tokenizer.json, etc.) + if (model.auxiliaryFiles) { + for (const aux of model.auxiliaryFiles) { + const auxPath = path.join(storageDir, aux.fileName); + try { + await fs.access(auxPath, fsConstants.R_OK); + continue; // Already exists + } catch { + await downloadFileWithProgress(aux.url, auxPath, () => undefined); + } + await downloadFileWithProgress(aux.url, auxPath, () => {}); + } + } + + sendModelDownloadProgress(webContents, { + modelId, status: "downloaded", progress: 100, - path: WHISPER_SMALL_MODEL_PATH, + path: primaryPath, }); - return WHISPER_SMALL_MODEL_PATH; + return primaryPath; } catch (error) { await fs.rm(tempPath, { force: true }).catch(() => undefined); - sendWhisperModelDownloadProgress(webContents, { + sendModelDownloadProgress(webContents, { + modelId, status: "error", progress: 0, path: null, - error: String(error), + error: error instanceof Error ? error.message : String(error), }); throw error; } } -export async function deleteWhisperSmallModel(): Promise { - await fs.rm(WHISPER_SMALL_MODEL_PATH, { force: true }); +// ─── Model Deletion ───────────────────────────────────────────────────── + +/** + * Delete a downloaded model and its auxiliary files. + */ +export async function deleteModel(modelId: string): Promise { + const model = getModelById(modelId); + if (!model) throw new Error(`Unknown model: ${modelId}`); + + const storageDir = getModelStorageDir(model, app.getPath("userData")); + await fs.rm(storageDir, { recursive: true, force: true }); } + + diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index 2c5cf8f17..e6de0cf39 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -17,9 +17,9 @@ export const AUTO_RECORDING_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000; export const ALLOW_RECORDLY_WINDOW_CAPTURE = Boolean(process.env["VITE_DEV_SERVER_URL"]); export const RECORDING_SESSION_MANIFEST_SUFFIX = ".recordly-session.json"; export const WHISPER_MODEL_DOWNLOAD_URL = - "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"; + "https://hf-mirror.com/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin"; export const WHISPER_MODEL_DIR = path.join(USER_DATA_PATH, "whisper"); -export const WHISPER_SMALL_MODEL_PATH = path.join(WHISPER_MODEL_DIR, "ggml-small.bin"); +export const WHISPER_SMALL_MODEL_PATH = path.join(WHISPER_MODEL_DIR, "ggml-large-v3.bin"); export const COMPANION_AUDIO_LAYOUTS = [ { platform: "mac" as const, systemSuffix: ".system.m4a", micSuffix: ".mic.m4a" }, { platform: "win" as const, systemSuffix: ".system.wav", micSuffix: ".mic.wav" }, diff --git a/electron/ipc/ffmpeg/binary.ts b/electron/ipc/ffmpeg/binary.ts index a0cf8468c..5126773df 100644 --- a/electron/ipc/ffmpeg/binary.ts +++ b/electron/ipc/ffmpeg/binary.ts @@ -120,11 +120,22 @@ export function getFfmpegBinaryPath(): string { const ffmpegStatic = loadFfmpegStatic(); if (ffmpegStatic && typeof ffmpegStatic === "string") { const bundledPath = app.isPackaged - ? ffmpegStatic.replace(/\.asar([/\\])/, ".asar.unpacked$1") + ? ffmpegStatic.replace(/\.asar([\/\\])/, ".asar.unpacked$1") : ffmpegStatic; if (existsSync(bundledPath)) { - return bundledPath; + // Verify the binary is actually executable (macOS 16+ blocks unsigned binaries) + const result = spawnSync(bundledPath, ["-version"], { + timeout: 3000, + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status === 0) { + return bundledPath; + } + console.warn( + `[ffmpeg] Static binary at ${bundledPath} is present but not executable (status=${result.status}, error=${result.error?.message ?? "none"}), falling back to system ffmpeg.`, + ); } } @@ -146,7 +157,18 @@ export function getFfprobeBinaryPath(): string { : ffprobeStatic; if (existsSync(bundledPath)) { - return bundledPath; + // Verify the binary is actually executable (macOS 16+ blocks unsigned binaries) + const result = spawnSync(bundledPath, ["-version"], { + timeout: 3000, + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status === 0) { + return bundledPath; + } + console.warn( + `[ffprobe] Static binary at ${bundledPath} is present but not executable (status=${result.status}), falling back to system ffprobe.`, + ); } } diff --git a/electron/ipc/register/captions.ts b/electron/ipc/register/captions.ts index fe93afd70..bcad79635 100644 --- a/electron/ipc/register/captions.ts +++ b/electron/ipc/register/captions.ts @@ -2,15 +2,16 @@ import path from "node:path"; import { dialog, ipcMain } from "electron"; import { generateAutoCaptionsFromVideo } from "../captions/generate"; import { - deleteWhisperSmallModel, - downloadWhisperSmallModel, - getWhisperSmallModelStatus, - sendWhisperModelDownloadProgress, + deleteModel, + downloadModel, + getModelStatus, + sendModelDownloadProgress, } from "../captions/whisper"; +import { CAPTION_MODELS } from "../captions/models"; import { LEGACY_PROJECT_FILE_EXTENSIONS, PROJECT_FILE_EXTENSION } from "../constants"; import { hasProjectFileExtension, loadProjectFromPath } from "../project/manager"; import { setCurrentProjectPath } from "../state"; -import { approveUserPath, getRecordingsDir } from "../utils"; +import { approveUserPath } from "../utils"; const VIDEO_FILE_EXTENSIONS = ["webm", "mp4", "mov", "avi", "mkv"]; const PROJECT_FILE_EXTENSIONS = [PROJECT_FILE_EXTENSION, ...LEGACY_PROJECT_FILE_EXTENSIONS]; @@ -22,29 +23,23 @@ type OpenVideoFilePickerOptions = { export function registerCaptionHandlers() { ipcMain.handle("open-video-file-picker", async (_, options?: OpenVideoFilePickerOptions) => { try { - const includeProjects = Boolean(options?.includeProjects); - const recordingsDir = await getRecordingsDir(); + const filters: Electron.FileFilter[] = [ + { + name: "Video Files", + extensions: VIDEO_FILE_EXTENSIONS, + }, + ]; + + if (options?.includeProjects) { + filters.unshift({ + name: "Recordly Projects", + extensions: PROJECT_FILE_EXTENSIONS, + }); + } + const result = await dialog.showOpenDialog({ - title: includeProjects ? "Import Media or Recordly Project" : "Select Video File", - defaultPath: recordingsDir, - filters: [ - ...(includeProjects - ? [ - { - name: "Media or Recordly Projects", - extensions: [ - ...VIDEO_FILE_EXTENSIONS, - ...PROJECT_FILE_EXTENSIONS, - ], - }, - ] - : []), - { name: "Video Files", extensions: VIDEO_FILE_EXTENSIONS }, - ...(includeProjects - ? [{ name: "Recordly Projects", extensions: PROJECT_FILE_EXTENSIONS }] - : []), - { name: "All Files", extensions: ["*"] }, - ], + title: "Open Video", + filters, properties: ["openFile"], }); @@ -52,43 +47,38 @@ export function registerCaptionHandlers() { return { success: false, canceled: true }; } - const selectedPath = result.filePaths[0]; - - if (includeProjects && hasProjectFileExtension(selectedPath)) { - const projectResult = await loadProjectFromPath(selectedPath); - return projectResult.success - ? { ...projectResult, kind: "project" } - : projectResult; + const filePath = result.filePaths[0]; + const extension = path.extname(filePath).slice(1).toLowerCase(); + + if (options?.includeProjects && hasProjectFileExtension(filePath)) { + try { + const project = await loadProjectFromPath(filePath); + setCurrentProjectPath(filePath); + return { success: true, canceled: false, path: filePath, extension, kind: "project", project }; + } catch (error) { + return { success: false, canceled: false, error: `Failed to load project: ${error}` }; + } } - approveUserPath(selectedPath); + approveUserPath(filePath); setCurrentProjectPath(null); - return { - success: true, - kind: "media", - path: selectedPath, - extension: path.extname(selectedPath).replace(/^\./, "").toLowerCase(), - }; + + return { success: true, canceled: false, path: filePath, extension, kind: "media" }; } catch (error) { - console.error("Failed to open file picker:", error); - return { - success: false, - message: "Failed to open file picker", - error: String(error), - }; + console.error("Failed to open video file picker:", error); + return { success: false, error: String(error) }; } }); ipcMain.handle("open-audio-file-picker", async () => { try { const result = await dialog.showOpenDialog({ - title: "Select Audio File", + title: "Open Audio File", filters: [ { name: "Audio Files", extensions: ["mp3", "wav", "aac", "m4a", "flac", "ogg"], }, - { name: "All Files", extensions: ["*"] }, ], properties: ["openFile"], }); @@ -97,18 +87,13 @@ export function registerCaptionHandlers() { return { success: false, canceled: true }; } - approveUserPath(result.filePaths[0]); - return { - success: true, - path: result.filePaths[0], - }; + const filePath = result.filePaths[0]; + approveUserPath(filePath); + + return { success: true, path: filePath }; } catch (error) { console.error("Failed to open audio file picker:", error); - return { - success: false, - message: "Failed to open audio file picker", - error: String(error), - }; + return { success: false, error: String(error) }; } }); @@ -119,9 +104,8 @@ export function registerCaptionHandlers() { filters: [ { name: "Executables", - extensions: process.platform === "win32" ? ["exe", "cmd", "bat"] : ["*"], + extensions: process.platform === "win32" ? ["exe"] : ["*"], }, - { name: "All Files", extensions: ["*"] }, ], properties: ["openFile"], }); @@ -130,8 +114,10 @@ export function registerCaptionHandlers() { return { success: false, canceled: true }; } - approveUserPath(result.filePaths[0]); - return { success: true, path: result.filePaths[0] }; + const filePath = result.filePaths[0]; + approveUserPath(filePath); + + return { success: true, path: filePath }; } catch (error) { console.error("Failed to open Whisper executable picker:", error); return { success: false, error: String(error) }; @@ -144,6 +130,7 @@ export function registerCaptionHandlers() { title: "Select Whisper Model", filters: [ { name: "Whisper Models", extensions: ["bin"] }, + { name: "ONNX Models", extensions: ["onnx"] }, { name: "All Files", extensions: ["*"] }, ], properties: ["openFile"], @@ -153,102 +140,104 @@ export function registerCaptionHandlers() { return { success: false, canceled: true }; } - approveUserPath(result.filePaths[0]); - return { success: true, path: result.filePaths[0] }; + const filePath = result.filePaths[0]; + approveUserPath(filePath); + + return { success: true, path: filePath }; } catch (error) { console.error("Failed to open Whisper model picker:", error); return { success: false, error: String(error) }; } }); - ipcMain.handle("get-whisper-small-model-status", async () => { + // ── Model registry queries ────────────────────────────────────────── + + ipcMain.handle("get-available-models", () => { + return CAPTION_MODELS.map((m) => ({ + id: m.id, + name: m.name, + engine: m.engine, + sizeLabel: m.sizeLabel, + languages: m.languages, + description: m.description, + })); + }); + + // ── Per-model status / download / delete ──────────────────────────── + + ipcMain.handle("get-model-status", async (_, modelId: string) => { try { - return await getWhisperSmallModelStatus(); + return await getModelStatus(modelId); } catch (error) { return { success: false, exists: false, path: null, error: String(error) }; } }); - ipcMain.handle("download-whisper-small-model", async (event) => { + ipcMain.handle("download-model", async (event, modelId: string) => { try { - const existing = await getWhisperSmallModelStatus(); + const existing = await getModelStatus(modelId); if (existing.exists) { - sendWhisperModelDownloadProgress(event.sender, { + sendModelDownloadProgress(event.sender, { + modelId, status: "downloaded", progress: 100, path: existing.path, }); - return { success: true, path: existing.path, alreadyDownloaded: true }; + return { success: true, path: existing.path }; } - const modelPath = await downloadWhisperSmallModel(event.sender); + const modelPath = await downloadModel(event.sender, modelId); return { success: true, path: modelPath }; } catch (error) { - console.error("Failed to download Whisper small model:", error); + console.error(`Failed to download model ${modelId}:`, error); return { success: false, error: String(error) }; } }); - ipcMain.handle("delete-whisper-small-model", async (event) => { + ipcMain.handle("delete-model", async (event, modelId: string) => { try { - await deleteWhisperSmallModel(); - sendWhisperModelDownloadProgress(event.sender, { + await deleteModel(modelId); + sendModelDownloadProgress(event.sender, { + modelId, status: "idle", progress: 0, - path: null, }); return { success: true }; } catch (error) { - console.error("Failed to delete Whisper small model:", error); - // Verify whether the file was actually removed despite the error - const status = await getWhisperSmallModelStatus(); + // If the file is actually gone despite the error, report success + const status = await getModelStatus(modelId); if (!status.exists) { - // File is gone — treat as success - sendWhisperModelDownloadProgress(event.sender, { + sendModelDownloadProgress(event.sender, { + modelId, status: "idle", progress: 0, - path: null, }); return { success: true }; } - sendWhisperModelDownloadProgress(event.sender, { - status: "error", - progress: 0, - path: null, - error: String(error), - }); + console.error(`Failed to delete model ${modelId}:`, error); return { success: false, error: String(error) }; } }); + // ── Caption generation ────────────────────────────────────────────── + ipcMain.handle( "generate-auto-captions", async ( _, options: { videoPath: string; - whisperExecutablePath: string; + whisperExecutablePath?: string; whisperModelPath: string; + modelId?: string; language?: string; }, ) => { try { - const result = await generateAutoCaptionsFromVideo(options); - return { - success: true, - cues: result.cues, - message: - result.audioSourceLabel === "recording" - ? `Generated ${result.cues.length} caption cues.` - : `Generated ${result.cues.length} caption cues from the ${result.audioSourceLabel}.`, - }; + return await generateAutoCaptionsFromVideo(options); } catch (error) { - console.error("Failed to generate auto captions:", error); - return { - success: false, - error: String(error), - message: "Failed to generate auto captions", - }; + const message = error instanceof Error ? error.message : String(error); + return { success: false, cues: [], error: message }; } }, ); diff --git a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor index e0e478f11..475db2880 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-arm64/recordly-system-cursors b/electron/native/bin/darwin-arm64/recordly-system-cursors index f4b41ab66..a51c8e1a7 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-system-cursors and b/electron/native/bin/darwin-arm64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-arm64/recordly-window-list b/electron/native/bin/darwin-arm64/recordly-window-list index 76a7dab4a..d80287162 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-window-list and b/electron/native/bin/darwin-arm64/recordly-window-list differ diff --git a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor index d577b1a6f..a2bd93ecd 100755 Binary files a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-x64/recordly-system-cursors b/electron/native/bin/darwin-x64/recordly-system-cursors index 545613624..d78520557 100755 Binary files a/electron/native/bin/darwin-x64/recordly-system-cursors and b/electron/native/bin/darwin-x64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-x64/recordly-window-list b/electron/native/bin/darwin-x64/recordly-window-list index e165257ae..85d6ef659 100755 Binary files a/electron/native/bin/darwin-x64/recordly-window-list and b/electron/native/bin/darwin-x64/recordly-window-list differ diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..f66a2c8a7 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -686,17 +686,22 @@ contextBridge.exposeInMainWorld("electronAPI", { openWhisperModelPicker: () => { return ipcRenderer.invoke("open-whisper-model-picker"); }, - getWhisperSmallModelStatus: () => { - return ipcRenderer.invoke("get-whisper-small-model-status"); + // ── Multi-model API ───────────────────────────────────────────── + getAvailableModels: () => { + return ipcRenderer.invoke("get-available-models"); }, - downloadWhisperSmallModel: () => { - return ipcRenderer.invoke("download-whisper-small-model"); + getModelStatus: (modelId: string) => { + return ipcRenderer.invoke("get-model-status", modelId); }, - deleteWhisperSmallModel: () => { - return ipcRenderer.invoke("delete-whisper-small-model"); + downloadModel: (modelId: string) => { + return ipcRenderer.invoke("download-model", modelId); }, - onWhisperSmallModelDownloadProgress: ( + deleteModel: (modelId: string) => { + return ipcRenderer.invoke("delete-model", modelId); + }, + onModelDownloadProgress: ( callback: (state: { + modelId: string; status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; @@ -706,19 +711,21 @@ contextBridge.exposeInMainWorld("electronAPI", { const listener = ( _event: Electron.IpcRendererEvent, payload: { + modelId: string; status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string; }, ) => callback(payload); - ipcRenderer.on("whisper-small-model-download-progress", listener); - return () => ipcRenderer.removeListener("whisper-small-model-download-progress", listener); + ipcRenderer.on("model-download-progress", listener); + return () => ipcRenderer.removeListener("model-download-progress", listener); }, generateAutoCaptions: (options: { videoPath: string; whisperExecutablePath?: string; whisperModelPath: string; + modelId?: string; language?: string; }) => { return ipcRenderer.invoke("generate-auto-captions", options); diff --git a/package-lock.json b/package-lock.json index 71432b2b1..69c7007e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,8 @@ "electron-updater": "^6.8.3", "ffmpeg-static": "^5.3.0", "ffprobe-static": "^3.1.0", + "sherpa-onnx": "^1.13.4", + "sherpa-onnx-darwin-arm64": "^1.13.4", "uiohook-napi": "^1.5.4" }, "devDependencies": { @@ -1394,24 +1396,6 @@ "node": ">=12" } }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", - "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, "node_modules/@esbuild/netbsd-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", @@ -1429,24 +1413,6 @@ "node": ">=12" } }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", - "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, "node_modules/@esbuild/openbsd-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", @@ -1464,24 +1430,6 @@ "node": ">=12" } }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", - "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, "node_modules/@esbuild/sunos-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", @@ -8942,6 +8890,24 @@ "node": ">=8" } }, + "node_modules/sherpa-onnx": { + "version": "1.13.4", + "resolved": "https://registry.npmmirror.com/sherpa-onnx/-/sherpa-onnx-1.13.4.tgz", + "integrity": "sha512-KnfQkA+LxbptrWX1gd7upGDyFkLslJVlOudUWPkwveHwYIXo5Qq97Tx02NF5aE0G3cgKpHBh2z+CR+s6ywZPPQ==", + "license": "Apache-2.0" + }, + "node_modules/sherpa-onnx-darwin-arm64": { + "version": "1.13.4", + "resolved": "https://registry.npmmirror.com/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.13.4.tgz", + "integrity": "sha512-QcYKzyrTzGSx6aKCD6hUODgRS1LetqfG57Z/+i5LCyfMlrgCvDc1lRcl9cdB+TozBsLha9QwLTlI0vmDcf5JKg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "os": [ + "darwin" + ] + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -9990,420 +9956,6 @@ } } }, - "node_modules/vitest/node_modules/@esbuild/aix-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", - "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/android-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", - "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/android-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", - "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/android-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", - "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/darwin-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", - "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/darwin-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", - "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/freebsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", - "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/freebsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", - "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", - "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", - "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", - "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-loong64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", - "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-mips64el": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", - "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", - "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-riscv64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", - "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-s390x": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", - "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/linux-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", - "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/netbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", - "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/openbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", - "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/sunos-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", - "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/win32-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", - "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/win32-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", - "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/vitest/node_modules/@esbuild/win32-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", - "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=18" - } - }, "node_modules/vitest/node_modules/@vitest/mocker": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", @@ -10431,50 +9983,6 @@ } } }, - "node_modules/vitest/node_modules/esbuild": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", - "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.1", - "@esbuild/android-arm": "0.28.1", - "@esbuild/android-arm64": "0.28.1", - "@esbuild/android-x64": "0.28.1", - "@esbuild/darwin-arm64": "0.28.1", - "@esbuild/darwin-x64": "0.28.1", - "@esbuild/freebsd-arm64": "0.28.1", - "@esbuild/freebsd-x64": "0.28.1", - "@esbuild/linux-arm": "0.28.1", - "@esbuild/linux-arm64": "0.28.1", - "@esbuild/linux-ia32": "0.28.1", - "@esbuild/linux-loong64": "0.28.1", - "@esbuild/linux-mips64el": "0.28.1", - "@esbuild/linux-ppc64": "0.28.1", - "@esbuild/linux-riscv64": "0.28.1", - "@esbuild/linux-s390x": "0.28.1", - "@esbuild/linux-x64": "0.28.1", - "@esbuild/netbsd-arm64": "0.28.1", - "@esbuild/netbsd-x64": "0.28.1", - "@esbuild/openbsd-arm64": "0.28.1", - "@esbuild/openbsd-x64": "0.28.1", - "@esbuild/openharmony-arm64": "0.28.1", - "@esbuild/sunos-x64": "0.28.1", - "@esbuild/win32-arm64": "0.28.1", - "@esbuild/win32-ia32": "0.28.1", - "@esbuild/win32-x64": "0.28.1" - } - }, "node_modules/vitest/node_modules/picomatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", diff --git a/package.json b/package.json index 6fd53cd35..05cfe882e 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "type": "module", "scripts": { "dev": "vite --config vite.config.ts", - "postinstall": "node scripts/postinstall.mjs", + "postinstall": "node scripts/postinstall.mjs && node scripts/download-bundled-models.mjs", "build": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run normalize:electron-main-cjs && npm run smoke:electron-main-cjs && electron-builder", "lint": "biome lint .", "lint:fix": "biome lint --write .", @@ -34,6 +34,7 @@ "build:mac": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run normalize:electron-main-cjs && npm run smoke:electron-main-cjs && electron-builder --mac", "build:win": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run normalize:electron-main-cjs && npm run smoke:electron-main-cjs && electron-builder --win", "build:linux": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && npm run normalize:electron-main-cjs && npm run smoke:electron-main-cjs && electron-builder --linux", + "download:models": "node scripts/download-bundled-models.mjs", "i18n:check": "node scripts/i18n-check.mjs", "benchmark:export-queues": "node scripts/benchmark-export-queues.mjs", "normalize:electron-main-cjs": "node scripts/normalize-electron-main-cjs.mjs", @@ -50,6 +51,8 @@ "electron-updater": "^6.8.3", "ffmpeg-static": "^5.3.0", "ffprobe-static": "^3.1.0", + "sherpa-onnx": "^1.13.4", + "sherpa-onnx-darwin-arm64": "^1.13.4", "uiohook-napi": "^1.5.4" }, "devDependencies": { diff --git a/scripts/download-bundled-models.mjs b/scripts/download-bundled-models.mjs new file mode 100755 index 000000000..c54264f62 --- /dev/null +++ b/scripts/download-bundled-models.mjs @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +/** + * Download bundled caption-generation models into the project's `models/` directory. + * + * In development: + * The models live at /models// + * These are NOT tracked by git (.gitignore entry is added). + * + * In production (packaged app): + * electron-builder's `extraResources` copies models/ into the app bundle. + * Process.resourcesPath points there at runtime. + */ + +import { createWriteStream, existsSync } from "node:fs"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { get as httpsGet } from "node:https"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = path.resolve(__dirname, ".."); +const MODELS_DIR = path.join(PROJECT_ROOT, "models"); + +const MODELS = [ + { + id: "sensevoice-small", + files: [ + { + url: "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/model.int8.onnx", + fileName: "model.int8.onnx", + sizeBytes: 239_233_841, + }, + { + url: "https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/tokens.txt", + fileName: "tokens.txt", + sizeBytes: 102_685, + }, + ], + }, +]; + +function downloadFile(url, destinationPath) { + return new Promise((resolve, reject) => { + const tryDownload = (currentUrl, remainingRedirects) => { + if (remainingRedirects <= 0) { + reject(new Error("Too many redirects.")); + return; + } + + const req = httpsGet(currentUrl, (response) => { + const statusCode = response.statusCode ?? 0; + + if (statusCode >= 300 && statusCode < 400 && response.headers.location) { + response.resume(); + const location = response.headers.location; + // Handle relative redirects + const nextUrl = location.startsWith("http") + ? location + : new URL(location, currentUrl).href; + tryDownload(nextUrl, remainingRedirects - 1); + return; + } + + if (statusCode !== 200) { + response.resume(); + reject(new Error(`Download failed with status ${statusCode}: ${currentUrl}`)); + return; + } + + const totalBytes = Number.parseInt(response.headers["content-length"] ?? "0", 10); + let downloadedBytes = 0; + const fileStream = createWriteStream(destinationPath); + + response.on("data", (chunk) => { + downloadedBytes += chunk.length; + if (totalBytes > 0) { + const pct = Math.round((downloadedBytes / totalBytes) * 100); + process.stdout.write( + `\r ${path.basename(destinationPath)}: ${pct}% (${(downloadedBytes / 1_000_000).toFixed(1)} / ${(totalBytes / 1_000_000).toFixed(1)} MB)`, + ); + } + }); + + response.pipe(fileStream); + fileStream.on("finish", () => { + fileStream.close(); + console.log(`\r ${path.basename(destinationPath)}: 100%`); + resolve(); + }); + fileStream.on("error", reject); + response.on("error", reject); + }); + + req.on("error", reject); + req.setTimeout(120_000, () => { + req.destroy(new Error("Download timed out.")); + }); + }; + + tryDownload(url, 5); + }); +} + +async function main() { + console.log("[download-bundled-models]"); + + for (const model of MODELS) { + const modelDir = path.join(MODELS_DIR, model.id); + await mkdir(modelDir, { recursive: true }); + + const allExist = model.files.every((f) => { + const fp = path.join(modelDir, f.fileName); + return existsSync(fp); + }); + + if (allExist) { + console.log(` ${model.id}: already downloaded, skipping.`); + continue; + } + + console.log(` ${model.id}: downloading...`); + for (const file of model.files) { + const dest = path.join(modelDir, file.fileName); + if (existsSync(dest)) { + console.log(` ${file.fileName}: exists, skipping.`); + continue; + } + const tempDest = `${dest}.download`; + try { + await downloadFile(file.url, tempDest); + // Rename temp -> final + await rm(tempDest, { force: true }); + await writeFile(tempDest, ""); // touch + await rm(tempDest); + // Re-download to proper path + await downloadFile(file.url, dest); + } catch (error) { + await rm(tempDest, { force: true }).catch(() => undefined); + console.error(` FAILED: ${file.fileName} - ${error.message}`); + // Don't fail the whole process, let the app download on demand + } + } + } + + // Write a manifest so the app knows which models are bundled + const manifest = MODELS.map((m) => ({ + id: m.id, + files: m.files.map((f) => f.fileName), + })); + await writeFile(path.join(MODELS_DIR, "manifest.json"), JSON.stringify(manifest, null, 2)); + + console.log("[download-bundled-models] Done."); +} + +main().catch((error) => { + // Don't fail the install if downloads fail — models can be fetched on demand + console.error("[download-bundled-models] Warning: download failed:", error.message); +}); diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index a90028e2b..3c6dc1355 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -830,16 +830,26 @@ interface SettingsPanelProps { autoCaptionSettings?: AutoCaptionSettings; whisperExecutablePath?: string | null; whisperModelPath?: string | null; - whisperModelDownloadStatus?: "idle" | "downloading" | "downloaded" | "error"; - whisperModelDownloadProgress?: number; isGeneratingCaptions?: boolean; + selectedModelId?: string; + availableModels?: Array<{ + id: string; + name: string; + engine: string; + sizeLabel?: string; + languages: string[]; + description: string; + }>; + modelStatuses?: Record; + modelDownloadProgress?: Record; onAutoCaptionSettingsChange?: (settings: AutoCaptionSettings) => void; onPickWhisperExecutable?: () => void; onPickWhisperModel?: () => void; + onSelectModel?: (modelId: string) => void; + onDownloadModel?: (modelId: string) => void; + onDeleteModel?: (modelId: string) => void; onGenerateAutoCaptions?: () => void; onClearAutoCaptions?: () => void; - onDownloadWhisperSmallModel?: () => void; - onDeleteWhisperSmallModel?: () => void; captionCurrentTimeMs?: number; selectedCaptionId?: string | null; onBeginCaptionEdit?: (id: string) => void; @@ -1273,15 +1283,17 @@ export function SettingsPanel({ autoCaptions = [], autoCaptionSettings = DEFAULT_AUTO_CAPTION_SETTINGS, whisperModelPath, - whisperModelDownloadStatus = "idle", - whisperModelDownloadProgress = 0, isGeneratingCaptions = false, + selectedModelId = "sensevoice-small", + availableModels = [], + modelStatuses = {}, + modelDownloadProgress = {}, onAutoCaptionSettingsChange, - onPickWhisperModel, + onSelectModel, + onDownloadModel, + onDeleteModel, onGenerateAutoCaptions, onClearAutoCaptions, - onDownloadWhisperSmallModel, - onDeleteWhisperSmallModel, captionCurrentTimeMs = 0, selectedCaptionId = null, onBeginCaptionEdit, @@ -2648,16 +2660,53 @@ export function SettingsPanel({
+ {/* ── Model Selector ─────────────────────────────────────── */}
-
+
+ + {/* ── Model description ────────────────────────────────── */} + {(() => { + const currentModel = availableModels.find((m) => m.id === selectedModelId); + if (!currentModel) return null; + return ( +
+ {currentModel.description} +
+ ); + })()} + + {/* ── Language ─────────────────────────────────────────── */}
{tSettings("captions.language", "Language")} @@ -2678,46 +2727,75 @@ export function SettingsPanel({
-
-
- {whisperModelDownloadStatus === "downloading" ? ( - - ) : whisperModelPath ? ( - - ) : ( + + {/* ── Download / Delete / Clear ─────────────────────────── */} +
+ {(() => { + const dlStatus = modelDownloadProgress[selectedModelId]; + const status = modelStatuses[selectedModelId]; + const isDownloaded = status?.exists; + const isDownloading = dlStatus?.status === "downloading"; + + if (isDownloading) { + return ( + + ); + } + if (isDownloaded) { + return ( + + ); + } + return ( - )} - -
+ ); + })()} +
+ + {/* ── Download progress bar ────────────────────────────── */} + {(() => { + const dlStatus = modelDownloadProgress[selectedModelId]; + if (dlStatus?.status !== "downloading") return null; + return ( +
+
+
+ ); + })()} + + {/* ── Generate button ──────────────────────────────────── */}
- {whisperModelDownloadStatus === "downloading" ? ( -
-
-
- ) : null}
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c2e16ed60..0b748fa62 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -576,13 +576,26 @@ export default function VideoEditor() { const [whisperModelPath, setWhisperModelPath] = useState( initialEditorPreferences.whisperModelPath, ); - const [downloadedWhisperModelPath, setDownloadedWhisperModelPath] = useState( - null, + // ── Multi-model state ────────────────────────────────────────────── + const [selectedModelId, setSelectedModelId] = useState( + initialEditorPreferences.selectedModelId ?? "sensevoice-small", ); - const [whisperModelDownloadStatus, setWhisperModelDownloadStatus] = useState< - "idle" | "downloading" | "downloaded" | "error" - >(initialEditorPreferences.whisperModelPath ? "downloaded" : "idle"); - const [whisperModelDownloadProgress, setWhisperModelDownloadProgress] = useState(0); + const [availableModels, setAvailableModels] = useState< + Array<{ + id: string; + name: string; + engine: string; + sizeLabel?: string; + languages: string[]; + description: string; + }> + >([]); + const [modelStatuses, setModelStatuses] = useState< + Record + >({}); + const [modelDownloadProgress, setModelDownloadProgress] = useState< + Record + >({}); const [isGeneratingCaptions, setIsGeneratingCaptions] = useState(false); const [isExporting, setIsExporting] = useState(false); const [exportProgress, setExportProgress] = useState(null); @@ -813,6 +826,7 @@ export default function VideoEditor() { autoCaptionSettings: { ...autoCaptionSettings }, whisperExecutablePath, whisperModelPath, + selectedModelId, }), [ wallpaper, @@ -870,6 +884,7 @@ export default function VideoEditor() { autoCaptionSettings, whisperExecutablePath, whisperModelPath, + selectedModelId, ], ); @@ -2734,16 +2749,28 @@ export default function VideoEditor() { whisperModelPath, ]); + // ── Multi-model: load available models & statuses ─────────────────── useEffect(() => { - const unsubscribe = window.electronAPI.onWhisperSmallModelDownloadProgress((state) => { - setWhisperModelDownloadStatus(state.status); - setWhisperModelDownloadProgress(state.progress); - if (state.status === "downloaded") { - setDownloadedWhisperModelPath(state.path ?? null); - setWhisperModelPath((currentPath) => currentPath ?? state.path ?? null); + const unsubscribe = window.electronAPI.onModelDownloadProgress((state) => { + setModelDownloadProgress((prev) => ({ + ...prev, + [state.modelId]: { status: state.status, progress: state.progress }, + })); + if (state.status === "downloaded" && state.path) { + setModelStatuses((prev) => ({ + ...prev, + [state.modelId]: { exists: true, path: state.path }, + })); + // If this is the selected model, also update whisperModelPath + if (state.modelId === selectedModelId) { + setWhisperModelPath(state.path); + } } if (state.status === "idle") { - setDownloadedWhisperModelPath(null); + setModelStatuses((prev) => ({ + ...prev, + [state.modelId]: { exists: false, path: null }, + })); } if (state.status === "error" && state.error) { toast.error(state.error); @@ -2751,22 +2778,32 @@ export default function VideoEditor() { }); void (async () => { - const result = await window.electronAPI.getWhisperSmallModelStatus(); - if (!result.success) { - return; + // Load available models + try { + const models = await window.electronAPI.getAvailableModels(); + setAvailableModels(models); + } catch { + // fallback: models list stays empty } - if (result.exists && result.path) { - setDownloadedWhisperModelPath(result.path); - setWhisperModelPath((currentPath) => currentPath ?? result.path ?? null); - setWhisperModelDownloadStatus("downloaded"); - setWhisperModelDownloadProgress(100); - return; + // Load status for all models (check which are already downloaded) + try { + const models = await window.electronAPI.getAvailableModels(); + for (const model of models) { + const result = await window.electronAPI.getModelStatus(model.id); + if (result.success) { + setModelStatuses((prev) => ({ + ...prev, + [model.id]: { exists: result.exists, path: result.path }, + })); + if (result.exists && result.path && model.id === selectedModelId) { + setWhisperModelPath(result.path); + } + } + } + } catch { + // ignore } - - setDownloadedWhisperModelPath(null); - setWhisperModelDownloadStatus("idle"); - setWhisperModelDownloadProgress(0); })(); return () => unsubscribe?.(); @@ -2782,25 +2819,74 @@ export default function VideoEditor() { toast.success("Whisper executable selected"); }, []); - const handleDownloadWhisperSmallModel = useCallback(async () => { - if (whisperModelDownloadStatus === "downloading") { - return; - } + const handleDownloadModel = useCallback( + async (modelId: string) => { + const current = modelDownloadProgress[modelId]; + if (current?.status === "downloading") return; - setWhisperModelDownloadStatus("downloading"); - setWhisperModelDownloadProgress(0); - const result = await window.electronAPI.downloadWhisperSmallModel(); - if (!result.success) { - setWhisperModelDownloadStatus("error"); - toast.error(result.error || "Failed to download Whisper small model"); - return; - } + setModelDownloadProgress((prev) => ({ + ...prev, + [modelId]: { status: "downloading", progress: 0 }, + })); + const result = await window.electronAPI.downloadModel(modelId); + if (!result.success) { + setModelDownloadProgress((prev) => ({ + ...prev, + [modelId]: { status: "error", progress: 0 }, + })); + toast.error(result.error || "Failed to download model"); + return; + } + if (result.path) { + setModelStatuses((prev) => ({ + ...prev, + [modelId]: { exists: true, path: result.path }, + })); + if (modelId === selectedModelId) { + setWhisperModelPath(result.path); + } + } + }, + [modelDownloadProgress, selectedModelId], + ); - if (result.path) { - setDownloadedWhisperModelPath(result.path); - setWhisperModelPath(result.path); - } - }, [whisperModelDownloadStatus]); + const handleDeleteModel = useCallback( + async (modelId: string) => { + const result = await window.electronAPI.deleteModel(modelId); + if (!result.success) { + toast.error(result.error || "Failed to delete model"); + setModelDownloadProgress((prev) => ({ + ...prev, + [modelId]: { status: "idle", progress: 0 }, + })); + return; + } + setModelStatuses((prev) => ({ + ...prev, + [modelId]: { exists: false, path: null }, + })); + setModelDownloadProgress((prev) => ({ + ...prev, + [modelId]: { status: "idle", progress: 0 }, + })); + // If deleted model was selected, clear whisperModelPath + if (modelId === selectedModelId) { + setWhisperModelPath(null); + } + toast.success("Model deleted"); + }, + [selectedModelId], + ); + + const handleSelectModel = useCallback( + (modelId: string) => { + setSelectedModelId(modelId); + // Update whisperModelPath to the selected model's path + const status = modelStatuses[modelId]; + setWhisperModelPath(status?.exists ? (status.path ?? null) : null); + }, + [modelStatuses], + ); const handlePickWhisperModel = useCallback(async () => { const result = await window.electronAPI.openWhisperModelPicker(); @@ -2812,25 +2898,6 @@ export default function VideoEditor() { toast.success("Whisper model selected"); }, []); - const handleDeleteWhisperSmallModel = useCallback(async () => { - const result = await window.electronAPI.deleteWhisperSmallModel(); - if (!result.success) { - toast.error(result.error || "Failed to delete Whisper small model"); - // Reset download state so re-download is not blocked - setWhisperModelDownloadStatus("idle"); - setWhisperModelDownloadProgress(0); - return; - } - - setWhisperModelPath((currentPath) => - currentPath === downloadedWhisperModelPath ? null : currentPath, - ); - setDownloadedWhisperModelPath(null); - setWhisperModelDownloadStatus("idle"); - setWhisperModelDownloadProgress(0); - toast.success("Whisper small model deleted"); - }, [downloadedWhisperModelPath]); - const handleGenerateAutoCaptions = useCallback(async () => { if (isGeneratingCaptions) { return; @@ -2868,7 +2935,7 @@ export default function VideoEditor() { await syncActiveVideoSource(sourcePath, webcam.sourcePath ?? null); if (!whisperModelPath) { - toast.error("Select a Whisper model or download the small model first"); + toast.error("Select a Whisper model or download the model first"); return; } @@ -2878,6 +2945,7 @@ export default function VideoEditor() { videoPath: sourcePath, whisperExecutablePath: whisperExecutablePath ?? undefined, whisperModelPath, + modelId: selectedModelId, language: autoCaptionSettings.language, }); @@ -6534,12 +6602,17 @@ export default function VideoEditor() { autoCaptionSettings={autoCaptionSettings} whisperExecutablePath={whisperExecutablePath} whisperModelPath={whisperModelPath} - whisperModelDownloadStatus={whisperModelDownloadStatus} - whisperModelDownloadProgress={whisperModelDownloadProgress} isGeneratingCaptions={isGeneratingCaptions} + selectedModelId={selectedModelId} + availableModels={availableModels} + modelStatuses={modelStatuses} + modelDownloadProgress={modelDownloadProgress} onAutoCaptionSettingsChange={setAutoCaptionSettings} onPickWhisperExecutable={handlePickWhisperExecutable} onPickWhisperModel={handlePickWhisperModel} + onSelectModel={handleSelectModel} + onDownloadModel={handleDownloadModel} + onDeleteModel={handleDeleteModel} onGenerateAutoCaptions={handleGenerateAutoCaptions} onClearAutoCaptions={handleClearAutoCaptions} captionCurrentTimeMs={Math.round(currentTime * 1000)} @@ -6550,8 +6623,6 @@ export default function VideoEditor() { onCaptionSplit={handleCaptionSplit} onCaptionMerge={handleCaptionMerge} onCaptionDelete={handleCaptionDelete} - onDownloadWhisperSmallModel={handleDownloadWhisperSmallModel} - onDeleteWhisperSmallModel={handleDeleteWhisperSmallModel} nativeCaptureUnavailableSession={sessionNativeCaptureUnavailable} onOpenNativeCaptureUnavailableModal={() => setNativeCaptureUnavailableModalOpen(true) diff --git a/src/components/video-editor/captionStyle.ts b/src/components/video-editor/captionStyle.ts index 6a4ed6c6f..36dd6d3ed 100644 --- a/src/components/video-editor/captionStyle.ts +++ b/src/components/video-editor/captionStyle.ts @@ -25,8 +25,8 @@ export function getCaptionScaledFontSize( export function getCaptionPadding(fontSize: number) { return { - x: fontSize * 1.1, - y: fontSize * 0.78, + x: Math.max(fontSize * 1.2, 20), + y: Math.max(fontSize * 0.6, 10), }; } diff --git a/src/components/video-editor/editorPreferences.ts b/src/components/video-editor/editorPreferences.ts index fbf354ab7..562cff8b0 100644 --- a/src/components/video-editor/editorPreferences.ts +++ b/src/components/video-editor/editorPreferences.ts @@ -73,6 +73,7 @@ export interface EditorPresetSnapshot extends PersistedEditorControls { autoCaptionSettings: PresetAutoCaptionSettings; whisperExecutablePath: string | null; whisperModelPath: string | null; + selectedModelId: string; } export interface EditorPreset { @@ -90,6 +91,7 @@ export interface EditorPreferences extends PersistedEditorControls { autoApplyFreshRecordingAutoZooms: boolean; whisperExecutablePath: string | null; whisperModelPath: string | null; + selectedModelId: string; } export const EDITOR_PREFERENCES_STORAGE_KEY = "recordly.editor.preferences"; @@ -155,6 +157,7 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = { autoApplyFreshRecordingAutoZooms: true, whisperExecutablePath: null, whisperModelPath: null, + selectedModelId: "sensevoice-small", }; function normalizeBoolean(value: unknown, fallback: boolean): boolean { @@ -221,6 +224,10 @@ function normalizeEditorPresetSnapshot(candidate: unknown): EditorPresetSnapshot normalizedPreferences.whisperExecutablePath, whisperModelPath: normalizeNullablePath(raw.whisperModelPath) ?? normalizedPreferences.whisperModelPath, + selectedModelId: + typeof raw.selectedModelId === "string" && raw.selectedModelId + ? raw.selectedModelId + : normalizedPreferences.selectedModelId, }; } @@ -444,6 +451,10 @@ export function normalizeEditorPreferences( whisperExecutablePath: normalizeNullablePath(raw.whisperExecutablePath) ?? fallback.whisperExecutablePath, whisperModelPath: normalizeNullablePath(raw.whisperModelPath) ?? fallback.whisperModelPath, + selectedModelId: + typeof raw.selectedModelId === "string" && raw.selectedModelId + ? raw.selectedModelId + : fallback.selectedModelId, }; } diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 60aa86221..7d25a99d3 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -168,6 +168,7 @@ "captions": { "selectOnTimeline": "Select a caption on the timeline to edit it.", "enabled": "Show", + "model": "Model", "timelineQuickAdd": "Hover to add on timeline", "language": "Language", "downloading": "Downloading...", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 1b34c58b6..4e1f60c08 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -137,6 +137,7 @@ "captions": { "selectOnTimeline": "在时间轴上选择字幕进行编辑。", "enabled": "显示", + "model": "模型", "timelineQuickAdd": "悬停以在时间轴上添加", "language": "语言", "downloading": "下载中...", diff --git a/vite.config.ts b/vite.config.ts index 3dd36633d..d0df64c32 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -70,7 +70,7 @@ export default defineConfig({ fileName: (_format, entryName) => `${entryName}.cjs`, }, rollupOptions: { - external: ["ffmpeg-static", "uiohook-napi"], + external: ["ffmpeg-static", "uiohook-napi", "sherpa-onnx"], output: { format: "cjs", inlineDynamicImports: true,