diff --git a/.gitignore b/.gitignore index 3e638691..de6c8fd6 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,4 @@ playwright-report/ test-results/ ROCM_TRIM.md Kokoro-FastAPI.code-workspace +experiments/* diff --git a/CHANGELOG.md b/CHANGELOG.md index b2906015..14893e69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,28 +7,26 @@ Per-PR attribution and contributor credits are published automatically on the co ## [Unreleased] ### Added - `/dev/ssml` endpoint, aligning with the SSML standard (experimental). Disable server-wide with `ENABLE_SSML=false`. -- Multi-speaker input (#294). Opt in per request with `allow_voice_tags: true`; disable server-wide with `ENABLE_VOICE_TAGS=false`. +- Multi-speaker input on `/v1/audio/speech` and `/dev/captioned_speech` (#294). Opt in per request with `allow_voice_tags: true`; disable server-wide with `ENABLE_VOICE_TAGS=false`. - Inline `[voice:name]` tags switch speaker mid-text. - `voice_aliases` mapping for named weighted voice mixes, with optional per-alias `rate`. -- `/v1/audio/speech`: - - Chunk timing sidecar (`return_timing`), writes per-chunk `{text, start, end}` JSON next to the download file. A lighter weight timestamps response (used in the web reader feature) -- `/dev/` endpoints: - - `POST /dev/dialogue` for ordered multi-speaker turns. - - `/dev/captioned_speech` timestamps include the speaking `voice` when tags are on. +- `return_timing` on `/v1/audio/speech`: per-chunk `{text, start, end}` JSON sidecar next to the download (powers the web reader). +- `POST /dev/dialogue` for ordered multi-speaker turns; `/dev/captioned_speech` timestamps now include the speaking `voice`. - Web UI: - - Voice alias/tag cast builder with import/export (re: parallel work by @radzrader, [#272](https://github.com/remsky/Kokoro-FastAPI/discussions/272)). + - Voice alias/tag cast builder with import/export and per-alias rate, synced with the editor (re: parallel work by @radzrader, [#272](https://github.com/remsky/Kokoro-FastAPI/discussions/272)). - Read-along mode: sentence highlighting synced to playback, bidirectional click to seek. - - Find/replace across pages, directly accessible page numbers. - - Download menu (audio / chunk timings / both). - - Alias rate in the cast builder, synced with the editor. + - Find/replace across pages, direct page-number entry, download menu (audio / timings / both). ### Changed -- Dropped unreachable list form of `voice` from the speech parser and unused `VoiceCombineRequest` schema. +- Docker images compile to bytecode at build, ~40% faster startup, all builds install frozen from uv.lock for consistency. - Speed bounds (0.25 to 4.0) shared across speed fields and SSML. - Unrecognized `.env` keys warn at startup instead of refusing to boot. - README config table, covering every setting. ### Removed +- Unused `ffmpeg` from all images (~600MB); audio encoding already runs through PyAV's bundled copy. +- Dead `pydub` dependency. +- Unreachable list form of `voice` from the speech parser and unused `VoiceCombineRequest` schema. - Legacy Gradio UI (`ui/`) code cruft; superseded by the web player since ~v0.2.0 - Legacy ONNX config compose vars, endpoints e.g `/debug/session_pools`. - `OUTPUT_DIR`, `OUTPUT_DIR_SIZE_LIMIT_MB`, `SAMPLE_RATE` settings, never read. diff --git a/api/src/core/config.py b/api/src/core/config.py index ac5894b3..1eaf3c6d 100644 --- a/api/src/core/config.py +++ b/api/src/core/config.py @@ -58,6 +58,7 @@ class Settings(BaseSettings): target_max_tokens: int = 250 # Target maximum tokens per chunk absolute_max_tokens: int = 450 # Absolute maximum tokens per chunk ssml_max_depth: int = 10 # Deepest SSML element nesting translated, real documents sit at 2-5 + max_pause_duration_s: float = 60.0 advanced_text_normalization: bool = True # Preproesses the text before misiki voice_weight_normalization: bool = ( True # Normalize the voice weights so they add up to 1 diff --git a/api/src/services/text_processing/ssml.py b/api/src/services/text_processing/ssml.py index 9af0994e..fb065f4b 100644 --- a/api/src/services/text_processing/ssml.py +++ b/api/src/services/text_processing/ssml.py @@ -66,7 +66,8 @@ def _break_seconds(el: ET.Element) -> float: match = TIME_ATTR.match(time_attr) if match: value = float(match.group(1)) - return value / 1000 if match.group(2).lower() == "ms" else value + seconds = value / 1000 if match.group(2).lower() == "ms" else value + return min(seconds, settings.max_pause_duration_s) return BREAK_STRENGTH_S.get(el.get("strength", "medium"), 0.5) diff --git a/api/src/services/text_processing/text_processor.py b/api/src/services/text_processing/text_processor.py index 56b3683c..a71081cd 100644 --- a/api/src/services/text_processing/text_processor.py +++ b/api/src/services/text_processing/text_processor.py @@ -186,7 +186,7 @@ async def smart_split( chunk_count = 0 logger.info(f"Starting smart split for {len(text)} chars") - # --- Step 1: Split by Pause Tags FIRST --- + # Split First by Pause Tags # This operates on the raw input text parts = PAUSE_TAG_PATTERN.split(text) logger.debug(f"Split raw text into {len(parts)} parts by pause tags.") @@ -196,7 +196,7 @@ async def smart_split( text_part_raw = parts[part_idx] # This part is raw text part_idx += 1 - # --- Process Text Part --- + # Processing Text Part if ( text_part_raw and text_part_raw.strip() ): # Only process if the part is not empty string @@ -219,7 +219,7 @@ async def smart_split( "Skipping text normalization as it is only supported for english" ) - # Process all sentences (original logic) + # Process all sentences sentences = get_sentence_info(processed_text, lang_code=lang_code) current_chunk = [] @@ -227,7 +227,7 @@ async def smart_split( current_count = 0 for sentence, tokens, count in sentences: - # Handle sentences that exceed max tokens (original logic) + # Handle sentences that exceed max tokens if count > max_tokens: # Yield current chunk if any if current_chunk: @@ -241,7 +241,7 @@ async def smart_split( current_tokens = [] current_count = 0 - # Split long sentence on commas (original logic) + # Split long sentence on commas clauses = re.split(r"([,])", sentence) clause_chunk = [] clause_tokens = [] @@ -340,7 +340,7 @@ async def smart_split( ) yield chunk_text, current_tokens, None - # --- Handle Pause Part --- + # Handle Pause # Check if the next part is a pause duration string if part_idx < len(parts): duration_str = parts[part_idx] @@ -348,7 +348,9 @@ async def smart_split( if re.fullmatch(r"\d+(?:\.\d+)?", duration_str): part_idx += 1 # Consume the duration string as it's been processed try: - duration = float(duration_str) + duration = min( + float(duration_str), settings.max_pause_duration_s + ) if duration > 0: chunk_count += 1 logger.info(f"Yielding pause chunk {chunk_count}: {duration}s") @@ -359,8 +361,8 @@ async def smart_split( f"Could not parse valid-looking pause duration: {duration_str}" ) - # --- End of parts loop --- + # End of parts loop total_time = time.time() - start_time - logger.info( + logger.debug( f"Split completed in {total_time * 1000:.2f}ms, produced {chunk_count} chunks (including pauses)" ) diff --git a/docker/cpu/Dockerfile.optimized b/docker/cpu/Dockerfile.optimized index 571ccdad..0f2edfb2 100644 --- a/docker/cpu/Dockerfile.optimized +++ b/docker/cpu/Dockerfile.optimized @@ -13,24 +13,21 @@ RUN apt-get update -y && \ WORKDIR /app # Copy dependency files -COPY pyproject.toml ./pyproject.toml +COPY pyproject.toml uv.lock ./ # Install dependencies with CPU extras -ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=3 +ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=3 UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy RUN --mount=type=cache,target=/root/.cache/uv \ uv venv --python 3.12 && \ - uv sync --extra cpu --no-install-project + uv sync --frozen --extra cpu --no-install-project # Stage 2: Runtime - slim image FROM python:3.12-slim -# Install runtime dependencies + uv (needed by entrypoint.sh) +# Install runtime dependencies RUN apt-get update -y && \ - apt-get install -y --no-install-recommends espeak-ng espeak-ng-data libsndfile1 ffmpeg curl && \ - curl -LsSf https://astral.sh/uv/install.sh | sh && \ - mv /root/.local/bin/uv /usr/local/bin/ && \ - mv /root/.local/bin/uvx /usr/local/bin/ && \ + apt-get install -y --no-install-recommends espeak-ng espeak-ng-data libsndfile1 curl && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ mkdir -p /usr/share/espeak-ng-data && \ ln -s /usr/lib/*/espeak-ng-data/* /usr/share/espeak-ng-data/ && \ @@ -76,7 +73,6 @@ USER appuser ENV PYTHONUNBUFFERED=1 \ PYTHONPATH=/app:/app/api \ PATH="/app/.venv/bin:$PATH" \ - UV_LINK_MODE=copy \ USE_GPU=false \ PHONEMIZER_ESPEAK_PATH=/usr/bin \ PHONEMIZER_ESPEAK_DATA=/usr/share/espeak-ng-data \ diff --git a/docker/gpu/Dockerfile.optimized b/docker/gpu/Dockerfile.optimized index 868075ab..9b435f35 100644 --- a/docker/gpu/Dockerfile.optimized +++ b/docker/gpu/Dockerfile.optimized @@ -18,13 +18,14 @@ RUN apt-get update -y && \ WORKDIR /app # Copy dependency files -COPY pyproject.toml ./pyproject.toml +COPY pyproject.toml uv.lock ./ # Install dependencies with GPU extras (--no-install-project since api/src doesn't exist yet) # UV_PYTHON_INSTALL_DIR keeps the uv-managed interpreter at a stage-shareable path so the # runtime stage can find the venv's python target via COPY --from=builder. ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=3 \ - UV_PYTHON_INSTALL_DIR=/opt/uv-python + UV_PYTHON_INSTALL_DIR=/opt/uv-python \ + UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy # GPU_EXTRA selects the torch wheel set: "gpu" (cu126, keeps Maxwell/Pascal # working) or "gpu-cu128" (cu128, adds Blackwell / RTX 50-series). Overridden @@ -34,17 +35,14 @@ ARG GPU_EXTRA=gpu # --managed-python: noble ships a system 3.12; without the flag uv would pick it and skip /opt/uv-python, breaking the runtime COPY RUN --mount=type=cache,target=/root/.cache/uv \ uv venv --python 3.12 --managed-python && \ - uv sync --extra ${GPU_EXTRA} --no-install-project + uv sync --frozen --extra ${GPU_EXTRA} --no-install-project # Stage 2: Runtime - `base` variant: cudart + NVIDIA_* env only; torch uses its pip CUDA/cuDNN copies, system ones were dead weight (#482) FROM --platform=$BUILDPLATFORM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-ubuntu24.04 -# Install runtime dependencies + uv (needed by entrypoint.sh) +# Install runtime dependencies RUN apt-get update -y && \ - apt-get install -y --no-install-recommends espeak-ng espeak-ng-data libsndfile1 ffmpeg curl && \ - curl -LsSf https://astral.sh/uv/install.sh | sh && \ - mv /root/.local/bin/uv /usr/local/bin/ && \ - mv /root/.local/bin/uvx /usr/local/bin/ && \ + apt-get install -y --no-install-recommends espeak-ng espeak-ng-data libsndfile1 curl && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ mkdir -p /usr/share/espeak-ng-data && \ ln -s /usr/lib/*/espeak-ng-data/* /usr/share/espeak-ng-data/ && \ @@ -80,7 +78,6 @@ RUN if [ "$INCLUDE_JAPANESE" = "true" ]; then \ fi # Copy project files -COPY --chown=appuser:appuser pyproject.toml ./pyproject.toml # model already baked above COPY --exclude=src/models --chown=appuser:appuser api ./api COPY --chown=appuser:appuser web ./web @@ -94,7 +91,6 @@ USER appuser ENV PATH="/app/.venv/bin:$PATH" \ PYTHONUNBUFFERED=1 \ PYTHONPATH=/app:/app/api \ - UV_LINK_MODE=copy \ USE_GPU=true \ PHONEMIZER_ESPEAK_PATH=/usr/bin \ PHONEMIZER_ESPEAK_DATA=/usr/share/espeak-ng-data \ diff --git a/docker/rocm/Dockerfile b/docker/rocm/Dockerfile index 819fcf28..b1157d45 100644 --- a/docker/rocm/Dockerfile +++ b/docker/rocm/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get update && apt upgrade -y && apt-get install -y --no-install-recommen git \ libsndfile1 \ curl \ - ffmpeg \ wget \ nano \ g++ \ @@ -40,16 +39,17 @@ USER appuser WORKDIR /app # Copy dependency files -COPY --chown=appuser:appuser pyproject.toml ./pyproject.toml +COPY --chown=appuser:appuser pyproject.toml uv.lock ./ ENV PHONEMIZER_ESPEAK_PATH=/usr/bin \ PHONEMIZER_ESPEAK_DATA=/usr/share/espeak-ng-data \ - ESPEAK_DATA_PATH=/usr/share/espeak-ng-data + ESPEAK_DATA_PATH=/usr/share/espeak-ng-data \ + UV_COMPILE_BYTECODE=1 # Install dependencies with GPU extras (using cache mounts) RUN --mount=type=cache,target=/root/.cache/uv \ uv venv --python 3.12 && \ - uv sync --extra rocm + uv sync --frozen --extra rocm # Japanese support requires the UniDic dictionary (~526MB on disk) for fugashi/MeCab. # Enabled by default; set --build-arg INCLUDE_JAPANESE=false to skip and shave the image. diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index a5784951..adee831c 100644 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -5,4 +5,4 @@ if [ "$DOWNLOAD_MODEL" = "true" ]; then python download_model.py --output api/src/models/v1_0 fi -exec uv run --extra $DEVICE --no-sync python -m uvicorn api.src.main:app --host 0.0.0.0 --port 8880 --log-level debug \ No newline at end of file +exec python -m uvicorn api.src.main:app --host 0.0.0.0 --port 8880 --log-level debug \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index 73b9be3b..13f64864 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,6 +118,7 @@ Names are the field names from `api/src/core/config.py`, uppercased. Unrecognize | `ABSOLUTE_MAX_TOKENS` | `450` | Hard ceiling per chunk, model limit is 510 | | `ENABLE_SSML` | `true` | Kill switch for the `/dev/ssml` router, both routes 403 when off | | `SSML_MAX_DEPTH` | `10` | Deepest SSML nesting translated, past it is a 400 | +| `MAX_PAUSE_DURATION_S` | `60.0` | Ceiling for a single `[pause:Ns]` tag or SSML ``, longer values are clamped | | `ADVANCED_TEXT_NORMALIZATION` | `true` | Master switch for number/URL/email expansion before phonemizing; English only, opt out per request with `normalization_options` | **Audio** diff --git a/pyproject.toml b/pyproject.toml index 0f9d5345..38b5869c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ dependencies = [ "tiktoken==0.8.0", "loguru==0.7.3", "openai>=1.59.6", - "pydub>=0.25.1", "mutagen>=1.47.0", "psutil>=6.1.1", "espeakng-loader==0.2.4", diff --git a/uv.lock b/uv.lock index d9b54998..7a499d28 100644 --- a/uv.lock +++ b/uv.lock @@ -1246,7 +1246,6 @@ dependencies = [ { name = "psutil" }, { name = "pydantic" }, { name = "pydantic-settings" }, - { name = "pydub" }, { name = "python-dotenv" }, { name = "regex" }, { name = "requests" }, @@ -1310,7 +1309,6 @@ requires-dist = [ { name = "psutil", specifier = ">=6.1.1" }, { name = "pydantic", specifier = "==2.10.4" }, { name = "pydantic-settings", specifier = "==2.7.0" }, - { name = "pydub", specifier = ">=0.25.1" }, { name = "pytest", marker = "extra == 'test'", specifier = "==8.3.5" }, { name = "pytest-asyncio", marker = "extra == 'test'", specifier = "==0.25.3" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = "==6.0.0" }, @@ -2491,15 +2489,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/00/57b4540deb5c3a39ba689bb519a4e03124b24ab8589e618be4aac2c769bd/pydantic_settings-2.7.0-py3-none-any.whl", hash = "sha256:e00c05d5fa6cbbb227c84bd7487c5c1065084119b750df7c8c1a554aed236eb5", size = 29549, upload-time = "2024-12-13T09:41:09.54Z" }, ] -[[package]] -name = "pydub" -version = "0.25.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/9a/e6bca0eed82db26562c73b5076539a4a08d3cffd19c3cc5913a3e61145fd/pydub-0.25.1.tar.gz", hash = "sha256:980a33ce9949cab2a569606b65674d748ecbca4f0796887fd6f46173a7b0d30f", size = 38326, upload-time = "2021-03-10T02:09:54.659Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/53/d78dc063216e62fc55f6b2eebb447f6a4b0a59f55c8406376f76bf959b08/pydub-0.25.1-py2.py3-none-any.whl", hash = "sha256:65617e33033874b59d87db603aa1ed450633288aefead953b30bded59cb599a6", size = 32327, upload-time = "2021-03-10T02:09:53.503Z" }, -] - [[package]] name = "pygments" version = "2.19.2" diff --git a/web/src/services/AudioService.js b/web/src/services/AudioService.js index a3fba007..3b70c33d 100644 --- a/web/src/services/AudioService.js +++ b/web/src/services/AudioService.js @@ -27,6 +27,7 @@ export class AudioService { this.preloadPromise = null; this.pendingSeek = null; this.pendingResume = false; + this.volume = 1; } supportsMSEMp3() { @@ -183,6 +184,7 @@ export class AudioService { const blobType = response.headers.get('content-type') || 'audio/mpeg'; const blob = new Blob(chunks, { type: blobType }); this.audio = new Audio(); + this.audio.volume = this.volume; this.attachAudioReadinessEvents(); this.objectUrl = URL.createObjectURL(blob); this.audio.src = this.objectUrl; @@ -215,6 +217,7 @@ export class AudioService { } this.audio = new Audio(); + this.audio.volume = this.volume; this.attachAudioReadinessEvents(); this.attachAudioErrorEvents('stream'); @@ -264,6 +267,11 @@ export class AudioService { play() { if (this.audio && !this.audio.error) { + const duration = this.audio.duration; + if (this.usingFileSource && Number.isFinite(duration) && + duration - this.audio.currentTime <= 0.1) { + this.audio.currentTime = 0; + } const playPromise = this.audio.play(); if (playPromise) { playPromise.catch(error => { @@ -444,8 +452,9 @@ export class AudioService { } setVolume(volume) { + this.volume = Math.max(0, Math.min(1, volume)); if (this.audio) { - this.audio.volume = Math.max(0, Math.min(1, volume)); + this.audio.volume = this.volume; } } @@ -455,6 +464,9 @@ export class AudioService { getDuration() { const duration = this.audio ? this.audio.duration : 0; + if (this.msePipeline && this.knownDuration) { + return this.knownDuration; + } if (Number.isFinite(duration) && duration > 0) { return duration; }