python_test.yml

name: Python tests

on:
  workflow_call:
    secrets:
      OPENAI_API_KEY:
        required: true
      ANTHROPIC_API_KEY:
        required: true
      CODECOV_TOKEN:
        required: false
    inputs:
      python-versions:
        description: "(Optional) Python versions to test"
        required: false
        type: string
        default: "['3.10', '3.11', '3.12', '3.13', '3.14']"
      ref:
        description: "(Optional) ref to checkout"
        required: false
        type: string
      nightly:
        description: "Whether run is from the nightly build"
        required: false
        type: boolean
        default: false
      runs-on:
        description: "Runner to use for the tests"
        required: false
        type: string
        default: "ubuntu-latest"
  workflow_dispatch:
    inputs:
      python-versions:
        description: "(Optional) Python versions to test"
        required: true
        type: string
        default: "['3.10', '3.11', '3.12', '3.13', '3.14']"
      runs-on:
        description: "Runner to use for the tests"
        required: false
        type: choice
        options:
          - ubuntu-latest
          - self-hosted
          - '["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral"]'
        default: ubuntu-latest

permissions:
  contents: read

env:
  POETRY_VERSION: "1.8.2"
  NODE_VERSION: "22"
  PYTEST_RUN_PATH: "src/backend/tests"

jobs:
  build:
    uses: ./.github/workflows/backend-unit-tests.yml
    with:
      python-versions: ${{ inputs.python-versions || '["3.10", "3.14"]' }}
      ref: ${{ inputs.ref || github.ref }}
      runs-on: ${{ inputs.runs-on || 'ubuntu-latest' }}
    secrets:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  bundle-guarded-tests:
    permissions:
      contents: read
    # The backend unit suite installs with a plain `uv sync`, which since #13869 no longer
    # brings in the opt-in lfx-bundles metapackage -- so every test carrying
    # `pytest.importorskip("lfx_bundles")` silently skipped in all of CI (issue #14554).
    # Most of those moved into src/bundles/lfx-bundles/tests/ and now run under
    # cross-bundle-test.yml. The stragglers cannot: they inherit tests/base.py's
    # version-compatibility harness (tests.constants + tests.integration.utils), which is
    # langflow-specific and cannot follow them into the bundle's lfx-only venv. This job is
    # their home -- same suite, bundles installed.
    name: Unit Tests (bundles installed) - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        # Cost control: oldest + newest supported Python, matching cross-bundle-test.yml.
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.14"]') }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.ref }}
          persist-credentials: false
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false

      - name: Install the project with the bundles extra
        run: uv sync --extra bundles

      - name: Fail if the bundles extra did not actually install lfx_bundles
        # Before the tests, not after: if lfx_bundles is missing, every guarded test
        # importorskips and the run below reports green having executed nothing -- the exact
        # failure mode this job exists to catch.
        run: uv run python -c "import lfx_bundles; print('lfx_bundles', lfx_bundles.__file__)"

      - name: Run the bundle-guarded backend unit tests
        run: |
          # Discovered, not hard-coded: a newly guarded test is picked up automatically
          # instead of quietly joining the set of tests nothing runs.
          # -E with ['\"] so a single-quoted importorskip is not silently missed -- a guarded
          # test this scan overlooks is exactly the bug being fixed.
          # `while read` rather than `mapfile`: mapfile needs bash 4+, and runs-on is an input
          # to this workflow, so the shell is not guaranteed to be a GitHub ubuntu runner.
          guarded=()
          while IFS= read -r f; do guarded+=("$f"); done < <(grep -rlE --include='*.py' "pytest\.importorskip\([[:space:]]*['\"]lfx_bundles" src/backend/tests || true)
          if [ ${#guarded[@]} -eq 0 ]; then
            echo "No lfx_bundles-guarded backend tests found -- nothing to run."
            exit 0
          fi
          printf 'Running %d bundle-guarded backend test file(s):\n' "${#guarded[@]}"
          printf '  %s\n' "${guarded[@]}"
          # -m "not api_key_required" matches `make unit_tests`, which the main backend job
          # uses: those tests make live provider calls and are deselected repo-wide by
          # policy. Deselecting is a deliberate exclusion, unlike the silent import-skip
          # this job exists to prevent.
          uv run pytest "${guarded[@]}" -v --timeout=300 -m "not api_key_required"

      - name: Run the bundle-guarded lfx unit tests
        # A separate invocation, and from src/lfx: src/backend/tests and src/lfx/tests both
        # provide a top-level `tests` package, so collecting both in one pytest run dies with
        # ImportPathMismatchError on tests.conftest. This mirrors the existing split between
        # the Unit Tests and LFX Tests jobs (`make lfx_tests` also runs from src/lfx).
        working-directory: src/lfx
        run: |
          guarded=()
          while IFS= read -r f; do guarded+=("$f"); done < <(grep -rlE --include='*.py' "pytest\.importorskip\([[:space:]]*['\"]lfx_bundles" tests || true)
          if [ ${#guarded[@]} -eq 0 ]; then
            echo "No lfx_bundles-guarded lfx tests found -- nothing to run."
            exit 0
          fi
          # The lfx package resolves its own venv, which does not carry lfx-bundles; install
          # the in-repo bundle so the guards pass here too.
          uv sync --dev --extra otel
          uv pip install "../bundles/lfx-bundles[all-no-torch]"
          # --no-sync from here on: the install above is deliberately outside the lock, and
          # a plain `uv run` re-syncs the environment to uv.lock. That reverts the lock-managed
          # packages the bundle install upgraded while leaving the injected ones alone, which
          # leaves an environment neither resolution ever produced. It bit us the day
          # langchain-openai 1.5.2 shipped: the bundle install brought openai 1.5.2 with
          # langchain-core 1.5.6, `uv run` put core back to the locked 1.5.1, and the pair
          # failed to import. Both resolutions were fine; only the mix was not.
          uv run --no-sync python -c "import lfx_bundles; print('lfx_bundles', lfx_bundles.__file__)"
          printf 'Running %d bundle-guarded lfx test file(s):\n' "${#guarded[@]}"
          printf '  %s\n' "${guarded[@]}"
          # No --timeout: pytest-timeout is not in lfx's dev group (pytest, pytest-asyncio,
          # pytest-cov only), and `make lfx_tests` does not pass one either.
          uv run --no-sync pytest "${guarded[@]}" -v -m "not api_key_required"

  integration-tests:
    name: Integration Tests - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }}
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v10.0.0
        with:
          version: latest-known
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false
      - name: Install the project
        run: uv sync
      - name: Run integration tests
        run: make integration_tests_no_api_keys
        env:
          PYLEAK_LOG_LEVEL: debug # enable pyleak logging
          DO_NOT_TRACK: true # disable telemetry reporting

  lfx-tests:
    name: LFX Tests - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }}
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v10.0.0
        with:
          version: latest-known
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false
      - name: Run lfx tests
        run: make lfx_tests
        env:
          DO_NOT_TRACK: true # disable telemetry reporting

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v7
        if: matrix.python-version == '3.10'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./src/lfx/coverage.xml
          flags: lfx
          name: lfx-coverage
          fail_ci_if_error: false
          directory: ./src/lfx/

      - name: Upload coverage artifacts
        uses: actions/upload-artifact@v7
        if: matrix.python-version == '3.10'
        with:
          name: lfx-coverage-report
          path: |
            src/lfx/coverage.xml
            src/lfx/htmlcov/
          retention-days: 30

  test-cli:
    name: Test CLI - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }}
    steps:
      - name: Check out the code at a specific ref
        uses: actions/checkout@v7
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v10.0.0
        with:
          version: latest-known
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false
          # This job intentionally exits before invoking uv when langflow-base
          # has not been published yet (including its first release).
          ignore-nothing-to-cache: true

      - name: Check Version
        id: check-version
        # The main wheel resolves its direct langflow-base dependency from PyPI.
        # Skip until the source base version (stable or canonical nightly) exists.
        run: |
          version=$(python -c "import pathlib, tomllib; print(tomllib.loads(pathlib.Path('src/backend/base/pyproject.toml').read_text())['project']['version'])")
          url="https://pypi.org/pypi/langflow-base/json"

          status=$(curl -sS -o "$RUNNER_TEMP/langflow-base-pypi.json" -w "%{http_code}" "$url")
          if [ "$status" = "404" ]; then
            last_released_version=""
          elif [ "$status" != "200" ]; then
            echo "Failed to fetch langflow-base releases: HTTP $status"
            cat "$RUNNER_TEMP/langflow-base-pypi.json"
            exit 1
          else
            last_released_version=$(jq -r '.releases | keys[]' "$RUNNER_TEMP/langflow-base-pypi.json" | sort -V | tail -n 1)
          fi

          if [ "$version" != "$last_released_version" ]; then
            echo "Version $version has not been released yet. Skipping the rest of the job."
            echo skipped=true >> "$GITHUB_OUTPUT"
            exit 0
          else
            echo "version=$version" >> "$GITHUB_OUTPUT"
            echo skipped=false >> "$GITHUB_OUTPUT"
          fi
      - name: Build wheel
        if: steps.check-version.outputs.skipped == 'false'
        run: |
          make build main=true
      - name: Install wheel and Test CLI
        if: steps.check-version.outputs.skipped == 'false'
        run: |
          uv venv new-venv
          source new-venv/bin/activate
          uv pip install dist/*.whl
      - name: Test CLI
        if: steps.check-version.outputs.skipped == 'false'
        env:
          LANGFLOW_SUPERUSER_PASSWORD: ci-startup-only-password  # pragma: allowlist secret
        run: |
          source new-venv/bin/activate
          python -m langflow run --host localhost --port 7860 --backend-only &
          SERVER_PID=$!
          # Wait for the server to start
          timeout 120 bash -c 'until curl -f http://localhost:7860/health_check; do sleep 5; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
          # Terminate the server
          kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
          sleep 20 # give the server some time to terminate
          # Check if the server is still running
          if kill -0 $SERVER_PID 2>/dev/null; then
            echo "Failed to terminate the server"
            exit 0
          else
            echo "Server terminated successfully"
          fi