test(e2e): cover a real adapter with a mocked Imgur API

.github/workflows/docker-build.yml

@@ -19,4 +19,4 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Build and test in Docker
-      run: docker compose run --rm test
+      run: docker compose run --rm gradle

.github/workflows/e2e.yml

@@ -0,0 +1,37 @@
+# Runs a complete offline-demo -> offline-demo transfer against the packaged
+# demo-server jar. See e2e/README.md.
+#
+# continue-on-error is deliberate and temporary: a flaky end-to-end job that
+# blocks merges is worse than no end-to-end job. Drop it once this has run
+# green on master for a while.
+
+name: End-to-end transfer
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  transfer:
+
+    runs-on: ubuntu-latest
+    continue-on-error: true
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Run the end-to-end transfer
+      run: ./e2e/run.sh
+
+    # The server log is the only record of what the transfer actually did, so
+    # keep it whether the run passed or failed -- a failure should be
+    # diagnosable without re-running it.
+    - name: Upload server log
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-server-log
+        path: e2e/.logs/
+        if-no-files-found: warn

.gitignore

@@ -50,6 +50,9 @@ testem.log
 # e2e
 /e2e/*.js
 /e2e/*.map
+# Run artifacts: server logs and mock request journals. The logs are already
+# covered by *.log above, the journals are not.
+/e2e/.logs/
 
 client-rest/dist/
 

Documentation/Developer.md

@@ -86,17 +86,20 @@ See [Running Locally](RunningLocally.md) for instructions.
 
 This is unrelated to the demo image built by `dockerize` above -- it's a separate, additive way to run
 the Java test suite without installing a JDK locally. The `Dockerfile` at the repo root pins
-`gradle:8.10.2-jdk11`; the source is bind-mounted at run time rather than baked into the image, so a
-rebuild isn't needed after every code change.
+`eclipse-temurin:11-jdk-jammy` and resolves the wrapper-pinned Gradle 6.9.2 into an image layer at
+build time; the source is bind-mounted at run time rather than baked into the image, so a rebuild
+isn't needed after every code change.
+
+The `gradle` service's `ENTRYPOINT` is `./gradlew`, so anything you pass it is a Gradle argument.
 
 Run the full check task against the current working tree:
 ```bash
-docker compose run --rm test
+docker compose run --rm gradle
 ```
 
 To run a specific Gradle task or test, override the default command:
 ```bash
-docker compose run --rm test test --tests SomeTest
+docker compose run --rm gradle test --tests SomeTest
 ```
 
 The `gradle-cache` named volume (defined in `docker-compose.yml`) persists resolved dependencies across
@@ -105,6 +108,31 @@ runs, so only the first run pays the full resolution cost.
 The JDK is pinned to 11 because the Gradle wrapper (6.9.2) can't parse Java 17 bytecode when compiling
 `build.gradle`/`settings.gradle` -- a `gradle:*-jdk17` image fails outright for this reason.
 
+## Running the end-to-end transfer test
+
+The `gradle` service above runs unit tests. To run a real transfer end to end -- job creation, auth,
+worker claim, export, import -- against the packaged `demo-server` jar:
+
+```bash
+./e2e/run.sh
+```
+
+No provider credentials, no local JDK and no local Python; about 35 seconds with a warm Gradle cache.
+It uses the `dtp` and `e2e` services in `docker-compose.yml`, and leaves the server log in
+`e2e/.logs/dtp.log` whether the run passes or fails.
+
+The same services are the shortest way to watch DTP actually do something without acquiring a single
+API key:
+
+```bash
+docker compose run --rm gradle --no-daemon \
+  :distributions:demo-server:shadowJar -PofflineData=true -PencryptionScheme=cleartext
+docker compose up dtp          # API on https://localhost:8080 (self-signed cert)
+```
+
+See [e2e/README.md](../e2e/README.md) for what a green run does and does not prove, and for how to
+add an adapter.
+
 ## Deploying in production
 
 A demo distribution for Google Cloud Platform is available at

docker-compose.yml

@@ -1,9 +1,132 @@
+# Three services; `gradle` and `dtp` share this repo's one Dockerfile.
+#
+# `gradle` runs the Gradle build (see Dockerfile) -- its ENTRYPOINT is
+# ./gradlew, so arguments passed to it are Gradle arguments. `dtp` runs the
+# packaged demo-server jar that build produces, and `e2e` drives a transfer
+# against it.
+#
+# `dtp` deliberately reuses `build: .` rather than defining an image of its
+# own -- the repo publishes no server image today, so the harness pins a
+# *contract* (an API on 8080 plus a log stream) rather than an artifact. When a
+# real server image exists, replace `build`/`entrypoint`/`command` on `dtp`
+# with `image:` and nothing else here changes.
+#
+#   docker compose run --rm gradle      # the build
+#   docker compose up dtp               # the server, on https://localhost:8080
+#   ./e2e/run.sh                        # the end-to-end transfer
 services:
-  test:
+  gradle:
     build: .
     volumes:
       - .:/workspace
       - gradle-cache:/home/gradle/.gradle
 
+  # The system under test: API + transfer worker in one JVM, as SingleVMMain
+  # requires (LocalJobStore shares state through private static maps, so the
+  # two only ever see the same job when co-located).
+  dtp:
+    build: .
+    # In every adapter profile: run.sh brings up a *fresh* dtp per adapter, so
+    # no suite can see another's jobs or temp files. LocalJobStore's static maps
+    # and LocalTempFileStore's files accumulate across jobs and nothing clears
+    # them, so a shared container would make test isolation a matter of luck.
+    profiles: ["offline-demo", "imgur"]
+    volumes:
+      - .:/workspace
+      - dtp-logs:/var/log/dtp
+    environment:
+      # LocalAppCredentialStore reads secrets from the environment and ApiMain
+      # builds a JWTTokenManager unconditionally, so it will not boot without
+      # these. The values are never checked against anything -- offline-demo
+      # bypasses OAuth entirely.
+      JWT_KEY: e2e-key
+      JWT_SECRET: e2e-secret
+      # Never checked either, but required: OAuth2ServiceExtension.initialize
+      # swallows the missing-credential IOException and returns *without*
+      # setting `initialized`, so the failure surfaces much later as
+      # "Cannot get OAuth2DataGenerator before initialization" on the first
+      # POST /api/transfer.
+      IMGUR_KEY: e2e-imgur-key
+      IMGUR_SECRET: e2e-imgur-secret
+    # Published so `docker compose up dtp` is also the credential-free way to
+    # poke at a running DTP by hand. JettyTransport hardcodes 8080.
+    ports:
+      - "8080:8080"
+    # The image's ENTRYPOINT is ./gradlew; override it to run the jar that
+    # `gradle` built. Not `:distributions:demo-server:run` -- that task reads
+    # build/resources/main but nothing makes it depend on createApiFile, so
+    # api.yaml is missing and ApiExtensionContext fails on a null baseUrl.
+    # shadowJar.dependsOn(createApiFile), so the jar is self-contained.
+    #
+    # Globbed because the jar name depends on the version, and root
+    # build.gradle sets `version = System.getenv('RELEASE_VERSION')` -- unset
+    # locally, so the artifact is plain `demo-server-all.jar`, but versioned in
+    # a release build. A glob needs a shell, hence bash -c.
+    #
+    # tee, not `>`: the e2e service reads the file, humans read `docker compose
+    # logs dtp`, and run.sh captures the same stream to e2e/.logs/ afterwards.
+    # Merging stderr matters -- ConsoleMonitor writes the server log to stderr
+    # while OfflineDemoImporter prints the delivered payload to stdout.
+    #
+    # `-cp <dir>:<jar>` rather than `-jar`, because `-jar` ignores -cp entirely.
+    # Config resolution is classpath-only -- TransferServiceConfig and
+    # ConfigUtils both use the *singular* getResourceAsStream, so first match
+    # per filename wins -- which makes a prepended directory the one supported
+    # way to feed an adapter a different base URL. e2e/config holds only files
+    # the jar does not ship, so this is purely additive and inert for adapters
+    # that have no file there.
+    #
+    # $$ escapes Compose's own interpolation so the shell gets $(...).
+    entrypoint: ["bash", "-c"]
+    command:
+      - "java -cp /workspace/e2e/config:$$(ls distributions/demo-server/build/libs/*-all.jar) org.datatransferproject.bootstrap.vm.SingleVMMain 2>&1 | tee /var/log/dtp/dtp.log"
+
+  # Stands in for api.imgur.com. Configured entirely by JSON -- mappings/ for
+  # the stubs, __files/ for the image bytes the exporter downloads during
+  # export. Its /__admin/requests journal is the assertion surface for what the
+  # importer actually sent, which is a far better one than a log line.
+  wiremock-imgur:
+    image: wiremock/wiremock:3.9.2
+    profiles: ["imgur"]
+    # Read-only on purpose. WireMock runs as root and will happily mkdir any of
+    # mappings/ or __files/ that is missing, leaving root-owned directories in
+    # your checkout -- the same papercut PYTHONDONTWRITEBYTECODE avoids for the
+    # driver. It never needs to write unless it is recording.
+    volumes:
+      - ./e2e/mocks/imgur:/home/wiremock:ro
+    # Published so run.sh can pull the request journal from the host, keeping
+    # the artifact owned by you rather than by root. The image ships no curl,
+    # so `compose exec` is not an option.
+    ports:
+      - "18080:8080"
+    command: ["--verbose"]
+
+  # Stock python image, no Dockerfile. Deps install into a cached volume.
+  e2e:
+    image: python:3.12-slim
+    working_dir: /workspace
+    # No depends_on: `dtp` is now profiled and run.sh starts it explicitly, one
+    # fresh container per adapter. The driver's await_ready() is the readiness
+    # gate regardless -- there has never been a healthcheck.
+    volumes:
+      - .:/workspace
+      - dtp-logs:/var/log/dtp:ro
+      - pip-cache:/root/.cache/pip
+    environment:
+      DTP_BASE_URL: https://dtp:8080
+      DTP_LOG: /var/log/dtp/dtp.log
+      WIREMOCK_IMGUR_URL: http://wiremock-imgur:8080
+      # Which adapter's suite to run, as a pytest marker. Empty runs all of
+      # them, which is what a bare `docker compose run --rm e2e` does.
+      E2E_MARKER: "${E2E_MARKER:-}"
+      # Keep the container from littering the bind-mounted repo with
+      # root-owned __pycache__ directories.
+      PYTHONDONTWRITEBYTECODE: "1"
+    entrypoint: ["bash", "-c"]
+    command:
+      - "pip install --quiet --root-user-action=ignore -r e2e/driver/requirements.txt && pytest e2e/driver -v -p no:cacheprovider $${E2E_MARKER:+-m $$E2E_MARKER}"
+
 volumes:
   gradle-cache:
+  dtp-logs:
+  pip-cache: