bene/flakiness tui

gradle/plugins/src/main/kotlin/app.testlens.gradle.component.application-cli.gradle.kts

@@ -38,7 +38,7 @@ sourceSets {
 
 val downloadOpenApiSpec =
     tasks.register("downloadOpenApiSpec") {
-        val uri = "https://api.testlens.app/v1/openapi"
+        val uri = "https://api.testlens.app/v1/openapi/dev"
         inputs.property("uri", uri)
         val specFile = layout.buildDirectory.file("openapi-spec.yml")
         outputs.file(specFile)

notes.md

@@ -0,0 +1,11 @@
+- later: Show environment data that we already capture: os, java version
+- later: ability to filter executions table
+- What's the sort order in the statistics table?! percentage? then we need to show two digits or the total number of executions
+- Footer line in statistics table need to be at the bottom of the view port
+- later later: cli mode should be plain text, no console control chars
+- cli mode for test executions shows `sha @ branch` where `branch` seems to be the target branch. it should be better visible whether this was an execution on a PR or on a (default) branch. Also think about gh merge queue branches and whether it makes sense to display them in a specific way.
+
+## Fuzzy matching of failures
+
+- later later later: determine whether failed runs for a sha have something in common compared to successful runs. e.g. for a single sha, do all failures have the same operating system?
+

src/main/java/app/testlens/cli/CommandSupport.java

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+package app.testlens.cli;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
+
+import app.testlens.cli.client.ApiClient;
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Pattern;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * Shared plumbing of the commands: repository resolution, token handling and API client setup. The convention
+ * established by {@link PrCommand}: {@code --repo} defaults to the GitHub remote {@code origin} of the current
+ * directory, {@code --token} to {@code $GITHUB_TOKEN} or {@code $GH_TOKEN}.
+ */
+public final class CommandSupport {
+
+    private static final Pattern GITHUB_REMOTE = Pattern.compile("github\\.com[:/]([^/]+)/(.+?)(?:\\.git)?/?$");
+
+    private CommandSupport() {}
+
+    static String @Nullable [] resolveRepo(@Nullable String repoOption) {
+        return repoOption != null ? repoOption.split("/", 2) : parseRemote(gitRemoteUrl());
+    }
+
+    /** Parses owner/repo from a GitHub remote URL. */
+    static String @Nullable [] parseRemote(@Nullable String url) {
+        if (url == null) {
+            return null;
+        }
+        var matcher = GITHUB_REMOTE.matcher(url.strip());
+        return matcher.find() ? new String[] { matcher.group(1), matcher.group(2) } : null;
+    }
+
+    static boolean isInteractiveTerminal() {
+        var console = System.console();
+        return console != null && console.isTerminal();
+    }
+
+    static ApiClient apiClient(String token, String baseUrl) {
+        var apiClient = new ApiClient();
+        apiClient.updateBaseUri(requireNonNull(baseUrl));
+        apiClient.setRequestInterceptor(request -> request.header("Authorization", "Bearer " + token));
+        return apiClient;
+    }
+
+    /** Display name of a test, skipping the first display name which denotes the test engine. */
+    static String testName(List<String> displayNames) {
+        return String.join(" > ", displayNames.subList(Math.min(1, displayNames.size()), displayNames.size()));
+    }
+
+    private static @Nullable String gitRemoteUrl() {
+        try {
+            var process = new ProcessBuilder("git", "remote", "get-url", "origin")
+                .redirectError(ProcessBuilder.Redirect.DISCARD)
+                .start();
+            var output = new String(process.getInputStream().readAllBytes(), UTF_8);
+            return process.waitFor() == 0 ? output : null;
+        } catch (IOException e) {
+            return null;
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return null;
+        }
+    }
+}

src/main/java/app/testlens/cli/ExecutionsLoader.java

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+package app.testlens.cli;
+
+import app.testlens.cli.client.ApiException;
+import app.testlens.cli.client.api.TestsApi;
+import app.testlens.cli.client.model.TestExecutionResponse;
+import java.util.List;
+import java.util.Map;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * Loads the pages of the executions endpoint for one test, excluding successful executions: the detail view is a
+ * triage view, and flaky tests execute successfully many times, crowding out the failures. The loaded pages follow the
+ * server's {@code rel="next"} links, one page per {@code next()} call.
+ */
+public final class ExecutionsLoader {
+
+    /**
+     * Raw outcome filter of the triage view: everything that is not a successful (or reused-successful) execution.
+     * Shared by the detail view's initial fetch and its page loads.
+     */
+    public static final List<String> NON_SUCCESSFUL_OUTCOMES = List.of("FAILED", "ABORTED", "MUTED", "SKIPPED");
+
+    /** One fetched page: the executions of that page and whether another page may follow. */
+    public record Fetched(List<TestExecutionResponse> executions, boolean hasMore) {}
+
+    private final TestsApi api;
+    private final String owner;
+    private final String repo;
+    private final long testId;
+    private int nextPage;
+    private boolean hasMore;
+
+    public ExecutionsLoader(TestsApi api, String owner, String repo, long testId) {
+        this.api = api;
+        this.owner = owner;
+        this.repo = repo;
+        this.testId = testId;
+    }
+
+    /** Fetches the first page; resets the loader, so it can also be re-called after a back-navigation. */
+    public Fetched first() throws ApiException {
+        return fetch(1);
+    }
+
+    /** Fetches the next page, or {@code null} when the last page was already loaded. */
+    public @Nullable Fetched next() throws ApiException {
+        return hasMore ? fetch(nextPage) : null;
+    }
+
+    private Fetched fetch(int page) throws ApiException {
+        nextPage = page + 1;
+        var response = api
+            .getTestExecutionsWithHttpInfo(
+                owner,
+                repo,
+                testId,
+                NON_SUCCESSFUL_OUTCOMES,
+                null,
+                null,
+                null,
+                page,
+                null,
+                Map.of()
+            );
+        hasMore = hasRelNext(response.getHeaders());
+        return new Fetched(response.getData() == null ? List.of() : response.getData(), hasMore);
+    }
+
+    public static boolean hasRelNext(Map<String, List<String>> headers) {
+        return headers.entrySet()
+            .stream()
+            .filter(header -> header.getKey().equalsIgnoreCase("Link"))
+            .flatMap(header -> header.getValue().stream())
+            .anyMatch(ExecutionsLoader::hasRelNext);
+    }
+
+    private static boolean hasRelNext(String linkHeader) {
+        return linkHeader.matches(".*rel\\s*=\\s*\"?next\"?.*");
+    }
+}

src/main/java/app/testlens/cli/FlakesCommand.java

@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+package app.testlens.cli;
+
+import static app.testlens.cli.CommandSupport.apiClient;
+import static app.testlens.cli.CommandSupport.isInteractiveTerminal;
+import static app.testlens.cli.CommandSupport.resolveRepo;
+import static app.testlens.cli.util.Console.clearProgress;
+import static app.testlens.cli.util.Console.showProgress;
+import static app.testlens.cli.util.Console.styled;
+import static java.util.Comparator.comparing;
+import static java.util.Objects.requireNonNull;
+
+import app.testlens.cli.TestCommand.Mode;
+import app.testlens.cli.client.ApiException;
+import app.testlens.cli.client.api.TestsApi;
+import app.testlens.cli.client.model.FlakeResponse;
+import app.testlens.cli.client.model.TestExecutionResponse;
+import app.testlens.cli.tui.FlakesTui;
+import app.testlens.cli.tui.controller.FlakesController;
+import app.testlens.cli.tui.model.TestInfo;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.jspecify.annotations.Nullable;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Help.Ansi;
+import picocli.CommandLine.Help.Ansi.Style;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+import picocli.CommandLine.Spec;
+
+/**
+ * {@code testlens flakes <id>} shows the flakes of one test: the builds in which it flipped its outcome after retries,
+ * each with all its executions in chronological order. Options mirror {@link TestCommand}.
+ */
+@Command(
+    name = "flakes",
+    description = "Show the flaky builds of a test with all their executions",
+    mixinStandardHelpOptions = true
+)
+public class FlakesCommand implements Callable<Integer> {
+
+    @Spec
+    @Nullable
+    CommandSpec spec;
+
+    @Parameters(index = "0", description = "Internal test ID, e.g. from testlens statistics flakiness")
+    long id;
+
+    @Option(names = "--branch", description = "Only flakes on this branch; PR builds count toward their target branch")
+    @Nullable
+    String branch;
+
+    @Option(names = { "--task", "--project" }, description = "Only flakes for this task/project")
+    @Nullable
+    String workUnit;
+
+    @Option(
+        names = "--repo",
+        description = "Repository as owner/repo (defaults to the GitHub remote 'origin' of the current directory)"
+    )
+    @Nullable
+    String repo;
+
+    @Option(
+        names = "--token",
+        description = "GitHub access token (defaults to $GITHUB_TOKEN or $GH_TOKEN)",
+        defaultValue = "${env:GITHUB_TOKEN:-${env:GH_TOKEN}}"
+    )
+    @Nullable
+    String token;
+
+    @Option(
+        names = "--mode",
+        description = "Output mode: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}, uses the TUI in an interactive terminal)",
+        defaultValue = "auto"
+    )
+    Mode mode = Mode.AUTO;
+
+    @Option(
+        names = "--base-url",
+        description = "API base URL",
+        defaultValue = "https://api.testlens.app",
+        hidden = true
+    )
+    @Nullable
+    String baseUrl;
+
+    @Override
+    public Integer call() throws Exception {
+        var out = requireNonNull(spec).commandLine().getOut();
+        var err = spec.commandLine().getErr();
+        if (token == null || token.isBlank()) {
+            err.println("No GitHub token provided (use --token or set GITHUB_TOKEN)");
+            return 2;
+        }
+        var ownerAndRepo = resolveRepo(repo);
+        if (ownerAndRepo == null || ownerAndRepo.length != 2) {
+            err.println("Cannot determine repository (use --repo owner/repo)");
+            return 2;
+        }
+        var owner = ownerAndRepo[0];
+        var repoName = ownerAndRepo[1];
+        var ansi = spec.commandLine().getColorScheme().ansi();
+        var api = new TestsApi(apiClient(token, requireNonNull(baseUrl)));
+        try {
+            var tui = mode == Mode.TUI || (mode == Mode.AUTO && isInteractiveTerminal());
+            showProgress(out, ansi, "Fetching test...");
+            var test = TestInfo.of(api.getTest(owner, repoName, id));
+            clearProgress(out, ansi);
+            showProgress(out, ansi, "Fetching flakes...");
+            var flakes = fetchFlakes(api, owner, repoName, id, branch, workUnit);
+            clearProgress(out, ansi);
+            if (tui) {
+                FlakesTui.run(test, branch, workUnit, flakes);
+            } else {
+                printFlakes(out, ansi, test, branch, workUnit, flakes);
+            }
+            return 0;
+        } catch (ApiException e) {
+            clearProgress(out, ansi);
+            return TestCommand.fail(err, e, "Test not found");
+        }
+    }
+
+    /** Fetches all pages of the flakes of a test, newest first. */
+    public static List<FlakeResponse> fetchFlakes(
+        TestsApi api,
+        String owner,
+        String repo,
+        long testId,
+        @Nullable String branch,
+        @Nullable String workUnit
+    ) throws ApiException {
+        var flakes = new ArrayList<FlakeResponse>();
+        for (int page = 1;; page++) {
+            var response =
+                api.getTestFlakesWithHttpInfo(owner, repo, testId, null, branch, workUnit, page, null, Map.of());
+            if (response.getData() != null) {
+                flakes.addAll(response.getData());
+            }
+            if (!ExecutionsLoader.hasRelNext(response.getHeaders())) {
+                return flakes;
+            }
+        }
+    }
+
+    static void printFlakes(
+        PrintWriter out,
+        Ansi ansi,
+        TestInfo test,
+        @Nullable String branch,
+        @Nullable String workUnit,
+        List<FlakeResponse> flakes
+    ) {
+        out.println(
+            styled(ansi, test.displayName(), Style.bold) + " (test " + test.id() + ")"
+                + (branch == null ? "" : " · branch " + branch)
+                + (workUnit == null ? "" : " · project/task " + workUnit)
+        );
+        out.println("   " + styled(ansi, test.uniqueId(), Style.faint));
+        out.println("   " + styled(ansi, "newest flake first, executions in chronological order", Style.faint));
+        if (flakes.isEmpty()) {
+            out.println();
+            out.println("No flakes in the last 30 days.");
+            return;
+        }
+        for (int i = 0; i < flakes.size(); i++) {
+            var flake = flakes.get(i);
+            out.println();
+            out.println(
+                "  " + (i + 1) + ". " + styled(
+                    ansi,
+                    FlakesController.abbreviatedSha(flake.getSha()) + " @ " + flake.getBranch(),
+                    Style.bold
+                ) + " · " + flake.getWorkUnit()
+            );
+            var executions = flake.getExecutions()
+                .stream()
+                .sorted(comparing(TestExecutionResponse::getStartTime))
+                .toList();
+            for (int j = 0; j < executions.size(); j++) {
+                TestCommand.printExecution(out, ansi, "      ", j + 1, executions.get(j), false);
+            }
+        }
+    }
+}