Issue 6039: Generate system test suite documentation

.github/workflows/java-status.yaml

@@ -8,6 +8,8 @@ on:
       # Check generated files are not edited directly
       - 'examples/**'
       - 'scripts/templates/**'
+      - 'docs/deployment/docker-images.md'
+      - 'docs/development/system-test-suites.md'
       - 'docs/usage/properties/**'
   push:
     branches:
@@ -60,16 +62,21 @@ jobs:
         working-directory: ./java
         run: |
           PROJECT_ROOT=$(cd .. && pwd)
-          mvn install -Pquick,skipShade -q -e -pl clients -am -DskipRust -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
+          mvn install -Pquick,skipShade -q -e -pl clients,system-test/system-test-suite -am -DskipRust \
+            -Dmaven.repo.local=${{ runner.temp }}/.m2/repository
           mvn exec:java -q -e -Dmaven.repo.local=${{ runner.temp }}/.m2/repository -pl clients \
             -Dexec.mainClass=sleeper.clients.deploy.documentation.GeneratePropertiesTemplates \
             -Dexec.args="$PROJECT_ROOT"
           mvn exec:java -q -e -Dmaven.repo.local=${{ runner.temp }}/.m2/repository -pl clients \
             -Dexec.mainClass=sleeper.clients.deploy.documentation.GenerateDockerImageDocumentation \
             -Dexec.args="$PROJECT_ROOT"
+          mvn exec:java -q -e -Dmaven.repo.local=${{ runner.temp }}/.m2/repository -pl system-test/system-test-suite \
+            -Dexec.classpathScope=test \
+            -Dexec.mainClass=sleeper.systemtest.suite.documentation.GenerateSystemTestSuiteDocumentation \
+            -Dexec.args="$PROJECT_ROOT"
           CHANGED_FILES=$(git status --porcelain)
           if [ ${#CHANGED_FILES} -gt 0 ]; then
-            echo "Found files changed by properties template generation."
+            echo "Found generated files or documentation that are out of date."
             echo "Please run scripts/dev/generateDocumentation.sh and commit."
             git status --verbose --verbose
             exit 1

docs/development/dev-scripts.md

@@ -64,9 +64,9 @@ Rust.
 
 #### `generateDocumentation.sh`
 
-This will regenerate the examples and templates for Sleeper configuration properties files. Use this if you've made any
-changes to Sleeper configuration properties. This will propagate any changes to property descriptions, ordering,
-grouping, etc.
+This will regenerate the examples and templates for Sleeper configuration properties files, Docker image
+documentation, and system test suite documentation. Use this if you've made changes to Sleeper configuration
+properties, Docker images, or the slow and expensive system test suite annotations.
 
 #### `publishDocker.sh`
 

docs/development/system-test-suites.md

@@ -1,20 +1,20 @@
 # Current Slow and Expensive test suites
-### This looks imbalanced but EksBulkImportST is a lot slower than others
+### This looks imbalanced but EksBulkImportST is a lot slower than the others.
 
 These tables show which system tests run in which suite. Each suite runs in parallel to the others to speed up the time
 it takes to complete the nightly system tests.
 
 | Slow1                      | Slow2                              | Slow3                           |
-| -------------------------- | -----------------------------------| ------------------------------- |
-| AutoStopEcsTaskST          | EksBulkImportST                    | CompactionCreationST            |
+|----------------------------|------------------------------------|---------------------------------|
 | AutoDeleteS3ObjectsST      | CompactionOnEC2ST                  | MultipleTablesST                |
-| RedeployOptionalStacksST   | ECSStateStoreCommitterThroughputST | StateStoreCommitterThroughputST |
-| EmrPersistentBulkImportST  | ECSStateStoreCommitterST
+| AutoStopEcsTaskST          | ECSStateStoreCommitterST           | StateStoreCommitterThroughputST |
+| CompactionCreationST       | ECSStateStoreCommitterThroughputST |
+| EmrPersistentBulkImportST  | EksBulkImportST                    |
 | OptionalFeaturesDisabledST |
-
+| RedeployOptionalStacksST   |
 
 | Expensive1                        | Expensive2                 | Expensive3            |
-| --------------------------------- | -------------------------- | --------------------- |
+|-----------------------------------|----------------------------|-----------------------|
 | CompactionDataFusionPerformanceST | CompactionPerformanceST    | IngestPerformanceST   |
-| CompactionVeryLargeST             | EmrBulkImportPerformanceST | ParallelCompactionsST |
-|                                   | EksBulkImportPerformanceST |                       |
+| CompactionVeryLargeST             | EksBulkImportPerformanceST | ParallelCompactionsST |
+|                                   | EmrBulkImportPerformanceST |

docs/development/system-tests.md

@@ -99,7 +99,7 @@ When adding a new Slow or Expensive System test add either the Slow1-n or Expens
 ### Current Slow and Expensive test suites
 The current system tests running in each suite can be seen in [system-test-suites](system-test-suites.md).
 
-When adding new slow or expensive system tests ensure this documentation is updated.
+When adding new slow or expensive system tests, run `scripts/dev/generateDocumentation.sh` to ensure documentation is updated.
 
 ### Running tests
 

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/documentation/GenerateSystemTestSuiteDocumentation.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2026 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package sleeper.systemtest.suite.documentation;
+
+import org.junit.platform.commons.support.ReflectionSupport;
+
+import sleeper.systemtest.suite.testutil.parallel.Expensive1;
+import sleeper.systemtest.suite.testutil.parallel.Expensive2;
+import sleeper.systemtest.suite.testutil.parallel.Expensive3;
+import sleeper.systemtest.suite.testutil.parallel.Slow1;
+import sleeper.systemtest.suite.testutil.parallel.Slow2;
+import sleeper.systemtest.suite.testutil.parallel.Slow3;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Generates documentation listing the system tests assigned to parallel suites.
+ */
+public class GenerateSystemTestSuiteDocumentation {
+
+    private static final String SYSTEM_TEST_PACKAGE = "sleeper.systemtest.suite";
+    private static final List<Class<? extends Annotation>> SLOW_SUITES = List.of(Slow1.class, Slow2.class, Slow3.class);
+    private static final List<Class<? extends Annotation>> EXPENSIVE_SUITES = List.of(
+            Expensive1.class, Expensive2.class, Expensive3.class);
+
+    private GenerateSystemTestSuiteDocumentation() {
+    }
+
+    /**
+     * Generates system test suite documentation from a template, and writes it to
+     * docs/development/system-test-suites.md.
+     *
+     * @param  args                     exactly one command line argument containing the project root
+     * @throws IllegalArgumentException if exactly one argument is not supplied, or the project root is not a directory
+     * @throws IOException              if the documentation cannot be read from or written to
+     */
+    public static void main(String[] args) throws IOException {
+        if (args.length != 1) {
+            throw new IllegalArgumentException(
+                    "Usage: GenerateSystemTestSuiteDocumentation <project-root>");
+        }
+        Path projectRoot = Path.of(args[0]).toAbsolutePath().normalize();
+        if (!Files.isDirectory(projectRoot)) {
+            throw new IllegalArgumentException(
+                    "Project root is not a directory: " + projectRoot);
+        }
+        generateDocumentation(projectRoot);
+    }
+
+    private static void generateDocumentation(Path projectRoot) throws IOException {
+        generateDocumentation(projectRoot, ReflectionSupport.findAllClassesInPackage(
+                SYSTEM_TEST_PACKAGE,
+                GenerateSystemTestSuiteDocumentation::isParallelSystemTest,
+                name -> true));
+    }
+
+    private static boolean isParallelSystemTest(Class<?> clazz) {
+        return isInSystemTestPackage(clazz.getPackageName())
+                && !clazz.isMemberClass()
+                && clazz.getSimpleName().endsWith("ST")
+                && (SLOW_SUITES.stream().anyMatch(clazz::isAnnotationPresent)
+                        || EXPENSIVE_SUITES.stream().anyMatch(clazz::isAnnotationPresent));
+    }
+
+    private static boolean isInSystemTestPackage(String packageName) {
+        return packageName.equals(SYSTEM_TEST_PACKAGE)
+                || packageName.startsWith(SYSTEM_TEST_PACKAGE + ".");
+    }
+
+    private static void generateDocumentation(Path projectRoot, List<Class<?>> systemTests) throws IOException {
+        List<Class<?>> sortedTests = systemTests.stream()
+                .sorted(Comparator.comparing(Class::getName))
+                .toList();
+        String output = loadTemplate()
+                .replace("%SLOW_SYSTEM_TEST_SUITES%", SystemTestSuiteTable.create(SLOW_SUITES, sortedTests))
+                .replace("%EXPENSIVE_SYSTEM_TEST_SUITES%", SystemTestSuiteTable.create(EXPENSIVE_SUITES, sortedTests));
+        writeOutput(projectRoot.resolve("docs/development/system-test-suites.md"), output);
+    }
+
+    /**
+     * Writes generated output to a temporary file. Then, when supported by the filesystem,
+     * requests an atomic move to overwrite the current documentation file.
+     * Has a fallback for filesystems that do not support atomic moves.
+     * Removes temporary file after moving.
+     *
+     * @param  destination the file to replace with the output
+     * @param  output      the output to write
+     * @throws IOException if the destination directory cannot be created, or if the output cannot be written or moved to
+     */
+    private static void writeOutput(Path destination, String output) throws IOException {
+        Files.createDirectories(destination.getParent());
+        Path temporary = Files.createTempFile(
+                destination.getParent(),
+                destination.getFileName().toString(),
+                ".tmp");
+        try {
+            Files.writeString(temporary, output, StandardCharsets.UTF_8);
+            try {
+                Files.move(
+                        temporary,
+                        destination,
+                        StandardCopyOption.REPLACE_EXISTING,
+                        StandardCopyOption.ATOMIC_MOVE);
+            } catch (AtomicMoveNotSupportedException e) {
+                Files.move(
+                        temporary,
+                        destination,
+                        StandardCopyOption.REPLACE_EXISTING);
+            }
+        } finally {
+            Files.deleteIfExists(temporary);
+        }
+    }
+
+    private static String loadTemplate() throws IOException {
+        String resource = "system-test-suites.template.md";
+        InputStream stream = GenerateSystemTestSuiteDocumentation.class
+                .getClassLoader()
+                .getResourceAsStream(resource);
+        if (stream == null) {
+            throw new IOException("Classpath resource not found: " + resource);
+        }
+        try (stream) {
+            return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
+        }
+    }
+
+}

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/documentation/GenerateSystemTestSuiteDocumentationIT.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2026 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package sleeper.systemtest.suite.documentation;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class GenerateSystemTestSuiteDocumentationIT {
+
+    @TempDir
+    private Path tempDir;
+
+    @ParameterizedTest(name = "{0} contains {1}")
+    @MethodSource("parallelSystemTests")
+    void shouldGenerateDocumentationForParallelSystemTests(String suiteName, String systemTestName) throws Exception {
+        // Given
+        Files.createDirectories(tempDir.resolve("docs/development"));
+
+        // When
+        GenerateSystemTestSuiteDocumentation.main(new String[]{tempDir.toString()});
+
+        // Then
+        assertThat(columnContaining(readDocumentation(), systemTestName)).isEqualTo(suiteName);
+    }
+
+    @Test
+    void shouldNotIncludeSystemTestsThatAreNotInParallelSuites() throws Exception {
+        // Given
+        Files.createDirectories(tempDir.resolve("docs/development"));
+
+        // When
+        GenerateSystemTestSuiteDocumentation.main(new String[]{tempDir.toString()});
+
+        // Then
+        assertThat(readDocumentation()).doesNotContain("QueryST");
+    }
+
+    @Test
+    void shouldOverwriteExistingDocumentation() throws Exception {
+        // Given
+        Path documentation = tempDir.resolve("docs/development/system-test-suites.md");
+        Files.createDirectories(documentation.getParent());
+        Files.writeString(documentation, "out of date");
+
+        // When
+        GenerateSystemTestSuiteDocumentation.main(new String[]{tempDir.toString()});
+
+        // Then
+        assertThat(readDocumentation())
+                .contains("# Current Slow and Expensive test suites")
+                .doesNotContain("out of date");
+    }
+
+    @Test
+    void shouldCreateDocumentationDirectory() throws Exception {
+        // When
+        GenerateSystemTestSuiteDocumentation.main(new String[]{tempDir.toString()});
+
+        // Then
+        assertThat(tempDir.resolve("docs/development/system-test-suites.md"))
+                .isRegularFile();
+    }
+
+    @ParameterizedTest
+    @ValueSource(ints = {0, 2})
+    void shouldRejectWrongNumberOfArguments(int argumentCount) {
+        // Given
+        String[] args = IntStream.range(0, argumentCount)
+                .mapToObj(index -> tempDir.toString())
+                .toArray(String[]::new);
+
+        // When/Then
+        assertThatThrownBy(() -> GenerateSystemTestSuiteDocumentation.main(args))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Usage: GenerateSystemTestSuiteDocumentation <project-root>");
+    }
+
+    @Test
+    void shouldRejectProjectRootThatIsNotADirectory() throws Exception {
+        // Given
+        Path file = Files.createFile(tempDir.resolve("not-a-directory"));
+
+        // When/Then
+        assertThatThrownBy(() -> GenerateSystemTestSuiteDocumentation.main(new String[]{file.toString()}))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Project root is not a directory: " + file.toAbsolutePath().normalize());
+    }
+
+    private String readDocumentation() throws Exception {
+        return Files.readString(tempDir.resolve("docs/development/system-test-suites.md"));
+    }
+
+    private static Stream<Arguments> parallelSystemTests() {
+        return Stream.of(
+                Arguments.of("Slow1", "AutoStopEcsTaskST"),
+                Arguments.of("Slow2", "EksBulkImportST"),
+                Arguments.of("Slow3", "MultipleTablesST"),
+                Arguments.of("Expensive1", "CompactionDataFusionPerformanceST"),
+                Arguments.of("Expensive2", "CompactionPerformanceST"),
+                Arguments.of("Expensive3", "IngestPerformanceST"));
+    }
+
+    private static String columnContaining(String documentation, String systemTestName) {
+        List<String[]> rows = documentation.lines()
+                .map(GenerateSystemTestSuiteDocumentationIT::splitRow)
+                .toList();
+        int rowIndex = IntStream.range(0, rows.size())
+                .filter(index -> Arrays.asList(rows.get(index)).contains(systemTestName))
+                .findFirst()
+                .orElseThrow();
+        String[] row = rows.get(rowIndex);
+        int columnIndex = IntStream.range(0, row.length)
+                .filter(index -> systemTestName.equals(row[index]))
+                .findFirst()
+                .orElseThrow();
+        for (int index = rowIndex - 1; index >= 0; index--) {
+            String[] cells = rows.get(index);
+            if (cells.length > columnIndex
+                    && (cells[columnIndex].startsWith("Slow") || cells[columnIndex].startsWith("Expensive"))) {
+                return cells[columnIndex];
+            }
+        }
+        throw new IllegalStateException("No suite header found for " + systemTestName);
+    }
+
+    private static String[] splitRow(String line) {
+        return Arrays.stream(line.split("\\|", -1))
+                .map(String::trim)
+                .toArray(String[]::new);
+    }
+}