From b509762764b50d40e053c71c74175e492642c27f Mon Sep 17 00:00:00 2001 From: Jamie Sinn Date: Fri, 5 Jun 2026 11:43:02 -0400 Subject: [PATCH] Setup github action to run the spec check. Signed-off-by: Jamie Sinn --- tools/repo_parser/README.md | 75 +++++++++++++++-- tools/repo_parser/action.yml | 83 +++++++++++++++++++ .../repo_parser/examples/spec-compliance.yml | 21 +++++ tools/repo_parser/spec_finder.py | 5 +- 4 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 tools/repo_parser/action.yml create mode 100644 tools/repo_parser/examples/spec-compliance.yml diff --git a/tools/repo_parser/README.md b/tools/repo_parser/README.md index fb7ef8c0c..1c83ae18a 100644 --- a/tools/repo_parser/README.md +++ b/tools/repo_parser/README.md @@ -1,22 +1,83 @@ # Repo Parser -This will parse the contents of an OpenFeature repo and determine how they adhere to the spec. This can be gamed and assumes everyone is participating in good faith. +This will parse the contents of an OpenFeature repo and determine how they adhere to the spec. This can be gamed and +assumes everyone is participating in good faith. -We look for a `.specrc` file in the root of a repository to figure out how to find test cases that are annotated with the spec number and the text of the spec. We can then produce a report which says "you're covered" or details about how you're not covered. The goal of this is to use that resulting report to power a spec-compliance matrix for end users to vet SDK quality. +We look for a `.specrc` file in the root of a repository to figure out how to find test cases that are annotated with +the spec number and the text of the spec. We can then produce a report which says "you're covered" or details about how +you're not covered. The goal of this is to use that resulting report to power a spec-compliance matrix for end users to +vet SDK quality. -## Usage +## GitHub Action + +The easiest way to run the compliance check is via the composite action. Add the following to your repo's workflow: + +Note - using the main tag is not recommended for security, but to avoid requiring to update this documentation to the +latest pinned tag, it's shown here for example purposes. Please pin to a specific tag in production use. + +```yaml +jobs: + spec-compliance: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: open-feature/spec/tools/repo_parser@main +``` + +The action always fetches the latest spec, runs the compliance check, and logs a summary. The job fails if any +requirements are missing, mismatched, or reference a non-existent spec entry. + +### Inputs + +| Input | Description | Default | +|-----------------|-----------------------------------------------------------------|----------| +| `image-version` | Tag of the `ghcr.io/open-feature/spec/repo_parser` image to use | `latest` | + +### Outputs + +All outputs are available to subsequent steps via `if: always()`. + +| Output | Type | Description | +|------------------------|-------------|--------------------------------------------------------------------| +| `compliant` | `string` | `'true'` if fully compliant, `'false'` otherwise | +| `missing` | JSON array | Requirement IDs not covered by the repo (e.g. `["1.1.1","1.2.3"]`) | +| `extra` | JSON array | Test reference IDs not found in the spec | +| `different-text` | JSON array | Requirement IDs whose annotated text does not match the spec | +| `missing-count` | `string` | Count of missing requirements | +| `extra-count` | `string` | Count of extra references | +| `different-text-count` | `string` | Count of text mismatches | +| `report` | JSON object | Full report: `{ missing, extra, different-text, good }` | + +### Consuming outputs + +```yaml + - uses: open-feature/spec/tools/repo_parser@main + id: compliance + + - if: always() + run: + echo "Missing requirements: ${{ steps.compliance.outputs.missing }}" +``` + +See [`examples/spec-compliance.yml`](examples/spec-compliance.yml) for a complete workflow. + +## Docker ``` $ docker build -t specfinder . -$ docker run --mount src=/path/tojava-sdk/,target=/appdir,type=bind -it specfinder \ - spec_finder.py --code-directory /appdir --diff --json-report +$ docker run --mount src=/path/to/java-sdk/,target=/appdir,type=bind -it specfinder \ + --code-directory /appdir --diff-output --json-report ``` -### `.specrc` +## `.specrc` This should be at the root of the repository. -`multiline_regex` captures the text which contains the test marker. In java, for instance, it's a specially crafted annotation. `number_subregex` and `text_subregex` which will match the substring found in the `multiline_regex` to parse the spec number and text found. These are multi-line regexes. +`multiline_regex` captures the text which contains the test marker. In java, for instance, it's a specially crafted +annotation. `number_subregex` and `text_subregex` which will match the substring found in the `multiline_regex` to parse +the spec number and text found. These are multi-line regexes. Example: diff --git a/tools/repo_parser/action.yml b/tools/repo_parser/action.yml new file mode 100644 index 000000000..ce2532420 --- /dev/null +++ b/tools/repo_parser/action.yml @@ -0,0 +1,83 @@ +name: OpenFeature Spec Compliance Check +description: Checks an OpenFeature SDK repo for spec compliance using the repo_parser tool. + +inputs: + image-version: + description: Tag of the ghcr.io/open-feature/spec/repo_parser image to use. + required: false + default: latest + +outputs: + report: + description: Full JSON compliance report ({ missing, extra, different-text, good }). + value: ${{ steps.check.outputs.report }} + compliant: + description: "'true' if fully compliant with the spec, 'false' otherwise." + value: ${{ steps.check.outputs.compliant }} + missing: + description: JSON array of spec requirement IDs not covered by the repo (e.g. ["1.1.1","1.2.3"]). + value: ${{ steps.check.outputs.missing }} + extra: + description: JSON array of test reference IDs not found in the spec. + value: ${{ steps.check.outputs.extra }} + different-text: + description: JSON array of spec requirement IDs whose text does not match the spec. + value: ${{ steps.check.outputs.different_text }} + missing-count: + description: Number of spec requirements not covered by the repo. + value: ${{ steps.check.outputs.missing_count }} + extra-count: + description: Number of test references not found in the spec. + value: ${{ steps.check.outputs.extra_count }} + different-text-count: + description: Number of spec requirements with mismatched text. + value: ${{ steps.check.outputs.different_text_count }} + +runs: + using: composite + steps: + - name: Check spec compliance + id: check + shell: bash + run: | + ARGS="--code-directory /appdir --json-report --refresh-spec --diff-output" + + set +e + docker run \ + --mount type=bind,src=${{ github.workspace }},target=/appdir \ + -w /appdir \ + ghcr.io/open-feature/spec/repo_parser:${{ inputs.image-version }} \ + $ARGS + EXIT_CODE=$? + set -e + + REPORT_FILE=$(ls "${{ github.workspace }}"/*-report.json 2>/dev/null | head -1) + if [[ -n "$REPORT_FILE" ]]; then + REPORT=$(cat "$REPORT_FILE") + rm "$REPORT_FILE" + + DELIMITER=$(openssl rand -hex 8) + echo "report<<$DELIMITER" >> "$GITHUB_OUTPUT" + echo "$REPORT" >> "$GITHUB_OUTPUT" + echo "$DELIMITER" >> "$GITHUB_OUTPUT" + + echo "missing=$(echo "$REPORT" | jq -c '.missing')" >> "$GITHUB_OUTPUT" + echo "extra=$(echo "$REPORT" | jq -c '.extra')" >> "$GITHUB_OUTPUT" + echo "different_text=$(echo "$REPORT" | jq -c '."different-text"')" >> "$GITHUB_OUTPUT" + + echo "missing_count=$(echo "$REPORT" | jq '.missing | length')" >> "$GITHUB_OUTPUT" + echo "extra_count=$(echo "$REPORT" | jq '.extra | length')" >> "$GITHUB_OUTPUT" + echo "different_text_count=$(echo "$REPORT" | jq '."different-text" | length')" >> "$GITHUB_OUTPUT" + fi + + echo "compliant=$([[ $EXIT_CODE -eq 0 ]] && echo 'true' || echo 'false')" >> "$GITHUB_OUTPUT" + exit $EXIT_CODE + + - name: Log compliance summary + if: always() + shell: bash + run: | + echo "Compliant: ${{ steps.check.outputs.compliant }}" + echo "Missing: ${{ steps.check.outputs.missing }}" + echo "Extra: ${{ steps.check.outputs.extra }}" + echo "Different text: ${{ steps.check.outputs.different_text }}" diff --git a/tools/repo_parser/examples/spec-compliance.yml b/tools/repo_parser/examples/spec-compliance.yml new file mode 100644 index 000000000..27c3d4870 --- /dev/null +++ b/tools/repo_parser/examples/spec-compliance.yml @@ -0,0 +1,21 @@ +name: Spec compliance + +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: { } + +jobs: + spec-compliance: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + # Binding to the main branch is not secure. Replace this with the specific commit SHA of the repo_parser image you want to use for a more secure setup. + - uses: open-feature/spec/tools/repo_parser@main diff --git a/tools/repo_parser/spec_finder.py b/tools/repo_parser/spec_finder.py index abf4a065d..e73fcf3da 100755 --- a/tools/repo_parser/spec_finder.py +++ b/tools/repo_parser/spec_finder.py @@ -62,10 +62,11 @@ def get_spec(force_refresh: bool = False, path_prefix: str = './') -> dict[str, with open(spec_path) as f: data = ''.join(f.readlines()) else: - # TODO: Status code check spec_response = urllib.request.urlopen( 'https://raw.githubusercontent.com/open-feature/spec/main/specification.json' ) + if spec_response.status not in (200, 304): + raise RuntimeError(f'Failed to fetch spec: HTTP {spec_response.status}') raw = [] for i in spec_response.readlines(): raw.append(i.decode('utf-8')) @@ -97,7 +98,7 @@ def specmap_from_file(actual_spec: dict[str, Any]) -> dict[str, str]: def find_covered_specs(config: Config, data: str) -> dict[str, str]: repo_specs = {} for match in re.findall(config['multiline_regex'], data, re.MULTILINE | re.DOTALL): - match = match.replace('\n', '').replace(config['inline_comment_prefix'], '') + match = match.replace('\n', '').replace(config.get('inline_comment_prefix') or '', '') # normalize whitespace match = re.sub(' {2,}', ' ', match.strip()) number = re.findall(config['number_subregex'], match)[0]