diff --git a/CHANGELOG.md b/CHANGELOG.md index e807b241..afaccc55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to Traust are documented here. +## [0.44.0] + +- Restore `portfolio inventory-export --format csv` through a registered Core Insight view. +- Preserve the eight-column JSON-array stdout, newest-first registration order, current + owners, deduplicated shipping products and product-definition contact semantics. +- Keep per-invocation `--pd-cache-dir` overrides and the no-files default; expose canonical + inventory JSON plus Core-owned Markdown/HTML/CSV through `dashboard view inventory-export`. +- Replace client-side inventory joins with bounded, batched Core reads. Refuse unsupported + scopes and over-budget workloads rather than silently ignoring scope or truncating rows. +- Reuse Core's shared CSV and human renderers to keep untrusted source text inert. + ## [0.42.0] - Publish the ranked PQC worklist as a Core-owned Insight report while preserving diff --git a/VERSION b/VERSION index 787ffc30..a8ab6c96 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.42.0 +0.44.0 diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ed39edae..4d1e25e6 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -456,18 +456,31 @@ judgements on existing core findings; it does not replace machine-report intake. ```bash traust portfolio onboard --product example --label Example --org example-org traust portfolio owner-hints --repo https://github.com/example-org/api -traust portfolio inventory-export [--pd-cache-dir ] +traust portfolio inventory-export [--format json|csv] [--pd-cache-dir ] traust products add example --label Example --ownership owned --business-unit Commerce traust products list ``` -`inventory-export` composes every registered repository from the portfolio's own lists: -`Repository`, `URL`, `Host`, `Organization/Group`, `Repo Name`, `Owner Team` (current owning -teams), `Products` (products shipping it) and `Accountable Contacts` (product-definitions -candidates for its URL through the recorded product map, with source field and embargo -eligibility), as JSON (no CSV until core registers its Insight view (a priority follow-up)). `--pd-cache-dir` reads a -product-definitions snapshot other than `locations.feeds_cache` for that run only. A contact -is a candidate, never an owner, and grants no authority to disclose findings. Declared +`inventory-export` publishes Core's global inventory Insight view: every registered +repository in newest-first registration order, with `Repository`, `URL`, `Host`, +`Organization/Group`, `Repo Name`, `Owner Team` (current owning teams), `Products` +(products shipping it) and `Accountable Contacts` (product-definition candidates for +its URL through the recorded product map, with source field and embargo eligibility). +Default stdout remains the same JSON array with raw values; `--format csv` emits Core's +eight-column CSV instead, joining list cells with semicolons and neutralizing spreadsheet +formulas and forged rows through the shared renderer. Neither form creates report files. + +`traust dashboard view inventory-export --out ` writes Core's Markdown, HTML +and CSV plus the canonical `{rows: [...]}` JSON wrapper. This view takes no product, team +or historical scope; unsupported options are refused. Reads use a cumulative 10,000-record +workload budget across inventory, related rows, mappings and product-definition work; +overflow fails rather than exporting a partial inventory. Building the report scans +nothing and changes no database rows or object-store bytes. + +`--pd-cache-dir` selects a different cached product-definition snapshot for this invocation +only. Missing optional cache data retains empty contacts, not a claim that no accountable +party exists. A contact is a candidate, never an owner, and grants no authority to disclose +findings. Declared per-repository inventory facts (service labels, managers, tracker routes, escalation contacts) are retired: traust-core has one ownership source, the onboarding and owner-import doors. `owner-hints` prints forge hints; it never stores them. Product boundary caveats are diff --git a/pyproject.toml b/pyproject.toml index 3267b45e..a951e315 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "traust" # version must equal the VERSION file — check_docs_consistency.py # version-sync check fails the pre-push when they drift (and fails # closed if this field is removed) -version = "0.42.0" +version = "0.44.0" license = "Apache-2.0" description = "AI agent tooling for automated security audits of Red Hat Hybrid Platforms source code" requires-python = ">=3.11" @@ -32,7 +32,7 @@ traust = "traust.cli.__main__:main" dev = ["ruff>=0.6", "pytest-xdist>=3", "psycopg[binary]>=3.1"] [tool.uv.sources] -traust-core = { git = "https://github.com/traust-security/traust-core.git", rev = "374341a673dea75cd92035b161a9b4be97edddee" } +traust-core = { git = "https://github.com/traust-security/traust-core.git", rev = "81e27c598e602594ae517aa2e0fc39e49637f962" } # Multi-skill CLIs live under src/traust/; harnessing// for single-skill # scripts. Install for deps + importable path helpers. diff --git a/src/traust/cli/dashboard_view.py b/src/traust/cli/dashboard_view.py index 9431c544..e4a6fdde 100644 --- a/src/traust/cli/dashboard_view.py +++ b/src/traust/cli/dashboard_view.py @@ -32,6 +32,10 @@ def main(argv: list[str] | None = None) -> int: _scope_flags(ap) ap.add_argument("--out", type=Path, required=True, help="the folder the view is written to") args = ap.parse_args(argv) + if args.view == ViewName.INVENTORY_EXPORT: + for flag in ("until", "sla_policy", "invoice"): + if getattr(args, flag) is not None: + ap.error(f"--{flag.replace('_', '-')} is not supported by inventory-export") if args.panel and args.view != ViewName.EXECUTIVE_SUMMARY: ap.error("--panel requires the executive-summary view") diff --git a/src/traust/cli/portfolio_onboard.py b/src/traust/cli/portfolio_onboard.py index 53bb4bdd..9be444a6 100644 --- a/src/traust/cli/portfolio_onboard.py +++ b/src/traust/cli/portfolio_onboard.py @@ -18,22 +18,20 @@ import argparse import re from pathlib import Path -from typing import Any -from urllib.parse import urlsplit +from typing import Any, cast import pydantic import yaml from traust_core.v1.context import Context from traust_core.v1.errors import NotFoundError, ValidationError from traust_core.v1.models.base import Dto, Model -from traust_core.v1.models.common import every_item +from traust_core.v1.models.insight import InventoryExport, ViewFilter, ViewName from traust_core.v1.models.inventory import ( OrgName, OwnerAssignment, OwnershipSource, RefName, ReleaseCommit, - RepositoryFilter, TeamName, VersionLabel, ) @@ -244,20 +242,9 @@ def release(version: str) -> Any: return run(call) -INVENTORY_COLUMNS = ( - "Repository", - "URL", - "Host", - "Organization/Group", - "Repo Name", - "Owner Team", - "Products", - "Accountable Contacts", -) - - def inventory_export(argv: list[str] | None = None) -> int: ap = parser("portfolio", "inventory-export", "Joined inventory, owners and contacts.") + ap.add_argument("--format", choices=("json", "csv"), default="json") ap.add_argument( "--pd-cache-dir", type=Path, @@ -265,53 +252,18 @@ def inventory_export(argv: list[str] | None = None) -> int: ) args = ap.parse_args(argv) - def call() -> Any: + def call() -> object: core = load(args.config_home) - rows = _inventory_rows(core, args.pd_cache_dir) - return [dict(zip(INVENTORY_COLUMNS, row, strict=True)) for row in rows] + if args.pd_cache_dir is not None: + locations = core.config.locations.model_copy( + update={"feeds_cache": str(args.pd_cache_dir.resolve())} + ) + core.config = core.config.model_copy(update={"locations": locations}) + rendered = core.services().insight.publish(ViewName.INVENTORY_EXPORT, ViewFilter()) + if args.format == "csv": + assert rendered.csv is not None + return rendered.csv.rstrip("\r\n") + model = cast(InventoryExport, rendered.model) + return [row.model_dump(mode="json") for row in model.rows] return run(call) - - -def _inventory_rows(core: Context, cache_dir: Path | None) -> list[tuple[object, ...]]: - """Every registered repository with its current owning teams, the products shipping it and - the product-definitions contacts for its URL: composed from the portfolio's own lists.""" - portfolio = core.services().portfolio - repositories = list( - every_item(lambda page: portfolio.list_repositories(RepositoryFilter(), page)) - ) - resolution = portfolio.resolve_products( - core.product_definitions(cache_dir).read(), - portfolio.product_mappings(), - repo_urls=[repo.url for repo in repositories], - ) - contacts = { - match.query.value: [ - f"{c.kerberos_id} ({c.field}; embargo_cleared={c.embargo_cleared})" - for c in match.contacts - ] - for match in resolution.results - if match.query is not None - } - rows = [] - # NEED(core: bulk owners/products for inventory export): PortfolioService at the pinned core - # answers owners (`list_owners`) and shipping products (`list_products_of`) one repository at - # a time, with no list or by-ids variant, so this export makes two store queries per - # repository until core has one. - for repo in repositories: - assert repo.id is not None - parts = urlsplit(repo.url) - organization, _, name = parts.path.strip("/").rpartition("/") - rows.append( - ( - f"{organization}/{name}" if organization else name, - repo.url, - parts.hostname or "", - organization, - name, - [team.name for team in portfolio.list_owners(repo.id)], - [str(product.slug) for product in portfolio.list_products_of(repo.id)], - contacts.get(repo.url, []), - ) - ) - return rows diff --git a/tests/test_cloud_config_doors.py b/tests/test_cloud_config_doors.py index faa6852e..d44d83cf 100644 --- a/tests/test_cloud_config_doors.py +++ b/tests/test_cloud_config_doors.py @@ -22,8 +22,9 @@ import yaml from traust_core.v1 import contracts from traust_core.v1.models.identity import IdentitySource, Principal +from traust_core.v1.models.inventory import RefName from traust_core.v1.models.operations import OrderRequest, WorkerId -from traust_core.v1.models.values import HttpsRepoUrl +from traust_core.v1.models.values import GitSha, HttpsRepoUrl from traust_core.v1.models.vocabulary import ActorKind from traust_core.v1.repositories import apply_schema, create_database_engine from traust_core.v1.repositories.sql import Dialect @@ -355,6 +356,8 @@ def test_record_binds_the_audit_to_its_leased_order(home: Path, tmp_path: Path) repo = services.portfolio.register_repository( HttpsRepoUrl.parse(REPO, load(home).config.forge_hosts), False ) + assert repo.id is not None + services.portfolio.mark_default_branch(repo.id, GitSha.parse(SHA), RefName("main")) operator = Principal( identity="operator@example.com", kind=ActorKind.HUMAN, diff --git a/tests/test_inventory_export.py b/tests/test_inventory_export.py index 09cd9827..ef41f440 100644 --- a/tests/test_inventory_export.py +++ b/tests/test_inventory_export.py @@ -4,16 +4,21 @@ from __future__ import annotations import contextlib +import csv import io import json +from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path import pytest import yaml from traust_core.v1.errors import ConfigError -from traust_core.v1.models.values import HttpsRepoUrl, ProductSlug +from traust_core.v1.models.inventory import Component, Release +from traust_core.v1.models.values import GitSha, HttpsRepoUrl, ProductSlug +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork +from tests.test_products_doors import _postgres from traust.cli.__main__ import main from traust.context import load from traust.inventory import repository_inventory @@ -29,12 +34,22 @@ def invoke(home: Path, *arguments: str) -> tuple[int, str, str]: return code, out.getvalue(), err.getvalue() -@pytest.fixture -def home(tmp_path: Path) -> Path: +@pytest.fixture(params=["sqlite", "postgres"]) +def home( + request: pytest.FixtureRequest, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Iterator[Path]: + monkeypatch.delenv("FEEDS_CACHE_DIR", raising=False) config = tmp_path / "config" config.mkdir() - (config / "traust.yaml").write_text(yaml.safe_dump({"home": str(tmp_path / "store")})) - return config + settings = { + "home": str(tmp_path / "store"), + "locations": {"feeds_cache": str(tmp_path / "configured-feeds")}, + } + with contextlib.ExitStack() as stack: + if request.param == "postgres": + settings["database"] = {"url": stack.enter_context(_postgres())} + (config / "traust.yaml").write_text(yaml.safe_dump(settings)) + yield config def test_export_joins_owners_and_products_as_json(home: Path) -> None: @@ -59,10 +74,147 @@ def test_export_joins_owners_and_products_as_json(home: Path) -> None: assert row["Owner Team"] == ["Gateway Team"] assert rows[OTHER]["Organization/Group"] == "example-group/sub" assert rows[OTHER]["Repo Name"] == "web" - assert rows[OTHER]["Owner Team"] == ["=HYPERLINK(1)"] # JSON keeps the text; no CSV yet + assert rows[OTHER]["Owner Team"] == ["=HYPERLINK(1)"] assert rows[OTHER]["Products"] == rows[OTHER]["Accountable Contacts"] == [] +def contact_cache(folder: Path, identity: str) -> None: + folder.mkdir() + (folder / "product_definitions.json").write_text( + json.dumps( + { + "ps_products": {"widget": {"ps_modules": ["widget-1"]}}, + "ps_modules": { + "widget-1": { + "ps_update_streams": ["widget-default"], + "private_tracker_cc": [identity], + } + }, + "ps_update_streams": { + "widget-default": { + "managed_service_components": [{"name": "api", "git_repo_url": URL}] + } + }, + } + ) + ) + + +def test_csv_export_preserves_products_and_cache_override_contacts( + home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + core = load(home) + portfolio = core.services().portfolio + repo = portfolio.register_repository(HttpsRepoUrl.parse(URL, core.config.forge_hosts), True) + assert repo.id is not None + portfolio.assign_owner(repo.id, "=HYPERLINK(1)", datetime(2026, 10, 1, tzinfo=UTC)) + product = portfolio.register_product(ProductSlug("widget"), "Widget") + ref = portfolio.register_ref(repo.id, GitSha.parse("a" * 40), None, None) + assert product.id is not None and ref.id is not None + with SqlTraustUnitOfWork(core.database) as uow: + release = uow.releases.add(Release(product_id=product.id, version="1.0")) + assert release.id is not None + uow.components.add(Component(release_id=release.id, repository_ref_id=ref.id)) + uow.commit() + contact_cache(tmp_path / "configured-feeds", "configuredperson") + contact_cache(tmp_path / "override-feeds", "overriddenperson") + code, output, error = invoke(home, "portfolio", "inventory-export") + assert code == 0, error + (original,) = json.loads(output) + assert original["Accountable Contacts"] == [ + "configuredperson (private_tracker_cc; embargo_cleared=True)" + ] + monkeypatch.chdir(tmp_path) + code, output, error = invoke( + home, "portfolio", "inventory-export", "--pd-cache-dir", "override-feeds" + ) + assert code == 0, error + (overridden,) = json.loads(output) + assert overridden["Products"] == ["widget"] + assert overridden["Owner Team"] == ["=HYPERLINK(1)"] + assert overridden["Accountable Contacts"] == [ + "overriddenperson (private_tracker_cc; embargo_cleared=True)" + ] + code, output, error = invoke( + home, + "portfolio", + "inventory-export", + "--format", + "csv", + "--pd-cache-dir", + "override-feeds", + ) + assert code == 0, error + (exported,) = list(csv.DictReader(io.StringIO(output))) + assert exported["URL"] == URL + assert exported["Products"] == "widget" + assert exported["Owner Team"] == "'=HYPERLINK(1)" + assert exported["Accountable Contacts"] == overridden["Accountable Contacts"][0] + + +def test_dashboard_exports_inventory_wrapper_and_refuses_historical_scope( + home: Path, tmp_path: Path +) -> None: + core = load(home) + portfolio = core.services().portfolio + portfolio.register_repository(HttpsRepoUrl.parse(URL, core.config.forge_hosts), True) + destination = tmp_path / "dashboard" + code, output, error = invoke( + home, "dashboard", "view", "inventory-export", "--out", str(destination) + ) + assert code == 0, error + written = [Path(path) for path in json.loads(output)["written"]] + assert {path.suffix for path in written} == {".json", ".md", ".html", ".csv"} + wrapper = json.loads(next(path for path in written if path.suffix == ".json").read_text()) + assert set(wrapper) == {"rows"} + assert [(row["URL"], row["Owner Team"], row["Products"]) for row in wrapper["rows"]] == [ + (URL, [], []) + ] + refused = tmp_path / "historical" + code, _, _ = invoke( + home, + "dashboard", + "view", + "inventory-export", + "--out", + str(refused), + "--as-of", + "2026-10-01T00:00:00Z", + ) + assert code == 2 + assert not refused.exists() + with pytest.raises(SystemExit) as rejected: + invoke( + home, + "dashboard", + "view", + "inventory-export", + "--out", + str(refused), + "--until", + "2026-10-01T00:00:00Z", + ) + assert rejected.value.code == 2 + assert not refused.exists() + + +def test_empty_inventory_csv_still_has_its_columns(home: Path) -> None: + code, output, error = invoke(home, "portfolio", "inventory-export", "--format", "csv") + assert code == 0, error + assert list(csv.reader(io.StringIO(output))) == [ + [ + "Repository", + "URL", + "Host", + "Organization/Group", + "Repo Name", + "Owner Team", + "Products", + "Accountable Contacts", + ] + ] + + def test_inventory_reader_lists_each_segment_in_descriptor_order(tmp_path: Path) -> None: inputs = tmp_path / "inputs" (inputs / "alpha" / "group").mkdir(parents=True) diff --git a/tests/test_pqc_worklist.py b/tests/test_pqc_worklist.py index e3a2fb93..c0a7a3d0 100644 --- a/tests/test_pqc_worklist.py +++ b/tests/test_pqc_worklist.py @@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta from html.parser import HTMLParser from pathlib import Path -from uuid import UUID, uuid4 +from uuid import uuid4 import pytest import sqlalchemy as sa @@ -69,7 +69,6 @@ def repository( sha="a" * 40, name="main", kind="branch", - created_at=NOW, ) ) if commit @@ -118,6 +117,7 @@ def test_native_ref_scope_distinguishes_all_from_empty_and_selects_one_commit_pe ) -> None: repo, first = repository(core, "https://github.com/example/widget") waiting, _ = repository(core, "https://github.com/example/waiting", commit=False) + assert first is not None with SqlTraustUnitOfWork(core.database) as uow: recent = uow.repository_refs.add( RepositoryRef( @@ -125,7 +125,6 @@ def test_native_ref_scope_distinguishes_all_from_empty_and_selects_one_commit_pe sha="b" * 40, name="newer", kind="branch", - created_at=NOW + timedelta(days=1), ) ) default = uow.repository_refs.add( @@ -135,19 +134,26 @@ def test_native_ref_scope_distinguishes_all_from_empty_and_selects_one_commit_pe name="default", kind="branch", is_default=True, - created_at=NOW - timedelta(days=1), ) ) tied = uow.repository_refs.add( RepositoryRef( - id=UUID("ffffffff-ffff-ffff-ffff-ffffffffffff"), repository_id=repo.id, sha="d" * 40, name="tie", kind="branch", - created_at=NOW + timedelta(days=1), ) ) + table = uow.repository_refs.table + for ref, created_at in ( + (first, NOW), + (recent, NOW + timedelta(days=1)), + (default, NOW - timedelta(days=1)), + (tied, NOW + timedelta(days=1)), + ): + uow.connection.execute( + sa.update(table).where(table.c.id == ref.id).values(created_at=created_at) + ) uow.commit() insight = core.services().insight global_view = insight.publish(ViewName.PQC_WORKLIST, ViewFilter()).model @@ -165,7 +171,9 @@ def test_native_ref_scope_distinguishes_all_from_empty_and_selects_one_commit_pe ViewName.PQC_WORKLIST, ViewFilter(repository_ref_ids=(recent.id, tied.id)), ).model - assert [entry.repository_ref_id for entry in tied_view.entries] == [tied.id] + assert [entry.repository_ref_id for entry in tied_view.entries] == [ + max((recent.id, tied.id), key=str) + ] assert ( insight.publish(ViewName.PQC_WORKLIST, ViewFilter(repository_ref_ids=())).model.entries == () diff --git a/uv.lock b/uv.lock index adababcb..09f205a4 100644 --- a/uv.lock +++ b/uv.lock @@ -1535,7 +1535,7 @@ wheels = [ [[package]] name = "traust" -version = "0.42.0" +version = "0.44.0" source = { editable = "." } dependencies = [ { name = "jsonschema" }, @@ -1572,9 +1572,9 @@ requires-dist = [ { name = "pyyaml" }, { name = "referencing" }, { name = "sigstore", marker = "extra == 'signing'", specifier = ">=4.0,<5.0" }, - { name = "traust-core", git = "https://github.com/traust-security/traust-core.git?rev=374341a673dea75cd92035b161a9b4be97edddee" }, - { name = "traust-core", extras = ["postgres"], marker = "extra == 'postgres'", git = "https://github.com/traust-security/traust-core.git?rev=374341a673dea75cd92035b161a9b4be97edddee" }, - { name = "traust-core", extras = ["s3"], marker = "extra == 's3'", git = "https://github.com/traust-security/traust-core.git?rev=374341a673dea75cd92035b161a9b4be97edddee" }, + { name = "traust-core", git = "https://github.com/traust-security/traust-core.git?rev=81e27c598e602594ae517aa2e0fc39e49637f962" }, + { name = "traust-core", extras = ["postgres"], marker = "extra == 'postgres'", git = "https://github.com/traust-security/traust-core.git?rev=81e27c598e602594ae517aa2e0fc39e49637f962" }, + { name = "traust-core", extras = ["s3"], marker = "extra == 's3'", git = "https://github.com/traust-security/traust-core.git?rev=81e27c598e602594ae517aa2e0fc39e49637f962" }, ] provides-extras = ["postgres", "s3", "signing"] @@ -1588,7 +1588,7 @@ dev = [ [[package]] name = "traust-contracts" version = "0.0.0" -source = { git = "https://github.com/traust-security/traust-contracts.git?rev=b65d590845cc5e5fe363481476df392ba991c056#b65d590845cc5e5fe363481476df392ba991c056" } +source = { git = "https://github.com/traust-security/traust-contracts.git?rev=1a0c02c9b139975b5ce94e94b44933e854117d50#1a0c02c9b139975b5ce94e94b44933e854117d50" } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] }, { name = "pydantic" }, @@ -1598,7 +1598,7 @@ dependencies = [ [[package]] name = "traust-core" version = "0.1.0" -source = { git = "https://github.com/traust-security/traust-core.git?rev=374341a673dea75cd92035b161a9b4be97edddee#374341a673dea75cd92035b161a9b4be97edddee" } +source = { git = "https://github.com/traust-security/traust-core.git?rev=81e27c598e602594ae517aa2e0fc39e49637f962#81e27c598e602594ae517aa2e0fc39e49637f962" } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] }, { name = "pydantic" }, diff --git a/version.args b/version.args index fd4220ca..7ec8fa2b 100644 --- a/version.args +++ b/version.args @@ -1 +1 @@ -VERSION=0.42.0 +VERSION=0.44.0