Shell API and CLI: Add option for staging additional runtime targets

This enables users to add additional functionality such as debug tooling in the shell sandbox,
without needing to modify the target element. This is achived through introducing a new option to
shell.

All existing API and UX is maintained, to not break existing scripts. An alternative design was
considered, to have the additional elements as positional arguments similar to the existing element,
but this would need manual parsing to handle the cases where `--` is present and not
present, splitting based on a `.bst` suffix. This UX could be re-visited in future.

Example usage:
  bst shell --with base.bst example.bst -- cat example.txt
Where:
- example.bst is a simple import element with no dependencies that imports a file called example.txt
- base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).

Changes:
- Introduces `test_with_other_targets` integration test to the shell test suite.
- Adds `--with` cli option to the shell subcommand and updates it's documentation.
  - option can be used multiple times by caller, providing a list of targets.
- Extends the shell top level calling interface in Buildstream core to accept a list of
other_targets
  - This is where the targets are loaded into elements and checked to make sure they are present
- Extends the shell element implementation to accept a list of other targets
  - This is where the other elements are staged and integrated into the sandbox

Author: nathanwilliams-ct

src/buildstream/_frontend/cli.py

@@ -378,8 +378,6 @@ def cli(context, **kwargs):
     user preferences configuration file.
     """
 
-    from .app import App
-
     # Create the App, giving it the main arguments
     context.obj = App.create(dict(kwargs))
     context.call_on_close(context.obj.cleanup)
@@ -687,6 +685,13 @@ def show(app, elements, deps, except_, order, format_):
     metavar="HOSTPATH PATH",
     help="Mount a file or directory into the sandbox",
 )
+@click.option(
+    "--with",
+    "other_targets",
+    type=click.Path(readable=False),
+    multiple=True,
+    help="A additional target to stage into an element's sandbox environment",
+)
 @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
 @click.option(
     "--use-buildtree",
@@ -726,6 +731,7 @@ def show(app, elements, deps, except_, order, format_):
 def shell(
     app: App,
     target,
+    other_targets,
     command,
     mount,
     isolate,
@@ -749,13 +755,38 @@ def shell(
     otherwise bst may respond to them instead. e.g.
 
     \b
-        bst shell example.bst -- df -h
+        bst shell base.bst -- df -h
 
     Use the --build option to create a temporary sysroot for
     building the element instead.
 
+    Use the --with option to stage the artifacts of other elements
+    into the temporary sysroot to make them available to run e.g.
+
+    \b
+        bst shell --with base.bst example.bst -- cat example.txt
+
     If no COMMAND is specified, the default is to attempt
     to run an interactive shell.
+
+    # Examples:
+
+    \b
+        # Attempt to run an interactive shell with example.bst
+        bst shell example.bst
+        # Attempt to run an df -h with example.bst
+        bst shell example.bst -- df h
+        # In a workspace directory, attempt to shell into the workspace element
+        bst shell
+        # Attempt to run cat from base.bst to read example.txt from example.bst
+        bst shell --with base.bst example.bst -- cat example.txt
+        # Attempt to run an interactive shell with the sources and all dependencies of example.bst
+        bst shell --build example.bst
+
+    For all examples on this page:
+    - example.bst is a simple import element with no dependencies
+       that imports a file called example.txt
+    - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).
     """
 
     # Buildtree can only be used with build shells
@@ -777,6 +808,7 @@ def shell(
                 target,
                 scope,
                 app.shell_prompt,
+                other_targets=other_targets,
                 mounts=mounts,
                 isolate=isolate,
                 command=command,

src/buildstream/_stream.py

@@ -243,6 +243,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     #    target: The name of the element to run the shell for
     #    scope: The scope for the shell, only BUILD or RUN are valid (_Scope)
     #    prompt: A function to return the prompt to display in the shell
+    #    other_targets (Iterable[str]): The name of other elements to stage in the shell
     #    unique_id: (str): A unique_id to use to lookup an Element instance
     #    mounts: Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
@@ -262,6 +263,7 @@ def shell(
         scope: _Scope,
         prompt: Callable[[Element], str],
         *,
+        other_targets: Iterable[str] = (),
         unique_id: Optional[str] = None,
         mounts: Optional[List[_HostMount]] = None,
         isolate: bool = False,
@@ -357,8 +359,46 @@ def shell(
             self._fetch([element])
             _pipeline.assert_sources_cached(self._context, [element])
 
+        # Load the other targets
+        try:
+            other_elements = self.load_selection(
+                other_targets,
+                selection=_PipelineSelection.RUN,
+                load_artifacts=True,
+                connect_artifact_cache=True,
+                connect_source_cache=True,
+                artifact_remotes=artifact_remotes,
+                source_remotes=source_remotes,
+                ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                ignore_project_source_remotes=ignore_project_source_remotes,
+            )
+        except StreamError as e:
+            if e.reason == "deps-not-supported":
+                raise StreamError(
+                    "Only buildtrees are supported with artifact names",
+                    detail="Use the --build and --use-buildtree options to shell into a cached build tree",
+                    reason="only-buildtrees-supported",
+                ) from e
+            raise
+
+        self.query_cache(other_elements)
+        self._pull_missing_artifacts(other_elements)
+        missing_deps = [dep for dep in _pipeline.dependencies(other_elements, _Scope.RUN) if not dep._cached()]
+        if missing_deps:
+            raise StreamError(
+                "Elements need to be built or downloaded before staging a shell environment",
+                detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+                reason="shell-missing-deps",
+            )
+
         return element._shell(
-            scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree
+            scope,
+            mounts=mounts,
+            isolate=isolate,
+            prompt=prompt(element),
+            command=command,
+            usebuildtree=usebuildtree,
+            other_elements=other_elements,
         )
 
     # build()

src/buildstream/element.py

@@ -62,6 +62,9 @@
 ---------------
 """
 
+# For 3.7+ support, not necessary and deprecated in 3.14+
+from __future__ import annotations
+
 import os
 import re
 import stat
@@ -74,6 +77,7 @@
 from threading import Lock
 from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence
 
+
 from pyroaring import BitMap  # pylint: disable=no-name-in-module
 
 from . import _yaml
@@ -2058,6 +2062,7 @@ def _push(self):
     #    prompt (str): A suitable prompt string for PS1
     #    command (list): An argv to launch in the sandbox
     #    usebuildtree (bool): Use the buildtree as its source
+    #    other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox
     #
     # Returns: Exit code
     def _shell(
@@ -2069,6 +2074,7 @@ def _shell(
         prompt: str | None = None,
         command: List[str] | None = None,
         usebuildtree: bool = False,
+        other_elements: List[Element] | None = None,
     ):
 
         with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox:
@@ -2085,6 +2091,17 @@ def _shell(
             if prompt is not None:
                 environment["PS1"] = prompt
 
+            with self.timed_activity("Staging other_targets", silent_nested=True), self.__collect_overlaps(sandbox):
+                self.stage_dependency_artifacts(sandbox, other_elements)
+
+            if other_elements:
+                # Stage artifacts from other_elements into the sandbox.
+                for element in other_elements:
+                    # Stage deps in the sandbox root
+                    with element.timed_activity("Integrating sandbox"), sandbox.batch():
+                        for dep in element._dependencies(_Scope.RUN):
+                            dep.integrate(sandbox)
+
             # Special configurations for non-isolated sandboxes
             if not isolate:
 

tests/integration/shell.py

@@ -16,12 +16,15 @@
 # pylint: disable=redefined-outer-name
 
 import os
+from typing import Dict, List, Tuple
 import uuid
 
+
 import pytest
 
 from buildstream import _yaml
 from buildstream._testing import cli_integration as cli  # pylint: disable=unused-import
+from buildstream._testing.runcli import CliIntegration
 from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN
 from buildstream.exceptions import ErrorDomain
 from buildstream import utils
@@ -46,18 +49,35 @@
 #    mount (tuple): A (host, target) tuple for the `--mount` option
 #    element (str): The element to build and run a shell with
 #    isolate (bool): Whether to pass --isolate to `bst shell`
-#
-def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False):
+#    other_elements (list(str)): Other elements to stage in the sandbox
+def execute_shell(
+    cli: CliIntegration,
+    project: str,
+    command: List[str],
+    *,
+    config: None | Dict = None,
+    mount: Tuple[str, str] | None = None,
+    element: str = "base.bst",
+    isolate: bool = False,
+    other_elements: List[str] | None = None
+):
     # Ensure the element is built
     result = cli.run_project_config(project=project, project_config=config, args=["build", element])
     assert result.exit_code == 0
+    if other_elements is not None:
+        for other_element in other_elements:
+            result = cli.run_project_config(project=project, project_config=config, args=["build", other_element])
+            assert result.exit_code == 0
 
     args = ["shell"]
     if isolate:
         args += ["--isolate"]
     if mount is not None:
         host_path, target_path = mount
         args += ["--mount", host_path, target_path]
+    if other_elements is not None:
+        for other_element in other_elements:
+            args += ["--with", other_element]
     args += [element, "--", *command]
 
     return cli.run_project_config(project=project, project_config=config, args=args)
@@ -86,6 +106,28 @@ def test_executable(cli, datafiles):
     assert result.output == "Horseys!\n"
 
 
+# Test staging and running additional targets in the shell of the main target for debugging.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_with_other_targets(cli, datafiles):
+    project = str(datafiles)
+
+    # Show we can't cat in a shell for manual/import-file.bst
+    result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst")
+    assert (
+        result.exit_code == -1
+    ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies"
+
+    # Show we can now cat with base.bst in a shell for manual/import-file.bst
+    result = execute_shell(
+        cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"]
+    )
+    assert (
+        result.exit_code == 0
+    ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat"
+    assert result.output == "This is a test\n"
+
+
 # Test shell environment variable explicit assignments
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)