From 9934d53959a42d40ae78108342efea1892b70187 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Tue, 8 Jul 2025 13:31:34 +0200 Subject: [PATCH 01/17] Implement bst inspect subcommand This adds a new subcommand to bst called `inspect` which dumps structured data to stdout of a given project. The data sent to stdout can be either JSON or YAML with JSON being the default. Having this command available will make writing external tools which need to inspect the state of a buildstream project more easy since JSON and YAML encoding are widely supported. This command may easily be extended in the future to support other inspectable elements related to bst project which isn't present in this first commit. --- src/buildstream/_frontend/app.py | 6 + src/buildstream/_frontend/cli.py | 78 +++++++++++- src/buildstream/_frontend/inspect.py | 184 +++++++++++++++++++++++++++ src/buildstream/types.py | 34 +++++ tests/frontend/inspect.py | 61 +++++++++ 5 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 src/buildstream/_frontend/inspect.py create mode 100644 tests/frontend/inspect.py diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index 577d80d4d..21a596608 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -40,6 +40,7 @@ from .profile import Profile from .status import Status from .widget import LogLine +from .inspect import Inspector # Intendation for all logging INDENT = 4 @@ -65,6 +66,7 @@ def __init__(self, main_options): self.logger = None # The LogLine object self.interactive = None # Whether we are running in interactive mode self.colors = None # Whether to use colors in logging + self.inspector = None # If inspection is required # # Private members @@ -302,6 +304,10 @@ def initialized(self, *, session_name=None): # self.stream.set_project(self.project) + # Initialize the inspector + if self._session_name == "Inspect": + self.inspector = Inspector(self.stream, self.project) + # Run the body of the session here, once everything is loaded try: yield diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 9d7619bae..85c9c2f35 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -21,7 +21,7 @@ from .. import _yaml from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled -from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope +from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope, _Encoding from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError @@ -531,6 +531,82 @@ def build( ) +################################################################## +# Inspect Command # +################################################################## +@cli.command(name="inspect", short_help="Inspect Project Information") +@click.option("-s", "--state", default=False, show_default=True, is_flag=True, help="Show information that requires inspecting remote state") +@click.option("-e", "--encoding", default=_Encoding.JSON, show_default=True, type=FastEnumType(_Encoding, [_Encoding.JSON, _Encoding.YAML])) +@click.option( + "--deps", + "-d", + default=_PipelineSelection.ALL, + show_default=True, + type=FastEnumType( + _PipelineSelection, + [ + _PipelineSelection.NONE, + _PipelineSelection.RUN, + _PipelineSelection.BUILD, + _PipelineSelection.ALL, + ], + ), + help="The dependencies to show", +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) +@click.pass_obj +def inspect(app, elements, state, encoding, deps): + """Access structured data about a given buildstream project and it's computed elements. + + Specifying no elements will result in showing the default targets + of the project. If no default targets are configured, all project + elements will be shown. + + When this command is executed from a workspace directory, the default + is to show the workspace element. + + By default this will show all of the dependencies of the + specified target element. + + Specify ``--deps`` to control which elements to show: + + \b + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies + + Use ``--encoding JSON|YAML`` to control the type of encoding written to stdout. + + If ``--state`` is toggled then pipeline elements which require remote state will be + shown in addition to information that is available on the local system. + + Examples: + + # Show all default elements with remote information + \n + bst inspect --state + + + # A specific target (glob pattern) + \n + bst inspect -s public/*.bst + + + # With a dependency target + \n + bst inspect -d run --state + + + # Show each remote file source (with the help of jq) + \n + bst inspect -d all | jq '.elements.[].sources | select( . != null ) | .[] | select( .medium == "remote-file") + + """ + with app.initialized(session_name="Inspect"): + app.inspector.dump_to_stdout(elements, selection=deps, encoding=encoding, with_state=state) + + ################################################################## # Show Command # ################################################################## diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py new file mode 100644 index 000000000..528919733 --- /dev/null +++ b/src/buildstream/_frontend/inspect.py @@ -0,0 +1,184 @@ +import json +import sys + +from dataclasses import dataclass + +from ruamel.yaml import YAML + +from ..types import _Encoding, _ElementState, _PipelineSelection, _Scope + +# Inspectable Elements as serialized to the terminal +@dataclass +class _InspectElement: + name: str + description: str + workspace: any + key: str + key_full: str + state: str + environment: dict[str, str] + variables: dict[str, str] + artifact: any + dependencies: list[str] + build_dependencies: list[str] + runtime_dependencies: list[str] + sources: list[dict[str, str]] + +@dataclass +class _ProjectOutput: + name: str + directory: str + +@dataclass +class _InspectOutput: + project: _ProjectOutput + elements: list[_InspectElement] + +# Inspect elements from a given Buildstream project +class Inspector: + def __init__(self, stream, project): + self.stream = stream + self.project = project + + def _read_state(self, element): + try: + if not element._has_all_sources_resolved(): + return _ElementState.NO_REFERENCE + else: + if element.get_kind() == "junction": + return _ElementState.JUNCTION + elif not element._can_query_cache(): + return _ElementState.WAITING + elif element._cached_failure(): + return _ElementState.FAILED + elif element._cached_success(): + return _ElementState.CACHED + elif not element._can_query_source_cache(): + return _ElementState.WAITING + elif element._fetch_needed(): + return _ElementState.FETCH_NEEDED + elif element._buildable(): + return _ElementState.BUILDABLE + else: + return _ElementState.WAITING + except BstError as e: + # Provide context to plugin error + e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),) + raise e + + def _elements(self, dependencies, with_state=False): + for element in dependencies: + + name = element._get_full_name() + description = " ".join(element._description.splitlines()) + workspace = element._get_workspace() + variables = dict(element._Element__variables) + environment = dict(element._Element__environment) + + sources = [] + for source in element.sources(): + source_infos = source.collect_source_info() + + if source_infos is not None: + serialized_sources = [] + for s in source_infos: + serialized = s.serialize() + serialized_sources.append(serialized) + + sources += serialized_sources + + # Show dependencies + dependencies = [e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False)] + + # Show build dependencies + build_dependencies = [e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False)] + + # Show runtime dependencies + runtime_dependencies = runtime_dependencies = [e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False)] + + # These operations require state and are only shown if requested + key = None + key_full = None + state = None + artifact = None + + if with_state: + key = element._get_display_key().brief + + key_full = element._get_display_key().full + + state = self._read_state(element).value + + # BUG: Due to the assersion within .get_artifact this will + # error but there is no other way to determine if an artifact + # exists and we only want to show this value for informational + # purposes. + try: + _artifact = element._get_artifact() + if _artifact.cached(): + artifact = { + "files": artifact.get_files(), + "digest": artifact_files._get_digest(), + } + except: + pass + + yield _InspectElement( + name=name, + description=description, + workspace=workspace, + key=key, + key_full=key_full, + state=state, + environment=environment, + variables=variables, + artifact=artifact, + dependencies=dependencies, + build_dependencies=build_dependencies, + runtime_dependencies=runtime_dependencies, + sources=sources, + ) + + + def _dump_project(self): + # TODO: What else do we want here? + return _ProjectOutput(name=self.project.name, directory=self.project.directory) + + + def _get_output(self, dependencies, with_state=False): + project = self._dump_project() + elements = [] + for element in self._elements(dependencies, with_state=with_state): + elements.append(element) + return _InspectOutput(project=project, elements=elements) + + + def _to_dict(self, dependencies, with_state=False): + output = self._get_output(dependencies, with_state) + + def _hide_null(element): + d = dict() + for key, value in element.__dict__.items(): + if value: + d[key] = value + return d + + return {"project": _hide_null(output.project), "elements": [_hide_null(element) for element in output.elements]} + + + def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_state=False, encoding=_Encoding.JSON): + if not elements: + elements = self.project.get_default_targets() + + dependencies = self.stream.load_selection( + elements, selection=selection, except_targets=[], need_state=with_state + ) + + if with_state: + self.stream.query_cache(dependencies, need_state=True) + + if encoding == _Encoding.JSON: + json.dump(self._to_dict(dependencies, with_state), sys.stdout) + elif encoding == _Encoding.YAML: + yaml = YAML() + yaml.dump(self._to_dict(dependencies, with_state), sys.stdout) diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 2ee92afa7..8e3a807fc 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -389,6 +389,40 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror": return cls(name, aliases) +# Used to indicate the state of a given element +class _ElementState(FastEnum): + # Cannot determine the element state + NO_REFERENCE = "no-reference" + + # The element has failed + FAILED = "failed" + + # The element is a junction + JUNCTION = "junction" + + # The element is waiting + WAITING = "waiting" + + # The element is cached + CACHED = "cached" + + # The element needs to be loaded from a remote source + FETCH_NEEDED = "fetch-needed" + + # The element my be built + BUILDABLE = "buildable" + + def __str__(self): + return str(self.value) + + +# The type of encoding used when outputing machine readable information +class _Encoding(FastEnum): + YAML = "yaml" + JSON = "json" + + def __str__(self): + return str(self.value) ######################################## # Type aliases # diff --git a/tests/frontend/inspect.py b/tests/frontend/inspect.py new file mode 100644 index 000000000..64d881799 --- /dev/null +++ b/tests/frontend/inspect.py @@ -0,0 +1,61 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os +import pytest +import json + +from buildstream._testing import cli # pylint: disable=unused-import + + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), +) + +def _element_by_name(elements, name): + for element in elements: + if element["name"] == name: + return element + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) +def test_inspect_basic(cli, datafiles): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["inspect"]) + result.assert_success() + output = json.loads(result.output) + assert(output["project"]["name"] == "test") + element = _element_by_name(output["elements"], "import-bin.bst") + source = element["sources"][0] + assert(source["kind"] == "local") + assert(source["url"] == "files/bin-files") + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) +def test_inspect_element_glob(cli, datafiles): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["inspect", "*.bst"]) + result.assert_success() + json.loads(result.output) + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-fetch")) +def test_inspect_with_state(cli, datafiles): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["inspect", "--state", "--deps", "all"]) + result.assert_success() + json.loads(result.output) From d12b0074ee15fd739a54ff91ac85a6b4c37e940d Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Wed, 9 Jul 2025 13:30:14 +0200 Subject: [PATCH 02/17] Clean up and parametrize tests --- tests/frontend/inspect.py | 77 ++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/tests/frontend/inspect.py b/tests/frontend/inspect.py index 64d881799..4bf03339d 100644 --- a/tests/frontend/inspect.py +++ b/tests/frontend/inspect.py @@ -18,6 +18,7 @@ import os import pytest import json +from dataclasses import dataclass from buildstream._testing import cli # pylint: disable=unused-import @@ -26,36 +27,70 @@ os.path.dirname(os.path.realpath(__file__)), ) +# check to see if a source exists in an element +@dataclass +class _Source: + name: str # element name + kind: str + version: str + def _element_by_name(elements, name): for element in elements: if element["name"] == name: return element +def _assert_has_elements(elements, expected): + n_elements = len(elements) + n_expected = len(expected) + if len(elements) != len(expected): + raise Exception(f"Expected {n_expected} elements, got {n_elements}") + for expected_name in expected: + if _element_by_name(elements, expected_name) is None: + raise Exception(f"Element {expected_name} is missing") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) -def test_inspect_basic(cli, datafiles): - project = str(datafiles) - result = cli.run(project=project, silent=True, args=["inspect"]) - result.assert_success() - output = json.loads(result.output) - assert(output["project"]["name"] == "test") - element = _element_by_name(output["elements"], "import-bin.bst") - source = element["sources"][0] - assert(source["kind"] == "local") - assert(source["url"] == "files/bin-files") - +def _assert_has_source(elements, expected: _Source): + element = _element_by_name(elements, expected.name) + if element is None: + raise Exception(f"Cannot find element {expected.name}") + if "sources" in element: + for source in element["sources"]: + kind = source["kind"] + version = source["version"] + if kind == expected.kind and version == expected.version: + return + raise Exception(f"Element {expected.name} does not contain the expected source") + +@pytest.mark.parametrize( + "flags,elements", + [ + ([], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), + (["*.bst", "**/*.bst"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), + (["--state"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), + (["--state", "--deps", "all"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), + (["subdir/*.bst"], ["import-dev.bst", "subdir/target.bst"]) + ], +) @pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) -def test_inspect_element_glob(cli, datafiles): +def test_inspect_simple(cli, datafiles, flags, elements): project = str(datafiles) - result = cli.run(project=project, silent=True, args=["inspect", "*.bst"]) - result.assert_success() - json.loads(result.output) + result = cli.run(project=project, silent=True, args=["inspect"] + flags) + output = json.loads(result.output) + _assert_has_elements(output["elements"], elements) -@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-fetch")) -def test_inspect_with_state(cli, datafiles): +@pytest.mark.parametrize( + "flags,sources", + [ + ([], [ + _Source(name="tar-custom-version.bst", kind="tar", version="9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501"), + _Source(name="extradata.bst", kind="extradata", version="1234567") + ]), + ], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +def test_inspect_sources(cli, datafiles, flags, sources): project = str(datafiles) - result = cli.run(project=project, silent=True, args=["inspect", "--state", "--deps", "all"]) - result.assert_success() - json.loads(result.output) + result = cli.run(project=project, silent=True, args=["inspect"] + flags) + output = json.loads(result.output) + [_assert_has_source(output["elements"], source) for source in sources] From fddf780bfc2852cc10a5db892c7dc0dfd46d0a57 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Wed, 9 Jul 2025 16:20:36 +0200 Subject: [PATCH 03/17] Migrate contrib/bst-graph to use bst inspect --- contrib/bst-graph | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/contrib/bst-graph b/contrib/bst-graph index 63c73bef6..ceb783b65 100755 --- a/contrib/bst-graph +++ b/contrib/bst-graph @@ -29,10 +29,10 @@ installed. import argparse import subprocess import re +import json import urllib.parse from graphviz import Digraph -from ruamel.yaml import YAML def parse_args(): '''Handle parsing of command line arguments. @@ -72,34 +72,25 @@ def unique_node_name(s): return urllib.parse.quote_plus(s) -def parse_graph(lines): +def parse_graph(output): '''Return nodes and edges of the parsed grpah. Args: - lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS' + output: json output Returns: Tuple of format (nodes,build_deps,runtime_deps) Each member of build_deps and runtime_deps is also a tuple. ''' - parser = YAML(typ="safe") + project = json.loads(output) nodes = set() build_deps = set() runtime_deps = set() - for line in lines: - line = line.strip() - if not line: - continue - # It is safe to split on '|' as it is not a valid character for - # element names. - name, build_dep, runtime_dep = line.split('|') - - build_dep = parser.load(build_dep) - runtime_dep = parser.load(runtime_dep) - + for element in project["elements"]: + name = element["name"] nodes.add(name) - [build_deps.add((name, dep)) for dep in build_dep if dep] - [runtime_deps.add((name, dep)) for dep in runtime_dep if dep] + [build_deps.add((name, dep)) for dep in element.get("build_dependencies", [])] + [runtime_deps.add((name, dep)) for dep in element.get("runtime_dependencies", [])] return nodes, build_deps, runtime_deps @@ -127,13 +118,13 @@ def generate_graph(nodes, build_deps, runtime_deps): def main(): args = parse_args() - cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}||'] + cmd = ['bst', 'inspect'] if 'element' in args: cmd += args.element - graph_lines = subprocess.check_output(cmd, universal_newlines=True) + json_output = subprocess.check_output(cmd, universal_newlines=True) # NOTE: We generate nodes and edges before giving them to graphviz as # the library does not de-deuplicate them. - nodes, build_deps, runtime_deps = parse_graph(re.split(r"\|\|", graph_lines)) + nodes, build_deps, runtime_deps = parse_graph(json_output) graph = generate_graph(nodes, build_deps, runtime_deps) print(graph.source) From 5ea5933fe73c087d18f951b7562d2eb8e81b9aa3 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 11 Jul 2025 14:00:35 +0200 Subject: [PATCH 04/17] Eliminate _Encoding type --- src/buildstream/_frontend/cli.py | 9 +++------ src/buildstream/_frontend/inspect.py | 11 ++++------- src/buildstream/types.py | 8 -------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 85c9c2f35..65aa87a37 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -21,7 +21,7 @@ from .. import _yaml from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled -from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope, _Encoding +from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError @@ -536,7 +536,6 @@ def build( ################################################################## @cli.command(name="inspect", short_help="Inspect Project Information") @click.option("-s", "--state", default=False, show_default=True, is_flag=True, help="Show information that requires inspecting remote state") -@click.option("-e", "--encoding", default=_Encoding.JSON, show_default=True, type=FastEnumType(_Encoding, [_Encoding.JSON, _Encoding.YAML])) @click.option( "--deps", "-d", @@ -555,7 +554,7 @@ def build( ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def inspect(app, elements, state, encoding, deps): +def inspect(app, elements, state, deps): """Access structured data about a given buildstream project and it's computed elements. Specifying no elements will result in showing the default targets @@ -576,8 +575,6 @@ def inspect(app, elements, state, encoding, deps): build: Build time dependencies, excluding the element itself all: All dependencies - Use ``--encoding JSON|YAML`` to control the type of encoding written to stdout. - If ``--state`` is toggled then pipeline elements which require remote state will be shown in addition to information that is available on the local system. @@ -604,7 +601,7 @@ def inspect(app, elements, state, encoding, deps): """ with app.initialized(session_name="Inspect"): - app.inspector.dump_to_stdout(elements, selection=deps, encoding=encoding, with_state=state) + app.inspector.dump_to_stdout(elements, selection=deps, with_state=state) ################################################################## diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 528919733..489a3c2d4 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -5,7 +5,7 @@ from ruamel.yaml import YAML -from ..types import _Encoding, _ElementState, _PipelineSelection, _Scope +from ..types import _ElementState, _PipelineSelection, _Scope # Inspectable Elements as serialized to the terminal @dataclass @@ -166,7 +166,7 @@ def _hide_null(element): return {"project": _hide_null(output.project), "elements": [_hide_null(element) for element in output.elements]} - def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_state=False, encoding=_Encoding.JSON): + def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_state=False): if not elements: elements = self.project.get_default_targets() @@ -177,8 +177,5 @@ def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_st if with_state: self.stream.query_cache(dependencies, need_state=True) - if encoding == _Encoding.JSON: - json.dump(self._to_dict(dependencies, with_state), sys.stdout) - elif encoding == _Encoding.YAML: - yaml = YAML() - yaml.dump(self._to_dict(dependencies, with_state), sys.stdout) + json.dump(self._to_dict(dependencies, with_state), sys.stdout) + diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 8e3a807fc..e72267223 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -416,14 +416,6 @@ def __str__(self): return str(self.value) -# The type of encoding used when outputing machine readable information -class _Encoding(FastEnum): - YAML = "yaml" - JSON = "json" - - def __str__(self): - return str(self.value) - ######################################## # Type aliases # ######################################## From c92bceba7194e0a91b4b349dd8f86df0a302a587 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 11 Jul 2025 14:02:00 +0200 Subject: [PATCH 05/17] Eliminate SessionName --- src/buildstream/_frontend/app.py | 3 +-- src/buildstream/_frontend/cli.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index 21a596608..cf4837889 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -305,8 +305,7 @@ def initialized(self, *, session_name=None): self.stream.set_project(self.project) # Initialize the inspector - if self._session_name == "Inspect": - self.inspector = Inspector(self.stream, self.project) + self.inspector = Inspector(self.stream, self.project) # Run the body of the session here, once everything is loaded try: diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 65aa87a37..12728ac1d 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -600,7 +600,7 @@ def inspect(app, elements, state, deps): bst inspect -d all | jq '.elements.[].sources | select( . != null ) | .[] | select( .medium == "remote-file") """ - with app.initialized(session_name="Inspect"): + with app.initialized(): app.inspector.dump_to_stdout(elements, selection=deps, with_state=state) From 8660c306918910017e256df27b95e52c9712ffe4 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 11 Jul 2025 14:07:01 +0200 Subject: [PATCH 06/17] Make _ElementState a private inspect.py type --- src/buildstream/_frontend/inspect.py | 27 ++++++++++++++++++++++++++- src/buildstream/types.py | 26 -------------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 489a3c2d4..0fd438972 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -1,11 +1,12 @@ import json import sys +from enum import StrEnum from dataclasses import dataclass from ruamel.yaml import YAML -from ..types import _ElementState, _PipelineSelection, _Scope +from ..types import _PipelineSelection, _Scope # Inspectable Elements as serialized to the terminal @dataclass @@ -34,6 +35,30 @@ class _InspectOutput: project: _ProjectOutput elements: list[_InspectElement] +# Used to indicate the state of a given element +class _ElementState(StrEnum): + # Cannot determine the element state + NO_REFERENCE = "no-reference" + + # The element has failed + FAILED = "failed" + + # The element is a junction + JUNCTION = "junction" + + # The element is waiting + WAITING = "waiting" + + # The element is cached + CACHED = "cached" + + # The element needs to be loaded from a remote source + FETCH_NEEDED = "fetch-needed" + + # The element my be built + BUILDABLE = "buildable" + + # Inspect elements from a given Buildstream project class Inspector: def __init__(self, stream, project): diff --git a/src/buildstream/types.py b/src/buildstream/types.py index e72267223..2ee92afa7 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -389,32 +389,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror": return cls(name, aliases) -# Used to indicate the state of a given element -class _ElementState(FastEnum): - # Cannot determine the element state - NO_REFERENCE = "no-reference" - - # The element has failed - FAILED = "failed" - - # The element is a junction - JUNCTION = "junction" - - # The element is waiting - WAITING = "waiting" - - # The element is cached - CACHED = "cached" - - # The element needs to be loaded from a remote source - FETCH_NEEDED = "fetch-needed" - - # The element my be built - BUILDABLE = "buildable" - - def __str__(self): - return str(self.value) - ######################################## # Type aliases # From aa370985a1fdd273cdd793d2da74bd2e404fcaf7 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 14 Jul 2025 13:52:39 +0200 Subject: [PATCH 07/17] Fixup tests --- tests/frontend/inspect.py | 19 +++++++++---------- .../inspect/elements/import-local-files.bst | 4 ++++ .../inspect/elements/import-remote-files.bst | 8 ++++++++ tests/frontend/inspect/elements/target.bst | 12 ++++++++++++ tests/frontend/inspect/files/greeting.c | 12 ++++++++++++ tests/frontend/inspect/project.conf | 7 +++++++ 6 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 tests/frontend/inspect/elements/import-local-files.bst create mode 100644 tests/frontend/inspect/elements/import-remote-files.bst create mode 100644 tests/frontend/inspect/elements/target.bst create mode 100644 tests/frontend/inspect/files/greeting.c create mode 100644 tests/frontend/inspect/project.conf diff --git a/tests/frontend/inspect.py b/tests/frontend/inspect.py index 4bf03339d..80ddb47ae 100644 --- a/tests/frontend/inspect.py +++ b/tests/frontend/inspect.py @@ -60,18 +60,17 @@ def _assert_has_source(elements, expected: _Source): return raise Exception(f"Element {expected.name} does not contain the expected source") - @pytest.mark.parametrize( "flags,elements", [ - ([], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), - (["*.bst", "**/*.bst"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), - (["--state"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), - (["--state", "--deps", "all"], ["import-dev.bst", "import-bin.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), - (["subdir/*.bst"], ["import-dev.bst", "subdir/target.bst"]) + ([], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["*.bst", "**/*.bst"],["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["--state"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["--state", "--deps", "all"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["import-*.bst"], ["import-local-files.bst", "import-remote-files.bst"]) ], ) -@pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "inspect")) def test_inspect_simple(cli, datafiles, flags, elements): project = str(datafiles) result = cli.run(project=project, silent=True, args=["inspect"] + flags) @@ -83,12 +82,12 @@ def test_inspect_simple(cli, datafiles, flags, elements): "flags,sources", [ ([], [ - _Source(name="tar-custom-version.bst", kind="tar", version="9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501"), - _Source(name="extradata.bst", kind="extradata", version="1234567") + _Source(name="import-remote-files.bst", kind="remote", version="d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1"), + _Source(name="import-remote-files.bst", kind="tar", version="d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1"), ]), ], ) -@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "inspect")) def test_inspect_sources(cli, datafiles, flags, sources): project = str(datafiles) result = cli.run(project=project, silent=True, args=["inspect"] + flags) diff --git a/tests/frontend/inspect/elements/import-local-files.bst b/tests/frontend/inspect/elements/import-local-files.bst new file mode 100644 index 000000000..4083fbff3 --- /dev/null +++ b/tests/frontend/inspect/elements/import-local-files.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files diff --git a/tests/frontend/inspect/elements/import-remote-files.bst b/tests/frontend/inspect/elements/import-remote-files.bst new file mode 100644 index 000000000..35d41e39e --- /dev/null +++ b/tests/frontend/inspect/elements/import-remote-files.bst @@ -0,0 +1,8 @@ +kind: import +sources: + - kind: remote + url: example:foo.bar.bin + ref: d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1 + - kind: tar + url: example:baz.qux.tar.gz + ref: d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1 diff --git a/tests/frontend/inspect/elements/target.bst b/tests/frontend/inspect/elements/target.bst new file mode 100644 index 000000000..3459480f9 --- /dev/null +++ b/tests/frontend/inspect/elements/target.bst @@ -0,0 +1,12 @@ +kind: stack +description: | + + Main stack target for the bst build test + +depends: +- import-local-files.bst +- import-remote-files.bst + +config: + build-commands: + - "cc greeting.c -o greeting" diff --git a/tests/frontend/inspect/files/greeting.c b/tests/frontend/inspect/files/greeting.c new file mode 100644 index 000000000..d685fa9d1 --- /dev/null +++ b/tests/frontend/inspect/files/greeting.c @@ -0,0 +1,12 @@ +#include +#include + +void do_greeting() { + __uid_t uid; + uid = getuid(); + printf("Hello, %d, nice to meet you! \n", uid); +} + +int main() { + do_greeting(); +} diff --git a/tests/frontend/inspect/project.conf b/tests/frontend/inspect/project.conf new file mode 100644 index 000000000..429bb5e00 --- /dev/null +++ b/tests/frontend/inspect/project.conf @@ -0,0 +1,7 @@ +# Project config for frontend build test +name: test +min-version: 2.0 +element-path: elements + +aliases: + example: https://example.org/ From 03e65e7241a71de0c044d7ff5ab0c8f82ffca255 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 14 Jul 2025 14:04:11 +0200 Subject: [PATCH 08/17] Fixup inspect completion test --- tests/frontend/completions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 38ad6c831..48075ee3c 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -22,7 +22,7 @@ # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "completions") -MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "shell ", "show ", "source ", "workspace "] +MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "inspect ", "shell ", "show ", "source ", "workspace "] MAIN_OPTIONS = [ "--builders ", @@ -236,6 +236,7 @@ def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir): [ # When running in the project directory ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], None), + ("project", "bst inspect ", 2, [e + " " for e in PROJECT_ELEMENTS], None), ( "project", "bst build com", @@ -335,7 +336,7 @@ def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expect ("bst he", 1, ["help "]), ("bst help ", 2, MAIN_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), - ("bst help in", 2, ["init "]), + ("bst help in", 2, ["init ", "inspect "]), ("bst help source ", 3, SOURCE_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), ("bst help w", 2, ["workspace "]), From ea24b15acc7ad75e6b042423dd81cbcb1d6a798c Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Tue, 15 Jul 2025 14:16:25 +0200 Subject: [PATCH 09/17] Continued bst inspect refactoring --- src/buildstream/_frontend/app.py | 2 +- src/buildstream/_frontend/inspect.py | 339 +++++++++++++++++++++------ 2 files changed, 266 insertions(+), 75 deletions(-) diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index cf4837889..76859296e 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -305,7 +305,7 @@ def initialized(self, *, session_name=None): self.stream.set_project(self.project) # Initialize the inspector - self.inspector = Inspector(self.stream, self.project) + self.inspector = Inspector(self.stream, self.project, self.context) # Run the body of the session here, once everything is loaded try: diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 0fd438972..45fb9203b 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -1,16 +1,14 @@ import json import sys - +from dataclasses import dataclass, fields, is_dataclass from enum import StrEnum -from dataclasses import dataclass - -from ruamel.yaml import YAML from ..types import _PipelineSelection, _Scope + # Inspectable Elements as serialized to the terminal @dataclass -class _InspectElement: +class _Element: name: str description: str workspace: any @@ -25,15 +23,76 @@ class _InspectElement: runtime_dependencies: list[str] sources: list[dict[str, str]] + +# Representation of a cache server +@dataclass +class _CacheServer: + url: str + instance: str + + def __init__(self, spec): + self.url = spec.url + self.instance = spec.instance_name + + +# User configuration +@dataclass +class _UserConfig: + configuration: str + cache_directory: str + log_directory: str + source_directory: str + build_directory: str + source_mirrors: str + build_area: str + strict_build_plan: bool + cache_directory: str + maximum_fetch_tasks: int + maximum_build_tasks: int + maximum_push_tasks: int + maximum_network_retries: int + cache_storage_service: _CacheServer | None + # remote specs + remote_execution_service: _CacheServer | None + remote_storage_service: _CacheServer | None + remote_action_cache_service: _CacheServer | None + + +# String representation of loaded plugins @dataclass -class _ProjectOutput: +class _Plugin: name: str - directory: str + full: str # class str + + +# Configuration of a given project +@dataclass +class _ProjectConfig: + name: str + directory: str | None + junction: str | None + variables: [(str, str)] + element_plugins: [_Plugin] + source_plugins: [_Plugin] + + +# A single project loaded from the current configuration +@dataclass +class _Project: + provenance: str + duplicates: [str] + declarations: [str] + config: _ProjectConfig + +# Wrapper object ecapsulating the entire output of `bst inspect` @dataclass class _InspectOutput: - project: _ProjectOutput - elements: list[_InspectElement] + project: [_Project] + # user configuration + user_config: _UserConfig + elements: list[_Element] + # Used to indicate the state of a given element class _ElementState(StrEnum): @@ -59,11 +118,91 @@ class _ElementState(StrEnum): BUILDABLE = "buildable" +# _make_dataclass() +# +# This is a helper class for extracting values from different objects used +# across Buildstream into JSON serializable output. +# +# If keys is a list of str then each attribute is copied directly to the +# dataclass. +# If keys is a tuple of str then the first value is extracted from the object +# and renamed to the second value. +# +# The field of kwarg is mapped directly onto the dataclass. If the value is +# callable then that function is called passing the object to it. +# +# Args: +# obj: Whichever object you are serializing +# _cls: The dataclass you are constructing +# keys: attributes to include directly from the obj +# kwargs: key values passed into the dataclass +def _make_dataclass(obj, _cls, keys: list[(str, str)] | list[str], **kwargs): + params = dict() + for key in keys: + name = None + rename = None + if isinstance(key, tuple): + name = key[0] + rename = key[1] + elif isinstance(key, str): + name = key + rename = None + else: + raise Exception("BUG: Keys may only be (str, str) or str") + value = None + if isinstance(obj, dict): + value = obj.get(name) + elif isinstance(obj, object): + try: + value = getattr(obj, name) + except AttributeError: + pass + else: + raise Exception("BUG: obj must be a dict or object") + if rename: + params[rename] = value + else: + params[name] = value + for key, helper in kwargs.items(): + if callable(helper): + params[key] = helper(obj) + else: + params[key] = helper + return _cls(**params) + + +# Recursively dump the dataclass into a serializable dictionary. Null values +# are dropped from the output. +def _dump_dataclass(_cls): + d = dict() + if not is_dataclass(_cls): + raise Exception("BUG: obj must be a dataclass") + for field in fields(_cls): + value = getattr(_cls, field.name) + if value is None: # hide null values + continue + if is_dataclass(value): + d[field.name] = _dump_dataclass(value) + elif isinstance(value, list): + items = [] + for item in value: + if is_dataclass(item): + # check if it's a list of dataclasses + items.append(_dump_dataclass(item)) + else: + items.append(item) + d[field.name] = items + else: + d[field.name] = value + return d + + # Inspect elements from a given Buildstream project class Inspector: - def __init__(self, stream, project): + def __init__(self, stream, project, context): self.stream = stream self.project = project + self.context = context def _read_state(self, element): try: @@ -94,33 +233,6 @@ def _read_state(self, element): def _elements(self, dependencies, with_state=False): for element in dependencies: - name = element._get_full_name() - description = " ".join(element._description.splitlines()) - workspace = element._get_workspace() - variables = dict(element._Element__variables) - environment = dict(element._Element__environment) - - sources = [] - for source in element.sources(): - source_infos = source.collect_source_info() - - if source_infos is not None: - serialized_sources = [] - for s in source_infos: - serialized = s.serialize() - serialized_sources.append(serialized) - - sources += serialized_sources - - # Show dependencies - dependencies = [e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False)] - - # Show build dependencies - build_dependencies = [e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False)] - - # Show runtime dependencies - runtime_dependencies = runtime_dependencies = [e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False)] - # These operations require state and are only shown if requested key = None key_full = None @@ -148,48 +260,127 @@ def _elements(self, dependencies, with_state=False): except: pass - yield _InspectElement( - name=name, - description=description, - workspace=workspace, + sources = [] + for source in element.sources(): + source_infos = source.collect_source_info() + + if source_infos is not None: + serialized_sources = [] + for s in source_infos: + serialized = s.serialize() + serialized_sources.append(serialized) + + sources += serialized_sources + + yield _make_dataclass( + element, + _Element, + [], + name=lambda element: element._get_full_name(), + description=lambda element: " ".join(element._description.splitlines()), + workspace=lambda element: element._get_workspace(), + variables=lambda element: dict(element._Element__variables), + environment=lambda element: dict(element._Element__environment), + sources=sources, + dependencies=lambda element: [ + dependency._get_full_name() for dependency in element._dependencies(_Scope.ALL, recurse=False) + ], + build_dependencies=lambda element: [ + dependency._get_full_name() for dependency in element._dependencies(_Scope.BUILD, recurse=False) + ], + runtime_dependencies=lambda element: [ + dependency._get_full_name() for dependency in element._dependencies(_Scope.RUN, recurse=False) + ], key=key, key_full=key_full, state=state, - environment=environment, - variables=variables, artifact=artifact, - dependencies=dependencies, - build_dependencies=build_dependencies, - runtime_dependencies=runtime_dependencies, - sources=sources, ) + def _get_projects(self) -> [_Project]: + projects = [] + for wrapper in self.project.loaded_projects(): + variables = dict() + wrapper.project.options.printable_variables(variables) + project_config = _make_dataclass( + wrapper.project, + _ProjectConfig, + ["name", "directory"], + variables=variables, + junction=lambda config: None if not config.junction else config.junction._get_full_name(), + element_plugins=lambda config: [ + _Plugin(name=plugin[0], full=str(plugin[1])) for plugin in config.element_factory.list_plugins() + ], + source_plugins=lambda config: [ + _Plugin(name=plugin[0], full=str(plugin[1])) for plugin in config.source_factory.list_plugins() + ], + ) + projects.append( + _make_dataclass( + wrapper, + _Project, + keys=["provenance"], + duplicates=lambda config: ( + [] if not hasattr(config, "duplicates") else [duplicate for duplicate in config.duplicates] + ), + declarations=lambda config: ( + [] + if not hasattr(config, "declarations") + else [declaration for declaration in config.declarations] + ), + config=project_config, + ) + ) + return projects + + def _get_user_config(self) -> _UserConfig: + + remote_execution_service = None + remote_storage_service = None + remote_action_cache_service = None + + if self.context.remote_execution_specs: + specs = self.context.remote_execution_specs + remote_execution_service = _CacheServer(specs.exec_spec) + storage_spec = specs.storage_spec or self.context.remote_cache_spec + remote_storage_spec = _CacheServer(storage_spec) + if specs.action_spec: + remote_action_cache_service = _CacheServer(specs.action_spec) + + return _make_dataclass( + self.context, + _UserConfig, + [ + ("cachedir", "cache_directory"), + ("logdir", "log_directory"), + ("sourcedir", "source_directory"), + ("builddir", "build_directory"), + ("sourcedir", "source_mirrors"), + ("builddir", "build_area"), + ("cachedir", "cache_directory"), + ("sched_fetchers", "maximum_fetch_tasks"), + ("sched_builders", "maximum_build_tasks"), + ("sched_pushers", "maximum_push_tasks"), + ("sched_network_retries", "maximum_network_retries"), + ], + strict_build_plan=lambda context: ( + "Default Configuration" if not context.config_origin else context.config_origin + ), + configuration=lambda context: "default" if not context.config_origin else context.config_origin, + cache_storage_service=lambda context: ( + None if not context.remote_execution_specs else _CacheServer(context.remote_execution_specs) + ), + remote_execution_service=remote_execution_service, + remote_storage_service=remote_storage_service, + remote_action_cache_service=remote_action_cache_service, + ) - def _dump_project(self): - # TODO: What else do we want here? - return _ProjectOutput(name=self.project.name, directory=self.project.directory) - - - def _get_output(self, dependencies, with_state=False): - project = self._dump_project() - elements = [] - for element in self._elements(dependencies, with_state=with_state): - elements.append(element) - return _InspectOutput(project=project, elements=elements) - - - def _to_dict(self, dependencies, with_state=False): - output = self._get_output(dependencies, with_state) - - def _hide_null(element): - d = dict() - for key, value in element.__dict__.items(): - if value: - d[key] = value - return d - - return {"project": _hide_null(output.project), "elements": [_hide_null(element) for element in output.elements]} - + def _get_output(self, dependencies, with_state=False) -> _InspectOutput: + return _InspectOutput( + project=self._get_projects(), + user_config=self._get_user_config(), + elements=[element for element in self._elements(dependencies, with_state=with_state)], + ) def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_state=False): if not elements: @@ -202,5 +393,5 @@ def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_st if with_state: self.stream.query_cache(dependencies, need_state=True) - json.dump(self._to_dict(dependencies, with_state), sys.stdout) - + output = self._get_output(dependencies, with_state) + json.dump(_dump_dataclass(output), sys.stdout) From a224502fdccf923d7e8415b1e90eb030185cea35 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Thu, 17 Jul 2025 15:40:22 +0200 Subject: [PATCH 10/17] Wip, add more project information --- src/buildstream/_frontend/inspect.py | 61 ++++++++++++++----- .../inspect/elements/import-local-files.bst | 2 + tests/frontend/inspect/project.conf | 5 +- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 45fb9203b..e87882898 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -3,7 +3,11 @@ from dataclasses import dataclass, fields, is_dataclass from enum import StrEnum +from .._project import ProjectConfig as _BsProjectConfig +from .._pluginfactory.pluginorigin import PluginType +from .._options import OptionPool from ..types import _PipelineSelection, _Scope +from ..node import MappingNode # Inspectable Elements as serialized to the terminal @@ -62,7 +66,8 @@ class _UserConfig: @dataclass class _Plugin: name: str - full: str # class str + description: str + plugin_type: PluginType # Configuration of a given project @@ -70,10 +75,16 @@ class _Plugin: class _ProjectConfig: name: str directory: str | None + # Original configuration from the project.conf + original: dict[str, any] junction: str | None - variables: [(str, str)] - element_plugins: [_Plugin] - source_plugins: [_Plugin] + # Interpolated options + options: [(str, str)] + aliases: dict[str, str] + element_overrides: any + source_overrides: any + # plugin information + plugins: [_Plugin] # A single project loaded from the current configuration @@ -197,6 +208,11 @@ def _dump_dataclass(_cls): return d +def _dump_option_pool(options: OptionPool): + opts = dict() + return options.export_variables(opts) + + # Inspect elements from a given Buildstream project class Inspector: def __init__(self, stream, project, context): @@ -300,20 +316,37 @@ def _elements(self, dependencies, with_state=False): def _get_projects(self) -> [_Project]: projects = [] for wrapper in self.project.loaded_projects(): - variables = dict() - wrapper.project.options.printable_variables(variables) + plugins = [] + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.ELEMENT.value) + for plugin in wrapper.project.element_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE.value) + for plugin in wrapper.project.source_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE_MIRROR.value) + for plugin in wrapper.project.source_factory.list_plugins() + ] + ) + project_config = _make_dataclass( wrapper.project, _ProjectConfig, ["name", "directory"], - variables=variables, - junction=lambda config: None if not config.junction else config.junction._get_full_name(), - element_plugins=lambda config: [ - _Plugin(name=plugin[0], full=str(plugin[1])) for plugin in config.element_factory.list_plugins() - ], - source_plugins=lambda config: [ - _Plugin(name=plugin[0], full=str(plugin[1])) for plugin in config.source_factory.list_plugins() - ], + options=lambda project: _dump_option_pool(project.options), + original=lambda project: project._project_conf.strip_node_info(), + aliases=lambda project: project.config._aliases.strip_node_info(), + source_overrides=lambda project: project.source_overrides.strip_node_info(), + element_overrides=lambda project: project.element_overrides.strip_node_info(), + junction=lambda project: None if not project.junction else project.junction._get_full_name(), + plugins=plugins, ) projects.append( _make_dataclass( diff --git a/tests/frontend/inspect/elements/import-local-files.bst b/tests/frontend/inspect/elements/import-local-files.bst index 4083fbff3..ea81d747a 100644 --- a/tests/frontend/inspect/elements/import-local-files.bst +++ b/tests/frontend/inspect/elements/import-local-files.bst @@ -2,3 +2,5 @@ kind: import sources: - kind: local path: files +config: + target: / diff --git a/tests/frontend/inspect/project.conf b/tests/frontend/inspect/project.conf index 429bb5e00..204b5f186 100644 --- a/tests/frontend/inspect/project.conf +++ b/tests/frontend/inspect/project.conf @@ -3,5 +3,8 @@ name: test min-version: 2.0 element-path: elements +variables: + schema: https + aliases: - example: https://example.org/ + example: "%{schema}://example.org/" From 8d32ba343261e0c5f34a392114b347ce788a3137 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 21 Jul 2025 12:28:19 +0200 Subject: [PATCH 11/17] Eliminate state entirely --- src/buildstream/_frontend/cli.py | 17 +--- src/buildstream/_frontend/inspect.py | 117 ++++----------------------- tests/frontend/inspect.py | 4 +- 3 files changed, 23 insertions(+), 115 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 12728ac1d..c22ae49cc 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -535,7 +535,6 @@ def build( # Inspect Command # ################################################################## @cli.command(name="inspect", short_help="Inspect Project Information") -@click.option("-s", "--state", default=False, show_default=True, is_flag=True, help="Show information that requires inspecting remote state") @click.option( "--deps", "-d", @@ -554,7 +553,7 @@ def build( ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def inspect(app, elements, state, deps): +def inspect(app, elements, deps): """Access structured data about a given buildstream project and it's computed elements. Specifying no elements will result in showing the default targets @@ -575,24 +574,16 @@ def inspect(app, elements, state, deps): build: Build time dependencies, excluding the element itself all: All dependencies - If ``--state`` is toggled then pipeline elements which require remote state will be - shown in addition to information that is available on the local system. - Examples: - # Show all default elements with remote information - \n - bst inspect --state - - # A specific target (glob pattern) \n - bst inspect -s public/*.bst + bst inspect public/*.bst # With a dependency target \n - bst inspect -d run --state + bst inspect -d run # Show each remote file source (with the help of jq) @@ -601,7 +592,7 @@ def inspect(app, elements, state, deps): """ with app.initialized(): - app.inspector.dump_to_stdout(elements, selection=deps, with_state=state) + app.inspector.dump_to_stdout(elements, selection=deps) ################################################################## diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index e87882898..c6c298e73 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -16,12 +16,8 @@ class _Element: name: str description: str workspace: any - key: str - key_full: str - state: str environment: dict[str, str] variables: dict[str, str] - artifact: any dependencies: list[str] build_dependencies: list[str] runtime_dependencies: list[str] @@ -90,7 +86,6 @@ class _ProjectConfig: # A single project loaded from the current configuration @dataclass class _Project: - provenance: str duplicates: [str] declarations: [str] config: _ProjectConfig @@ -105,30 +100,6 @@ class _InspectOutput: elements: list[_Element] -# Used to indicate the state of a given element -class _ElementState(StrEnum): - # Cannot determine the element state - NO_REFERENCE = "no-reference" - - # The element has failed - FAILED = "failed" - - # The element is a junction - JUNCTION = "junction" - - # The element is waiting - WAITING = "waiting" - - # The element is cached - CACHED = "cached" - - # The element needs to be loaded from a remote source - FETCH_NEEDED = "fetch-needed" - - # The element my be built - BUILDABLE = "buildable" - - # _make_dataclass() # # This is a helper class for extracting values from different objects used @@ -213,6 +184,12 @@ def _dump_option_pool(options: OptionPool): return options.export_variables(opts) +def _maybe_strip_node_info(obj): + out = dict() + if obj and hasattr(obj, "strip_node_info"): + return obj.strip_node_info() + + # Inspect elements from a given Buildstream project class Inspector: def __init__(self, stream, project, context): @@ -220,62 +197,9 @@ def __init__(self, stream, project, context): self.project = project self.context = context - def _read_state(self, element): - try: - if not element._has_all_sources_resolved(): - return _ElementState.NO_REFERENCE - else: - if element.get_kind() == "junction": - return _ElementState.JUNCTION - elif not element._can_query_cache(): - return _ElementState.WAITING - elif element._cached_failure(): - return _ElementState.FAILED - elif element._cached_success(): - return _ElementState.CACHED - elif not element._can_query_source_cache(): - return _ElementState.WAITING - elif element._fetch_needed(): - return _ElementState.FETCH_NEEDED - elif element._buildable(): - return _ElementState.BUILDABLE - else: - return _ElementState.WAITING - except BstError as e: - # Provide context to plugin error - e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),) - raise e - - def _elements(self, dependencies, with_state=False): + def _elements(self, dependencies): for element in dependencies: - # These operations require state and are only shown if requested - key = None - key_full = None - state = None - artifact = None - - if with_state: - key = element._get_display_key().brief - - key_full = element._get_display_key().full - - state = self._read_state(element).value - - # BUG: Due to the assersion within .get_artifact this will - # error but there is no other way to determine if an artifact - # exists and we only want to show this value for informational - # purposes. - try: - _artifact = element._get_artifact() - if _artifact.cached(): - artifact = { - "files": artifact.get_files(), - "digest": artifact_files._get_digest(), - } - except: - pass - sources = [] for source in element.sources(): source_infos = source.collect_source_info() @@ -307,10 +231,6 @@ def _elements(self, dependencies, with_state=False): runtime_dependencies=lambda element: [ dependency._get_full_name() for dependency in element._dependencies(_Scope.RUN, recurse=False) ], - key=key, - key_full=key_full, - state=state, - artifact=artifact, ) def _get_projects(self) -> [_Project]: @@ -341,10 +261,10 @@ def _get_projects(self) -> [_Project]: _ProjectConfig, ["name", "directory"], options=lambda project: _dump_option_pool(project.options), - original=lambda project: project._project_conf.strip_node_info(), - aliases=lambda project: project.config._aliases.strip_node_info(), - source_overrides=lambda project: project.source_overrides.strip_node_info(), - element_overrides=lambda project: project.element_overrides.strip_node_info(), + original=lambda project: _maybe_strip_node_info(project._project_conf), + aliases=lambda project: _maybe_strip_node_info(project.config._aliases), + source_overrides=lambda project: _maybe_strip_node_info(project.source_overrides), + element_overrides=lambda project: _maybe_strip_node_info(project.element_overrides), junction=lambda project: None if not project.junction else project.junction._get_full_name(), plugins=plugins, ) @@ -352,7 +272,7 @@ def _get_projects(self) -> [_Project]: _make_dataclass( wrapper, _Project, - keys=["provenance"], + keys=[], duplicates=lambda config: ( [] if not hasattr(config, "duplicates") else [duplicate for duplicate in config.duplicates] ), @@ -408,23 +328,20 @@ def _get_user_config(self) -> _UserConfig: remote_action_cache_service=remote_action_cache_service, ) - def _get_output(self, dependencies, with_state=False) -> _InspectOutput: + def _get_output(self, dependencies) -> _InspectOutput: return _InspectOutput( project=self._get_projects(), user_config=self._get_user_config(), - elements=[element for element in self._elements(dependencies, with_state=with_state)], + elements=[element for element in self._elements(dependencies)], ) - def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE, with_state=False): + def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE): if not elements: elements = self.project.get_default_targets() dependencies = self.stream.load_selection( - elements, selection=selection, except_targets=[], need_state=with_state + elements, selection=selection, except_targets=[] ) - if with_state: - self.stream.query_cache(dependencies, need_state=True) - - output = self._get_output(dependencies, with_state) + output = self._get_output(dependencies) json.dump(_dump_dataclass(output), sys.stdout) diff --git a/tests/frontend/inspect.py b/tests/frontend/inspect.py index 80ddb47ae..c87b37715 100644 --- a/tests/frontend/inspect.py +++ b/tests/frontend/inspect.py @@ -65,8 +65,8 @@ def _assert_has_source(elements, expected: _Source): [ ([], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), (["*.bst", "**/*.bst"],["import-local-files.bst", "import-remote-files.bst", "target.bst"]), - (["--state"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), - (["--state", "--deps", "all"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + ([], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["--deps", "all"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), (["import-*.bst"], ["import-local-files.bst", "import-remote-files.bst"]) ], ) From c72aa10d8a608cf0821a3773404dce02ab2b693d Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 21 Jul 2025 14:54:29 +0200 Subject: [PATCH 12/17] Move Inspector out of app.py --- src/buildstream/_frontend/app.py | 5 ----- src/buildstream/_frontend/cli.py | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index 76859296e..577d80d4d 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -40,7 +40,6 @@ from .profile import Profile from .status import Status from .widget import LogLine -from .inspect import Inspector # Intendation for all logging INDENT = 4 @@ -66,7 +65,6 @@ def __init__(self, main_options): self.logger = None # The LogLine object self.interactive = None # Whether we are running in interactive mode self.colors = None # Whether to use colors in logging - self.inspector = None # If inspection is required # # Private members @@ -304,9 +302,6 @@ def initialized(self, *, session_name=None): # self.stream.set_project(self.project) - # Initialize the inspector - self.inspector = Inspector(self.stream, self.project, self.context) - # Run the body of the session here, once everything is loaded try: yield diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index c22ae49cc..dfd1e9c07 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -24,6 +24,7 @@ from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError +from .inspect import Inspector ################################################################## @@ -592,7 +593,8 @@ def inspect(app, elements, deps): """ with app.initialized(): - app.inspector.dump_to_stdout(elements, selection=deps) + inspector = Inspector(app.stream, app.project, app.context) + inspector.dump_to_stdout(elements, selection=deps) ################################################################## From d3cf1c714c0bc38d5959519c7909406c77375a99 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 21 Jul 2025 15:01:30 +0200 Subject: [PATCH 13/17] Add support for --except flag --- src/buildstream/_frontend/cli.py | 7 +++++-- src/buildstream/_frontend/inspect.py | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index dfd1e9c07..fb0d68dee 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -536,6 +536,9 @@ def build( # Inspect Command # ################################################################## @cli.command(name="inspect", short_help="Inspect Project Information") +@click.option( + "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" +) @click.option( "--deps", "-d", @@ -554,7 +557,7 @@ def build( ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def inspect(app, elements, deps): +def inspect(app, except_, elements, deps): """Access structured data about a given buildstream project and it's computed elements. Specifying no elements will result in showing the default targets @@ -594,7 +597,7 @@ def inspect(app, elements, deps): """ with app.initialized(): inspector = Inspector(app.stream, app.project, app.context) - inspector.dump_to_stdout(elements, selection=deps) + inspector.dump_to_stdout(elements, except_=except_, selection=deps) ################################################################## diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index c6c298e73..10e8e790e 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -335,13 +335,13 @@ def _get_output(self, dependencies) -> _InspectOutput: elements=[element for element in self._elements(dependencies)], ) - def dump_to_stdout(self, elements=[], selection=_PipelineSelection.NONE): + def dump_to_stdout(self, elements=[], except_=[], selection=_PipelineSelection.NONE): if not elements: elements = self.project.get_default_targets() - dependencies = self.stream.load_selection( - elements, selection=selection, except_targets=[] - ) + elements = [element for element in filter(lambda name: name not in except_, elements)] + + dependencies = self.stream.load_selection(elements, selection=selection, except_targets=[]) output = self._get_output(dependencies) json.dump(_dump_dataclass(output), sys.stdout) From 2a90fe24623404eb2c2017f39c183e3bda523500 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Mon, 21 Jul 2025 15:06:00 +0200 Subject: [PATCH 14/17] Eliminate original field --- src/buildstream/_frontend/inspect.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 10e8e790e..797287677 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -71,8 +71,6 @@ class _Plugin: class _ProjectConfig: name: str directory: str | None - # Original configuration from the project.conf - original: dict[str, any] junction: str | None # Interpolated options options: [(str, str)] @@ -261,7 +259,6 @@ def _get_projects(self) -> [_Project]: _ProjectConfig, ["name", "directory"], options=lambda project: _dump_option_pool(project.options), - original=lambda project: _maybe_strip_node_info(project._project_conf), aliases=lambda project: _maybe_strip_node_info(project.config._aliases), source_overrides=lambda project: _maybe_strip_node_info(project.source_overrides), element_overrides=lambda project: _maybe_strip_node_info(project.element_overrides), From 830351965d7143aab770116155b79bd511092a20 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 25 Jul 2025 11:11:49 +0200 Subject: [PATCH 15/17] Drop UserConfig entirely --- src/buildstream/_frontend/inspect.py | 69 ---------------------------- 1 file changed, 69 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 797287677..15b16ae58 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -34,30 +34,6 @@ def __init__(self, spec): self.url = spec.url self.instance = spec.instance_name - -# User configuration -@dataclass -class _UserConfig: - configuration: str - cache_directory: str - log_directory: str - source_directory: str - build_directory: str - source_mirrors: str - build_area: str - strict_build_plan: bool - cache_directory: str - maximum_fetch_tasks: int - maximum_build_tasks: int - maximum_push_tasks: int - maximum_network_retries: int - cache_storage_service: _CacheServer | None - # remote specs - remote_execution_service: _CacheServer | None - remote_storage_service: _CacheServer | None - remote_action_cache_service: _CacheServer | None - - # String representation of loaded plugins @dataclass class _Plugin: @@ -93,8 +69,6 @@ class _Project: @dataclass class _InspectOutput: project: [_Project] - # user configuration - user_config: _UserConfig elements: list[_Element] @@ -283,52 +257,9 @@ def _get_projects(self) -> [_Project]: ) return projects - def _get_user_config(self) -> _UserConfig: - - remote_execution_service = None - remote_storage_service = None - remote_action_cache_service = None - - if self.context.remote_execution_specs: - specs = self.context.remote_execution_specs - remote_execution_service = _CacheServer(specs.exec_spec) - storage_spec = specs.storage_spec or self.context.remote_cache_spec - remote_storage_spec = _CacheServer(storage_spec) - if specs.action_spec: - remote_action_cache_service = _CacheServer(specs.action_spec) - - return _make_dataclass( - self.context, - _UserConfig, - [ - ("cachedir", "cache_directory"), - ("logdir", "log_directory"), - ("sourcedir", "source_directory"), - ("builddir", "build_directory"), - ("sourcedir", "source_mirrors"), - ("builddir", "build_area"), - ("cachedir", "cache_directory"), - ("sched_fetchers", "maximum_fetch_tasks"), - ("sched_builders", "maximum_build_tasks"), - ("sched_pushers", "maximum_push_tasks"), - ("sched_network_retries", "maximum_network_retries"), - ], - strict_build_plan=lambda context: ( - "Default Configuration" if not context.config_origin else context.config_origin - ), - configuration=lambda context: "default" if not context.config_origin else context.config_origin, - cache_storage_service=lambda context: ( - None if not context.remote_execution_specs else _CacheServer(context.remote_execution_specs) - ), - remote_execution_service=remote_execution_service, - remote_storage_service=remote_storage_service, - remote_action_cache_service=remote_action_cache_service, - ) - def _get_output(self, dependencies) -> _InspectOutput: return _InspectOutput( project=self._get_projects(), - user_config=self._get_user_config(), elements=[element for element in self._elements(dependencies)], ) From 4a26d9ed2878b4ec6f6a91c7ed60a5ab941cd36a Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 25 Jul 2025 16:11:42 +0200 Subject: [PATCH 16/17] Rework element dependencies --- src/buildstream/_frontend/inspect.py | 74 ++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 15b16ae58..39014ef07 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -3,11 +3,25 @@ from dataclasses import dataclass, fields, is_dataclass from enum import StrEnum -from .._project import ProjectConfig as _BsProjectConfig +from .._project import ProjectConfig as _BsProjectConfig, Project as _BsProject from .._pluginfactory.pluginorigin import PluginType from .._options import OptionPool -from ..types import _PipelineSelection, _Scope +from ..types import _PipelineSelection, _Scope, _ProjectInformation from ..node import MappingNode +from ..element import Element + + +class _DependencyKind(StrEnum): + ALL = "all" + RUNTIME = "runtime" + BUILD = "build" + + +@dataclass +class _Dependency: + name: str + junction: str | None + kind: _DependencyKind # Inspectable Elements as serialized to the terminal @@ -18,9 +32,7 @@ class _Element: workspace: any environment: dict[str, str] variables: dict[str, str] - dependencies: list[str] - build_dependencies: list[str] - runtime_dependencies: list[str] + dependencies: list[_Dependency] sources: list[dict[str, str]] @@ -34,6 +46,7 @@ def __init__(self, spec): self.url = spec.url self.instance = spec.instance_name + # String representation of loaded plugins @dataclass class _Plugin: @@ -164,12 +177,12 @@ def _maybe_strip_node_info(obj): # Inspect elements from a given Buildstream project class Inspector: - def __init__(self, stream, project, context): + def __init__(self, stream, project: _BsProject, context): self.stream = stream self.project = project self.context = context - def _elements(self, dependencies): + def _elements(self, dependencies: list[Element]): for element in dependencies: sources = [] @@ -184,6 +197,43 @@ def _elements(self, dependencies): sources += serialized_sources + junction_name = None + project = element._get_project() + if project: + if hasattr(project, "junction") and project.junction: + junction_name = project.junction.name + + + named_by_kind = { + str(_DependencyKind.ALL): {}, + str(_DependencyKind.BUILD): {}, + str(_DependencyKind.RUNTIME): {}, + } + + dependencies = [] + for dependency in element._dependencies(_Scope.ALL, recurse=True): + named_by_kind[str(_DependencyKind.ALL)][dependency.name] = dependency + # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) + for dependency in element._dependencies(_Scope.BUILD, recurse=True): + named_by_kind[str(_DependencyKind.BUILD)][dependency.name] = dependency + # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD)) + for dependency in element._dependencies(_Scope.RUN, recurse=True): + named_by_kind[str(_DependencyKind.RUNTIME)][dependency.name] = dependency + # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME)) + + for dependency in named_by_kind[str(_DependencyKind.ALL)].values(): + dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) + + # Filter out dependencies covered by ALL + + for (name, dependency) in named_by_kind[str(_DependencyKind.BUILD)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD)) + + for (name, dependency) in named_by_kind[str(_DependencyKind.RUNTIME)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME)) + yield _make_dataclass( element, _Element, @@ -194,15 +244,7 @@ def _elements(self, dependencies): variables=lambda element: dict(element._Element__variables), environment=lambda element: dict(element._Element__environment), sources=sources, - dependencies=lambda element: [ - dependency._get_full_name() for dependency in element._dependencies(_Scope.ALL, recurse=False) - ], - build_dependencies=lambda element: [ - dependency._get_full_name() for dependency in element._dependencies(_Scope.BUILD, recurse=False) - ], - runtime_dependencies=lambda element: [ - dependency._get_full_name() for dependency in element._dependencies(_Scope.RUN, recurse=False) - ], + dependencies=dependencies, ) def _get_projects(self) -> [_Project]: From d1e1836d7e80703926e5291201bc5cae3823c1f1 Mon Sep 17 00:00:00 2001 From: Kevin Schoon Date: Fri, 25 Jul 2025 17:33:37 +0200 Subject: [PATCH 17/17] Continued updates per feedback, WIP --- src/buildstream/_frontend/inspect.py | 308 +++++++++++---------------- 1 file changed, 121 insertions(+), 187 deletions(-) diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py index 39014ef07..6cd097b7b 100644 --- a/src/buildstream/_frontend/inspect.py +++ b/src/buildstream/_frontend/inspect.py @@ -6,10 +6,14 @@ from .._project import ProjectConfig as _BsProjectConfig, Project as _BsProject from .._pluginfactory.pluginorigin import PluginType from .._options import OptionPool +from .._stream import Stream from ..types import _PipelineSelection, _Scope, _ProjectInformation from ..node import MappingNode from ..element import Element +from .. import _yaml +from .. import _site + class _DependencyKind(StrEnum): ALL = "all" @@ -55,87 +59,27 @@ class _Plugin: plugin_type: PluginType -# Configuration of a given project +# A single project loaded from the current configuration @dataclass -class _ProjectConfig: +class _Project: name: str - directory: str | None junction: str | None - # Interpolated options options: [(str, str)] - aliases: dict[str, str] - element_overrides: any - source_overrides: any - # plugin information plugins: [_Plugin] + elements: [_Element] -# A single project loaded from the current configuration +# Default values defined for each element within @dataclass -class _Project: - duplicates: [str] - declarations: [str] - config: _ProjectConfig +class _Defaults: + environment: dict[str, str] # Wrapper object ecapsulating the entire output of `bst inspect` @dataclass class _InspectOutput: - project: [_Project] - elements: list[_Element] - - -# _make_dataclass() -# -# This is a helper class for extracting values from different objects used -# across Buildstream into JSON serializable output. -# -# If keys is a list of str then each attribute is copied directly to the -# dataclass. -# If keys is a tuple of str then the first value is extracted from the object -# and renamed to the second value. -# -# The field of kwarg is mapped directly onto the dataclass. If the value is -# callable then that function is called passing the object to it. -# -# Args: -# obj: Whichever object you are serializing -# _cls: The dataclass you are constructing -# keys: attributes to include directly from the obj -# kwargs: key values passed into the dataclass -def _make_dataclass(obj, _cls, keys: list[(str, str)] | list[str], **kwargs): - params = dict() - for key in keys: - name = None - rename = None - if isinstance(key, tuple): - name = key[0] - rename = key[1] - elif isinstance(key, str): - name = key - rename = None - else: - raise Exception("BUG: Keys may only be (str, str) or str") - value = None - if isinstance(obj, dict): - value = obj.get(name) - elif isinstance(obj, object): - try: - value = getattr(obj, name) - except AttributeError: - pass - else: - raise Exception("BUG: obj must be a dict or object") - if rename: - params[rename] = value - else: - params[name] = value - for key, helper in kwargs.items(): - if callable(helper): - params[key] = helper(obj) - else: - params[key] = helper - return _cls(**params) + projects: [_Project] + defaults: _Defaults # Recursively dump the dataclass into a serializable dictionary. Null values @@ -177,132 +121,122 @@ def _maybe_strip_node_info(obj): # Inspect elements from a given Buildstream project class Inspector: - def __init__(self, stream, project: _BsProject, context): + def __init__(self, stream: Stream, project: _BsProject, context): self.stream = stream self.project = project self.context = context + # Load config defaults so we can only show them once instead of + # for each element unless they are distinct + _default_config = _yaml.load(_site.default_project_config, shortname="projectconfig.yaml") + self.default_environment = _default_config.get_mapping("environment").strip_node_info() + + def _get_element(self, element: Element): + sources = [] + for source in element.sources(): + source_infos = source.collect_source_info() + + if source_infos is not None: + serialized_sources = [] + for s in source_infos: + serialized = s.serialize() + serialized_sources.append(serialized) + + sources += serialized_sources + + junction_name = None + project = element._get_project() + if project: + if hasattr(project, "junction") and project.junction: + junction_name = project.junction.name + + named_by_kind = { + str(_DependencyKind.ALL): {}, + str(_DependencyKind.BUILD): {}, + str(_DependencyKind.RUNTIME): {}, + } + + dependencies = [] + for dependency in element._dependencies(_Scope.ALL, recurse=True): + named_by_kind[str(_DependencyKind.ALL)][dependency.name] = dependency + for dependency in element._dependencies(_Scope.BUILD, recurse=True): + named_by_kind[str(_DependencyKind.BUILD)][dependency.name] = dependency + for dependency in element._dependencies(_Scope.RUN, recurse=True): + named_by_kind[str(_DependencyKind.RUNTIME)][dependency.name] = dependency + + for dependency in named_by_kind[str(_DependencyKind.ALL)].values(): + dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) + + # Filter out dependencies covered by ALL + + for name, dependency in named_by_kind[str(_DependencyKind.BUILD)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append( + _Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD) + ) - def _elements(self, dependencies: list[Element]): - for element in dependencies: - - sources = [] - for source in element.sources(): - source_infos = source.collect_source_info() - - if source_infos is not None: - serialized_sources = [] - for s in source_infos: - serialized = s.serialize() - serialized_sources.append(serialized) - - sources += serialized_sources - - junction_name = None - project = element._get_project() - if project: - if hasattr(project, "junction") and project.junction: - junction_name = project.junction.name - - - named_by_kind = { - str(_DependencyKind.ALL): {}, - str(_DependencyKind.BUILD): {}, - str(_DependencyKind.RUNTIME): {}, - } - - dependencies = [] - for dependency in element._dependencies(_Scope.ALL, recurse=True): - named_by_kind[str(_DependencyKind.ALL)][dependency.name] = dependency - # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) - for dependency in element._dependencies(_Scope.BUILD, recurse=True): - named_by_kind[str(_DependencyKind.BUILD)][dependency.name] = dependency - # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD)) - for dependency in element._dependencies(_Scope.RUN, recurse=True): - named_by_kind[str(_DependencyKind.RUNTIME)][dependency.name] = dependency - # dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME)) - - for dependency in named_by_kind[str(_DependencyKind.ALL)].values(): - dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) - - # Filter out dependencies covered by ALL - - for (name, dependency) in named_by_kind[str(_DependencyKind.BUILD)].items(): - if not name in named_by_kind[str(_DependencyKind.ALL)]: - dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD)) - - for (name, dependency) in named_by_kind[str(_DependencyKind.RUNTIME)].items(): - if not name in named_by_kind[str(_DependencyKind.ALL)]: - dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME)) - - yield _make_dataclass( - element, - _Element, - [], - name=lambda element: element._get_full_name(), - description=lambda element: " ".join(element._description.splitlines()), - workspace=lambda element: element._get_workspace(), - variables=lambda element: dict(element._Element__variables), - environment=lambda element: dict(element._Element__environment), - sources=sources, - dependencies=dependencies, - ) - - def _get_projects(self) -> [_Project]: - projects = [] - for wrapper in self.project.loaded_projects(): - plugins = [] - plugins.extend( - [ - _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.ELEMENT.value) - for plugin in wrapper.project.element_factory.list_plugins() - ] - ) - plugins.extend( - [ - _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE.value) - for plugin in wrapper.project.source_factory.list_plugins() - ] - ) - plugins.extend( - [ - _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE_MIRROR.value) - for plugin in wrapper.project.source_factory.list_plugins() - ] - ) - - project_config = _make_dataclass( - wrapper.project, - _ProjectConfig, - ["name", "directory"], - options=lambda project: _dump_option_pool(project.options), - aliases=lambda project: _maybe_strip_node_info(project.config._aliases), - source_overrides=lambda project: _maybe_strip_node_info(project.source_overrides), - element_overrides=lambda project: _maybe_strip_node_info(project.element_overrides), - junction=lambda project: None if not project.junction else project.junction._get_full_name(), - plugins=plugins, - ) - projects.append( - _make_dataclass( - wrapper, - _Project, - keys=[], - duplicates=lambda config: ( - [] if not hasattr(config, "duplicates") else [duplicate for duplicate in config.duplicates] - ), - declarations=lambda config: ( - [] - if not hasattr(config, "declarations") - else [declaration for declaration in config.declarations] - ), - config=project_config, + for name, dependency in named_by_kind[str(_DependencyKind.RUNTIME)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append( + _Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME) ) - ) - return projects - def _get_output(self, dependencies) -> _InspectOutput: + environment = dict() + for key, value in element.get_environment().items(): + if key in self.default_environment and self.default_environment[key] == value: + continue + environment[key] = value + + return _Element( + name=element._get_full_name(), + description=" ".join(element._description.splitlines()), + workspace=element._get_workspace(), + variables=dict(element._Element__variables), + environment=environment, + sources=sources, + dependencies=dependencies, + ) + + def _get_project(self, info: _ProjectInformation, project: _BsProject, elements: [Element]) -> _Project: + plugins = [] + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.ELEMENT.value) + for plugin in project.element_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE.value) + for plugin in project.source_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE_MIRROR.value) + for plugin in project.source_mirror_factory.list_plugins() + ] + ) + + options = _dump_option_pool(project.options) + + junction = None + if hasattr(project, "junction") and project.junction: + junction = project.junction._get_full_name() + + return _Project( + name=project.name, + junction=junction, + options=options, + plugins=plugins, + elements=[self._get_element(element) for element in elements], + ) + + def _get_output(self, elements: [Element]) -> _InspectOutput: return _InspectOutput( - project=self._get_projects(), - elements=[element for element in self._elements(dependencies)], + projects=[ + self._get_project(wrapper, wrapper.project, elements) for wrapper in self.project.loaded_projects() + ], + defaults=_Defaults(environment=self.default_environment), ) def dump_to_stdout(self, elements=[], except_=[], selection=_PipelineSelection.NONE):