Generic Source Provenance

src/buildstream/_project.py

@@ -125,6 +125,10 @@ def __init__(
         self.sandbox: Optional[MappingNode] = None
         self.splits: Optional[MappingNode] = None
 
+        self.source_provenance_attributes: Optional[
+            MappingNode
+        ] = None  # Source provenance attributes and their description
+
         #
         # Private members
         #
@@ -726,6 +730,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False):
                 "sources",
                 "source-caches",
                 "junctions",
+                "source-provenance-attributes",
                 "(@)",
                 "(?)",
             ]
@@ -1006,6 +1011,13 @@ def _load_second_pass(self):
 
             self._shell_host_files.append(mount)
 
+        # We don't want to combine the source provenance attributes that a project defines with the defaults
+        # If the project config defines source-provenance-attributes use that, otherwise fall back to the defaults
+        # This is purely to maintain backwards compatibility with homepage and issue-tracker being available
+        self.source_provenance_attributes = project_conf_second_pass.get_mapping(
+            "source-provenance-attributes", None
+        ) or config.get_mapping("source-provenance-attributes")
+
     # _load_pass():
     #
     # Loads parts of the project configuration that are different

src/buildstream/data/projectconfig.yaml

@@ -175,6 +175,12 @@ shell:
   #
   command: [ 'sh', '-i' ]
 
+# Define the set of fields accepted in `provenance` dictionaries of sources.
+#
+source-provenance-attributes:
+  homepage: "The project homepage URL"
+  issue-tracker: "The project's issue tracking URL"
+
 # Defaults for bst commands
 #
 defaults:

src/buildstream/element.py

@@ -85,12 +85,12 @@
 from . import utils
 from . import _cachekey
 from . import _site
-from .node import Node
+from .node import Node, MappingNode, ScalarNode
 from .plugin import Plugin
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -102,7 +102,7 @@
 
 if TYPE_CHECKING:
     from typing import Tuple
-    from .node import MappingNode, ScalarNode, SequenceNode
+    from .node import SequenceNode
     from .types import SourceRef
 
     # pylint: disable=cyclic-import
@@ -2635,19 +2635,32 @@ def __load_sources(self, load_element):
                     del source[Symbol.DIRECTORY]
 
                 # Provenance is optional
-                provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None)
-                provenance = None
+                provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None)
                 if provenance_node:
                     del source[Symbol.PROVENANCE]
-                    provenance = _SourceProvenance.new_from_node(provenance_node)
+                    try:
+                        provenance_node.validate_keys(project.source_provenance_attributes.keys())
+                    except LoadError as E:
+                        raise LoadError(
+                            f"Specified source provenance attribute not defined in project config\n {E}",
+                            LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE,
+                        )
+
+                    # make sure everything is a string
+                    for key, value in provenance_node.items():
+                        if not isinstance(value, ScalarNode):
+                            raise LoadError(
+                                f"{value}: Expected string for the value of provenance attribute '{key}'",
+                                LoadErrorReason.INVALID_DATA,
+                            )
 
                 meta_source = MetaSource(
                     self.name,
                     index,
                     self.get_kind(),
                     kind.as_str(),
                     directory,
-                    provenance,
+                    provenance_node,
                     source,
                     load_element.first_pass,
                 )

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/node.pyi

@@ -65,6 +65,7 @@ class MappingNode(Node, Generic[TNode]):
     def items(self) -> Iterable[Tuple[str, Any]]: ...
     def safe_del(self, key: str) -> None: ...
     def validate_keys(self, valid_keys: List[str]): ...
+    def values(self) -> List[Node]: ...
     @overload
     def get_scalar(self, key: str) -> ScalarNode: ...
     @overload

src/buildstream/source.py

@@ -38,15 +38,11 @@
   to provide additional source provenance related metadata which will later
   be reported in :class:`.SourceInfo` objects.
 
-  The ``provenance`` dictionary supports the following fields:
+  The ``provenance`` dictionary itself does not have any specific required keys.
 
-  * Homepage
-
-    The ``homepage`` attribute can be used to specify the project homepage URL
-
-  * Issue Tracker
-
-    The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL
+  Any attribute used in the ``provenance`` dictionary of a source must be
+  defined in the project.conf using the ``source-provenance-attributes`` dictionary
+  to define the attribute and its significance.
 
   *Since: 2.5*
 
@@ -371,16 +367,26 @@
 
 import os
 from contextlib import contextmanager
-from typing import Iterable, Iterator, Optional, Tuple, Dict, Any, Set, TYPE_CHECKING, Union
+from typing import (
+    Iterable,
+    Iterator,
+    Optional,
+    Tuple,
+    Dict,
+    Any,
+    Set,
+    TYPE_CHECKING,
+    Union,
+)
 from dataclasses import dataclass
 
 from . import _yaml, utils
-from .node import MappingNode
+from .node import MappingNode, ScalarNode
 from .plugin import Plugin
 from .sourcemirror import SourceMirror
-from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance
-from ._exceptions import BstError, ImplError, PluginError
-from .exceptions import ErrorDomain
+from .types import SourceRef, CoreWarnings, FastEnum
+from ._exceptions import BstError, ImplError, PluginError, LoadError
+from .exceptions import ErrorDomain, LoadErrorReason
 from ._loader.metasource import MetaSource
 from ._projectrefs import ProjectRefStorage
 from ._cachekey import generate_key
@@ -396,6 +402,8 @@
 
     # pylint: enable=cyclic-import
 
+SourceProvenance = MappingNode
+
 
 class SourceError(BstError):
     """This exception should be raised by :class:`.Source` implementations
@@ -555,6 +563,7 @@ def __init__(
         url: str,
         homepage: Optional[str],
         issue_tracker: Optional[str],
+        provenance: Optional[SourceProvenance],
         medium: Union[SourceInfoMedium, str],
         version_type: Union[SourceVersionType, str],
         version: str,
@@ -579,7 +588,12 @@ def __init__(
 
         self.issue_tracker: Optional[str] = issue_tracker
         """
-        The project issue tracking URL
+        The issue tracker for the project
+        """
+
+        self.provenance = provenance
+        """
+        The optional YAML node with source provenance attributes
         """
 
         self.medium: Union[SourceInfoMedium, str] = medium
@@ -642,10 +656,14 @@ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]:
             "url": self.url,
         }
 
-        if self.homepage is not None:
-            version_info["homepage"] = self.homepage
-        if self.issue_tracker is not None:
-            version_info["issue-tracker"] = self.issue_tracker
+        if self.provenance is not None:
+            # need to keep homepage/issue-tracker [also] at the top-level for backward compat
+            if (homepage := self.provenance.get_str("homepage", None)) is not None:
+                version_info["homepage"] = homepage
+            if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None:
+                version_info["issue-tracker"] = issue_tracker
+
+            version_info["provenance"] = self.provenance.strip_node_info()
 
         version_info["medium"] = medium_str
         version_info["version-type"] = version_type_str
@@ -759,6 +777,14 @@ class Source(Plugin):
     # The defaults from the project
     __defaults: Optional[Dict[str, Any]] = None
 
+    BST_CUSTOM_SOURCE_PROVENANCE = False
+    """Whether multiple sources' provenance information are provided
+
+    Used primarily to override the usage of top-level source
+    provenance of a source where individual sub-source's
+    provenance should instead be provided
+    """
+
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
     """Whether access to previous sources is required during track
 
@@ -825,8 +851,14 @@ def __init__(
         self._directory = meta.directory  # Staging relative directory
         self.__variables = variables  # The variables used to resolve the source's config
         self.__provenance: Optional[
-            _SourceProvenance
-        ] = meta.provenance  # The _SourceProvenance for general user provided SourceInfo
+            SourceProvenance
+        ] = meta.provenance  # The source provenance for general user provided SourceInfo
+
+        if self.__provenance is not None and self.BST_CUSTOM_SOURCE_PROVENANCE:
+            raise SourceError(
+                f"{self._get_provenance()} Custom source provenance plugin: Refusing to use top level source provenance",
+                reason="top-level-provenance-on-custom-implementation",
+            )
 
         self.__key = None  # Cache key for source
 
@@ -1368,6 +1400,7 @@ def create_source_info(
         *,
         version_guess: Optional[str] = None,
         extra_data: Optional[Dict[str, str]] = None,
+        provenance_node: Optional[MappingNode] = None,
     ) -> SourceInfo:
         """Create a :class:`.SourceInfo` object
 
@@ -1387,20 +1420,49 @@ def create_source_info(
            version: A string which represents a unique version of this source input
            version_guess: An optional string representing the guessed human readable version
            extra_data: Additional plugin defined key/values
+           provenance_node: An optional :class:`Node <buildstream.node.Node>` with source provenance attributes,
+                            defaults to the provenance specified at the top level of the source.
 
         *Since: 2.5*
         """
+        project = self._get_project()
+
+        source_provenance: SourceProvenance | None
+        if provenance_node is not None:
+            # Ensure provenance node keys are valid and values are all strings
+            try:
+                provenance_node.validate_keys(project.source_provenance_attributes.keys())
+            except LoadError as E:
+                raise LoadError(
+                    "Specified source provenance attribute not defined in project config\n {}".format(E),
+                    LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE,
+                )
+
+            # Make sure everything is a string
+            for key, value in provenance_node.items():
+                if not isinstance(value, ScalarNode):
+                    raise LoadError(
+                        f"{value}: Expected string for the value of provenance attribute '{key}'",
+                        LoadErrorReason.INVALID_DATA,
+                    )
+
+            source_provenance = provenance_node
+        else:
+            source_provenance = self.__provenance
+
         homepage = None
         issue_tracker = None
-        if self.__provenance is not None:
-            homepage = self.__provenance.homepage
-            issue_tracker = self.__provenance.issue_tracker
+
+        if source_provenance is not None:
+            homepage = source_provenance.get_str("homepage", None)
+            issue_tracker = source_provenance.get_str("issue-tracker", None)
 
         return SourceInfo(
             self.get_kind(),
             url,
             homepage,
             issue_tracker,
+            source_provenance,
             medium,
             version_type,
             version,

src/buildstream/types.py

@@ -390,42 +390,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror":
         return cls(name, aliases)
 
 
-# _SourceProvenance()
-#
-# A simple object describing user provided source provenance information
-#
-# Args:
-#    homepage: The project homepage URL
-#    issue_tracker: The project issue reporting URL
-#
-class _SourceProvenance:
-    def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]):
-        self.homepage: Optional[str] = homepage
-        self.issue_tracker: Optional[str] = issue_tracker
-
-    # new_from_node():
-    #
-    # Creates a _SourceProvenance() from a YAML loaded node.
-    #
-    # Args:
-    #    node: The configuration node describing the spec.
-    #
-    # Returns:
-    #    The described _SourceProvenance instance.
-    #
-    # Raises:
-    #    LoadError: If the node is malformed.
-    #
-    @classmethod
-    def new_from_node(cls, node: MappingNode) -> "_SourceProvenance":
-        node.validate_keys(["homepage", "issue-tracker"])
-
-        homepage: Optional[str] = node.get_str("homepage", None)
-        issue_tracker: Optional[str] = node.get_str("issue-tracker", None)
-
-        return cls(homepage, issue_tracker)
-
-
 ########################################
 #           Type aliases               #
 ########################################

tests/frontend/show.py

@@ -814,3 +814,32 @@ def test_source_info_workspace(cli, datafiles, tmpdir):
 
     # There is no version guessing for a workspace
     assert source_info.get_str("version-guess", None) is None
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info"))
+def test_multi_source_info(cli, datafiles):
+    project = str(datafiles)
+    result = cli.run(
+        project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "multisource.bst"]
+    )
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    sources = loaded.get_sequence("multisource.bst")
+
+    source_info = sources.mapping_at(0)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/ponies"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+
+    source_info = sources.mapping_at(1)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/happy"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+    assert source_info.get_str("homepage") == "http://happy.ponyfarm.com"

tests/frontend/source-info/elements/multisource.bst

@@ -0,0 +1,4 @@
+kind: import
+
+sources:
+- kind: multisource