From cc7185725594322fa2f99bbb941d159c5eeca0ed Mon Sep 17 00:00:00 2001 From: Abderrahim Kitouni Date: Thu, 20 Nov 2025 09:27:33 +0100 Subject: [PATCH] _elementsources: cache intermediate combined sources (in-memory) When tracking an element that has multiple sources that depend on previous sources, buildstream stages all previous sources each time, and since they aren't cached because we only cache sources that don't depend on previous sources or combined elements this ends up staging things from original sources again and again. This change keeps the a digest of the intermediate staged sources in memory, and speeds up these cases a lot --- src/buildstream/_elementsources.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py index 39241493a..9ed26499d 100644 --- a/src/buildstream/_elementsources.py +++ b/src/buildstream/_elementsources.py @@ -47,6 +47,8 @@ def __init__(self, context: Context, project: "Project", plugin: Plugin): self._cache_key = None # Our cached cache key self._proto = None # The cached Source proto + self._cache = {} + # get_project(): # # Return the project associated with this object @@ -462,6 +464,10 @@ def _stage(self, *, stop=None): if source == stop: break + if source in self._cache: + vdir = CasBasedDirectory(cas, digest=self._cache[source]) + continue + if source._directory: vsubdir = vdir.open_directory(source._directory.lstrip(os.path.sep), create=True) else: @@ -483,6 +489,8 @@ def _stage(self, *, stop=None): source_dir = self._sourcecache.export(source) vsubdir.import_files(source_dir, collect_result=False) + self._cache[source] = vsubdir._get_digest() + return vdir # Context manager that stages sources in a cas based or temporary file