chore: merge release into master

.github/dependabot.yml

@@ -3,8 +3,7 @@ updates:
   - package-ecosystem: maven
     directory: "/"
     schedule:
-      interval: daily
-      time: "04:00"
+      interval: "weekly"
     open-pull-requests-limit: 10
     assignees:
       - awildturtok
@@ -16,8 +15,7 @@ updates:
   - package-ecosystem: npm
     directory: "/frontend"
     schedule:
-      interval: daily
-      time: "04:00"
+      interval: "weekly"
     open-pull-requests-limit: 0
     assignees:
       - Kadrian
@@ -34,3 +32,14 @@ updates:
     labels:
       - gh-actions
       - dependencies
+  - package-ecosystem: "docker"
+    # Should match both dockerfiles according to https://github.com/dependabot/dependabot-core/commit/4d797fd82bbc9e99c80c0deea72ab698ceb64320
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    assignees:
+      - awildturtok
+      - thoniTUB
+    labels:
+      - docker
+      - dependencies

.github/workflows/prepare-merge-release.yml

@@ -20,6 +20,8 @@ jobs:
         id: check_pr_count
         run: >
           echo "pr_count=$(gh pr list --head release --limit 1 --base $MAIN_BRANCH_NAME --state open --json state --jq 'length')" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create pull-request
         if: ${{ steps.check_pr_count.outputs.pr_count == 0 }}

.github/workflows/release_backend.yml

@@ -25,7 +25,7 @@ jobs:
           java-version: '21'
           overwrite-settings: false
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/run_autodoc.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/test_backend.yml

@@ -20,7 +20,7 @@ jobs:
       with:
         submodules: true
     - name: Cache local Maven repository
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -56,7 +56,7 @@ jobs:
         with:
           submodules: true
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/test_cypress.yml

@@ -20,7 +20,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -52,7 +52,7 @@ jobs:
       - name: Cypress run
         # This is a preconfigured action, maintained by cypress, to run e2e tests
         # https://github.com/cypress-io/github-action
-        uses: cypress-io/github-action@v6
+        uses: cypress-io/github-action@v7
         with:
           working-directory: .
           start: bash ./scripts/run_e2e_all.sh

backend.Dockerfile

@@ -1,5 +1,5 @@
 # Version Extractor
-FROM alpine/git:v2.49.1 AS version-extractor
+FROM alpine/git:2.52.0 AS version-extractor
 
 WORKDIR /app
 COPY .git .
@@ -37,14 +37,14 @@
 RUN ln -s /usr/lib/libfontconfig.so.1 /usr/lib/libfontconfig.so && \
     ln -s /lib/libuuid.so.1 /usr/lib/libuuid.so.1 && \
     ln -s /lib/libc.musl-x86_64.so.1 /usr/lib/libc.musl-x86_64.so.1
 ENV LD_LIBRARY_PATH /usr/lib
 
 WORKDIR /app
 COPY --from=builder /app/executable/target/executable*jar ./conquery.jar
 
 ENV CLUSTER_PORT=${CLUSTER_PORT:-8082}
 ENV ADMIN_PORT=${ADMIN_PORT:-8081}
 ENV API_PORT=${API_PORT:-8080}
 
 RUN mkdir /app/logs
 VOLUME /app/logs

backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java

@@ -20,6 +20,7 @@
 import com.bakdata.conquery.models.worker.Worker;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Throwables;
 import io.dropwizard.core.setup.Environment;
 import io.dropwizard.lifecycle.Managed;
 import io.dropwizard.util.Duration;
@@ -174,8 +175,10 @@ public void sessionIdle(IoSession session, IdleStatus status) {
 	}
 
 	@Override
-	public void exceptionCaught(IoSession session, Throwable cause) {
-		log.error("Exception caught", cause);
+	public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
+		// Rethrow
+		Throwables.throwIfInstanceOf(cause, Exception.class);
+		throw new RuntimeException("Encountered problem in %s".formatted(session.toString()), cause);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java

@@ -246,6 +246,19 @@ public String getMessageTemplate(ErrorMessages errorMessages) {
 		}
 	}
 
+	/**
+	 * Result size was too large
+	 */
+	@CPSType(base = ConqueryError.class, id = "CQ_EXECUTION_RESULT_SIZE")
+	@RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
+	public static class ExecutionProcessingResultSizeError extends ConqueryError {
+
+		@Override
+		public String getMessageTemplate(ErrorMessages errorMessages) {
+			return errorMessages.resultSizeTooLarge();
+		}
+	}
+
 	@CPSType(base = ConqueryError.class, id = "CQ_EXECUTION_NO_SECONDARY_ID")
 	@RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
 	public static class NoSecondaryIdSelectedError extends ConqueryError {

backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java

@@ -53,6 +53,10 @@ public interface ErrorMessages {
 	@De("Die Anfrage lief zu lange und wurde abgebrochen.")
 	String executionTimeout();
 
+	@En("Query result was too large.")
+	@De("Das Anfrageergebnis ist zu groß.")
+	String resultSizeTooLarge();
+
 	@En("No secondaryId could be selected.")
 	@De("Die ausgewählte Analyseebenen konnte in keinem der ausgewählten Konzepten gefunden werden.")
 	String noSecondaryIdSelected();

backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java

@@ -61,9 +61,8 @@ public void react(Worker worker) throws Exception {
 				query.createQueryPlan(new QueryPlanContext(worker.getStorage(), queryExecutor.getSecondaryIdSubPlanLimit()));
 			}
 			catch (Exception e) {
-				ConqueryError err = asConqueryError(e);
-				log.warn("Failed to create query plans for {}.", formId, err);
-				queryExecutor.sendFailureToManagerNode(result, err, worker);
+				log.warn("Failed to create query plans for {}.", formId, e);
+				queryExecutor.sendFailureToManagerNode(e, formId);
 				return;
 			}
 

backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java

@@ -56,9 +56,8 @@ public void react(Worker worker) throws Exception {
 			log.trace("Created query plan in {}", stopwatch);
 		}
 		catch (Exception e) {
-			ConqueryError err = asConqueryError(e);
-			log.warn("Failed to create query plans for {}.", id, err);
-			queryExecutor.sendFailureToManagerNode(result, err, worker);
+			log.warn("Failed to create query plans for {}.", id, e);
+			queryExecutor.sendFailureToManagerNode(e, id);
 			return;
 		}
 

backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java

@@ -19,6 +19,7 @@
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.CBlock;
+import com.bakdata.conquery.models.identifiable.ids.specific.CBlockId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId;
 import com.bakdata.conquery.models.jobs.Job;
@@ -73,13 +74,9 @@ public void execute() throws Exception {
 					subJobs =
 					concepts.stream()
 							.collect(Collectors.toMap(Functions.identity(),
-													  concept -> CompletableFuture.runAsync(() -> {
-														  final Concept<?> resolved = concept.resolve();
-														  final Map<ConceptElementId<?>, MatchingStats.Entry>
-																  matchingStats =
-																  new HashMap<>(resolved.countElements());
+													  conceptId -> CompletableFuture.runAsync(() -> {
 
-														  calculateConceptMatches(resolved, matchingStats, worker);
+														  Map<ConceptElementId<?>, MatchingStats.Entry> matchingStats = calculateConceptMatches(conceptId, worker);
 
 														  final WriteFuture writeFuture = worker.send(new UpdateElementMatchingStats(worker.getInfo().getId(), matchingStats));
 
@@ -127,18 +124,24 @@ public String getLabel() {
 			return String.format("Calculate Matching Stats for %s", worker.getInfo().getDataset());
 		}
 
-		private static void calculateConceptMatches(Concept<?> concept, Map<ConceptElementId<?>, MatchingStats.Entry> results, Worker worker) {
-			log.debug("BEGIN calculating for `{}`", concept.getId());
+		private static Map<ConceptElementId<?>, MatchingStats.Entry> calculateConceptMatches(ConceptId conceptId, Worker worker) {
 
-			try(Stream<CBlock> allCBlocks = worker.getStorage().getAllCBlocks()) {
+			Concept<?> concept = conceptId.resolve();
+
+			final Map<ConceptElementId<?>, MatchingStats.Entry>	matchingStats =	new HashMap<>(concept.countElements());
+
+			log.debug("BEGIN calculating for `{}`", conceptId);
+
+			try (Stream<CBlockId> allCBlocks = worker.getStorage().getAllCBlockIds()) {
 
-				for (CBlock cBlock : allCBlocks.toList()) {
+				for (CBlockId cBlockId : allCBlocks.toList()) {
 
-					if (!cBlock.getConnector().getConcept().equals(concept.getId())) {
+					if (!cBlockId.getConnector().getConcept().equals(conceptId)) {
 						continue;
 					}
 
 					try {
+						CBlock cBlock = cBlockId.resolve();
 						final Bucket bucket = cBlock.getBucket().resolve();
 						final Table table = bucket.getTable().resolve();
 
@@ -152,7 +155,7 @@ private static void calculateConceptMatches(Concept<?> concept, Map<ConceptEleme
 
 
 								if (!(concept instanceof TreeConcept) || localIds == null) {
-									results.computeIfAbsent(concept.getId(), (ignored) -> new MatchingStats.Entry()).addEvent(table, bucket, event, entity);
+									matchingStats.computeIfAbsent(conceptId, (ignored) -> new MatchingStats.Entry()).addEvent(table, bucket, event, entity);
 									continue;
 								}
 
@@ -163,24 +166,24 @@ private static void calculateConceptMatches(Concept<?> concept, Map<ConceptEleme
 								ConceptElement<?> element = ((TreeConcept) concept).getElementByLocalIdPath(localIds);
 
 								while (element != null) {
-									results.computeIfAbsent(element.getId(), (ignored) -> new MatchingStats.Entry())
-										   .addEvent(table, bucket, event, entity);
+									matchingStats.computeIfAbsent(element.getId(), (ignored) -> new MatchingStats.Entry())
+												 .addEvent(table, bucket, event, entity);
 									element = element.getParent();
 								}
 							}
 						}
 
 					}
 					catch (Exception e) {
-						log.error("Failed to collect the matching stats for {}", cBlock, e);
+						log.error("Failed to collect the matching stats for {}", cBlockId, e);
 					}
 				}
 			}
 
-			log.trace("DONE calculating for `{}`", concept.getId());
+			log.trace("DONE calculating for `{}`", conceptId);
+
+			return matchingStats;
 		}
 
 	}
-
-
 }

backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java

@@ -85,15 +85,19 @@ public void doCancelQuery(ManagedExecutionId executionId) {
 	@SneakyThrows
 	public <R extends ShardResult, E extends ManagedExecution & InternalExecution> void handleQueryResult(R result, E execution) {
 
-
-		log.debug("Received Result[size={}] for Query[{}]", result.getResults().size(), result.getQueryId());
-		log.trace("Received Result\n{}", result.getResults());
-
 		if (execution == null) {
 			log.debug("Ignoring result {} because the corresponding execution was 'null' (probably deleted)", result);
 			return;
 		}
 
+		if (result.getError() != null) {
+			execution.fail(result.getError());
+			return;
+		}
+
+		log.debug("Received Result[size={}] for Query[{}]", result.getResults().size(), result.getExecutionId());
+		log.trace("Received Result\n{}", result.getResults());
+
 		Optional<ExecutionInfo> optInfo = tryGetExecutionInfo(execution.getId());
 
 		if (optInfo.isEmpty()) {
@@ -114,18 +118,13 @@ public <R extends ShardResult, E extends ManagedExecution & InternalExecution> v
 			return;
 		}
 
-		if (result.getError().isPresent()) {
-			execution.fail(result.getError().get());
-		}
-		else {
-			distributedInfo.addShardResult(result);
+		distributedInfo.addShardResult(result);
 
-			// If all known workers have returned a result, the query is DONE.
-			if (distributedInfo.allResultsArrived(getWorkerHandler(execution.getDataset()).getAllWorkerIds())) {
+		// If all known workers have returned a result, the query is DONE.
+		if (distributedInfo.allResultsArrived(getWorkerHandler(execution.getDataset()).getAllWorkerIds())) {
 
-				execution.finish(ExecutionState.DONE);
+			execution.finish(ExecutionState.DONE);
 
-			}
 		}
 
 		// State changed to DONE or FAILED
@@ -139,7 +138,7 @@ public <R extends ShardResult, E extends ManagedExecution & InternalExecution> v
 
 			/* This log is here to prevent an NPE which could occur when no strong reference to result.getResults()
 			 existed anymore after the query finished and immediately was reset */
-			log.trace("Collected metrics for execution {}. Last result received: {}:", result.getQueryId(), result.getResults());
+			log.trace("Collected metrics for execution {}. Last result received: {}:", result.getExecutionId(), result.getResults());
 		}
 
 	}