diff --git a/CMakeLists.txt b/CMakeLists.txt index 47ee518..ef28a70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.9) set(CMAKE_CXX_STANDARD 17) + project(duck-query-lambda LANGUAGES CXX) find_package(aws-lambda-runtime) diff --git a/Dockerfile b/Dockerfile index 8426027..beba576 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ ARG TARGETPLATFORM ARG TARGETARCH -FROM public.ecr.aws/sam/build-provided.al2023:latest +# Ensuring GLIBCXX_3.4.29 as available in Lambda provided.al2023 +FROM public.ecr.aws/sam/build-provided.al2023:1.132 RUN mkdir src WORKDIR /src diff --git a/Makefile b/Makefile index 7320473..ae7d68b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PROJECT_NAME = duck-query-lambda OUTPUT_DIR = output EXTENSIONS = httpfs -DUCKDB_VERSION = 1.1.3 +DUCKDB_VERSION = 1.2.2 .PHONY: docker_build binary build-bootstrap build-DuckFunction build-DuckFunctionArm diff --git a/README.md b/README.md index cbebbce..2ee5025 100644 --- a/README.md +++ b/README.md @@ -130,13 +130,14 @@ Here is an example of how to use the Lambda Layer in an AWS Step Function: ### Invoking the Lambda Function and getting the query results back synchronously -By default, the Lambda function will not return the query results. This is because it's not trivial to convert all results back to JSON in a way that every user expects. If you do want to get the results back synchronously, you can write them to a temporary file in the Lambda and the function will then return the contents of that file, base64 encoded. +By default, the Lambda function will not return the query results. This is because it's not trivial to convert all results back to JSON in a way that every user expects. If you do want to get the results back synchronously, you can write them to a temporary file in the Lambda and the function will then return the contents of that file. This data is base64 encoded by default, but you can also return the raw JSON from the output file using the `outputFormat` parameter. Here is an example of how to do this: ```json { "query": "COPY (SELECT * FROM 'https://github.com/Teradata/kylo/raw/refs/heads/master/samples/sample-data/parquet/userdata1.parquet' LIMIT 10) TO '/tmp/output.json'", - "outputFile": "/tmp/output.json" + "outputFile": "/tmp/output.json", + "outputFormat": "json" } ``` diff --git a/bootstrap.cpp b/bootstrap.cpp index d367d34..6f67a33 100644 --- a/bootstrap.cpp +++ b/bootstrap.cpp @@ -75,12 +75,28 @@ static invocation_response query_handler(invocation_request const &req, Connecti // property, it will be base64 encoded and returned as the result. This temporary file is also deleted before // the function completes. auto outputFile = view.GetString("outputFile"); + if (view.KeyExists("outputFormat")) + { + auto outputFormat = view.GetString("outputFormat"); + if (outputFormat == "json") + { + // Read file as JSON string + ifstream fileStream(outputFile); + if (!fileStream) + { + throw std::runtime_error("Failed to open file"); + } + std::string jsonString((std::istreambuf_iterator(fileStream)), std::istreambuf_iterator()); + fileStream.close(); + remove(outputFile.c_str()); + return invocation_response::success(jsonString, "application/json"); + } + } + // Vanilla base64 encoded response auto encodedOutput = b64encode(outputFile); // Delete the file after encoding remove(outputFile.c_str()); - - return invocation_response::success(encodedOutput, - "application/base64"); + return invocation_response::success(encodedOutput, "application/base64"); } return invocation_response::success(result->ToString(), "text/plain"); diff --git a/examples/sam/Makefile b/examples/sam/Makefile new file mode 100644 index 0000000..c6b05e5 --- /dev/null +++ b/examples/sam/Makefile @@ -0,0 +1,3 @@ +build-DuckQueryFunction: + echo "This file is intentionally empty. Everything required is provided by the Lambda layer" > $(ARTIFACTS_DIR)/empty + diff --git a/template.yaml b/template.yaml index 42375bf..2c5fe5d 100644 --- a/template.yaml +++ b/template.yaml @@ -27,7 +27,7 @@ Resources: CompatibleArchitectures: - arm64 CompatibleRuntimes: - - provided + - provided.al2023 LicenseInfo: Apache-2.0 RetentionPolicy: Retain