From cb119a12d84f53f7324c7f469c12670d9a91c023 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 08:31:49 +0100
Subject: [PATCH 01/13] run only fixture tests of files deployed

---
 .gitlab/ci_cd.yaml   | 4 +++-
 scripts/exec_test.sh | 9 ++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index f96e963..58a31a4 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -82,6 +82,8 @@ variables:
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
         tb --semver ${VERSION} deploy ${CI_FLAGS}
+        DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee /dev/tty)
+        DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
         tb release ls
       fi
 
@@ -182,7 +184,7 @@ variables:
 
     - |
       if [ -f ./scripts/exec_test.sh ]; then
-        ./scripts/exec_test.sh $VERSION
+        ./scripts/exec_test.sh $VERSION $DEPLOY_RESOURCE_NAMES
       fi
 
     # Run data quality tests
diff --git a/scripts/exec_test.sh b/scripts/exec_test.sh
index 9b2c0d6..4d2abe7 100755
--- a/scripts/exec_test.sh
+++ b/scripts/exec_test.sh
@@ -3,6 +3,7 @@ set -euxo pipefail
 
 export TB_VERSION_WARNING=0
 export VERSION=$1
+export RESOURCE_NAMES=$2
 
 run_test() {
     t=$1
@@ -55,7 +56,13 @@ run_test() {
 export -f run_test
 
 fail=0
-find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1
+if [ -v RESOURCE_NAMES ]; then
+    for name in $RESOURCE_NAMES; do
+        find ./tests -type f -name "*.test" -exec grep -q "$name" {} \; -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1
+    done
+else
+    find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1
+fi
 
 if [ $fail == 1 ]; then
   exit -1;

From 121611b032dbe416d643cca066f5839121b01497 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 08:41:26 +0100
Subject: [PATCH 02/13] remove /dev/tty

---
 .gitlab/ci_cd.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 58a31a4..ac2eb8a 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -82,7 +82,7 @@ variables:
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
         tb --semver ${VERSION} deploy ${CI_FLAGS}
-        DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee /dev/tty)
+        DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee)
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
         tb release ls
       fi

From 29bd44d73a86a677e62255fc7189ecf4649d264c Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:00:11 +0100
Subject: [PATCH 03/13] remove deploy

---
 .gitlab/ci_cd.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index ac2eb8a..c094390 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -81,7 +81,6 @@ variables:
       DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
-        tb --semver ${VERSION} deploy ${CI_FLAGS}
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee)
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
         tb release ls

From 3bbaba70dfca384b8264adac494821a781f44d1b Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:12:55 +0100
Subject: [PATCH 04/13] generate artifacts

---
 .gitlab/ci_cd.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index c094390..f1aa7ae 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -23,10 +23,14 @@ variables:
   interruptible: true
   before_script:
     - apt-get update && apt-get install -y --no-install-recommends git
+  artifacts:
+    paths:
+      - deploy.env
   script:
     - *validate_input
     # set envvars
     - |
+      touch deploy.env
       _ENV_FLAGS="${ENV_FLAGS:=--last-partition --wait}"
       cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
@@ -83,6 +87,7 @@ variables:
         echo "$DEPLOY_FILE not found, running default tb deploy command"
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee)
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
+        echo "DEPLOY_RESOURCE_NAMES=$DEPLOY_RESOURCE_NAMES" > deploy.env
         tb release ls
       fi
 
@@ -114,6 +119,7 @@ variables:
       cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
       source .tinyenv
+      source deploy.env
 
     # Create Python Virtual Environment
     - |

From d70290d764dd4739cdb794167bde8138d17dac29 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:19:43 +0100
Subject: [PATCH 05/13] print

---
 .gitlab/ci_cd.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index f1aa7ae..056d5f7 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -85,8 +85,10 @@ variables:
       DEPLOY_FILE=./deploy/${VERSION}/deploy.sh
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
-        DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS} | tee)
-        DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
+        DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS})
+        echo -e "$DEPLOY_OUTPUT"
+        DEPLOY_RESOURCE_NAMES=$(echo -e "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
+        echo -e "$DEPLOY_RESOURCE_NAMES"
         echo "DEPLOY_RESOURCE_NAMES=$DEPLOY_RESOURCE_NAMES" > deploy.env
         tb release ls
       fi

From 541bb69fd8d8d5926ab3940ad9b7c33194f8bb89 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:28:13 +0100
Subject: [PATCH 06/13] echo

---
 .gitlab/ci_cd.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 056d5f7..94e9635 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -86,9 +86,10 @@ variables:
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS})
-        echo -e "$DEPLOY_OUTPUT"
-        DEPLOY_RESOURCE_NAMES=$(echo -e "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
-        echo -e "$DEPLOY_RESOURCE_NAMES"
+        echo "after command"
+        echo "$DEPLOY_OUTPUT"
+        DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
+        echo "$DEPLOY_RESOURCE_NAMES"
         echo "DEPLOY_RESOURCE_NAMES=$DEPLOY_RESOURCE_NAMES" > deploy.env
         tb release ls
       fi

From b62b353e32fa87c9d96246dd520928e296475371 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:34:22 +0100
Subject: [PATCH 07/13] more echo

---
 .gitlab/ci_cd.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 94e9635..464a4a7 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -89,8 +89,10 @@ variables:
         echo "after command"
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
-        echo "$DEPLOY_RESOURCE_NAMES"
+        echo "resource names: $DEPLOY_RESOURCE_NAMES"
         echo "DEPLOY_RESOURCE_NAMES=$DEPLOY_RESOURCE_NAMES" > deploy.env
+        echo "deploy.env"
+        cat deploy.env
         tb release ls
       fi
 

From f4f9579300af56909c4f34337b0bc2b6208a3dc5 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:47:56 +0100
Subject: [PATCH 08/13] avoid multiline

---
 .gitlab/ci_cd.yaml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 464a4a7..b47b446 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -86,13 +86,9 @@ variables:
       if [ ! -f "$DEPLOY_FILE" ]; then
         echo "$DEPLOY_FILE not found, running default tb deploy command"
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS})
-        echo "after command"
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
-        echo "resource names: $DEPLOY_RESOURCE_NAMES"
-        echo "DEPLOY_RESOURCE_NAMES=$DEPLOY_RESOURCE_NAMES" > deploy.env
-        echo "deploy.env"
-        cat deploy.env
+        echo "$DEPLOY_RESOURCE_NAMES" > deploy.env
         tb release ls
       fi
 
@@ -124,7 +120,9 @@ variables:
       cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
       source .tinyenv
-      source deploy.env
+      if [ -f deploy.env ]; then
+          export DEPLOY_RESOURCE_NAMES=$(cat deploy.env)
+      fi
 
     # Create Python Virtual Environment
     - |

From 93078455e59ec28434b027871ec67654766134e3 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:52:03 +0100
Subject: [PATCH 09/13] cat

---
 .gitlab/ci_cd.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index b47b446..810db6f 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -89,6 +89,7 @@ variables:
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
         echo "$DEPLOY_RESOURCE_NAMES" > deploy.env
+        cat deploy.env
         tb release ls
       fi
 

From d562cc2be2a462354dd748da27d28cb7f85414cc Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 09:53:04 +0100
Subject: [PATCH 10/13] echo

---
 .gitlab/ci_cd.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 810db6f..118d89d 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -122,7 +122,9 @@ variables:
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
       source .tinyenv
       if [ -f deploy.env ]; then
-          export DEPLOY_RESOURCE_NAMES=$(cat deploy.env)
+        export DEPLOY_RESOURCE_NAMES=$(cat deploy.env)
+      else
+        echo "deploy.env not found"
       fi
 
     # Create Python Virtual Environment

From aba6ec715c4e5fc929a8007a89ebb255b2d4ec69 Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 10:18:09 +0100
Subject: [PATCH 11/13] more echo

---
 .gitlab/ci_cd.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 118d89d..2397458 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -89,6 +89,7 @@ variables:
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
         echo "$DEPLOY_RESOURCE_NAMES" > deploy.env
+        echo "cat deploy.env"
         cat deploy.env
         tb release ls
       fi

From a428205fd136b1b422679631402c3ec2f1a5a54b Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 10:48:46 +0100
Subject: [PATCH 12/13] use absolute path

---
 .gitlab/ci_cd.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index 2397458..c1ad795 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -25,12 +25,12 @@ variables:
     - apt-get update && apt-get install -y --no-install-recommends git
   artifacts:
     paths:
-      - deploy.env
+      - /tmp/deploy.env
   script:
     - *validate_input
     # set envvars
     - |
-      touch deploy.env
+      touch /tmp/deploy.env
       _ENV_FLAGS="${ENV_FLAGS:=--last-partition --wait}"
       cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
@@ -88,9 +88,9 @@ variables:
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS})
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
-        echo "$DEPLOY_RESOURCE_NAMES" > deploy.env
-        echo "cat deploy.env"
-        cat deploy.env
+        echo "$DEPLOY_RESOURCE_NAMES" > /tmp/deploy.env
+        echo "cat /tmp/deploy.env"
+        cat /tmp/deploy.env
         tb release ls
       fi
 

From a1eafdfec93b004a0eb03c5b5a6651fb4afbd0be Mon Sep 17 00:00:00 2001
From: alrocar <alrocar@tinybird.co>
Date: Thu, 14 Mar 2024 10:57:01 +0100
Subject: [PATCH 13/13] use subpath

---
 .gitlab/ci_cd.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml
index c1ad795..0610179 100644
--- a/.gitlab/ci_cd.yaml
+++ b/.gitlab/ci_cd.yaml
@@ -25,12 +25,12 @@ variables:
     - apt-get update && apt-get install -y --no-install-recommends git
   artifacts:
     paths:
-      - /tmp/deploy.env
+      - $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env
   script:
     - *validate_input
     # set envvars
     - |
-      touch /tmp/deploy.env
+      touch $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env
       _ENV_FLAGS="${ENV_FLAGS:=--last-partition --wait}"
       cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR
       _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
@@ -88,9 +88,9 @@ variables:
         DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS})
         echo "$DEPLOY_OUTPUT"
         DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'")
-        echo "$DEPLOY_RESOURCE_NAMES" > /tmp/deploy.env
-        echo "cat /tmp/deploy.env"
-        cat /tmp/deploy.env
+        echo "$DEPLOY_RESOURCE_NAMES" > $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env
+        echo "cat $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env"
+        cat $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env
         tb release ls
       fi