diff --git a/.gitlab/ci_cd.yaml b/.gitlab/ci_cd.yaml index f96e963..0610179 100644 --- a/.gitlab/ci_cd.yaml +++ b/.gitlab/ci_cd.yaml @@ -23,10 +23,14 @@ variables: interruptible: true before_script: - apt-get update && apt-get install -y --no-install-recommends git + artifacts: + paths: + - $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env script: - *validate_input # set envvars - | + touch $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env _ENV_FLAGS="${ENV_FLAGS:=--last-partition --wait}" cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_') @@ -81,7 +85,12 @@ variables: DEPLOY_FILE=./deploy/${VERSION}/deploy.sh if [ ! -f "$DEPLOY_FILE" ]; then echo "$DEPLOY_FILE not found, running default tb deploy command" - tb --semver ${VERSION} deploy ${CI_FLAGS} + DEPLOY_OUTPUT=$(tb --semver ${VERSION} deploy ${CI_FLAGS}) + echo "$DEPLOY_OUTPUT" + DEPLOY_RESOURCE_NAMES=$(echo "$DEPLOY_OUTPUT" | grep -o 'Running .*' | awk '{print $2}' | tr -d "'") + echo "$DEPLOY_RESOURCE_NAMES" > $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env + echo "cat $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env" + cat $CI_PROJECT_DIR/$DATA_PROJECT_DIR/deploy.env tb release ls fi @@ -113,6 +122,11 @@ variables: cd $CI_PROJECT_DIR/$DATA_PROJECT_DIR _NORMALIZED_BRANCH_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_') source .tinyenv + if [ -f deploy.env ]; then + export DEPLOY_RESOURCE_NAMES=$(cat deploy.env) + else + echo "deploy.env not found" + fi # Create Python Virtual Environment - | @@ -182,7 +196,7 @@ variables: - | if [ -f ./scripts/exec_test.sh ]; then - ./scripts/exec_test.sh $VERSION + ./scripts/exec_test.sh $VERSION $DEPLOY_RESOURCE_NAMES fi # Run data quality tests diff --git a/scripts/exec_test.sh b/scripts/exec_test.sh index 9b2c0d6..4d2abe7 100755 --- a/scripts/exec_test.sh +++ b/scripts/exec_test.sh @@ -3,6 +3,7 @@ set -euxo pipefail export TB_VERSION_WARNING=0 export VERSION=$1 +export RESOURCE_NAMES=$2 run_test() { t=$1 @@ -55,7 +56,13 @@ run_test() { export -f run_test fail=0 -find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1 +if [ -v RESOURCE_NAMES ]; then + for name in $RESOURCE_NAMES; do + find ./tests -type f -name "*.test" -exec grep -q "$name" {} \; -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1 + done +else + find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1 +fi if [ $fail == 1 ]; then exit -1;