rework env traversal

gitlab/gitlab-templates.yaml

@@ -5,11 +5,21 @@
   services:
   - docker:dind
   before_script:
+  # wait for docker to be ready
   - until docker info >/dev/null; do sleep 1; echo "Waiting for docker startup"; done
+  # login to internal gitlab images registry
   - docker login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
+  # add a bash shell
   - apk add --no-cache bash #TODO why dont we build our own base tools image?
+  # get user-specified and required tool versions
   - source versions.source
 
+.artifacts:
+  artifacts:
+    paths:
+    - "images/${IMAGE_NAME}/build.env"
+    expire_in: 1 week
+
 # -----------------  Stage Templates for standard images -------------------- #
 
 
@@ -17,6 +27,7 @@
   stage: build-python
   extends:
   - .docker-base
+  - .artifacts
   script:
   - ./scripts/docker-build-python-builder.sh
   rules:
@@ -43,7 +54,7 @@
 
 # -----------------  Stage Templates for python images  -------------------- #
 
-
+# prerequisite to build python package; later embed the result in a distroless image.
 .build-python-builder:
   stage: build-python-builder
   extends:
@@ -70,10 +81,12 @@
       - .gitlab-ci.yml
       - gitlab/*
 
+# embed our python stuff in a distroless image for a smaller footprint.
 .build-python-distroless:
   stage: build-python-distroless
   extends:
   - .docker-base
+  - .artifacts
   script:
   - ./scripts/docker-build-python-distroless.sh
   rules:
@@ -95,14 +108,22 @@
       - gitlab/*
 
 
-# -----------------  Scan and publish for all public images  -------------------- #
+# -----------------  Scan and publish for all images  -------------------- #
 
 .scan:
   stage: scan-image
   extends:
   - .docker-base
+  - .artifacts
   script:
   - ./scripts/scan.sh
+  needs:
+  - job: .build-python-distroless
+    artifacts: true
+    optional: true
+  - job: .build-image
+    artifacts: true
+    optional: true
 
 .publish:
   stage: publish-image
@@ -112,3 +133,6 @@
   - ./scripts/publish.sh
   # only:
   # - master
+  needs:
+  - job: .scan
+    artifacts: true
\ No newline at end of file

scripts/docker-build-python-builder.sh

 #!/usr/bin/env bash
 
-[[ -z ${DEBUG:-} ]] && set -x
-
 set -eouE pipefail
 
+[[ -z ${DEBUG:-} ]] || set -x
+
+
 pushd "$(dirname "${BASH_SOURCE[0]}")/../" >/dev/null || exit
 
 # shellcheck disable=SC1091
@@ -20,8 +21,9 @@ docker build \
     -t "${PYTHON_BUILDER_IMAGE}" \
     -f "${DOCKERFILE}" .
 
+# we dont validate the internal image, as it's for internal use only.
 if [[ ${CI_SERVER:-} == "yes" ]]; then
-    docker push "${PYTHON_BUILDER_IMAGE}"
+    docker push ${PYTHON_BUILDER_IMAGE}
 fi
 
 popd > /dev/null || exit

scripts/docker-build-python-distroless.sh

 #!/usr/bin/env bash
 
-[[ -z ${DEBUG:-} ]] && set -x
-
+[[ -z ${DEBUG:-} ]] || set -x
+set -x
 set -eouE pipefail
 
 pushd "$(dirname "${BASH_SOURCE[0]}")/../" >/dev/null || exit
@@ -10,13 +10,22 @@ pushd "$(dirname "${BASH_SOURCE[0]}")/../" >/dev/null || exit
 . ./scripts/vars.sh
 
 cd "images/${IMAGE_NAME}"
+
+# traverse these variables to next CI steps
+cat >build.env<<EOF
+PYTHON_IMAGE=yes
+FINAL_IMAGE=${PYTHON_DISTROLESS_IMAGE}
+DRAFT_IMAGE=${PYTHON_DISTROLESS_IMAGE}-draft-${CI_PIPELINE_ID}
+EOF
+. build.env
+
 DOCKERFILE=${DOCKERFILE:-Dockerfile}
 
 docker build \
     --build-arg PYTHON_BUILDER_IMAGE="${PYTHON_BUILDER_IMAGE}" \
     --build-arg GOOGLE_DISTROLESS_BASE_IMAGE="${GOOGLE_DISTROLESS_BASE_IMAGE}" \
     --build-arg CHIPSET_ARCH="${CHIPSET_ARCH}" \
-    -t "${PYTHON_DISTROLESS_IMAGE}-${CI_PIPELINE_ID}" \
+    -t "${DRAFT_IMAGE}" \
     -f "${DOCKERFILE}" .
 
 # debug tooling is inherited from within the google distroless image
@@ -24,7 +33,13 @@ docker build \
     --build-arg PYTHON_BUILDER_IMAGE="${PYTHON_BUILDER_IMAGE}" \
     --build-arg GOOGLE_DISTROLESS_BASE_IMAGE="${GOOGLE_DISTROLESS_BASE_IMAGE}:debug" \
     --build-arg CHIPSET_ARCH="${CHIPSET_ARCH}" \
-    -t "${PYTHON_DISTROLESS_IMAGE}-debug-${CI_PIPELINE_ID}" \
+    -t "${DRAFT_IMAGE}-debug" \
     -f "${DOCKERFILE}" .
 
+# push temporary images to be validated by the next step - scanning
+if [[ ${CI_SERVER:-} == "yes" ]]; then
+    docker push "${DRAFT_IMAGE}"
+    docker push "${DRAFT_IMAGE}-debug"
+fi
+
 popd > /dev/null || exit

scripts/docker-build.sh

 #!/usr/bin/env bash
 
-[[ -z ${DEBUG:-} ]] && set -x
+[[ -z ${DEBUG:-} ]] || set -x
 
 set -eouE pipefail
 
@@ -10,8 +10,7 @@ set -eouE pipefail
 : "$DIRENV_VERSION"
 : "$GLENV_VERSION"
 
-cd "images/${IMAGE_NAME}"
-DOCKERFILE=${DOCKERFILE:-Dockerfile}
+#- 
 
 parse_dockerfile_build_args() {
     BUILD_ARGS=''
@@ -34,7 +33,7 @@ parse_dockerfile_build_args() {
     echo "Using build args from ${DOCKERFILE}: ${BUILD_ARGS}"
 }
 
-TAG=""
+#- 
 
 if [[ ${CI_SERVER:-} != "yes" ]]; then
     # https://github.com/kiwicom/glenv
@@ -59,12 +58,30 @@ else
 fi
 
 
+cd "images/${IMAGE_NAME}"
+
+TAG=""
+DOCKERFILE=${DOCKERFILE:-Dockerfile}
+
+# traverse these variables to next CI steps
+cat >>.env<<EOF
+PYTHON_IMAGE=yes
+FINAL_IMAGE=${CONTAINER_IMAGE}
+DRAFT_IMAGE=${PYTHON_DISTROLESS_IMAGE}-draft-${CI_PIPELINE_ID}
+EOF
+. .env
+
 echo "Commit ref is ${CI_COMMIT_REF_NAME} - ci build tag is ${CI_BUILD_TAG}"
 echo "Container image: ${CONTAINER_IMAGE}, tag: ${TAG}"
 
 parse_dockerfile_build_args
-docker build "${BUILD_ARGS}" -t "${CONTAINER_IMAGE}:${TAG}" -f "${DOCKERFILE}" .
+docker build "${BUILD_ARGS}" -t "${DRAFT_IMAGE}" -f "${DOCKERFILE}" .
+
+if [[ ${CI_SERVER:-} == "yes" ]]; then
+    docker push "${DRAFT_IMAGE}"
+fi
 
 export TAG
+export DRAFT_IMAGE
 
 popd > /dev/null || exit

scripts/publish.sh

 #!/usr/bin/env bash
 
+[[ -z ${DEBUG:-} ]] || set -x
+
 set -eouE pipefail
 
 pushd "$(dirname "${BASH_SOURCE[0]}")/../" >/dev/null || exit
 
+if [[ -z ${IMAGE_NAME:-} ]]; then
+    echo "-> [ERROR] IMAGE_NAME not set - aborting"
+    exit 1
+fi
 # shellcheck disable=SC1091
 . ./scripts/vars.sh
 
+. "images/${IMAGE_NAME}/build.env"
+if [[ -z ${DRAFT_IMAGE:-} ]]; then
+    echo "-> [ERROR] DRAFT_IMAGE not set - aborting"
+    exit 1
+fi
+
+if [[ -z ${FINAL_IMAGE:-} ]]; then
+    echo "-> [ERROR] FINAL_IMAGE not set - aborting"
+    exit 1
+fi
+
+
 if [[ ${CI_SERVER:-} == "yes" ]]; then
-    docker push "${CONTAINER_IMAGE}:${TAG}"
-    echo "Pushed ${CONTAINER_IMAGE}:${TAG}"
+    docker pull "${DRAFT_IMAGE}"
+    if [[ -n ${PYTHON_IMAGE:-} ]]; then
+        docker pull "${DRAFT_IMAGE}-debug"
+    fi
+
+    docker tag "${DRAFT_IMAGE}" "${FINAL_IMAGE}"
+    # remove temp tag
+    docker rmi "${DRAFT_IMAGE}"
+    docker push "${FINAL_IMAGE}"
+    echo "Pushed ${FINAL_IMAGE}"
+    if [[ -n ${PYTHON_IMAGE:-} ]]; then
+        docker tag "${DRAFT_IMAGE}-debug" "${FINAL_IMAGE}-debug"
+        # remove temporary scanning tag
+        docker rmi "${DRAFT_IMAGE}-debug"
+        docker push "${FINAL_IMAGE}-debug"
+    fi
     if [[ -e "images/${IMAGE_NAME}/tests.yaml" ]]; then
         apk add --no-cache curl && curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 \
          && chmod +x container-structure-test-linux-amd64 \
-         && ./container-structure-test-linux-amd64 test --image  ${CONTAINER_IMAGE}:${TAG} --config "images/${IMAGE_NAME}/tests.yaml"
+         && ./container-structure-test-linux-amd64 test --image  ${FINAL_IMAGE} --config "images/${IMAGE_NAME}/tests.yaml"
     fi
 fi
 

scripts/scan.sh

 #!/usr/bin/env bash
+
+[[ -z ${DEBUG:-} ]] || set -x
+set -x
 set -oeuE pipefail
 
 pushd "$(dirname "${BASH_SOURCE[0]}")/../" >/dev/null || exit
 
+if [[ -z ${IMAGE_NAME:-} ]]; then
+    echo "-> [ERROR] IMAGE_NAME not set - aborting"
+    exit 1
+fi
 # shellcheck disable=SC1091
 . ./scripts/vars.sh
 
+. "images/${IMAGE_NAME}/build.env"
+if [[ -z ${DRAFT_IMAGE:-} ]]; then
+    echo "-> [ERROR] DRAFT_IMAGE not set - aborting"
+    exit 1
+fi
+
 # install trivy if missing
 if [[ ! $(which trivy) ]]; then
     wget https://github.com/aquasecurity/trivy/releases/download/v"${TRIVY_VERSION}"/trivy_"${TRIVY_VERSION}"_Linux-64bit.tar.gz && \
-        tar zxvf trivy_"${TRIVY_VERSION}"_Linux-64bit.tar.gz && \
+        tar zxf trivy_"${TRIVY_VERSION}"_Linux-64bit.tar.gz && \
         mv trivy /usr/local/bin/trivy
 fi
 
 # scan images locally (before push)
-echo; echo "-> Trivy scan for image: ${CONTAINER_IMAGE}"; echo
+docker pull "${DRAFT_IMAGE}"
+echo; echo "-> Trivy scan for image: ${DRAFT_IMAGE}"; echo
 trivy image --clear-cache
-trivy image --exit-code 1 --scanners vuln --severity CRITICAL,HIGH --no-progress "${CONTAINER_IMAGE}"
+trivy image --exit-code 1 --scanners vuln --severity CRITICAL,HIGH --no-progress "${DRAFT_IMAGE}"
 
 popd > /dev/null || exit

scripts/vars.sh

@@ -14,7 +14,7 @@ if [[ "$(uname -m)" == "arm64" ]]; then
 fi
 
 if [[ ${ARCH} == "-arm64" ]]; then
-    CHIPSET_ARCH=aarch64-linux-gnu
+    CHIPSET_ARCH=aarch64-linux-gnu # linux vm in a arm64 mac
 else
     CHIPSET_ARCH=x86_64-linux-gnu
 fi
@@ -34,30 +34,35 @@ if [[ -z ${IMAGE_NAME:-} ]]; then
     exit 1
 fi
 
-# use the C (glibc) distroless - required by common packages like grpcio + numpy
+if [[ -z ${CI_PIPELINE_ID:-} ]]; then
+    CI_PIPELINE_ID=non-ci-$(git rev-parse --short HEAD)
+    NAMESPACE="automation-images"
+else
+    NAMESPACE="${CI_REGISTRY}/${CI_PROJECT_PATH}"
+fi
+CONTAINER_IMAGE="${NAMESPACE}/${IMAGE_NAME}"
+
+# use the C (glibc) distroless
 GOOGLE_DISTROLESS_BASE_IMAGE=gcr.io/distroless/cc-${OS_VERSION}
 # Cut patch version from semver Python version for streamlined image tags: 3.12.0 -> 3.12
 #shellcheck disable=SC2001,SC2068
 PYTHON_MINOR="$(echo $PYTHON_VERSION | sed -e "s#^\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\)#\1.\2#")"
-TAG=${PYTHON_MINOR}-${OS_VERSION}
-PYTHON_BUILDER_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/python-builder${ARCH}:${PYTHON_MINOR}-${OS_VERSION}"
-PYTHON_DISTROLESS_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/python-distroless${ARCH}:${PYTHON_MINOR}-${OS_VERSION}"
+PYTHON_TAG=${PYTHON_MINOR}-${OS_VERSION}
+PYTHON_BUILDER_IMAGE="${NAMESPACE}/python-builder${ARCH}:${PYTHON_MINOR}-${OS_VERSION}"
+PYTHON_DISTROLESS_IMAGE="${NAMESPACE}/python-distroless${ARCH}:${PYTHON_MINOR}-${OS_VERSION}"
 
+# '--debug' passed via the CLI
 if [[ $(echo "${@:-}" | grep -c -- '--debug') -gt 0 ]]; then
     PYTHON_DISTROLESS_IMAGE=${PYTHON_DISTROLESS_IMAGE}-debug
 fi
 
-if [[ -z ${CI_PIPELINE_ID:-} ]]; then
-    CI_PIPELINE_ID=non-ci-$(git rev-parse --short HEAD)
-fi
-CONTAINER_IMAGE=${CI_REGISTRY}/${CI_PROJECT_PATH}/${IMAGE_NAME}
 
 export PYTHON_VERSION
 export PYTHON_MINOR
 export OS_VERSION
 export ARCH
 export CHIPSET_ARCH
-export TAG
+export PYTHON_TAG
 export PYTHON_BUILDER_IMAGE
 export PYTHON_DISTROLESS_IMAGE
 export GOOGLE_DISTROLESS_BASE_IMAGE