From b118cd58127c71c3b2c346926e9728837b98bb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 23 Dec 2019 17:53:19 +0100 Subject: [PATCH] Building the Docker image with Github Actions --- .github/workflows/push.yml | 31 ++++++++++++++++++ .github/workflows/release.yml | 39 +++++++++++++++++++++++ build.sh | 20 ++++++++---- docker-compose.test.yml | 45 ++++++++++++++++++++++++++ docker-compose.yml | 2 -- docker/docker-entrypoint.sh | 27 ++++++++++++---- hooks/build | 6 ---- hooks/common | 60 ----------------------------------- hooks/push | 14 -------- hooks/test | 15 --------- test.sh | 22 ++++++++----- 11 files changed, 164 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 docker-compose.test.yml delete mode 100755 hooks/build delete mode 100755 hooks/common delete mode 100755 hooks/push delete mode 100755 hooks/test diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..3290e27 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,31 @@ +on: + push: + branches-ignore: + - release + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + docker_from: + - '' # use the default of the DOCKERFILE + - python:3.7-alpine + - python:3.8-alpine + - python:3.9-rc-alpine + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + DOCKER_FROM: ${{ matrix.docker_from }} + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..72a0bdb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +on: + push: + branches: + - release + schedule: + - cron: '45 5 * * *' + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + - name: Login to the Docker Registry + run: | + echo "::add-mask::$DOCKERHUB_USERNAME" + echo "::add-mask::$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}" + env: + DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} + DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} + - name: Push the image + run: ${{ matrix.build_cmd }} --push-only + - name: Logout of the Docker Registry + run: docker logout "${DOCKER_REGISTRY}" diff --git a/build.sh b/build.sh index 01e841a..94cbe6a 100755 --- a/build.sh +++ b/build.sh @@ -63,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " DRY_RUN Prints all build statements instead of running them." echo " Default: undefined" + echo " GH_ACTION If defined, special 'echo' statements are enabled that set the" + echo " following environment variables in Github Actions:" + echo " - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable" + echo " Default: undefined" echo "" echo "Examples:" echo " ${0} master" @@ -92,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi ### -# Determining the build command to use +# Enabling dry-run mode ### if [ -z "${DRY_RUN}" ]; then DRY="" @@ -102,7 +106,7 @@ else fi ### -# variables for fetching the source +# Variables for fetching the source ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" @@ -111,7 +115,7 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" ### -# fetching the source +# Fetching the source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" @@ -150,7 +154,7 @@ if [ ! -f "${DOCKERFILE}" ]; then fi ### -# variables for labelling the docker image +# Variables for labelling the docker image ### BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" @@ -158,7 +162,7 @@ if [ -d ".git" ]; then GIT_REF="$(git rev-parse HEAD)" fi -# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 +# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" # Get the Git information from the netbox directory @@ -169,7 +173,7 @@ if [ -d "${NETBOX_PATH}/.git" ]; then fi ### -# variables for tagging the docker image +# Variables for tagging the docker image ### DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" @@ -193,6 +197,7 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" ### # Build each target ### +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1} for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do echo "🏗 Building the target '${DOCKER_TARGET}'" @@ -203,6 +208,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi + if [ -n "${GH_ACTION}" ]; then + echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + fi ### # composing the additional DOCKER_SHORT_TAG, diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..534a025 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,45 @@ +version: '3.4' +services: + netbox: + image: ${IMAGE-netboxcommunity/netbox:latest} + depends_on: + - postgres + - redis + env_file: env/netbox.env + user: '101' + volumes: + - ./startup_scripts:/opt/netbox/startup_scripts:z,ro + - ./initializers:/opt/netbox/initializers:z,ro + - ./configuration:/etc/netbox/config:z,ro + - ./reports:/etc/netbox/reports:z,ro + - ./scripts:/etc/netbox/scripts:z,ro + - netbox-nginx-config:/etc/netbox-nginx:z + - netbox-static-files:/opt/netbox/netbox/static:z + - netbox-media-files:/opt/netbox/netbox/media:z + nginx: + command: nginx -c /etc/netbox-nginx/nginx.conf + image: nginx:1.17-alpine + depends_on: + - netbox + ports: + - 8080 + volumes: + - netbox-static-files:/opt/netbox/netbox/static:ro + - netbox-nginx-config:/etc/netbox-nginx/:ro + postgres: + image: postgres:11-alpine + env_file: env/postgres.env + redis: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env +volumes: + netbox-static-files: + driver: local + netbox-nginx-config: + driver: local + netbox-media-files: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index e4961d6..10b4ad9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,8 +57,6 @@ volumes: driver: local netbox-media-files: driver: local - netbox-report-files: - driver: local netbox-postgres-data: driver: local netbox-redis-data: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 8b7481e..f555695 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,13 +1,27 @@ #!/bin/bash +# Runs on every start of the Netbox Docker container + +# Stop when an error occures set -e + +# Allows Netbox to be run as non-root users umask 002 -# wait shortly and then run db migrations (retry on error) -while ! ./manage.py migrate 2>&1; do - echo "âŗ Waiting on DB..." - sleep 3 +# Try to connect to the DB +DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} +MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} +CUR_DB_WAIT_TIME=0 +while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do + echo "âŗ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)" + sleep "${DB_WAIT_TIMEOUT}" + CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT )) done +if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then + echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready." + exit 1 +fi +# Create Superuser if required if [ "$SKIP_SUPERUSER" == "true" ]; then echo "↩ī¸ Skip creating the superuser" else @@ -43,18 +57,19 @@ END echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}" fi +# Run the startup scripts (and initializers) if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩ī¸ Skipping startup scripts" else echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi -# copy static files +# Copy static files ./manage.py collectstatic --no-input echo "✅ Initialisation is done." -# launch whatever is passed by docker +# Launch whatever is passed by docker # (i.e. the RUN instruction in the Dockerfile) # # shellcheck disable=SC2068 diff --git a/hooks/build b/hooks/build deleted file mode 100755 index 80c4165..0000000 --- a/hooks/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. hooks/common - -# shellcheck disable=SC2119 -run_build diff --git a/hooks/common b/hooks/common deleted file mode 100755 index ffa1115..0000000 --- a/hooks/common +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -ensure_jq() { - if [ ! -x "$(command -v jq)" ]; then - if [ -x "$(command -v apt-get)" ]; then - echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" - apt-get update && apt-get install -y jq - else - echo "⚠ī¸âš ī¸âš ī¸ apt-get not found, unable to automatically install 'jq'." - fi - fi -} - -# Passes args to the scripts -run_build() { - echo "đŸŗđŸŗđŸŗ Building '${BUILD}' images" - case ${BUILD} in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ - ;; - this) # Pull Requests - # only build the 'master' branch of netbox - # (resulting in the 'latest' docker tag) - # and the 'main' target. - DOCKER_TARGET=main ./build.sh master - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠ī¸ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac -} - -echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG=netboxcommunity -export DOCKER_REPO=netbox -export DOCKER_REGISTRY=docker.io -# shellcheck disable=SC2153 -export BUILD="${DOCKER_TAG}" - -unset DOCKER_TAG - -ensure_jq diff --git a/hooks/push b/hooks/push deleted file mode 100755 index 6513b24..0000000 --- a/hooks/push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -. hooks/common - -if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "release" ]; then - echo "⚠ī¸âš ī¸âš ī¸ Would exit, but DEBUG is '${DEBUG}'". - fi - - run_build --push-only -else - echo "⚠ī¸âš ī¸âš ī¸ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" - exit 0 -fi diff --git a/hooks/test b/hooks/test deleted file mode 100755 index a66db12..0000000 --- a/hooks/test +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -. hooks/common - -# test on builds of 'branches' -if [ "${BUILD}" == "branches" ] \ - || [ "${DEBUG}" == "true" ]; then - ./test.sh latest - ./test.sh snapshot -# test on bulds of 'this' (i.e. pull request) -elif [ "${BUILD}" == "this" ]; then - ./test.sh latest -else - echo "đŸŗđŸŗđŸŗ No tests are implemented for build '${BUILD}'." -fi diff --git a/test.sh b/test.sh index d6d784a..627eef1 100755 --- a/test.sh +++ b/test.sh @@ -5,11 +5,14 @@ set -e # version is used by `docker-compose.yml` do determine the tag # of the Docker Image that is to be used -export VERSION=${VERSION-latest} +export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" + +# The docker compose command to use +doco="docker-compose -f docker-compose.test.yml" test_netbox_unit_tests() { echo "⏱ Running Netbox Unit Tests" - docker-compose run --rm netbox ./manage.py test + $doco run --rm netbox ./manage.py test } test_initializers() { @@ -25,17 +28,20 @@ test_initializers() { mv initializers initializers_original mv initializers_test initializers - docker-compose run --rm netbox ./manage.py check + $doco run --rm netbox ./manage.py check } test_cleanup() { echo "đŸ’Ŗ Cleaning Up" - docker-compose down -v - rm -rf initializers - mv initializers_original initializers + $doco down -v + + if [ -d initializers_original ]; then + rm -rf initializers + mv initializers_original initializers + fi } -echo "đŸŗđŸŗđŸŗ Start testing '${VERSION}'" +echo "đŸŗđŸŗđŸŗ Start testing '${IMAGE}'" # Make sure the cleanup script is executed trap test_cleanup EXIT ERR @@ -43,4 +49,4 @@ trap test_cleanup EXIT ERR test_netbox_unit_tests test_initializers -echo "đŸŗđŸŗđŸŗ Done testing '${VERSION}'" +echo "đŸŗđŸŗđŸŗ Done testing '${IMAGE}'"