From d0ebb344320d9c7de8d2cb6708a3b1128adda226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 10 Oct 2019 12:42:10 +0200 Subject: [PATCH 01/15] Refactor to multistage builds This commit introduces a huge change in the build process. What changed: - Dockerfile.ldap was integrated into Dockerfile as a seperate [build stage][multistage-build]. - All the build scripts were refactored according to this. - The `docker-compose.yml` file was adjusted likewise. - The main build script, `/build.sh`, now always builds all targets (formerly called variants). - The minimal requirements for Docker and docker-compose have increased. - The build on hub.docker.com must be adjusted. This change should also fix #156 permanently. [multistage-build]: https://docs.docker.com/develop/develop-images/multistage-build/ --- DOCKER_HUB.md | 31 ++---- Dockerfile | 14 ++- Dockerfile.ldap | 9 -- README.md | 4 +- build-all.sh | 78 +++++---------- build-branches.sh | 21 ++++- build-latest.sh | 56 +++++++++-- build.sh | 231 +++++++++++++++++++++++++-------------------- docker-compose.yml | 9 +- hooks/common | 46 ++------- hooks/test | 6 +- 11 files changed, 259 insertions(+), 246 deletions(-) delete mode 100644 Dockerfile.ldap diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index 6149c3c..f378336 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -12,28 +12,16 @@ Repository Links: Enable for Base Image Build Rules: - Source Type: Branch Source: master - Docker Tag: branches-main + Docker Tag: branches Dockerfile location: Dockerfile - Source Type: Branch Source: master - Docker Tag: branches-ldap - Dockerfile location: Dockerfile.ldap -- Source Type: Branch - Source: master - Docker Tag: prerelease-main + Docker Tag: prerelease Dockerfile location: Dockerfile - Source Type: Branch Source: master - Docker Tag: prerelease-ldap - Dockerfile location: Dockerfile.ldap -- Source Type: Branch - Source: master - Docker Tag: release-main + Docker Tag: release Dockerfile location: Dockerfile -- Source Type: Branch - Source: master - Docker Tag: release-ldap - Dockerfile location: Dockerfile.ldap Build Environment Variables: # Create an app on Github and use it's OATH credentials here - Key: GITHUB_OAUTH_CLIENT_ID @@ -51,16 +39,15 @@ The build system of cloud.docker.com is not made for this kind of project. But we found a way to make it work, and this is how: 1. The docker hub build system [allows to overwrite the scripts that get executed - for `build`, `test` and `push`](overwrite). See `hooks/*`. -2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`. -3. The `build` script runs `run_build()` from `hooks/common`. - This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`. + for `build`, `test` and `push`](overwrite). See `/hooks/*`. +2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`. +3. The `build` script runs `run_build()` from `/hooks/common`. + This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`. 4. The `test` script just invokes `docker-compose` commands. 5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. This causes the `build.sh` script to not re-build the Docker image, but just the just built image. -The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_). - -The _Dockerfile location_ configuration setting is completely ignored by the build scripts. +The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`. +Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose. [overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands diff --git a/Dockerfile b/Dockerfile index bc7db56..481613e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.7-alpine3.10 +ARG FROM=python:3.7-alpine +FROM ${FROM} as main RUN apk add --no-cache \ bash \ @@ -71,3 +72,14 @@ LABEL SRC_URL="$URL" ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" + +##### +## LDAP specific tasks +##### + +FROM main as ldap + +RUN pip install django_auth_ldap + +COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py +COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/Dockerfile.ldap b/Dockerfile.ldap deleted file mode 100644 index 303c3a8..0000000 --- a/Dockerfile.ldap +++ /dev/null @@ -1,9 +0,0 @@ -ARG DOCKER_ORG=netboxcommunity -ARG DOCKER_REPO=netbox -ARG FROM_TAG=latest -FROM $DOCKER_ORG/$DOCKER_REPO:$FROM_TAG - -RUN pip install django_auth_ldap - -COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py -COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/README.md b/README.md index 1e40a8a..89b39a1 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ Default credentials: This project relies only on *Docker* and *docker-compose* meeting this requirements: -* The *Docker version* must be at least `1.13.0`. -* The *docker-compose version* must be at least `1.10.0`. +* The *Docker version* must be at least `17.05`. +* The *docker-compose version* must be at least `1.17.0`. To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. diff --git a/build-all.sh b/build-all.sh index d88c5af..0ba18ed 100755 --- a/build-all.sh +++ b/build-all.sh @@ -12,8 +12,6 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}") echo "⚙️ Configured builds: ${BUILDS[*]}" -VARIANTS=("main" "ldap") - if [ -n "${DEBUG}" ]; then export DEBUG fi @@ -22,63 +20,35 @@ ERROR=0 # Don't build if not on `master` and don't build if on a pull request, # but build when DEBUG is not empty -for VARIANT in "${VARIANTS[@]}"; do - export VARIANT - # Checking which VARIANT to build - if [ "${VARIANT}" == "main" ]; then - DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}" - else - DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}" - - # Fail fast - if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." - ERROR=1 +for BUILD in "${BUILDS[@]}"; do + echo "🛠 Building '$BUILD' from '$DOCKERFILE'" + case $BUILD in + release) + # build the latest release + # shellcheck disable=SC2068 + ./build-latest.sh $@ || ERROR=1 + ;; + prerelease) + # build the latest pre-release + # shellcheck disable=SC2068 + PRERELEASE=true ./build-latest.sh $@ || ERROR=1 + ;; + branches) + # build all branches + # shellcheck disable=SC2068 + ./build-branches.sh $@ || ERROR=1 + ;; + *) + echo "🚨 Unrecognized build '$BUILD'." if [ -z "$DEBUG" ]; then - continue + exit 1 else - echo "⚠️ Would skip this, but DEBUG is enabled." + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." fi - fi - fi - - for BUILD in "${BUILDS[@]}"; do - echo "🛠 Building '$BUILD' from '$DOCKERFILE'" - case $BUILD in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ || ERROR=1 - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ || ERROR=1 - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ || ERROR=1 - ;; - special) - # special build - # shellcheck disable=SC2068 - #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1 - echo "✅ No special builds today." - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac - done + ;; + esac done exit $ERROR diff --git a/build-branches.sh b/build-branches.sh index 622e0ec..a6bc736 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -3,6 +3,10 @@ echo "▶️ $0 $*" +### +# Checking for the presence of GITHUB_OAUTH_CLIENT_ID +# and GITHUB_OAUTH_CLIENT_SECRET +### if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" @@ -11,18 +15,33 @@ else GITHUB_OAUTH_PARAMS="" fi +### +# Calling Github to get the all branches +### ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}" +# Composing the JQ commans to extract the most recent version number +JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string' + CURL="curl -sS" -BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string') +# Querying the Github API to fetch all branches +BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES") +### +# Building each branch +### + +# keeping track whether an error occured ERROR=0 +# calling build.sh for each branch for BRANCH in $BRANCHES; do # shellcheck disable=SC2068 ./build.sh "${BRANCH}" $@ || ERROR=1 done + +# returning whether an error occured exit $ERROR diff --git a/build-latest.sh b/build-latest.sh index b86c280..ef97569 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -3,6 +3,10 @@ echo "▶️ $0 $*" +### +# Checking for the presence of GITHUB_OAUTH_CLIENT_ID +# and GITHUB_OAUTH_CLIENT_SECRET +### if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" @@ -11,17 +15,38 @@ else GITHUB_OAUTH_PARAMS="" fi +### +# Checking if PRERELEASE is either unset, 'true' or 'false' +### +if [ -n "$PRERELEASE" ] && + { [ "$PRERELEASE" != "true" ] && [ "$PRERELEASE" != "false" ]; }; then + + if [ -z "$DEBUG" ]; then + echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was \"$PRERELEASE\"!" + exit 1 + else + echo "⚠️ Would exit here with code '0', but DEBUG is enabled." + fi +fi + +### +# Calling Github to get the latest version +### ORIGINAL_GITHUB_REPO="netbox-community/netbox" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}" +# Composing the JQ commans to extract the most recent version number JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name" CURL="curl -sS" +# Querying the Github API to fetch the most recent version number VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}") +### # Check if the prerelease version is actually higher than stable version +### if [ "${PRERELEASE}" == "true" ]; then JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name" STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}") @@ -35,8 +60,9 @@ if [ "${PRERELEASE}" == "true" ]; then # shellcheck disable=SC2003 MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)') - if ( [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ] ) \ + if { [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ]; } \ || [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then + echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')." if [ -z "$DEBUG" ]; then exit 0 @@ -46,28 +72,38 @@ if [ "${PRERELEASE}" == "true" ]; then fi fi -# Check if that version is not already available on docker hub: +### +# Compose DOCKER_TAG to build +### +if [ -z "$DOCKER_TARGET" ] || [ "$DOCKER_TARGET" == "main" ]; then + DOCKER_TAG="${VERSION}" +else + DOCKER_TAG="${VERSION}-${DOCKER_TARGET}" +fi + +### +# Check if the version received is not already available on Docker Hub: +### ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}" DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}" + +# Bearer Token URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull" BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)" +# Actual API call URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list" AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}" - -if [ -z "$VARIANT" ] || [ "$VARIANT" == "main" ]; then - DOCKER_TAG="${VERSION}" -else - DOCKER_TAG="${VERSION}-${VARIANT}" -fi - ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")" +### +# Only build the image if it's not already been built before +### if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then if [ -n "$DEBUG" ]; then echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled." fi - + # shellcheck disable=SC2068 ./build.sh "${VERSION}" $@ exit $? diff --git a/build.sh b/build.sh index 3255fd5..0c3de72 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds the Dockerfile[.variant] and injects tgz'ed Netbox code from Github +# Builds the Dockerfile and injects tgz'ed Netbox code from Github echo "▶️ $0 $*" @@ -12,12 +12,6 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " --push-only Does not build. Only pushes the Docker image to the registry." echo "" echo "You can use the following ENV variables to customize the build:" - echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." - echo " DRY_RUN Prints all build statements instead of running them." - echo " DOCKER_OPTS Add parameters to Docker." - echo " Default:" - echo " When starts with 'v': \"\"" - echo " Else: \"--no-cache\"" echo " BRANCH The branch to build." echo " Also used for tagging the image." echo " TAG The version part of the docker tag." @@ -25,18 +19,27 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " When =master: latest" echo " When =develop: snapshot" echo " Else: same as " - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//) " + echo " DOCKER_OPTS Add parameters to Docker." + echo " Default:" + echo " When starts with 'v': \"\"" + echo " Else: \"--no-cache\"" + echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//)" echo " Also used for tagging the image." echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r//) " + echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r//)" echo " Also used for tagging the image." echo " Default: netbox" + echo " DOCKER_FROM The base image to use." + echo " Default: Whatever is defined as default in the Dockerfile." echo " DOCKER_TAG The name of the tag which is applied to the image." echo " Useful for pushing into another registry than hub.docker.com." echo " Default: /:" - echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the image." - echo " This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5" + echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" + echo " image. This is used to tag all patch releases to their" + echo " containing version e.g. v2.5.1 -> v2.5" echo " Default: /:\$MAJOR.\$MINOR" + echo " DOCKERFILE The name of Dockerfile to use." + echo " Default: Dockerfile" echo " SRC_ORG Which fork of netbox to use (i.e. github.com//)." echo " Default: netbox-community" echo " SRC_REPO The name of the netbox for to use (i.e. github.com//)." @@ -44,18 +47,6 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " URL Where to fetch the package from." echo " Must be a tar.gz file of the source code." echo " Default: https://github.com///archive/\$BRANCH.tar.gz" - echo " VARIANT The variant to build." - echo " The value will be used as a suffix to the \$TAG and for the Dockerfile" - echo " selection. The TAG being build must exist for the base variant and" - echo " corresponding Dockerfile must start with the following lines:" - echo " ARG DOCKER_ORG=netboxcommunity" - echo " ARG DOCKER_REPO=netbox" - echo " ARG FROM_TAG=latest" - echo " FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG" - echo " Example: VARIANT=ldap will result in the tag 'latest-ldap' and the" - echo " Dockerfile './Dockerfile.ldap' being used." - echo " Exception: VARIANT=main will use the './Dockerfile' Dockerfile" - echo " Default: main" echo " HTTP_PROXY The proxy to use for http requests." echo " Example: http://proxy.domain.tld:3128" echo " Default: empty" @@ -68,6 +59,8 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." echo " Example: .domain1.tld,.domain2.tld" echo " Default: empty" + echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." + echo " DRY_RUN Prints all build statements instead of running them." if [ "${1}x" == "x" ]; then exit 1 @@ -76,27 +69,27 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi fi -# read the project version and trim it +### +# read the project version from the `VERSION` file and trim it # see https://stackoverflow.com/a/3232433/172132 +### NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" +### # variables for fetching the source +### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" -# Checking which VARIANT to build -VARIANT="${VARIANT-main}" -if [ "$VARIANT" == "main" ]; then - DOCKERFILE="Dockerfile" -else - DOCKERFILE="Dockerfile.${VARIANT}" -fi - -# Fail fast +### +# Determining the value for DOCKERFILE +# and checking whether it exists +### +DOCKERFILE="${DOCKERFILE-Dockerfile}" if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile ${DOCKERFILE} for variant '${VARIANT}' doesn't exist." + echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist." if [ -z "$DEBUG" ]; then exit 1 @@ -105,7 +98,9 @@ if [ ! -f "${DOCKERFILE}" ]; then fi fi +### # variables for tagging the docker image +### DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" case "${BRANCH}" in @@ -117,81 +112,109 @@ case "${BRANCH}" in TAG="${TAG-$BRANCH}";; esac -DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" -if [ "$VARIANT" != "main" ]; then - DOCKER_TAG="${DOCKER_TAG}-${VARIANT}" -fi - -if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then - MAJOR=${BASH_REMATCH[1]} - MINOR=${BASH_REMATCH[2]} - - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" - - if [ "$VARIANT" != "main" ]; then - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${VARIANT}" +DOCKER_TARGETS=("main" "ldap") +for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do + ### + # composing the final DOCKER_TAG + ### + DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" + if [ "$DOCKER_TARGET" != "main" ]; then + DOCKER_TAG="${DOCKER_TAG}-${DOCKER_TARGET}" fi -fi -DOCKER_OPTS=("${DOCKER_OPTS[@]}") + ### + # composing the additional DOCKER_SHORT_TAG, + # i.e. "v2.6.1" becomes "v2.6", + # which is only relevant for version tags + ### + if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then + MAJOR=${BASH_REMATCH[1]} + MINOR=${BASH_REMATCH[2]} -# caching is only ok for version tags -case "${TAG}" in - v*) ;; - *) DOCKER_OPTS+=( "--no-cache" ) ;; -esac + DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" -DOCKER_OPTS+=( "--pull" ) - -# Build args -DOCKER_BUILD_ARGS=( - --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "FROM_TAG=${TAG}" - --build-arg "BRANCH=${BRANCH}" - --build-arg "URL=${URL}" - --build-arg "DOCKER_ORG=${DOCKER_ORG}" - --build-arg "DOCKER_REPO=${DOCKER_REPO}" -) -if [ -n "$HTTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) -fi -if [ -n "$HTTPS_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) -fi -if [ -n "$FTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" ) -fi -if [ -n "$NO_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) -fi - -if [ -z "$DRY_RUN" ]; then - DOCKER_CMD="docker" -else - echo "⚠️ DRY_RUN MODE ON ⚠️" - DOCKER_CMD="echo docker" -fi - -if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." - $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . - echo "✅ Finished building the Docker images '${DOCKER_TAG}'" - - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}" - echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" + if [ "$DOCKER_TARGET" != "main" ]; then + DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" + fi fi -fi -if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then - echo "⏫ Pushing '${DOCKER_TAG}" - $DOCKER_CMD push "${DOCKER_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." + ### + # Composing global Docker CLI arguments + ### + DOCKER_OPTS=("${DOCKER_OPTS[@]}") - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DOCKER_CMD push "${DOCKER_SHORT_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + # caching is only ok for version tags + case "${TAG}" in + v*) ;; + *) DOCKER_OPTS+=( --no-cache ) ;; + esac + + DOCKER_OPTS+=( --pull ) + DOCKER_OPTS+=( --target "$DOCKER_TARGET" ) + + ### + # Composing arguments for `docker build` CLI + ### + DOCKER_BUILD_ARGS=( + --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" + --build-arg "BRANCH=${BRANCH}" + --build-arg "URL=${URL}" + --build-arg "DOCKER_ORG=${DOCKER_ORG}" + --build-arg "DOCKER_REPO=${DOCKER_REPO}" + ) + if [ -n "$DOCKER_FROM" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) fi -fi + if [ -n "$HTTP_PROXY" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) + fi + if [ -n "$HTTPS_PROXY" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) + fi + if [ -n "$FTP_PROXY" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" ) + fi + if [ -n "$NO_PROXY" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) + fi + + ### + # Determining the build command to use + ### + if [ -z "$DRY_RUN" ]; then + DOCKER_CMD="docker" + else + echo "⚠️ DRY_RUN MODE ON ⚠️" + DOCKER_CMD="echo docker" + fi + + ### + # Building the docker images, except if `--push-only` is passed + ### + if [ "${2}" != "--push-only" ] ; then + echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." + $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + echo "✅ Finished building the Docker images '${DOCKER_TAG}'" + + if [ -n "$DOCKER_SHORT_TAG" ]; then + echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." + $DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}" + echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" + fi + fi + + ### + # Pushing the docker images if either `--push` or `--push-only` are passed + ### + if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then + echo "⏫ Pushing '${DOCKER_TAG}" + $DOCKER_CMD push "${DOCKER_TAG}" + echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." + + if [ -n "$DOCKER_SHORT_TAG" ]; then + echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" + $DOCKER_CMD push "${DOCKER_SHORT_TAG}" + echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + fi + fi +done diff --git a/docker-compose.yml b/docker-compose.yml index 6c80b72..f57de21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,11 @@ -version: '3' +version: '3.4' services: netbox: &netbox build: - context: . - args: - - BRANCH=${VERSION-master} + context: . + target: ${DOCKER_TARGET-main} + args: + - BRANCH=${VERSION-master} image: netboxcommunity/netbox:${VERSION-latest} depends_on: - postgres diff --git a/hooks/common b/hooks/common index dc60799..7f60a7c 100755 --- a/hooks/common +++ b/hooks/common @@ -1,37 +1,19 @@ #!/bin/bash ensure_jq() { - echo "🛠🛠🛠 Installing JQ via apt-get" - [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq ) -} - -ensure_dockerfile_present() { - if [ "${VARIANT}" == "main" ]; then - DOCKERFILE="Dockerfile" - else - DOCKERFILE="Dockerfile.${VARIANT}" - - # Fail fast - if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would skip this, but DEBUG is enabled." - fi - fi - - if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then - echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'." - echo " This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'." + if [ ! -x "$(command -v jq)" ]; then + if [ ! -x "$(command -v apt-get)" ]; then + echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" + apt-get update && apt-get install -y jq + else + echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'." fi fi } # Passes args to the scripts run_build() { - echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant" + echo "🐳🐳🐳 Building '${BUILD}' images" case $BUILD in release) # build the latest release @@ -48,12 +30,6 @@ run_build() { # shellcheck disable=SC2068 ./build-branches.sh $@ ;; - special) - # special build - # shellcheck disable=SC2068 - #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ - echo "✅ No special builds today." - ;; *) echo "🚨 Unrecognized build '$BUILD'." @@ -70,13 +46,9 @@ echo "🤖🤖🤖 Preparing build" export DOCKER_ORG="index.docker.io/netboxcommunity" export DOCKER_REPO=netbox export DOCKERHUB_REPO=netboxcommunity/netbox - -# mis-using the "${DOCKER_TAG}" variable as "branch to build" -export BUILD="${DOCKER_TAG%-*}" -export VARIANT="${DOCKER_TAG#*-}" +# shellcheck disable=SC2153 +export BUILD="$DOCKER_TAG" unset DOCKER_TAG -ensure_dockerfile_present - ensure_jq diff --git a/hooks/test b/hooks/test index 6c89c87..9ef0d8a 100755 --- a/hooks/test +++ b/hooks/test @@ -2,11 +2,13 @@ . hooks/common -if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then +if [ "${BUILD}" == "BRANCHES" ]; then echo "🐳🐳🐳 Testing" + export DOCKER_TARGET="main" + docker-compose pull --parallel docker-compose build docker-compose run netbox ./manage.py test else - echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'." + echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." fi From d0c9dfe2e5c6839ca5ad762afb5aff94bf9d4dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 10 Oct 2019 14:08:31 +0200 Subject: [PATCH 02/15] Try to get PR builds working --- hooks/common | 4 ++++ hooks/test | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hooks/common b/hooks/common index 7f60a7c..26add15 100755 --- a/hooks/common +++ b/hooks/common @@ -30,6 +30,10 @@ run_build() { # shellcheck disable=SC2068 ./build-branches.sh $@ ;; + this) # Pull Requests + # build all branches, but never push + ./build-branches.sh + ;; *) echo "🚨 Unrecognized build '$BUILD'." diff --git a/hooks/test b/hooks/test index 9ef0d8a..14ad88f 100755 --- a/hooks/test +++ b/hooks/test @@ -2,13 +2,13 @@ . hooks/common -if [ "${BUILD}" == "BRANCHES" ]; then +# test 'branches' and 'this' (i.e. pull request) builds +if [ "${BUILD}" == "branches" ] || [ "${BUILD}" == "this" ]; then echo "🐳🐳🐳 Testing" export DOCKER_TARGET="main" - docker-compose pull --parallel - docker-compose build docker-compose run netbox ./manage.py test + docker-compose down -v else echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." fi From fadac8c5c36263b6d36f604628b916ee80186f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 10 Oct 2019 14:48:45 +0200 Subject: [PATCH 03/15] Only build the master branch to speed up the test --- build.sh | 22 +++++++++++++++++++--- hooks/common | 6 ++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 0c3de72..4345aa5 100755 --- a/build.sh +++ b/build.sh @@ -40,6 +40,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: /:\$MAJOR.\$MINOR" echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" + echo " DOCKER_TARGET A specific target to build." + echo " It's currently not possible to pass multiple targets." + echo " Default: main ldap" echo " SRC_ORG Which fork of netbox to use (i.e. github.com//)." echo " Default: netbox-community" echo " SRC_REPO The name of the netbox for to use (i.e. github.com//)." @@ -112,8 +115,19 @@ case "${BRANCH}" in TAG="${TAG-$BRANCH}";; esac -DOCKER_TARGETS=("main" "ldap") +### +# Determine targets to build +### +DEFAULT_DOCKER_TARGETS=("main" "ldap") +DOCKER_TARGETS=( "${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}") +echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" + +### +# Build each target +### for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do + echo "🏗 Building the target '$DOCKER_TARGET'" + ### # composing the final DOCKER_TAG ### @@ -143,10 +157,12 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### DOCKER_OPTS=("${DOCKER_OPTS[@]}") - # caching is only ok for version tags + # caching is only ok for version tags, + # but turning the cache off is only required for the + # first build target, usually "main". case "${TAG}" in v*) ;; - *) DOCKER_OPTS+=( --no-cache ) ;; + *) [ "$DOCKER_TARGET" == "${DOCKER_TARGETS[0]}" ] && DOCKER_OPTS+=( --no-cache ) ;; esac DOCKER_OPTS+=( --pull ) diff --git a/hooks/common b/hooks/common index 26add15..92a4724 100755 --- a/hooks/common +++ b/hooks/common @@ -31,8 +31,10 @@ run_build() { ./build-branches.sh $@ ;; this) # Pull Requests - # build all branches, but never push - ./build-branches.sh + # only build the 'master' branch + # (resulting in the 'latest' docker tag) + # and the 'main' target. + DOCKER_TARGET=main ./build.sh master ;; *) echo "🚨 Unrecognized build '$BUILD'." From 052b53aa5c66b30d267121cdc4a4bd09dd59df34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 10 Oct 2019 14:51:06 +0200 Subject: [PATCH 04/15] Fix apt-get detection --- hooks/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/common b/hooks/common index 92a4724..fcf84a3 100755 --- a/hooks/common +++ b/hooks/common @@ -2,7 +2,7 @@ ensure_jq() { if [ ! -x "$(command -v jq)" ]; then - if [ ! -x "$(command -v apt-get)" ]; then + if [ -x "$(command -v apt-get)" ]; then echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" apt-get update && apt-get install -y jq else From ab4b8720d162a7297f3848823e5016659fff4525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 10 Oct 2019 15:24:53 +0200 Subject: [PATCH 05/15] Fix caching in build.sh --- build.sh | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index 4345aa5..6ebea6f 100755 --- a/build.sh +++ b/build.sh @@ -19,10 +19,6 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " When =master: latest" echo " When =develop: snapshot" echo " Else: same as " - echo " DOCKER_OPTS Add parameters to Docker." - echo " Default:" - echo " When starts with 'v': \"\"" - echo " Else: \"--no-cache\"" echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//)" echo " Also used for tagging the image." echo " Default: netboxcommunity" @@ -129,11 +125,11 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do echo "🏗 Building the target '$DOCKER_TARGET'" ### - # composing the final DOCKER_TAG + # composing the final TARGET_DOCKER_TAG ### - DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" + TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" if [ "$DOCKER_TARGET" != "main" ]; then - DOCKER_TAG="${DOCKER_TAG}-${DOCKER_TARGET}" + TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi ### @@ -155,7 +151,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### # Composing global Docker CLI arguments ### - DOCKER_OPTS=("${DOCKER_OPTS[@]}") + DOCKER_OPTS=( + --pull + --target "$DOCKER_TARGET" + ) # caching is only ok for version tags, # but turning the cache off is only required for the @@ -165,9 +164,6 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do *) [ "$DOCKER_TARGET" == "${DOCKER_TARGETS[0]}" ] && DOCKER_OPTS+=( --no-cache ) ;; esac - DOCKER_OPTS+=( --pull ) - DOCKER_OPTS+=( --target "$DOCKER_TARGET" ) - ### # Composing arguments for `docker build` CLI ### @@ -208,13 +204,13 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # Building the docker images, except if `--push-only` is passed ### if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." - $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . - echo "✅ Finished building the Docker images '${DOCKER_TAG}'" + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}' from the url '${URL}'." + $DOCKER_CMD build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" if [ -n "$DOCKER_SHORT_TAG" ]; then echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}" + $DOCKER_CMD tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" fi fi @@ -223,9 +219,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # Pushing the docker images if either `--push` or `--push-only` are passed ### if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then - echo "⏫ Pushing '${DOCKER_TAG}" - $DOCKER_CMD push "${DOCKER_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." + echo "⏫ Pushing '${TARGET_DOCKER_TAG}" + $DOCKER_CMD push "${TARGET_DOCKER_TAG}" + echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." if [ -n "$DOCKER_SHORT_TAG" ]; then echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" From f3b9c34e3b89e8c8b273ead44a0701af937c8fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Sun, 13 Oct 2019 16:00:42 +0200 Subject: [PATCH 06/15] externalize netbox download with wget --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 69 +++++++++++++++++++++++++--------------------- README.md | 11 ++++---- build.sh | 68 ++++++++++++++++++++++++++++----------------- docker-compose.yml | 5 ---- 6 files changed, 88 insertions(+), 67 deletions(-) diff --git a/.dockerignore b/.dockerignore index ce32f24..1b2bacc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ .travis.yml build* *.env +.git diff --git a/.gitignore b/.gitignore index 53a4e81..cbaffa8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.sql.gz +.netbox diff --git a/Dockerfile b/Dockerfile index 481613e..dab0723 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG FROM=python:3.7-alpine -FROM ${FROM} as main +FROM ${FROM} as builder RUN apk add --no-cache \ bash \ @@ -8,51 +8,52 @@ RUN apk add --no-cache \ cyrus-sasl-dev \ graphviz \ jpeg-dev \ + libevent-dev \ libffi-dev \ - libxml2-dev \ libxslt-dev \ openldap-dev \ - postgresql-dev \ - ttf-ubuntu-font-family \ - wget + postgresql-dev -RUN pip install \ +WORKDIR /install + +RUN pip install --install-option="--prefix=/install" \ # gunicorn is used for launching netbox gunicorn \ + greenlet \ + eventlet \ # napalm is used for gathering information from network devices napalm \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ -# pinning django to the version required by netbox -# adding it here, to install the correct version of -# django-rq - 'Django>=2.2,<2.3' \ -# django-rq is used for webhooks - django-rq +# django_auth_ldap is required for ldap + django_auth_ldap -ARG BRANCH=master +COPY .netbox/netbox/requirements.txt / +RUN pip install --install-option="--prefix=/install" -r /requirements.txt -WORKDIR /tmp +### +# Main stage +### -# As the requirements don't change very often, -# and as they take some time to compile, -# we try to cache them very agressively. -ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt -ADD ${REQUIREMENTS_URL} requirements.txt -RUN pip install -r requirements.txt +ARG FROM +FROM ${FROM} as main -# Cache bust when the upstream branch changes: -# ADD will fetch the file and check if it has changed -# If not, Docker will use the existing build cache. -# If yes, Docker will bust the cache and run every build step from here on. -ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH -ADD ${REF_URL} version.json +RUN apk add --no-cache \ + bash \ + ca-certificates \ + graphviz \ + libevent \ + libffi \ + libjpeg-turbo \ + libressl \ + libxslt \ + postgresql-libs \ + ttf-ubuntu-font-family WORKDIR /opt -ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz -RUN wget -q -O - "${URL}" | tar xz \ - && mv netbox* netbox +COPY --from=builder /install /usr/local +COPY .netbox/netbox /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY configuration/gunicorn_config.py /etc/netbox/config/ @@ -73,13 +74,19 @@ LABEL SRC_URL="$URL" ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" +ARG NETBOX_BRANCH=custom_build +LABEL NETBOX_BRANCH="$NETBOX_BRANCH" + ##### -## LDAP specific tasks +## LDAP specific configuration ##### FROM main as ldap -RUN pip install django_auth_ldap +RUN apk add --no-cache \ + libsasl \ + libldap \ + util-linux COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/README.md b/README.md index 89b39a1..6b3c8b1 100644 --- a/README.md +++ b/README.md @@ -69,18 +69,17 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.2.6 +export VERSION=v2.6.6 docker-compose pull netbox docker-compose up -d ``` -You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid -[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github]. -Most commonly you will specify a tag or branch name. +You can also build a specific version of the Netbox Docker image yourself. +`VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=develop -docker-compose build --no-cache netbox +export VERSION=v2.6.6 +./build.sh $VERSION docker-compose up -d ``` diff --git a/build.sh b/build.sh index 6ebea6f..72428ae 100755 --- a/build.sh +++ b/build.sh @@ -68,6 +68,16 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi fi +### +# Determining the build command to use +### +if [ -z "$DRY_RUN" ]; then + DRY="" +else + echo "⚠️ DRY_RUN MODE ON ⚠️" + DRY="echo" +fi + ### # read the project version from the `VERSION` file and trim it # see https://stackoverflow.com/a/3232433/172132 @@ -82,6 +92,26 @@ SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" +### +# fetching the source +### +if [ "${2}" != "--push-only" ] ; then + echo "🗑️ Preparing" + $DRY rm -rf .netbox + $DRY mkdir .netbox + echo "✅ Done preparing" + + echo "🌐 Downloading netbox from the url '${URL}'" + ( + $DRY cd .netbox + + $DRY wget -qO netbox.tgz "${URL}" && \ + $DRY tar -xzf netbox.tgz && \ + $DRY mv netbox-* netbox + ) + echo "✅ Downloaded netbox" +fi + ### # Determining the value for DOCKERFILE # and checking whether it exists @@ -156,21 +186,12 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do --target "$DOCKER_TARGET" ) - # caching is only ok for version tags, - # but turning the cache off is only required for the - # first build target, usually "main". - case "${TAG}" in - v*) ;; - *) [ "$DOCKER_TARGET" == "${DOCKER_TARGETS[0]}" ] && DOCKER_OPTS+=( --no-cache ) ;; - esac - ### # Composing arguments for `docker build` CLI ### DOCKER_BUILD_ARGS=( --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "BRANCH=${BRANCH}" - --build-arg "URL=${URL}" + --build-arg "NETBOX_BRANCH=${BRANCH}" --build-arg "DOCKER_ORG=${DOCKER_ORG}" --build-arg "DOCKER_REPO=${DOCKER_REPO}" ) @@ -190,27 +211,17 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) fi - ### - # Determining the build command to use - ### - if [ -z "$DRY_RUN" ]; then - DOCKER_CMD="docker" - else - echo "⚠️ DRY_RUN MODE ON ⚠️" - DOCKER_CMD="echo docker" - fi - ### # Building the docker images, except if `--push-only` is passed ### if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}' from the url '${URL}'." - $DOCKER_CMD build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." + $DRY docker build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" if [ -n "$DOCKER_SHORT_TAG" ]; then echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DOCKER_CMD tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" + $DRY docker tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" fi fi @@ -220,13 +231,20 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then echo "⏫ Pushing '${TARGET_DOCKER_TAG}" - $DOCKER_CMD push "${TARGET_DOCKER_TAG}" + $DRY docker push "${TARGET_DOCKER_TAG}" echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." if [ -n "$DOCKER_SHORT_TAG" ]; then echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DOCKER_CMD push "${DOCKER_SHORT_TAG}" + $DRY docker push "${DOCKER_SHORT_TAG}" echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." fi fi done + +### +# Cleaning up +### +echo "🗑️ Cleaning up" +$DRY rm -rf .netbox +echo "✅ Cleaned up" diff --git a/docker-compose.yml b/docker-compose.yml index f57de21..f70a93f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,6 @@ version: '3.4' services: netbox: &netbox - build: - context: . - target: ${DOCKER_TARGET-main} - args: - - BRANCH=${VERSION-master} image: netboxcommunity/netbox:${VERSION-latest} depends_on: - postgres From 20109c3392bef62ae50e6de6d4a5369eff1ec66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 21:54:49 +0200 Subject: [PATCH 07/15] Checkout the repository with git This changes the build process even further. Instead f using `wget` to fetch the current code, `git` is used. This allows for faster switching between branches, because only the differences between them have to be fetched from the server. But the main advantage is that the build cache can finally be used as designed by Docker. Repetitive builds are very fast now. This is also true between branches and tags, as long as the `requirements.txt` file doesn't change. --- Dockerfile | 19 ++-- README.md | 3 - build-all.sh | 3 - build-latest.sh | 42 +------- build.sh | 248 ++++++++++++++++++++++++++---------------------- 5 files changed, 149 insertions(+), 166 deletions(-) diff --git a/Dockerfile b/Dockerfile index dab0723..c605e99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,8 @@ RUN pip install --install-option="--prefix=/install" \ # django_auth_ldap is required for ldap django_auth_ldap -COPY .netbox/netbox/requirements.txt / +ARG NETBOX_PATH +COPY ${NETBOX_PATH}/requirements.txt / RUN pip install --install-option="--prefix=/install" -r /requirements.txt ### @@ -53,7 +54,9 @@ RUN apk add --no-cache \ WORKDIR /opt COPY --from=builder /install /usr/local -COPY .netbox/netbox /opt/netbox + +ARG NETBOX_PATH +COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY configuration/gunicorn_config.py /etc/netbox/config/ @@ -69,13 +72,11 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] -LABEL SRC_URL="$URL" - -ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot -LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" - -ARG NETBOX_BRANCH=custom_build -LABEL NETBOX_BRANCH="$NETBOX_BRANCH" +LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \ + NETBOX_BRANCH="custom build" \ + ORIGINAL_DOCKER_TAG="custom build" \ + NETBOX_GIT_COMMIT="not built from git" \ + NETBOX_GIT_URL="not built from git" ##### ## LDAP specific configuration diff --git a/README.md b/README.md index 6b3c8b1..e375bea 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,6 @@ export VERSION=v2.6.6 docker-compose up -d ``` -Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary. -This can increase the build speed if you're just adjusting the config, for example. - [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References [netbox-github]: https://github.com/netbox-community/netbox/releases diff --git a/build-all.sh b/build-all.sh index 0ba18ed..462a83a 100755 --- a/build-all.sh +++ b/build-all.sh @@ -18,9 +18,6 @@ fi ERROR=0 -# Don't build if not on `master` and don't build if on a pull request, -# but build when DEBUG is not empty - for BUILD in "${BUILDS[@]}"; do echo "🛠 Building '$BUILD' from '$DOCKERFILE'" case $BUILD in diff --git a/build-latest.sh b/build-latest.sh index ef97569..f16e211 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -72,42 +72,6 @@ if [ "${PRERELEASE}" == "true" ]; then fi fi -### -# Compose DOCKER_TAG to build -### -if [ -z "$DOCKER_TARGET" ] || [ "$DOCKER_TARGET" == "main" ]; then - DOCKER_TAG="${VERSION}" -else - DOCKER_TAG="${VERSION}-${DOCKER_TARGET}" -fi - -### -# Check if the version received is not already available on Docker Hub: -### -ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}" -DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}" - -# Bearer Token -URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull" -BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)" - -# Actual API call -URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list" -AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}" -ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")" - -### -# Only build the image if it's not already been built before -### -if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then - if [ -n "$DEBUG" ]; then - echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled." - fi - - # shellcheck disable=SC2068 - ./build.sh "${VERSION}" $@ - exit $? -else - echo "✅ ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}" - exit 0 -fi +# shellcheck disable=SC2068 +./build.sh "${VERSION}" $@ +exit $? diff --git a/build.sh b/build.sh index 72428ae..c9ee110 100755 --- a/build.sh +++ b/build.sh @@ -8,58 +8,78 @@ set -e if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo "Usage: ${0} [--push|--push-only]" echo " branch The branch or tag to build. Required." - echo " --push Pushes built the Docker image to the registry." - echo " --push-only Does not build. Only pushes the Docker image to the registry." + echo " --push Pushes the built Docker image to the registry." + echo " --push-only Only pushes the Docker image to the registry, but does not build it." echo "" echo "You can use the following ENV variables to customize the build:" - echo " BRANCH The branch to build." - echo " Also used for tagging the image." - echo " TAG The version part of the docker tag." - echo " Default:" - echo " When =master: latest" - echo " When =develop: snapshot" - echo " Else: same as " - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//)" - echo " Also used for tagging the image." - echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r//)" - echo " Also used for tagging the image." - echo " Default: netbox" + echo " SRC_ORG Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox-community" + echo " SRC_REPO The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox" + echo " URL Where to fetch the code from." + echo " Must be a git repository. Can be private." + echo " Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git" + echo " NETBOX_PATH The path where netbox will be checkout out." + echo " Must not be outside of the netbox-docker repository (because of Docker)!" + echo " Default: .netbox" + echo " SKIP_GIT If defined, git is not invoked and \${NETBOX_PATH} will not be altered." + echo " This may be useful, if you are manually managing the NETBOX_PATH." + echo " Default: undefined" + echo " TAG The version part of the docker tag." + echo " Default:" + echo " When \${BRANCH}=master: latest" + echo " When \${BRANCH}=develop: snapshot" + echo " Else: same as \${BRANCH}" + echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netboxcommunity" + echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netbox" echo " DOCKER_FROM The base image to use." - echo " Default: Whatever is defined as default in the Dockerfile." - echo " DOCKER_TAG The name of the tag which is applied to the image." - echo " Useful for pushing into another registry than hub.docker.com." - echo " Default: /:" + echo " Default: Whatever is defined as default in the Dockerfile." + echo " DOCKER_TAG The name of the tag which is applied to the image." + echo " Useful for pushing into another registry than hub.docker.com." + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" - echo " image. This is used to tag all patch releases to their" - echo " containing version e.g. v2.5.1 -> v2.5" - echo " Default: /:\$MAJOR.\$MINOR" - echo " DOCKERFILE The name of Dockerfile to use." - echo " Default: Dockerfile" + echo " image. This is used to tag all patch releases to their" + echo " containing version e.g. v2.5.1 -> v2.5" + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:." + echo " DOCKERFILE The name of Dockerfile to use." + echo " Default: Dockerfile" echo " DOCKER_TARGET A specific target to build." - echo " It's currently not possible to pass multiple targets." - echo " Default: main ldap" - echo " SRC_ORG Which fork of netbox to use (i.e. github.com//)." - echo " Default: netbox-community" - echo " SRC_REPO The name of the netbox for to use (i.e. github.com//)." - echo " Default: netbox" - echo " URL Where to fetch the package from." - echo " Must be a tar.gz file of the source code." - echo " Default: https://github.com///archive/\$BRANCH.tar.gz" - echo " HTTP_PROXY The proxy to use for http requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " HTTPS_PROXY The proxy to use for https requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " FTP_PROXY The proxy to use for ftp requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." - echo " Example: .domain1.tld,.domain2.tld" - echo " Default: empty" - echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." - echo " DRY_RUN Prints all build statements instead of running them." + echo " It's currently not possible to pass multiple targets." + echo " Default: main ldap" + echo " HTTP_PROXY The proxy to use for http requests." + echo " Example: http://proxy.domain.tld:3128" + echo " Default: undefined" + echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." + echo " Example: .domain1.tld,.domain2.tld" + echo " Default: undefined" + echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." + echo " Default: undefined" + echo " DRY_RUN Prints all build statements instead of running them." + echo " Default: undefined" + echo "" + echo "Examples:" + echo " ${0} master" + echo " This will fetch the latest 'master' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:latest'." + echo " ${0} develop" + echo " This will fetch the latest 'develop' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:snapshot'." + echo " ${0} v2.6.6" + echo " This will fetch the 'v2.6.6' tag, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'." + echo " ${0} develop-2.7" + echo " This will fetch the 'develop-2.7' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:develop-2.7'." + echo " SRC_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'." + echo " SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'cimnine/netbox:feature-x'." if [ "${1}x" == "x" ]; then exit 1 @@ -71,7 +91,7 @@ fi ### # Determining the build command to use ### -if [ -z "$DRY_RUN" ]; then +if [ -z "${DRY_RUN}" ]; then DRY="" else echo "⚠️ DRY_RUN MODE ON ⚠️" @@ -90,26 +110,31 @@ NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:s SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" -URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" +URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" ### # fetching the source ### if [ "${2}" != "--push-only" ] ; then - echo "🗑️ Preparing" - $DRY rm -rf .netbox - $DRY mkdir .netbox - echo "✅ Done preparing" + NETBOX_PATH="${NETBOX_PATH-.netbox}" + echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + if [ ! -d "${NETBOX_PATH}" ]; then + $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + fi - echo "🌐 Downloading netbox from the url '${URL}'" ( - $DRY cd .netbox + $DRY cd "${NETBOX_PATH}" - $DRY wget -qO netbox.tgz "${URL}" && \ - $DRY tar -xzf netbox.tgz && \ - $DRY mv netbox-* netbox + if [ -n "${HTTP_PROXY}" ]; then + git config http.proxy "${HTTP_PROXY}" + fi + + $DRY git remote set-url origin "${URL}" + $DRY git fetch -qpP --depth 10 origin "${BRANCH}" + $DRY git checkout -qf FETCH_HEAD + $DRY git prune ) - echo "✅ Downloaded netbox" + echo "✅ Checked out netbox" fi ### @@ -120,7 +145,7 @@ DOCKERFILE="${DOCKERFILE-Dockerfile}" if [ ! -f "${DOCKERFILE}" ]; then echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist." - if [ -z "$DEBUG" ]; then + if [ -z "${DEBUG}" ]; then exit 1 else echo "⚠️ Would exit here with code '1', but DEBUG is enabled." @@ -152,13 +177,13 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" # Build each target ### for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do - echo "🏗 Building the target '$DOCKER_TARGET'" + echo "🏗 Building the target '${DOCKER_TARGET}'" ### # composing the final TARGET_DOCKER_TAG ### TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" - if [ "$DOCKER_TARGET" != "main" ]; then + if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi @@ -173,57 +198,63 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" - if [ "$DOCKER_TARGET" != "main" ]; then + if [ "${DOCKER_TARGET}" != "main" ]; then DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" fi fi ### - # Composing global Docker CLI arguments - ### - DOCKER_OPTS=( - --pull - --target "$DOCKER_TARGET" - ) - - ### - # Composing arguments for `docker build` CLI - ### - DOCKER_BUILD_ARGS=( - --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "NETBOX_BRANCH=${BRANCH}" - --build-arg "DOCKER_ORG=${DOCKER_ORG}" - --build-arg "DOCKER_REPO=${DOCKER_REPO}" - ) - if [ -n "$DOCKER_FROM" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) - fi - if [ -n "$HTTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) - fi - if [ -n "$HTTPS_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) - fi - if [ -n "$FTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" ) - fi - if [ -n "$NO_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) - fi - - ### - # Building the docker images, except if `--push-only` is passed + # Proceeding to buils stage, except if `--push-only` is passed ### if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." - $DRY docker build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . - echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" - - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DRY docker tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" - echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" + ### + # Composing all arguments for `docker build` + ### + DOCKER_BUILD_ARGS=( + --pull + --target "${DOCKER_TARGET}" + -f "${DOCKERFILE}" + -t "${TARGET_DOCKER_TAG}" + ) + if [ -n "${DOCKER_SHORT_TAG}" ]; then + DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" ) fi + + # --label + DOCKER_BUILD_ARGS+=( + --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" + --label "NETBOX_BRANCH=${BRANCH}" + --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" + ) + if [ -d "${NETBOX_PATH}/.git" ]; then + DOCKER_BUILD_ARGS+=( + --label "NETBOX_GIT_COMMIT=$($DRY cd ${NETBOX_PATH}; $DRY git rev-parse HEAD)" + --label "NETBOX_GIT_URL=$($DRY cd ${NETBOX_PATH}; $DRY git remote get-url origin)" + ) + fi + + # --build-arg + DOCKER_BUILD_ARGS+=( + --build-arg "NETBOX_PATH=${NETBOX_PATH}" + --build-arg "DOCKER_REPO=${DOCKER_REPO}" + ) + if [ -n "${DOCKER_FROM}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) + fi + if [ -n "${HTTP_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) + DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) + fi + if [ -n "${NO_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) + fi + + ### + # Building the docker image + ### + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." + $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . + echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" fi ### @@ -241,10 +272,3 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do fi fi done - -### -# Cleaning up -### -echo "🗑️ Cleaning up" -$DRY rm -rf .netbox -echo "✅ Cleaned up" From 123fd981e9eeaed6295a299886090481937d7857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 22:20:09 +0200 Subject: [PATCH 08/15] Fix build The Docker Hub build system runs on Ubuntu Xenial containers. Xenial's git is 2.7.4, which does not know the `-P` flag yet. --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index c9ee110..e14fc17 100755 --- a/build.sh +++ b/build.sh @@ -130,7 +130,7 @@ if [ "${2}" != "--push-only" ] ; then fi $DRY git remote set-url origin "${URL}" - $DRY git fetch -qpP --depth 10 origin "${BRANCH}" + $DRY git fetch -qp --depth 10 origin "${BRANCH}" $DRY git checkout -qf FETCH_HEAD $DRY git prune ) From 42642c94c3bb568aec49eb82d507218e77d71ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 23:10:42 +0200 Subject: [PATCH 09/15] Only push on 'main' branch --- hooks/push | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hooks/push b/hooks/push index 0c0b65f..c1e3ccb 100755 --- a/hooks/push +++ b/hooks/push @@ -2,4 +2,13 @@ . hooks/common -run_build --push-only +if [ "${SOURCE_BRANCH}" == "main" ] || [ "${DEBUG}" == "true" ]; then + if [ "${SOURCE_BRANCH}" != "main" ]; then + echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". + fi + + run_build --push-only +else + echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" + exit 0 +fi From 064908397e1526cbd66cf3a0942fea209f5646b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 23:16:51 +0200 Subject: [PATCH 10/15] Improved Build Job descriptions --- DOCKER_HUB.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index f378336..10c4154 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -14,14 +14,23 @@ Build Rules: Source: master Docker Tag: branches Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master Docker Tag: prerelease Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master Docker Tag: release Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on Build Environment Variables: # Create an app on Github and use it's OATH credentials here - Key: GITHUB_OAUTH_CLIENT_ID @@ -30,6 +39,7 @@ Build Environment Variables: Value: Build Triggers: - Name: Cron Trigger + Trigger URL: # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds ``` From 26b1f59d666c48a232113f681b05073f5cc552ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Tue, 15 Oct 2019 00:40:48 +0200 Subject: [PATCH 11/15] Improve formating and fix typo --- build-latest.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/build-latest.sh b/build-latest.sh index f16e211..4dea605 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -18,14 +18,14 @@ fi ### # Checking if PRERELEASE is either unset, 'true' or 'false' ### -if [ -n "$PRERELEASE" ] && - { [ "$PRERELEASE" != "true" ] && [ "$PRERELEASE" != "false" ]; }; then +if [ -n "${PRERELEASE}" ] && + { [ "${PRERELEASE}" != "true" ] && [ "${PRERELEAS}E" != "false" ]; }; then - if [ -z "$DEBUG" ]; then - echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was \"$PRERELEASE\"!" + if [ -z "${DEBUG}" ]; then + echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!" exit 1 else - echo "⚠️ Would exit here with code '0', but DEBUG is enabled." + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." fi fi @@ -60,10 +60,11 @@ if [ "${PRERELEASE}" == "true" ]; then # shellcheck disable=SC2003 MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)') - if { [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ]; } \ - || [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then + if { [ "${MAJOR_STABLE}" -eq "${MAJOR_UNSTABLE}" ] \ + && [ "${MINOR_STABLE}" -ge "${MINOR_UNSTABLE}" ]; + } || [ "${MAJOR_STABLE}" -gt "${MAJOR_UNSTABLE}" ]; then - echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')." + echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'." if [ -z "$DEBUG" ]; then exit 0 else From 0f4a872082165100bb83b90fb2be279efe29fffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Tue, 15 Oct 2019 00:47:19 +0200 Subject: [PATCH 12/15] More tests --- hooks/common | 2 +- hooks/test | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hooks/common b/hooks/common index fcf84a3..49cd507 100755 --- a/hooks/common +++ b/hooks/common @@ -53,7 +53,7 @@ export DOCKER_ORG="index.docker.io/netboxcommunity" export DOCKER_REPO=netbox export DOCKERHUB_REPO=netboxcommunity/netbox # shellcheck disable=SC2153 -export BUILD="$DOCKER_TAG" +export BUILD="${DOCKER_TAG}" unset DOCKER_TAG diff --git a/hooks/test b/hooks/test index 14ad88f..f741c45 100755 --- a/hooks/test +++ b/hooks/test @@ -3,11 +3,12 @@ . hooks/common # test 'branches' and 'this' (i.e. pull request) builds -if [ "${BUILD}" == "branches" ] || [ "${BUILD}" == "this" ]; then +if [ "${BUILD}" == "branches" ] \ + || [ "${BUILD}" == "this" ] \ + || [ "${DEBUG}" == "true" ]; then echo "🐳🐳🐳 Testing" - export DOCKER_TARGET="main" - - docker-compose run netbox ./manage.py test + VERSION=latest docker-compose run netbox ./manage.py test + VERSION=snapshot docker-compose run netbox ./manage.py test docker-compose down -v else echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." From fb608410471921b0d6d636164ac8f982457f4a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Tue, 15 Oct 2019 00:53:36 +0200 Subject: [PATCH 13/15] Only test 'latest' in PRs Because it's the only tag being built --- hooks/test | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hooks/test b/hooks/test index f741c45..1dd5538 100755 --- a/hooks/test +++ b/hooks/test @@ -2,14 +2,21 @@ . hooks/common -# test 'branches' and 'this' (i.e. pull request) builds -if [ "${BUILD}" == "branches" ] \ - || [ "${BUILD}" == "this" ] \ - || [ "${DEBUG}" == "true" ]; then - echo "🐳🐳🐳 Testing" - VERSION=latest docker-compose run netbox ./manage.py test - VERSION=snapshot docker-compose run netbox ./manage.py test +run_test() { + echo "🐳🐳🐳 Testing '${1}'" + VERSION="${1}" docker-compose run netbox ./manage.py test docker-compose down -v + echo "🐳🐳🐳 Done testing '${1}'" +} + +# test on builds of 'branches' +if [ "${BUILD}" == "branches" ] \ + || [ "${DEBUG}" == "true" ]; then + run_test latest + run_test snapshot +# test on bulds of 'this' (i.e. pull request) +elif [ "${BUILD}" == "this" ]; then + run_test latest else echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." fi From 0a38220497233b83408572d8a9a868202c04a6f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Tue, 15 Oct 2019 01:00:21 +0200 Subject: [PATCH 14/15] Update purpose text in build.sh --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index e14fc17..a268669 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds the Dockerfile and injects tgz'ed Netbox code from Github +# Clones the Netbox repository with git from Github and builds the Dockerfile echo "▶️ $0 $*" From e060f86b9aec1e31412312aa74c32076723029de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 15 Oct 2019 10:03:39 +0200 Subject: [PATCH 15/15] Corrections for shellcheck warnings --- build-latest.sh | 2 +- build.sh | 15 ++++++--------- hooks/build | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/build-latest.sh b/build-latest.sh index 4dea605..31a0f76 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -19,7 +19,7 @@ fi # Checking if PRERELEASE is either unset, 'true' or 'false' ### if [ -n "${PRERELEASE}" ] && - { [ "${PRERELEASE}" != "true" ] && [ "${PRERELEAS}E" != "false" ]; }; then + { [ "${PRERELEASE}" != "true" ] && [ "${PRERELEASE}" != "false" ]; }; then if [ -z "${DEBUG}" ]; then echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!" diff --git a/build.sh b/build.sh index a268669..123cd17 100755 --- a/build.sh +++ b/build.sh @@ -222,22 +222,19 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # --label DOCKER_BUILD_ARGS+=( - --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --label "NETBOX_BRANCH=${BRANCH}" - --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" + --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" + --label "NETBOX_BRANCH=${BRANCH}" + --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" ) if [ -d "${NETBOX_PATH}/.git" ]; then DOCKER_BUILD_ARGS+=( - --label "NETBOX_GIT_COMMIT=$($DRY cd ${NETBOX_PATH}; $DRY git rev-parse HEAD)" - --label "NETBOX_GIT_URL=$($DRY cd ${NETBOX_PATH}; $DRY git remote get-url origin)" + --label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)" + --label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)" ) fi # --build-arg - DOCKER_BUILD_ARGS+=( - --build-arg "NETBOX_PATH=${NETBOX_PATH}" - --build-arg "DOCKER_REPO=${DOCKER_REPO}" - ) + DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) if [ -n "${DOCKER_FROM}" ]; then DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) fi diff --git a/hooks/build b/hooks/build index 85bad75..80c4165 100755 --- a/hooks/build +++ b/hooks/build @@ -2,4 +2,5 @@ . hooks/common +# shellcheck disable=SC2119 run_build