diff --git a/.dockerignore b/.dockerignore index ce32f24..1b2bacc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ .travis.yml build* *.env +.git diff --git a/.gitignore b/.gitignore index 53a4e81..cbaffa8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.sql.gz +.netbox diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index 6149c3c..10c4154 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -12,28 +12,25 @@ Repository Links: Enable for Base Image Build Rules: - Source Type: Branch Source: master - Docker Tag: branches-main + Docker Tag: branches Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master - Docker Tag: branches-ldap - Dockerfile location: Dockerfile.ldap -- Source Type: Branch - Source: master - Docker Tag: prerelease-main + Docker Tag: prerelease Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master - Docker Tag: prerelease-ldap - Dockerfile location: Dockerfile.ldap -- Source Type: Branch - Source: master - Docker Tag: release-main + Docker Tag: release Dockerfile location: Dockerfile -- Source Type: Branch - Source: master - Docker Tag: release-ldap - Dockerfile location: Dockerfile.ldap + Build Context: / + Autobuild: on + Build Caching: on Build Environment Variables: # Create an app on Github and use it's OATH credentials here - Key: GITHUB_OAUTH_CLIENT_ID @@ -42,6 +39,7 @@ Build Environment Variables: Value: Build Triggers: - Name: Cron Trigger + Trigger URL: # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds ``` @@ -51,16 +49,15 @@ The build system of cloud.docker.com is not made for this kind of project. But we found a way to make it work, and this is how: 1. The docker hub build system [allows to overwrite the scripts that get executed - for `build`, `test` and `push`](overwrite). See `hooks/*`. -2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`. -3. The `build` script runs `run_build()` from `hooks/common`. - This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`. + for `build`, `test` and `push`](overwrite). See `/hooks/*`. +2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`. +3. The `build` script runs `run_build()` from `/hooks/common`. + This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`. 4. The `test` script just invokes `docker-compose` commands. 5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. This causes the `build.sh` script to not re-build the Docker image, but just the just built image. -The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_). - -The _Dockerfile location_ configuration setting is completely ignored by the build scripts. +The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`. +Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose. [overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands diff --git a/Dockerfile b/Dockerfile index bc7db56..c605e99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.7-alpine3.10 +ARG FROM=python:3.7-alpine +FROM ${FROM} as builder RUN apk add --no-cache \ bash \ @@ -7,51 +8,55 @@ RUN apk add --no-cache \ cyrus-sasl-dev \ graphviz \ jpeg-dev \ + libevent-dev \ libffi-dev \ - libxml2-dev \ libxslt-dev \ openldap-dev \ - postgresql-dev \ - ttf-ubuntu-font-family \ - wget + postgresql-dev -RUN pip install \ +WORKDIR /install + +RUN pip install --install-option="--prefix=/install" \ # gunicorn is used for launching netbox gunicorn \ + greenlet \ + eventlet \ # napalm is used for gathering information from network devices napalm \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ -# pinning django to the version required by netbox -# adding it here, to install the correct version of -# django-rq - 'Django>=2.2,<2.3' \ -# django-rq is used for webhooks - django-rq +# django_auth_ldap is required for ldap + django_auth_ldap -ARG BRANCH=master +ARG NETBOX_PATH +COPY ${NETBOX_PATH}/requirements.txt / +RUN pip install --install-option="--prefix=/install" -r /requirements.txt -WORKDIR /tmp +### +# Main stage +### -# As the requirements don't change very often, -# and as they take some time to compile, -# we try to cache them very agressively. -ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt -ADD ${REQUIREMENTS_URL} requirements.txt -RUN pip install -r requirements.txt +ARG FROM +FROM ${FROM} as main -# Cache bust when the upstream branch changes: -# ADD will fetch the file and check if it has changed -# If not, Docker will use the existing build cache. -# If yes, Docker will bust the cache and run every build step from here on. -ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH -ADD ${REF_URL} version.json +RUN apk add --no-cache \ + bash \ + ca-certificates \ + graphviz \ + libevent \ + libffi \ + libjpeg-turbo \ + libressl \ + libxslt \ + postgresql-libs \ + ttf-ubuntu-font-family WORKDIR /opt -ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz -RUN wget -q -O - "${URL}" | tar xz \ - && mv netbox* netbox +COPY --from=builder /install /usr/local + +ARG NETBOX_PATH +COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY configuration/gunicorn_config.py /etc/netbox/config/ @@ -67,7 +72,22 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] -LABEL SRC_URL="$URL" +LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \ + NETBOX_BRANCH="custom build" \ + ORIGINAL_DOCKER_TAG="custom build" \ + NETBOX_GIT_COMMIT="not built from git" \ + NETBOX_GIT_URL="not built from git" -ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot -LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" +##### +## LDAP specific configuration +##### + +FROM main as ldap + +RUN apk add --no-cache \ + libsasl \ + libldap \ + util-linux + +COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py +COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/Dockerfile.ldap b/Dockerfile.ldap deleted file mode 100644 index 303c3a8..0000000 --- a/Dockerfile.ldap +++ /dev/null @@ -1,9 +0,0 @@ -ARG DOCKER_ORG=netboxcommunity -ARG DOCKER_REPO=netbox -ARG FROM_TAG=latest -FROM $DOCKER_ORG/$DOCKER_REPO:$FROM_TAG - -RUN pip install django_auth_ldap - -COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py -COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/README.md b/README.md index 1e40a8a..e375bea 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,8 @@ Default credentials: This project relies only on *Docker* and *docker-compose* meeting this requirements: -* The *Docker version* must be at least `1.13.0`. -* The *docker-compose version* must be at least `1.10.0`. +* The *Docker version* must be at least `17.05`. +* The *docker-compose version* must be at least `1.17.0`. To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. @@ -69,24 +69,20 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.2.6 +export VERSION=v2.6.6 docker-compose pull netbox docker-compose up -d ``` -You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid -[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github]. -Most commonly you will specify a tag or branch name. +You can also build a specific version of the Netbox Docker image yourself. +`VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=develop -docker-compose build --no-cache netbox +export VERSION=v2.6.6 +./build.sh $VERSION docker-compose up -d ``` -Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary. -This can increase the build speed if you're just adjusting the config, for example. - [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References [netbox-github]: https://github.com/netbox-community/netbox/releases diff --git a/build-all.sh b/build-all.sh index d88c5af..462a83a 100755 --- a/build-all.sh +++ b/build-all.sh @@ -12,73 +12,40 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}") echo "⚙️ Configured builds: ${BUILDS[*]}" -VARIANTS=("main" "ldap") - if [ -n "${DEBUG}" ]; then export DEBUG fi ERROR=0 -# Don't build if not on `master` and don't build if on a pull request, -# but build when DEBUG is not empty -for VARIANT in "${VARIANTS[@]}"; do - export VARIANT - - # Checking which VARIANT to build - if [ "${VARIANT}" == "main" ]; then - DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}" - else - DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}" - - # Fail fast - if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." - ERROR=1 +for BUILD in "${BUILDS[@]}"; do + echo "🛠 Building '$BUILD' from '$DOCKERFILE'" + case $BUILD in + release) + # build the latest release + # shellcheck disable=SC2068 + ./build-latest.sh $@ || ERROR=1 + ;; + prerelease) + # build the latest pre-release + # shellcheck disable=SC2068 + PRERELEASE=true ./build-latest.sh $@ || ERROR=1 + ;; + branches) + # build all branches + # shellcheck disable=SC2068 + ./build-branches.sh $@ || ERROR=1 + ;; + *) + echo "🚨 Unrecognized build '$BUILD'." if [ -z "$DEBUG" ]; then - continue + exit 1 else - echo "⚠️ Would skip this, but DEBUG is enabled." + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." fi - fi - fi - - for BUILD in "${BUILDS[@]}"; do - echo "🛠 Building '$BUILD' from '$DOCKERFILE'" - case $BUILD in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ || ERROR=1 - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ || ERROR=1 - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ || ERROR=1 - ;; - special) - # special build - # shellcheck disable=SC2068 - #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1 - echo "✅ No special builds today." - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac - done + ;; + esac done exit $ERROR diff --git a/build-branches.sh b/build-branches.sh index 622e0ec..a6bc736 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -3,6 +3,10 @@ echo "▶️ $0 $*" +### +# Checking for the presence of GITHUB_OAUTH_CLIENT_ID +# and GITHUB_OAUTH_CLIENT_SECRET +### if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" @@ -11,18 +15,33 @@ else GITHUB_OAUTH_PARAMS="" fi +### +# Calling Github to get the all branches +### ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}" +# Composing the JQ commans to extract the most recent version number +JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string' + CURL="curl -sS" -BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string') +# Querying the Github API to fetch all branches +BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES") +### +# Building each branch +### + +# keeping track whether an error occured ERROR=0 +# calling build.sh for each branch for BRANCH in $BRANCHES; do # shellcheck disable=SC2068 ./build.sh "${BRANCH}" $@ || ERROR=1 done + +# returning whether an error occured exit $ERROR diff --git a/build-latest.sh b/build-latest.sh index b86c280..31a0f76 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -3,6 +3,10 @@ echo "▶️ $0 $*" +### +# Checking for the presence of GITHUB_OAUTH_CLIENT_ID +# and GITHUB_OAUTH_CLIENT_SECRET +### if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" @@ -11,17 +15,38 @@ else GITHUB_OAUTH_PARAMS="" fi +### +# Checking if PRERELEASE is either unset, 'true' or 'false' +### +if [ -n "${PRERELEASE}" ] && + { [ "${PRERELEASE}" != "true" ] && [ "${PRERELEASE}" != "false" ]; }; then + + if [ -z "${DEBUG}" ]; then + echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!" + exit 1 + else + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." + fi +fi + +### +# Calling Github to get the latest version +### ORIGINAL_GITHUB_REPO="netbox-community/netbox" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}" +# Composing the JQ commans to extract the most recent version number JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name" CURL="curl -sS" +# Querying the Github API to fetch the most recent version number VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}") +### # Check if the prerelease version is actually higher than stable version +### if [ "${PRERELEASE}" == "true" ]; then JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name" STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}") @@ -35,9 +60,11 @@ if [ "${PRERELEASE}" == "true" ]; then # shellcheck disable=SC2003 MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)') - if ( [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ] ) \ - || [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then - echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')." + if { [ "${MAJOR_STABLE}" -eq "${MAJOR_UNSTABLE}" ] \ + && [ "${MINOR_STABLE}" -ge "${MINOR_UNSTABLE}" ]; + } || [ "${MAJOR_STABLE}" -gt "${MAJOR_UNSTABLE}" ]; then + + echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'." if [ -z "$DEBUG" ]; then exit 0 else @@ -46,32 +73,6 @@ if [ "${PRERELEASE}" == "true" ]; then fi fi -# Check if that version is not already available on docker hub: -ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}" -DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}" -URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull" -BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)" - -URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list" -AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}" - -if [ -z "$VARIANT" ] || [ "$VARIANT" == "main" ]; then - DOCKER_TAG="${VERSION}" -else - DOCKER_TAG="${VERSION}-${VARIANT}" -fi - -ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")" - -if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then - if [ -n "$DEBUG" ]; then - echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled." - fi - - # shellcheck disable=SC2068 - ./build.sh "${VERSION}" $@ - exit $? -else - echo "✅ ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}" - exit 0 -fi +# shellcheck disable=SC2068 +./build.sh "${VERSION}" $@ +exit $? diff --git a/build.sh b/build.sh index 3255fd5..123cd17 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds the Dockerfile[.variant] and injects tgz'ed Netbox code from Github +# Clones the Netbox repository with git from Github and builds the Dockerfile echo "▶️ $0 $*" @@ -8,66 +8,78 @@ set -e if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo "Usage: ${0} [--push|--push-only]" echo " branch The branch or tag to build. Required." - echo " --push Pushes built the Docker image to the registry." - echo " --push-only Does not build. Only pushes the Docker image to the registry." + echo " --push Pushes the built Docker image to the registry." + echo " --push-only Only pushes the Docker image to the registry, but does not build it." echo "" echo "You can use the following ENV variables to customize the build:" - echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." - echo " DRY_RUN Prints all build statements instead of running them." - echo " DOCKER_OPTS Add parameters to Docker." - echo " Default:" - echo " When starts with 'v': \"\"" - echo " Else: \"--no-cache\"" - echo " BRANCH The branch to build." - echo " Also used for tagging the image." - echo " TAG The version part of the docker tag." - echo " Default:" - echo " When =master: latest" - echo " When =develop: snapshot" - echo " Else: same as " - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//) " - echo " Also used for tagging the image." - echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r//) " - echo " Also used for tagging the image." - echo " Default: netbox" - echo " DOCKER_TAG The name of the tag which is applied to the image." - echo " Useful for pushing into another registry than hub.docker.com." - echo " Default: /:" - echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the image." - echo " This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5" - echo " Default: /:\$MAJOR.\$MINOR" - echo " SRC_ORG Which fork of netbox to use (i.e. github.com//)." - echo " Default: netbox-community" - echo " SRC_REPO The name of the netbox for to use (i.e. github.com//)." - echo " Default: netbox" - echo " URL Where to fetch the package from." - echo " Must be a tar.gz file of the source code." - echo " Default: https://github.com///archive/\$BRANCH.tar.gz" - echo " VARIANT The variant to build." - echo " The value will be used as a suffix to the \$TAG and for the Dockerfile" - echo " selection. The TAG being build must exist for the base variant and" - echo " corresponding Dockerfile must start with the following lines:" - echo " ARG DOCKER_ORG=netboxcommunity" - echo " ARG DOCKER_REPO=netbox" - echo " ARG FROM_TAG=latest" - echo " FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG" - echo " Example: VARIANT=ldap will result in the tag 'latest-ldap' and the" - echo " Dockerfile './Dockerfile.ldap' being used." - echo " Exception: VARIANT=main will use the './Dockerfile' Dockerfile" - echo " Default: main" - echo " HTTP_PROXY The proxy to use for http requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " HTTPS_PROXY The proxy to use for https requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " FTP_PROXY The proxy to use for ftp requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." - echo " Example: .domain1.tld,.domain2.tld" - echo " Default: empty" + echo " SRC_ORG Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox-community" + echo " SRC_REPO The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox" + echo " URL Where to fetch the code from." + echo " Must be a git repository. Can be private." + echo " Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git" + echo " NETBOX_PATH The path where netbox will be checkout out." + echo " Must not be outside of the netbox-docker repository (because of Docker)!" + echo " Default: .netbox" + echo " SKIP_GIT If defined, git is not invoked and \${NETBOX_PATH} will not be altered." + echo " This may be useful, if you are manually managing the NETBOX_PATH." + echo " Default: undefined" + echo " TAG The version part of the docker tag." + echo " Default:" + echo " When \${BRANCH}=master: latest" + echo " When \${BRANCH}=develop: snapshot" + echo " Else: same as \${BRANCH}" + echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netboxcommunity" + echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netbox" + echo " DOCKER_FROM The base image to use." + echo " Default: Whatever is defined as default in the Dockerfile." + echo " DOCKER_TAG The name of the tag which is applied to the image." + echo " Useful for pushing into another registry than hub.docker.com." + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" + echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" + echo " image. This is used to tag all patch releases to their" + echo " containing version e.g. v2.5.1 -> v2.5" + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:." + echo " DOCKERFILE The name of Dockerfile to use." + echo " Default: Dockerfile" + echo " DOCKER_TARGET A specific target to build." + echo " It's currently not possible to pass multiple targets." + echo " Default: main ldap" + echo " HTTP_PROXY The proxy to use for http requests." + echo " Example: http://proxy.domain.tld:3128" + echo " Default: undefined" + echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." + echo " Example: .domain1.tld,.domain2.tld" + echo " Default: undefined" + echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." + echo " Default: undefined" + echo " DRY_RUN Prints all build statements instead of running them." + echo " Default: undefined" + echo "" + echo "Examples:" + echo " ${0} master" + echo " This will fetch the latest 'master' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:latest'." + echo " ${0} develop" + echo " This will fetch the latest 'develop' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:snapshot'." + echo " ${0} v2.6.6" + echo " This will fetch the 'v2.6.6' tag, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'." + echo " ${0} develop-2.7" + echo " This will fetch the 'develop-2.7' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:develop-2.7'." + echo " SRC_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'." + echo " SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'cimnine/netbox:feature-x'." if [ "${1}x" == "x" ]; then exit 1 @@ -76,36 +88,73 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi fi -# read the project version and trim it +### +# Determining the build command to use +### +if [ -z "${DRY_RUN}" ]; then + DRY="" +else + echo "⚠️ DRY_RUN MODE ON ⚠️" + DRY="echo" +fi + +### +# read the project version from the `VERSION` file and trim it # see https://stackoverflow.com/a/3232433/172132 +### NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" +### # variables for fetching the source +### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" -URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" +URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" -# Checking which VARIANT to build -VARIANT="${VARIANT-main}" -if [ "$VARIANT" == "main" ]; then - DOCKERFILE="Dockerfile" -else - DOCKERFILE="Dockerfile.${VARIANT}" +### +# fetching the source +### +if [ "${2}" != "--push-only" ] ; then + NETBOX_PATH="${NETBOX_PATH-.netbox}" + echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + if [ ! -d "${NETBOX_PATH}" ]; then + $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + fi + + ( + $DRY cd "${NETBOX_PATH}" + + if [ -n "${HTTP_PROXY}" ]; then + git config http.proxy "${HTTP_PROXY}" + fi + + $DRY git remote set-url origin "${URL}" + $DRY git fetch -qp --depth 10 origin "${BRANCH}" + $DRY git checkout -qf FETCH_HEAD + $DRY git prune + ) + echo "✅ Checked out netbox" fi -# Fail fast +### +# Determining the value for DOCKERFILE +# and checking whether it exists +### +DOCKERFILE="${DOCKERFILE-Dockerfile}" if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile ${DOCKERFILE} for variant '${VARIANT}' doesn't exist." + echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist." - if [ -z "$DEBUG" ]; then + if [ -z "${DEBUG}" ]; then exit 1 else echo "⚠️ Would exit here with code '1', but DEBUG is enabled." fi fi +### # variables for tagging the docker image +### DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" case "${BRANCH}" in @@ -117,81 +166,106 @@ case "${BRANCH}" in TAG="${TAG-$BRANCH}";; esac -DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" -if [ "$VARIANT" != "main" ]; then - DOCKER_TAG="${DOCKER_TAG}-${VARIANT}" -fi +### +# Determine targets to build +### +DEFAULT_DOCKER_TARGETS=("main" "ldap") +DOCKER_TARGETS=( "${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}") +echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" -if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then - MAJOR=${BASH_REMATCH[1]} - MINOR=${BASH_REMATCH[2]} +### +# Build each target +### +for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do + echo "🏗 Building the target '${DOCKER_TARGET}'" - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" - - if [ "$VARIANT" != "main" ]; then - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${VARIANT}" + ### + # composing the final TARGET_DOCKER_TAG + ### + TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" + if [ "${DOCKER_TARGET}" != "main" ]; then + TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi -fi -DOCKER_OPTS=("${DOCKER_OPTS[@]}") + ### + # composing the additional DOCKER_SHORT_TAG, + # i.e. "v2.6.1" becomes "v2.6", + # which is only relevant for version tags + ### + if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then + MAJOR=${BASH_REMATCH[1]} + MINOR=${BASH_REMATCH[2]} -# caching is only ok for version tags -case "${TAG}" in - v*) ;; - *) DOCKER_OPTS+=( "--no-cache" ) ;; -esac + DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" -DOCKER_OPTS+=( "--pull" ) - -# Build args -DOCKER_BUILD_ARGS=( - --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "FROM_TAG=${TAG}" - --build-arg "BRANCH=${BRANCH}" - --build-arg "URL=${URL}" - --build-arg "DOCKER_ORG=${DOCKER_ORG}" - --build-arg "DOCKER_REPO=${DOCKER_REPO}" -) -if [ -n "$HTTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) -fi -if [ -n "$HTTPS_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) -fi -if [ -n "$FTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" ) -fi -if [ -n "$NO_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) -fi - -if [ -z "$DRY_RUN" ]; then - DOCKER_CMD="docker" -else - echo "⚠️ DRY_RUN MODE ON ⚠️" - DOCKER_CMD="echo docker" -fi - -if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." - $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . - echo "✅ Finished building the Docker images '${DOCKER_TAG}'" - - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}" - echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" + if [ "${DOCKER_TARGET}" != "main" ]; then + DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" + fi fi -fi -if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then - echo "⏫ Pushing '${DOCKER_TAG}" - $DOCKER_CMD push "${DOCKER_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." + ### + # Proceeding to buils stage, except if `--push-only` is passed + ### + if [ "${2}" != "--push-only" ] ; then + ### + # Composing all arguments for `docker build` + ### + DOCKER_BUILD_ARGS=( + --pull + --target "${DOCKER_TARGET}" + -f "${DOCKERFILE}" + -t "${TARGET_DOCKER_TAG}" + ) + if [ -n "${DOCKER_SHORT_TAG}" ]; then + DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" ) + fi - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DOCKER_CMD push "${DOCKER_SHORT_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + # --label + DOCKER_BUILD_ARGS+=( + --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" + --label "NETBOX_BRANCH=${BRANCH}" + --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" + ) + if [ -d "${NETBOX_PATH}/.git" ]; then + DOCKER_BUILD_ARGS+=( + --label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)" + --label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)" + ) + fi + + # --build-arg + DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) + if [ -n "${DOCKER_FROM}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) + fi + if [ -n "${HTTP_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) + DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) + fi + if [ -n "${NO_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) + fi + + ### + # Building the docker image + ### + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." + $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . + echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" fi -fi + + ### + # Pushing the docker images if either `--push` or `--push-only` are passed + ### + if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then + echo "⏫ Pushing '${TARGET_DOCKER_TAG}" + $DRY docker push "${TARGET_DOCKER_TAG}" + echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." + + if [ -n "$DOCKER_SHORT_TAG" ]; then + echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" + $DRY docker push "${DOCKER_SHORT_TAG}" + echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + fi + fi +done diff --git a/docker-compose.yml b/docker-compose.yml index 6c80b72..f70a93f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,6 @@ -version: '3' +version: '3.4' services: netbox: &netbox - build: - context: . - args: - - BRANCH=${VERSION-master} image: netboxcommunity/netbox:${VERSION-latest} depends_on: - postgres diff --git a/hooks/build b/hooks/build index 85bad75..80c4165 100755 --- a/hooks/build +++ b/hooks/build @@ -2,4 +2,5 @@ . hooks/common +# shellcheck disable=SC2119 run_build diff --git a/hooks/common b/hooks/common index dc60799..49cd507 100755 --- a/hooks/common +++ b/hooks/common @@ -1,37 +1,19 @@ #!/bin/bash ensure_jq() { - echo "🛠🛠🛠 Installing JQ via apt-get" - [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq ) -} - -ensure_dockerfile_present() { - if [ "${VARIANT}" == "main" ]; then - DOCKERFILE="Dockerfile" - else - DOCKERFILE="Dockerfile.${VARIANT}" - - # Fail fast - if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would skip this, but DEBUG is enabled." - fi - fi - - if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then - echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'." - echo " This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'." + if [ ! -x "$(command -v jq)" ]; then + if [ -x "$(command -v apt-get)" ]; then + echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" + apt-get update && apt-get install -y jq + else + echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'." fi fi } # Passes args to the scripts run_build() { - echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant" + echo "🐳🐳🐳 Building '${BUILD}' images" case $BUILD in release) # build the latest release @@ -48,11 +30,11 @@ run_build() { # shellcheck disable=SC2068 ./build-branches.sh $@ ;; - special) - # special build - # shellcheck disable=SC2068 - #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ - echo "✅ No special builds today." + this) # Pull Requests + # only build the 'master' branch + # (resulting in the 'latest' docker tag) + # and the 'main' target. + DOCKER_TARGET=main ./build.sh master ;; *) echo "🚨 Unrecognized build '$BUILD'." @@ -70,13 +52,9 @@ echo "🤖🤖🤖 Preparing build" export DOCKER_ORG="index.docker.io/netboxcommunity" export DOCKER_REPO=netbox export DOCKERHUB_REPO=netboxcommunity/netbox - -# mis-using the "${DOCKER_TAG}" variable as "branch to build" -export BUILD="${DOCKER_TAG%-*}" -export VARIANT="${DOCKER_TAG#*-}" +# shellcheck disable=SC2153 +export BUILD="${DOCKER_TAG}" unset DOCKER_TAG -ensure_dockerfile_present - ensure_jq diff --git a/hooks/push b/hooks/push index 0c0b65f..c1e3ccb 100755 --- a/hooks/push +++ b/hooks/push @@ -2,4 +2,13 @@ . hooks/common -run_build --push-only +if [ "${SOURCE_BRANCH}" == "main" ] || [ "${DEBUG}" == "true" ]; then + if [ "${SOURCE_BRANCH}" != "main" ]; then + echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". + fi + + run_build --push-only +else + echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" + exit 0 +fi diff --git a/hooks/test b/hooks/test index 6c89c87..1dd5538 100755 --- a/hooks/test +++ b/hooks/test @@ -2,11 +2,21 @@ . hooks/common -if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then - echo "🐳🐳🐳 Testing" - docker-compose pull --parallel - docker-compose build - docker-compose run netbox ./manage.py test +run_test() { + echo "🐳🐳🐳 Testing '${1}'" + VERSION="${1}" docker-compose run netbox ./manage.py test + docker-compose down -v + echo "🐳🐳🐳 Done testing '${1}'" +} + +# test on builds of 'branches' +if [ "${BUILD}" == "branches" ] \ + || [ "${DEBUG}" == "true" ]; then + run_test latest + run_test snapshot +# test on bulds of 'this' (i.e. pull request) +elif [ "${BUILD}" == "this" ]; then + run_test latest else - echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'." + echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." fi