Merge pull request #159 from netbox-community/multistage-build

Refactor to multistage builds
This commit is contained in:
Christian Mäder 2019-10-15 11:47:09 +02:00 committed by GitHub
commit ef989284c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 411 additions and 350 deletions

View File

@ -2,3 +2,4 @@
.travis.yml .travis.yml
build* build*
*.env *.env
.git

1
.gitignore vendored
View File

@ -1 +1,2 @@
*.sql.gz *.sql.gz
.netbox

View File

@ -12,28 +12,25 @@ Repository Links: Enable for Base Image
Build Rules: Build Rules:
- Source Type: Branch - Source Type: Branch
Source: master Source: master
Docker Tag: branches-main Docker Tag: branches
Dockerfile location: Dockerfile Dockerfile location: Dockerfile
Build Context: /
Autobuild: on
Build Caching: on
- Source Type: Branch - Source Type: Branch
Source: master Source: master
Docker Tag: branches-ldap Docker Tag: prerelease
Dockerfile location: Dockerfile.ldap
- Source Type: Branch
Source: master
Docker Tag: prerelease-main
Dockerfile location: Dockerfile Dockerfile location: Dockerfile
Build Context: /
Autobuild: on
Build Caching: on
- Source Type: Branch - Source Type: Branch
Source: master Source: master
Docker Tag: prerelease-ldap Docker Tag: release
Dockerfile location: Dockerfile.ldap
- Source Type: Branch
Source: master
Docker Tag: release-main
Dockerfile location: Dockerfile Dockerfile location: Dockerfile
- Source Type: Branch Build Context: /
Source: master Autobuild: on
Docker Tag: release-ldap Build Caching: on
Dockerfile location: Dockerfile.ldap
Build Environment Variables: Build Environment Variables:
# Create an app on Github and use it's OATH credentials here # Create an app on Github and use it's OATH credentials here
- Key: GITHUB_OAUTH_CLIENT_ID - Key: GITHUB_OAUTH_CLIENT_ID
@ -42,6 +39,7 @@ Build Environment Variables:
Value: <secret> Value: <secret>
Build Triggers: Build Triggers:
- Name: Cron Trigger - Name: Cron Trigger
Trigger URL: <generated>
# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
``` ```
@ -51,16 +49,15 @@ The build system of cloud.docker.com is not made for this kind of project.
But we found a way to make it work, and this is how: But we found a way to make it work, and this is how:
1. The docker hub build system [allows to overwrite the scripts that get executed 1. The docker hub build system [allows to overwrite the scripts that get executed
for `build`, `test` and `push`](overwrite). See `hooks/*`. for `build`, `test` and `push`](overwrite). See `/hooks/*`.
2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`. 2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`.
3. The `build` script runs `run_build()` from `hooks/common`. 3. The `build` script runs `run_build()` from `/hooks/common`.
This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`. This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`.
4. The `test` script just invokes `docker-compose` commands. 4. The `test` script just invokes `docker-compose` commands.
5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. 5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
This causes the `build.sh` script to not re-build the Docker image, but just the just built image. This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_). The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`.
Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose.
The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands [overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands

View File

@ -1,4 +1,5 @@
FROM python:3.7-alpine3.10 ARG FROM=python:3.7-alpine
FROM ${FROM} as builder
RUN apk add --no-cache \ RUN apk add --no-cache \
bash \ bash \
@ -7,51 +8,55 @@ RUN apk add --no-cache \
cyrus-sasl-dev \ cyrus-sasl-dev \
graphviz \ graphviz \
jpeg-dev \ jpeg-dev \
libevent-dev \
libffi-dev \ libffi-dev \
libxml2-dev \
libxslt-dev \ libxslt-dev \
openldap-dev \ openldap-dev \
postgresql-dev \ postgresql-dev
ttf-ubuntu-font-family \
wget
RUN pip install \ WORKDIR /install
RUN pip install --install-option="--prefix=/install" \
# gunicorn is used for launching netbox # gunicorn is used for launching netbox
gunicorn \ gunicorn \
greenlet \
eventlet \
# napalm is used for gathering information from network devices # napalm is used for gathering information from network devices
napalm \ napalm \
# ruamel is used in startup_scripts # ruamel is used in startup_scripts
'ruamel.yaml>=0.15,<0.16' \ 'ruamel.yaml>=0.15,<0.16' \
# pinning django to the version required by netbox # django_auth_ldap is required for ldap
# adding it here, to install the correct version of django_auth_ldap
# django-rq
'Django>=2.2,<2.3' \
# django-rq is used for webhooks
django-rq
ARG BRANCH=master ARG NETBOX_PATH
COPY ${NETBOX_PATH}/requirements.txt /
RUN pip install --install-option="--prefix=/install" -r /requirements.txt
WORKDIR /tmp ###
# Main stage
###
# As the requirements don't change very often, ARG FROM
# and as they take some time to compile, FROM ${FROM} as main
# we try to cache them very agressively.
ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt
ADD ${REQUIREMENTS_URL} requirements.txt
RUN pip install -r requirements.txt
# Cache bust when the upstream branch changes: RUN apk add --no-cache \
# ADD will fetch the file and check if it has changed bash \
# If not, Docker will use the existing build cache. ca-certificates \
# If yes, Docker will bust the cache and run every build step from here on. graphviz \
ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH libevent \
ADD ${REF_URL} version.json libffi \
libjpeg-turbo \
libressl \
libxslt \
postgresql-libs \
ttf-ubuntu-font-family
WORKDIR /opt WORKDIR /opt
ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz COPY --from=builder /install /usr/local
RUN wget -q -O - "${URL}" | tar xz \
&& mv netbox* netbox ARG NETBOX_PATH
COPY ${NETBOX_PATH} /opt/netbox
COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
COPY configuration/gunicorn_config.py /etc/netbox/config/ COPY configuration/gunicorn_config.py /etc/netbox/config/
@ -67,7 +72,22 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
LABEL SRC_URL="$URL" LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \
NETBOX_BRANCH="custom build" \
ORIGINAL_DOCKER_TAG="custom build" \
NETBOX_GIT_COMMIT="not built from git" \
NETBOX_GIT_URL="not built from git"
ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot #####
LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" ## LDAP specific configuration
#####
FROM main as ldap
RUN apk add --no-cache \
libsasl \
libldap \
util-linux
COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

View File

@ -1,9 +0,0 @@
ARG DOCKER_ORG=netboxcommunity
ARG DOCKER_REPO=netbox
ARG FROM_TAG=latest
FROM $DOCKER_ORG/$DOCKER_REPO:$FROM_TAG
RUN pip install django_auth_ldap
COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

View File

@ -49,8 +49,8 @@ Default credentials:
This project relies only on *Docker* and *docker-compose* meeting this requirements: This project relies only on *Docker* and *docker-compose* meeting this requirements:
* The *Docker version* must be at least `1.13.0`. * The *Docker version* must be at least `17.05`.
* The *docker-compose version* must be at least `1.10.0`. * The *docker-compose version* must be at least `1.17.0`.
To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above.
@ -69,24 +69,20 @@ To use this feature, set the environment-variable `VERSION` before launching `do
[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
```bash ```bash
export VERSION=v2.2.6 export VERSION=v2.6.6
docker-compose pull netbox docker-compose pull netbox
docker-compose up -d docker-compose up -d
``` ```
You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid You can also build a specific version of the Netbox Docker image yourself.
[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github]. `VERSION` can be any valid [git ref][git-ref] in that case.
Most commonly you will specify a tag or branch name.
```bash ```bash
export VERSION=develop export VERSION=v2.6.6
docker-compose build --no-cache netbox ./build.sh $VERSION
docker-compose up -d docker-compose up -d
``` ```
Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary.
This can increase the build speed if you're just adjusting the config, for example.
[git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
[netbox-github]: https://github.com/netbox-community/netbox/releases [netbox-github]: https://github.com/netbox-community/netbox/releases

View File

@ -12,38 +12,12 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
echo "⚙️ Configured builds: ${BUILDS[*]}" echo "⚙️ Configured builds: ${BUILDS[*]}"
VARIANTS=("main" "ldap")
if [ -n "${DEBUG}" ]; then if [ -n "${DEBUG}" ]; then
export DEBUG export DEBUG
fi fi
ERROR=0 ERROR=0
# Don't build if not on `master` and don't build if on a pull request,
# but build when DEBUG is not empty
for VARIANT in "${VARIANTS[@]}"; do
export VARIANT
# Checking which VARIANT to build
if [ "${VARIANT}" == "main" ]; then
DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
else
DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"
# Fail fast
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
ERROR=1
if [ -z "$DEBUG" ]; then
continue
else
echo "⚠️ Would skip this, but DEBUG is enabled."
fi
fi
fi
for BUILD in "${BUILDS[@]}"; do for BUILD in "${BUILDS[@]}"; do
echo "🛠 Building '$BUILD' from '$DOCKERFILE'" echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
case $BUILD in case $BUILD in
@ -62,12 +36,6 @@ for VARIANT in "${VARIANTS[@]}"; do
# shellcheck disable=SC2068 # shellcheck disable=SC2068
./build-branches.sh $@ || ERROR=1 ./build-branches.sh $@ || ERROR=1
;; ;;
special)
# special build
# shellcheck disable=SC2068
#SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
echo "✅ No special builds today."
;;
*) *)
echo "🚨 Unrecognized build '$BUILD'." echo "🚨 Unrecognized build '$BUILD'."
@ -79,6 +47,5 @@ for VARIANT in "${VARIANTS[@]}"; do
;; ;;
esac esac
done done
done
exit $ERROR exit $ERROR

View File

@ -3,6 +3,10 @@
echo "▶️ $0 $*" echo "▶️ $0 $*"
###
# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
# and GITHUB_OAUTH_CLIENT_SECRET
###
if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
echo "🗝 Performing authenticated Github API calls." echo "🗝 Performing authenticated Github API calls."
GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@ -11,18 +15,33 @@ else
GITHUB_OAUTH_PARAMS="" GITHUB_OAUTH_PARAMS=""
fi fi
###
# Calling Github to get the all branches
###
ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}" ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
# Composing the JQ commans to extract the most recent version number
JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
CURL="curl -sS" CURL="curl -sS"
BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string') # Querying the Github API to fetch all branches
BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
###
# Building each branch
###
# keeping track whether an error occured
ERROR=0 ERROR=0
# calling build.sh for each branch
for BRANCH in $BRANCHES; do for BRANCH in $BRANCHES; do
# shellcheck disable=SC2068 # shellcheck disable=SC2068
./build.sh "${BRANCH}" $@ || ERROR=1 ./build.sh "${BRANCH}" $@ || ERROR=1
done done
# returning whether an error occured
exit $ERROR exit $ERROR

View File

@ -3,6 +3,10 @@
echo "▶️ $0 $*" echo "▶️ $0 $*"
###
# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
# and GITHUB_OAUTH_CLIENT_SECRET
###
if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
echo "🗝 Performing authenticated Github API calls." echo "🗝 Performing authenticated Github API calls."
GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@ -11,17 +15,38 @@ else
GITHUB_OAUTH_PARAMS="" GITHUB_OAUTH_PARAMS=""
fi fi
###
# Checking if PRERELEASE is either unset, 'true' or 'false'
###
if [ -n "${PRERELEASE}" ] &&
{ [ "${PRERELEASE}" != "true" ] && [ "${PRERELEASE}" != "false" ]; }; then
if [ -z "${DEBUG}" ]; then
echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!"
exit 1
else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
fi
###
# Calling Github to get the latest version
###
ORIGINAL_GITHUB_REPO="netbox-community/netbox" ORIGINAL_GITHUB_REPO="netbox-community/netbox"
GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}" GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}" URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}"
# Composing the JQ commans to extract the most recent version number
JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name" JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name"
CURL="curl -sS" CURL="curl -sS"
# Querying the Github API to fetch the most recent version number
VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}") VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}")
###
# Check if the prerelease version is actually higher than stable version # Check if the prerelease version is actually higher than stable version
###
if [ "${PRERELEASE}" == "true" ]; then if [ "${PRERELEASE}" == "true" ]; then
JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name" JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name"
STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}") STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}")
@ -35,9 +60,11 @@ if [ "${PRERELEASE}" == "true" ]; then
# shellcheck disable=SC2003 # shellcheck disable=SC2003
MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)') MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)')
if ( [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ] ) \ if { [ "${MAJOR_STABLE}" -eq "${MAJOR_UNSTABLE}" ] \
|| [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then && [ "${MINOR_STABLE}" -ge "${MINOR_UNSTABLE}" ];
echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')." } || [ "${MAJOR_STABLE}" -gt "${MAJOR_UNSTABLE}" ]; then
echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'."
if [ -z "$DEBUG" ]; then if [ -z "$DEBUG" ]; then
exit 0 exit 0
else else
@ -46,32 +73,6 @@ if [ "${PRERELEASE}" == "true" ]; then
fi fi
fi fi
# Check if that version is not already available on docker hub:
ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}"
DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}"
URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull"
BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)"
URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list"
AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}"
if [ -z "$VARIANT" ] || [ "$VARIANT" == "main" ]; then
DOCKER_TAG="${VERSION}"
else
DOCKER_TAG="${VERSION}-${VARIANT}"
fi
ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")"
if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then
if [ -n "$DEBUG" ]; then
echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled."
fi
# shellcheck disable=SC2068 # shellcheck disable=SC2068
./build.sh "${VERSION}" $@ ./build.sh "${VERSION}" $@
exit $? exit $?
else
echo "${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}"
exit 0
fi

292
build.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# Builds the Dockerfile[.variant] and injects tgz'ed Netbox code from Github # Clones the Netbox repository with git from Github and builds the Dockerfile
echo "▶️ $0 $*" echo "▶️ $0 $*"
@ -8,66 +8,78 @@ set -e
if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
echo "Usage: ${0} <branch> [--push|--push-only]" echo "Usage: ${0} <branch> [--push|--push-only]"
echo " branch The branch or tag to build. Required." echo " branch The branch or tag to build. Required."
echo " --push Pushes built the Docker image to the registry." echo " --push Pushes the built Docker image to the registry."
echo " --push-only Does not build. Only pushes the Docker image to the registry." echo " --push-only Only pushes the Docker image to the registry, but does not build it."
echo "" echo ""
echo "You can use the following ENV variables to customize the build:" echo "You can use the following ENV variables to customize the build:"
echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." echo " SRC_ORG Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
echo " DRY_RUN Prints all build statements instead of running them." echo " Default: netbox-community"
echo " DOCKER_OPTS Add parameters to Docker." echo " SRC_REPO The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
echo " Default:" echo " Default: netbox"
echo " When <TAG> starts with 'v': \"\"" echo " URL Where to fetch the code from."
echo " Else: \"--no-cache\"" echo " Must be a git repository. Can be private."
echo " BRANCH The branch to build." echo " Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git"
echo " Also used for tagging the image." echo " NETBOX_PATH The path where netbox will be checkout out."
echo " Must not be outside of the netbox-docker repository (because of Docker)!"
echo " Default: .netbox"
echo " SKIP_GIT If defined, git is not invoked and \${NETBOX_PATH} will not be altered."
echo " This may be useful, if you are manually managing the NETBOX_PATH."
echo " Default: undefined"
echo " TAG The version part of the docker tag." echo " TAG The version part of the docker tag."
echo " Default:" echo " Default:"
echo " When <BRANCH>=master: latest" echo " When \${BRANCH}=master: latest"
echo " When <BRANCH>=develop: snapshot" echo " When \${BRANCH}=develop: snapshot"
echo " Else: same as <BRANCH>" echo " Else: same as \${BRANCH}"
echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) " echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
echo " Also used for tagging the image." echo " Also used for tagging the image."
echo " Default: netboxcommunity" echo " Default: netboxcommunity"
echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) " echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
echo " Also used for tagging the image." echo " Also used for tagging the image."
echo " Default: netbox" echo " Default: netbox"
echo " DOCKER_FROM The base image to use."
echo " Default: Whatever is defined as default in the Dockerfile."
echo " DOCKER_TAG The name of the tag which is applied to the image." echo " DOCKER_TAG The name of the tag which is applied to the image."
echo " Useful for pushing into another registry than hub.docker.com." echo " Useful for pushing into another registry than hub.docker.com."
echo " Default: <DOCKER_ORG>/<DOCKER_REPO>:<BRANCH>" echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}"
echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the image." echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the"
echo " This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5" echo " image. This is used to tag all patch releases to their"
echo " Default: <DOCKER_ORG>/<DOCKER_REPO>:\$MAJOR.\$MINOR" echo " containing version e.g. v2.5.1 -> v2.5"
echo " SRC_ORG Which fork of netbox to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)." echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:<MAJOR>.<MINOR>"
echo " Default: netbox-community" echo " DOCKERFILE The name of Dockerfile to use."
echo " SRC_REPO The name of the netbox for to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)." echo " Default: Dockerfile"
echo " Default: netbox" echo " DOCKER_TARGET A specific target to build."
echo " URL Where to fetch the package from." echo " It's currently not possible to pass multiple targets."
echo " Must be a tar.gz file of the source code." echo " Default: main ldap"
echo " Default: https://github.com/<SRC_ORG>/<SRC_REPO>/archive/\$BRANCH.tar.gz"
echo " VARIANT The variant to build."
echo " The value will be used as a suffix to the \$TAG and for the Dockerfile"
echo " selection. The TAG being build must exist for the base variant and"
echo " corresponding Dockerfile must start with the following lines:"
echo " ARG DOCKER_ORG=netboxcommunity"
echo " ARG DOCKER_REPO=netbox"
echo " ARG FROM_TAG=latest"
echo " FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG"
echo " Example: VARIANT=ldap will result in the tag 'latest-ldap' and the"
echo " Dockerfile './Dockerfile.ldap' being used."
echo " Exception: VARIANT=main will use the './Dockerfile' Dockerfile"
echo " Default: main"
echo " HTTP_PROXY The proxy to use for http requests." echo " HTTP_PROXY The proxy to use for http requests."
echo " Example: http://proxy.domain.tld:3128" echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty" echo " Default: undefined"
echo " HTTPS_PROXY The proxy to use for https requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty"
echo " FTP_PROXY The proxy to use for ftp requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty"
echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for."
echo " Example: .domain1.tld,.domain2.tld" echo " Example: .domain1.tld,.domain2.tld"
echo " Default: empty" echo " Default: undefined"
echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied."
echo " Default: undefined"
echo " DRY_RUN Prints all build statements instead of running them."
echo " Default: undefined"
echo ""
echo "Examples:"
echo " ${0} master"
echo " This will fetch the latest 'master' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:latest'."
echo " ${0} develop"
echo " This will fetch the latest 'develop' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:snapshot'."
echo " ${0} v2.6.6"
echo " This will fetch the 'v2.6.6' tag, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'."
echo " ${0} develop-2.7"
echo " This will fetch the 'develop-2.7' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:develop-2.7'."
echo " SRC_ORG=cimnine ${0} feature-x"
echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
echo " build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'."
echo " SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x"
echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
echo " build a Docker Image and tag it 'cimnine/netbox:feature-x'."
if [ "${1}x" == "x" ]; then if [ "${1}x" == "x" ]; then
exit 1 exit 1
@ -76,36 +88,73 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
fi fi
fi fi
# read the project version and trim it ###
# Determining the build command to use
###
if [ -z "${DRY_RUN}" ]; then
DRY=""
else
echo "⚠️ DRY_RUN MODE ON ⚠️"
DRY="echo"
fi
###
# read the project version from the `VERSION` file and trim it
# see https://stackoverflow.com/a/3232433/172132 # see https://stackoverflow.com/a/3232433/172132
###
NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
###
# variables for fetching the source # variables for fetching the source
###
SRC_ORG="${SRC_ORG-netbox-community}" SRC_ORG="${SRC_ORG-netbox-community}"
SRC_REPO="${SRC_REPO-netbox}" SRC_REPO="${SRC_REPO-netbox}"
BRANCH="${1}" BRANCH="${1}"
URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
# Checking which VARIANT to build ###
VARIANT="${VARIANT-main}" # fetching the source
if [ "$VARIANT" == "main" ]; then ###
DOCKERFILE="Dockerfile" if [ "${2}" != "--push-only" ] ; then
else NETBOX_PATH="${NETBOX_PATH-.netbox}"
DOCKERFILE="Dockerfile.${VARIANT}" echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
if [ ! -d "${NETBOX_PATH}" ]; then
$DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}"
fi fi
# Fail fast (
if [ ! -f "${DOCKERFILE}" ]; then $DRY cd "${NETBOX_PATH}"
echo "🚨 The Dockerfile ${DOCKERFILE} for variant '${VARIANT}' doesn't exist."
if [ -z "$DEBUG" ]; then if [ -n "${HTTP_PROXY}" ]; then
git config http.proxy "${HTTP_PROXY}"
fi
$DRY git remote set-url origin "${URL}"
$DRY git fetch -qp --depth 10 origin "${BRANCH}"
$DRY git checkout -qf FETCH_HEAD
$DRY git prune
)
echo "✅ Checked out netbox"
fi
###
# Determining the value for DOCKERFILE
# and checking whether it exists
###
DOCKERFILE="${DOCKERFILE-Dockerfile}"
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist."
if [ -z "${DEBUG}" ]; then
exit 1 exit 1
else else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled." echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi fi
fi fi
###
# variables for tagging the docker image # variables for tagging the docker image
###
DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}"
DOCKER_REPO="${DOCKER_REPO-netbox}" DOCKER_REPO="${DOCKER_REPO-netbox}"
case "${BRANCH}" in case "${BRANCH}" in
@ -117,81 +166,106 @@ case "${BRANCH}" in
TAG="${TAG-$BRANCH}";; TAG="${TAG-$BRANCH}";;
esac esac
DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" ###
if [ "$VARIANT" != "main" ]; then # Determine targets to build
DOCKER_TAG="${DOCKER_TAG}-${VARIANT}" ###
DEFAULT_DOCKER_TARGETS=("main" "ldap")
DOCKER_TARGETS=( "${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}")
echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
###
# Build each target
###
for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
echo "🏗 Building the target '${DOCKER_TARGET}'"
###
# composing the final TARGET_DOCKER_TAG
###
TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
if [ "${DOCKER_TARGET}" != "main" ]; then
TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
fi fi
###
# composing the additional DOCKER_SHORT_TAG,
# i.e. "v2.6.1" becomes "v2.6",
# which is only relevant for version tags
###
if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
MAJOR=${BASH_REMATCH[1]} MAJOR=${BASH_REMATCH[1]}
MINOR=${BASH_REMATCH[2]} MINOR=${BASH_REMATCH[2]}
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
if [ "$VARIANT" != "main" ]; then if [ "${DOCKER_TARGET}" != "main" ]; then
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${VARIANT}" DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
fi fi
fi fi
DOCKER_OPTS=("${DOCKER_OPTS[@]}") ###
# Proceeding to buils stage, except if `--push-only` is passed
# caching is only ok for version tags ###
case "${TAG}" in if [ "${2}" != "--push-only" ] ; then
v*) ;; ###
*) DOCKER_OPTS+=( "--no-cache" ) ;; # Composing all arguments for `docker build`
esac ###
DOCKER_OPTS+=( "--pull" )
# Build args
DOCKER_BUILD_ARGS=( DOCKER_BUILD_ARGS=(
--build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" --pull
--build-arg "FROM_TAG=${TAG}" --target "${DOCKER_TARGET}"
--build-arg "BRANCH=${BRANCH}" -f "${DOCKERFILE}"
--build-arg "URL=${URL}" -t "${TARGET_DOCKER_TAG}"
--build-arg "DOCKER_ORG=${DOCKER_ORG}"
--build-arg "DOCKER_REPO=${DOCKER_REPO}"
) )
if [ -n "$HTTP_PROXY" ]; then if [ -n "${DOCKER_SHORT_TAG}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" )
fi fi
if [ -n "$HTTPS_PROXY" ]; then
# --label
DOCKER_BUILD_ARGS+=(
--label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}"
--label "NETBOX_BRANCH=${BRANCH}"
--label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}"
)
if [ -d "${NETBOX_PATH}/.git" ]; then
DOCKER_BUILD_ARGS+=(
--label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)"
--label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)"
)
fi
# --build-arg
DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" )
if [ -n "${DOCKER_FROM}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" )
fi
if [ -n "${HTTP_PROXY}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" )
DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" )
fi fi
if [ -n "$FTP_PROXY" ]; then if [ -n "${NO_PROXY}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" )
fi
if [ -n "$NO_PROXY" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" )
fi fi
if [ -z "$DRY_RUN" ]; then ###
DOCKER_CMD="docker" # Building the docker image
else ###
echo "⚠️ DRY_RUN MODE ON ⚠️" echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
DOCKER_CMD="echo docker" $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
fi echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
if [ "${2}" != "--push-only" ] ; then
echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
$DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
if [ -n "$DOCKER_SHORT_TAG" ]; then
echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'."
$DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}"
echo "✅ Tagged image '${DOCKER_SHORT_TAG}'"
fi
fi fi
###
# Pushing the docker images if either `--push` or `--push-only` are passed
###
if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
echo "⏫ Pushing '${DOCKER_TAG}" echo "⏫ Pushing '${TARGET_DOCKER_TAG}"
$DOCKER_CMD push "${DOCKER_TAG}" $DRY docker push "${TARGET_DOCKER_TAG}"
echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'."
if [ -n "$DOCKER_SHORT_TAG" ]; then if [ -n "$DOCKER_SHORT_TAG" ]; then
echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" echo "⏫ Pushing '${DOCKER_SHORT_TAG}'"
$DOCKER_CMD push "${DOCKER_SHORT_TAG}" $DRY docker push "${DOCKER_SHORT_TAG}"
echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'."
fi fi
fi fi
done

View File

@ -1,10 +1,6 @@
version: '3' version: '3.4'
services: services:
netbox: &netbox netbox: &netbox
build:
context: .
args:
- BRANCH=${VERSION-master}
image: netboxcommunity/netbox:${VERSION-latest} image: netboxcommunity/netbox:${VERSION-latest}
depends_on: depends_on:
- postgres - postgres

View File

@ -2,4 +2,5 @@
. hooks/common . hooks/common
# shellcheck disable=SC2119
run_build run_build

View File

@ -1,37 +1,19 @@
#!/bin/bash #!/bin/bash
ensure_jq() { ensure_jq() {
echo "🛠🛠🛠 Installing JQ via apt-get" if [ ! -x "$(command -v jq)" ]; then
[ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq ) if [ -x "$(command -v apt-get)" ]; then
} echo "🛠🛠🛠 Installing 'jq' via 'apt-get'"
apt-get update && apt-get install -y jq
ensure_dockerfile_present() {
if [ "${VARIANT}" == "main" ]; then
DOCKERFILE="Dockerfile"
else else
DOCKERFILE="Dockerfile.${VARIANT}" echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'."
# Fail fast
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
if [ -z "$DEBUG" ]; then
exit 1
else
echo "⚠️ Would skip this, but DEBUG is enabled."
fi
fi
if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then
echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'."
echo " This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'."
fi fi
fi fi
} }
# Passes args to the scripts # Passes args to the scripts
run_build() { run_build() {
echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant" echo "🐳🐳🐳 Building '${BUILD}' images"
case $BUILD in case $BUILD in
release) release)
# build the latest release # build the latest release
@ -48,11 +30,11 @@ run_build() {
# shellcheck disable=SC2068 # shellcheck disable=SC2068
./build-branches.sh $@ ./build-branches.sh $@
;; ;;
special) this) # Pull Requests
# special build # only build the 'master' branch
# shellcheck disable=SC2068 # (resulting in the 'latest' docker tag)
#SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ # and the 'main' target.
echo "✅ No special builds today." DOCKER_TARGET=main ./build.sh master
;; ;;
*) *)
echo "🚨 Unrecognized build '$BUILD'." echo "🚨 Unrecognized build '$BUILD'."
@ -70,13 +52,9 @@ echo "🤖🤖🤖 Preparing build"
export DOCKER_ORG="index.docker.io/netboxcommunity" export DOCKER_ORG="index.docker.io/netboxcommunity"
export DOCKER_REPO=netbox export DOCKER_REPO=netbox
export DOCKERHUB_REPO=netboxcommunity/netbox export DOCKERHUB_REPO=netboxcommunity/netbox
# shellcheck disable=SC2153
# mis-using the "${DOCKER_TAG}" variable as "branch to build" export BUILD="${DOCKER_TAG}"
export BUILD="${DOCKER_TAG%-*}"
export VARIANT="${DOCKER_TAG#*-}"
unset DOCKER_TAG unset DOCKER_TAG
ensure_dockerfile_present
ensure_jq ensure_jq

View File

@ -2,4 +2,13 @@
. hooks/common . hooks/common
if [ "${SOURCE_BRANCH}" == "main" ] || [ "${DEBUG}" == "true" ]; then
if [ "${SOURCE_BRANCH}" != "main" ]; then
echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'".
fi
run_build --push-only run_build --push-only
else
echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'"
exit 0
fi

View File

@ -2,11 +2,21 @@
. hooks/common . hooks/common
if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then run_test() {
echo "🐳🐳🐳 Testing" echo "🐳🐳🐳 Testing '${1}'"
docker-compose pull --parallel VERSION="${1}" docker-compose run netbox ./manage.py test
docker-compose build docker-compose down -v
docker-compose run netbox ./manage.py test echo "🐳🐳🐳 Done testing '${1}'"
}
# test on builds of 'branches'
if [ "${BUILD}" == "branches" ] \
|| [ "${DEBUG}" == "true" ]; then
run_test latest
run_test snapshot
# test on bulds of 'this' (i.e. pull request)
elif [ "${BUILD}" == "this" ]; then
run_test latest
else else
echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'." echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'."
fi fi