From 6568dff8e1e196e7774448e58801d7bd8a4c3a08 Mon Sep 17 00:00:00 2001 From: Stefan de Kooter Date: Wed, 6 Nov 2019 10:26:29 +0100 Subject: [PATCH 01/67] Add SSLMODE to database connection properties Defaulting to SSLMODE=prefer, optional override using environment variable: DB_SSLMODE --- configuration/configuration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/configuration.py b/configuration/configuration.py index 14c5d83..23170e8 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -37,6 +37,8 @@ DATABASE = { # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE } # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. From caaa68234c66578cb89d012af621246e8a43c058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 17 Nov 2019 17:07:02 +0100 Subject: [PATCH 02/67] Adds more labels to the Dockerfile It also updates the README and the bug_report template to reflect the changes. Additionally, in the README some relevant shields from shields.io are added. The labels follow [label-schema.org][lsorg] and the [OpenContainer image spec, section annotations][ocis], specifications. [lsorg]: http://label-schema.org/rc1/ [ocis]: https://github.com/opencontainers/image-spec/blob/master/annotations.md --- .github/ISSUE_TEMPLATE/bug_report.md | 14 +++- Dockerfile | 33 ++++++++-- README.md | 46 ++++++++++++-- build.sh | 95 ++++++++++++++++++---------- 4 files changed, 142 insertions(+), 46 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 88437e0..a4771fa 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -45,12 +45,22 @@ The output of `docker version`: `XXXXX` The output of `git rev-parse HEAD`: `XXXXX` The command you used to start the project: `XXXXX` + +The output of `docker inspect netboxcommunity/netbox:latest --format "{{json .ContainerConfig.Labels}}"`: + +```json +{ + "JSON JSON JSON": + "--> Please paste formatted json. (Use e.g. `jq` or https://jsonformatter.curiousconcept.com/)" +} +``` + The output of `docker-compose logs netbox`: -``` +```text LOG LOG LOG ``` @@ -60,6 +70,6 @@ Only if you have gotten a 5xx http error, else delete this section. If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com --> -``` +```text LOG LOG LOG ``` diff --git a/Dockerfile b/Dockerfile index 7ee1216..d7a9af9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,11 +72,34 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] -LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \ - NETBOX_BRANCH="custom build" \ - ORIGINAL_DOCKER_TAG="custom build" \ - NETBOX_GIT_COMMIT="not built from git" \ - NETBOX_GIT_URL="not built from git" +LABEL ORIGINAL_TAG="" \ + NETBOX_GIT_BRANCH="" \ + NETBOX_GIT_REF="" \ + NETBOX_GIT_URL="" \ +# See http://label-schema.org/rc1/#build-time-labels +# Also https://microbadger.com/labels + org.label-schema.schema-version="1.0" \ + org.label-schema.build-date="" \ + org.label-schema.name="Netbox Docker" \ + org.label-schema.description="A container based distribution of Netbox, the free and open IPAM and DCIM solution." \ + org.label-schema.vendor="The netbox-docker contributors." \ + org.label-schema.url="https://github.com/netbox-community/netbox-docker" \ + org.label-schema.usage="https://github.com/netbox-community/netbox-docker/wiki" \ + org.label-schema.vcs-url="https://github.com/netbox-community/netbox-docker.git" \ + org.label-schema.vcs-ref="" \ + org.label-schema.version="snapshot" \ +# See https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys + org.opencontainers.image.created="" \ + org.opencontainers.image.title="Netbox Docker" \ + org.opencontainers.image.description="A container based distribution of Netbox, the free and open IPAM and DCIM solution." \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.authors="The netbox-docker contributors." \ + org.opencontainers.image.vendor="The netbox-docker contributors." \ + org.opencontainers.image.url="https://github.com/netbox-community/netbox-docker" \ + org.opencontainers.image.documentation="https://github.com/netbox-community/netbox-docker/wiki" \ + org.opencontainers.image.source="https://github.com/netbox-community/netbox-docker.git" \ + org.opencontainers.image.revision="" \ + org.opencontainers.image.version="snapshot" ##### ## LDAP specific configuration diff --git a/README.md b/README.md index e375bea..13f7241 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,45 @@ # netbox-docker +[![GitHub release (latest by date)](https://img.shields.io/github/v/release/netbox-community/netbox-docker)][github-release] +[![GitHub stars](https://img.shields.io/github/stars/netbox-community/netbox-docker)][github-stargazers] +![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/netbox-community/netbox-docker) +![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/netboxcommunity/netbox) +![Docker Pulls](https://img.shields.io/docker/pulls/netboxcommunity/netbox) +[![MicroBadger Layers](https://img.shields.io/microbadger/layers/netboxcommunity/netbox)][netbox-docker-microbadger] +[![MicroBadger Size](https://img.shields.io/microbadger/image-size/netboxcommunity/netbox)][netbox-docker-microbadger] +[![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license] + [The Github repository](netbox-docker-github) houses the components needed to build Netbox as a Docker container. Images are built using this code and are released to [Docker Hub][netbox-dockerhub] once a day. Do you have any questions? Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel. +[github-stargazers]: https://github.com/netbox-community/netbox-docker/stargazers +[github-release]: https://github.com/netbox-community/netbox-docker/releases +[netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox [netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/tags/ [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/ [ntc-slack]: http://slack.networktocode.com/ [netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E +[netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/master/LICENSE + +## Docker Tags + +* `vX.Y.Z`: Release builds, built from [releases of Netbox][netbox-releases]. +* `latest`: Release builds, built from [`master` branch of Netbox][netbox-master]. +* `snapshot`: Pre-release builds, built from the [`develop` branch of Netbox][netbox-develop]. +* `develop-X.Y`: Pre-release builds, built from the corresponding [branch of Netbox][netbox-branches]. + +Then there is currently one extra tags for each of the above labels: + +* `-ldap`: Contains additional dependencies and configurations for connecting Netbox to an LDAP directroy. + [Learn more about that in our wiki][netbox-docker-ldap]. + +[netbox-releases]: https://github.com/netbox-community/netbox/releases +[netbox-master]: https://github.com/netbox-community/netbox/tree/master +[netbox-develop]: https://github.com/netbox-community/netbox/tree/develop +[netbox-branches]: https://github.com/netbox-community/netbox/branches +[netbox-docker-ldap]: https://github.com/netbox-community/netbox-docker/wiki/LDAP ## Quickstart @@ -54,12 +85,12 @@ This project relies only on *Docker* and *docker-compose* meeting this requireme To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. -## Reference Documentation +## Documentation -Please refer [to the wiki][wiki] for further information on how to use this Netbox Docker image properly. +Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly. It covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration. -[wiki]: https://github.com/netbox-community/netbox-docker/wiki/ +[netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/ ## Netbox Version @@ -69,7 +100,7 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.6.6 +export VERSION=v2.6.7 docker-compose pull netbox docker-compose up -d ``` @@ -78,7 +109,7 @@ You can also build a specific version of the Netbox Docker image yourself. `VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=v2.6.6 +export VERSION=v2.6.7 ./build.sh $VERSION docker-compose up -d ``` @@ -90,8 +121,9 @@ docker-compose up -d From time to time it might become necessary to re-engineer the structure of this setup. Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence. -Since April 2018 each image built from this repo contains a `NETBOX_DOCKER_PROJECT_VERSION` label. -You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.3.1 --format "{{json .ContainerConfig.Labels}}"`. +Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label. +(The images contained labels since April 2018, although in November 2019 the labels' names changed.) +You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.7 --format "{{json .ContainerConfig.Labels}}"`. Please read [the release notes][releases] carefully when updating to a new image version. diff --git a/build.sh b/build.sh index 123cd17..7588332 100755 --- a/build.sh +++ b/build.sh @@ -30,23 +30,26 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " When \${BRANCH}=master: latest" echo " When \${BRANCH}=develop: snapshot" echo " Else: same as \${BRANCH}" - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" - echo " Also used for tagging the image." + echo " DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." + echo " Default: docker.io" + echo " DOCKER_ORG The Docker repository's organisation (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" - echo " Also used for tagging the image." + echo " DOCKER_REPO The Docker repository's name (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." echo " Default: netbox" - echo " DOCKER_FROM The base image to use." - echo " Default: Whatever is defined as default in the Dockerfile." echo " DOCKER_TAG The name of the tag which is applied to the image." echo " Useful for pushing into another registry than hub.docker.com." - echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" + echo " Default: \${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" echo " image. This is used to tag all patch releases to their" echo " containing version e.g. v2.5.1 -> v2.5" - echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:." + echo " Default: \${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}:." echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" + echo " DOCKER_FROM The base image to use." + echo " Default: Whatever is defined as default in the Dockerfile." echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." echo " Default: main ldap" @@ -98,12 +101,6 @@ else DRY="echo" fi -### -# read the project version from the `VERSION` file and trim it -# see https://stackoverflow.com/a/3232433/172132 -### -NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" - ### # variables for fetching the source ### @@ -152,9 +149,29 @@ if [ ! -f "${DOCKERFILE}" ]; then fi fi +### +# variables for labelling the docker image +### +BUILD_DATE="$(date --utc --iso-8601=minutes)" + +if [ -d ".git" ]; then + GIT_REF="$(git rev-parse HEAD)" +fi + +# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 +PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" + +# Get the Git information from the netbox directory +if [ -d "${NETBOX_PATH}/.git" ]; then + NETBOX_GIT_REF=$(cd ${NETBOX_PATH}; git rev-parse HEAD) + NETBOX_GIT_BRANCH=$(cd ${NETBOX_PATH}; git rev-parse --abbrev-ref HEAD) + NETBOX_GIT_URL=$(cd ${NETBOX_PATH}; git remote get-url origin) +fi + ### # variables for tagging the docker image ### +DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" case "${BRANCH}" in @@ -182,7 +199,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### # composing the final TARGET_DOCKER_TAG ### - TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" + TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi @@ -196,10 +213,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do MAJOR=${BASH_REMATCH[1]} MINOR=${BASH_REMATCH[2]} - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" + TARGET_DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" if [ "${DOCKER_TARGET}" != "main" ]; then - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" + TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}" fi fi @@ -216,25 +233,39 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do -f "${DOCKERFILE}" -t "${TARGET_DOCKER_TAG}" ) - if [ -n "${DOCKER_SHORT_TAG}" ]; then - DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" ) + if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then + DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_SHORT_TAG}" ) fi # --label - DOCKER_BUILD_ARGS+=( - --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --label "NETBOX_BRANCH=${BRANCH}" - --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" - ) - if [ -d "${NETBOX_PATH}/.git" ]; then + if [ "${DOCKER_TARGET}" == "main" ]; then DOCKER_BUILD_ARGS+=( - --label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)" - --label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)" + --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}" + + --label "org.label-schema.build-date=${BUILD_DATE}" + --label "org.opencontainers.image.created=${BUILD_DATE}" + + --label "org.label-schema.version=${PROJECT_VERSION}" + --label "org.opencontainers.image.version=${PROJECT_VERSION}" ) + if [ -d ".git" ]; then + DOCKER_BUILD_ARGS+=( + --label "org.label-schema.vcs-ref=${GIT_REF}" + --label "org.opencontainers.image.revision=${GIT_REF}" + ) + fi + if [ -d "${NETBOX_PATH}/.git" ]; then + DOCKER_BUILD_ARGS+=( + --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}" + --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}" + --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}" + ) + fi fi # --build-arg - DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) + DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) + if [ -n "${DOCKER_FROM}" ]; then DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) fi @@ -262,10 +293,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do $DRY docker push "${TARGET_DOCKER_TAG}" echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DRY docker push "${DOCKER_SHORT_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then + echo "⏫ Pushing '${TARGET_DOCKER_SHORT_TAG}'" + $DRY docker push "${TARGET_DOCKER_SHORT_TAG}" + echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_SHORT_TAG}'." fi fi done From 552676cc9d596c44ef05b02db01350134d944d85 Mon Sep 17 00:00:00 2001 From: Mark Honeychurch Date: Mon, 18 Nov 2019 12:20:39 +1300 Subject: [PATCH 03/67] Ignore local override file Adding an ignore rule for docker-compose.override.yml should allow people to clone this repo and use an override file to match their environment, without that file causing issues every time they pull a new version of the repo. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cbaffa8..4b029d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.sql.gz .netbox +docker-compose.override.yml From 02a5171e37f65cf9b5d3a59ba9efd102603a4778 Mon Sep 17 00:00:00 2001 From: "sylvain.petit" Date: Wed, 20 Nov 2019 10:09:02 +0100 Subject: [PATCH 04/67] Update Nginx 1.17.6-alpine docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 60a6a90..6a70af8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - rqworker nginx: command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.15-alpine + image: nginx:1.17.6-alpine depends_on: - netbox ports: From 08cda559a33b51d823d4cee4ead2ed65077ed408 Mon Sep 17 00:00:00 2001 From: EdenPark59 <35762460+edenpark59@users.noreply.github.com> Date: Wed, 20 Nov 2019 11:00:09 +0100 Subject: [PATCH 05/67] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6a70af8..f00700a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - rqworker nginx: command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.17.6-alpine + image: nginx:1.17-alpine depends_on: - netbox ports: From f2fd7dbbe7e6d9f1d2a36a8f14abaee17a6ede71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 20 Nov 2019 11:09:37 +0100 Subject: [PATCH 06/67] Adds a link to the Getting Started wiki section to the README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e375bea..5c0ff33 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,15 @@ $ xdg-open "http://$(docker-compose port nginx 8080)/" &>/dev/null & Alternatively, use something like [Reception][docker-reception] to connect to _docker-compose_ projects. -Default credentials: +The default credentials are: * Username: **admin** * Password: **admin** * API Token: **0123456789abcdef0123456789abcdef01234567** +There is a more complete [Getting Started guide on our Wiki][wiki-getting-started]. + +[wiki-getting-started]: https://github.com/netbox-community/netbox-docker/wiki/Getting-Started [docker-reception]: https://github.com/nxt-engineering/reception ## Dependencies From 34ce5be006e6eb034c8de408e82dec8739cb48ea Mon Sep 17 00:00:00 2001 From: Matt Olenik Date: Wed, 20 Nov 2019 13:25:31 -0800 Subject: [PATCH 07/67] Respect SKIP_GIT environment variable when building Fixes issue #190 --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 123cd17..7efb23e 100755 --- a/build.sh +++ b/build.sh @@ -111,12 +111,12 @@ SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" +NETBOX_PATH="${NETBOX_PATH-.netbox}" ### # fetching the source ### -if [ "${2}" != "--push-only" ] ; then - NETBOX_PATH="${NETBOX_PATH-.netbox}" +if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" if [ ! -d "${NETBOX_PATH}" ]; then $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" From 29185fb9fdfa1fb6a8b557d16cd4dd78899e9a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 25 Nov 2019 17:09:37 +0100 Subject: [PATCH 08/67] reorganized docker ignore file .. and added some more files to it. --- .dockerignore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 1b2bacc..23492a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ +.git .github .travis.yml +*.md +env build* -*.env -.git +docker-compose.override.yml From c083baf640d0fe4f5a160e3435a2c3ec3e721107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 25 Nov 2019 17:11:37 +0100 Subject: [PATCH 09/67] Preparation for 0.19.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b72b05e..c0b8d59 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.19.3 +0.19.4 From 01c4137dc9d6e3f8e0e44f56328222d994d27e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 17 Nov 2019 18:03:59 +0100 Subject: [PATCH 10/67] Adds netbox user --- Dockerfile | 10 ++++++++++ docker-compose.yml | 1 + 2 files changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index d7a9af9..da15ecb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,10 @@ RUN apk add --no-cache \ postgresql-libs \ ttf-ubuntu-font-family +RUN addgroup -S -g 101 netbox \ + && adduser -DHS -u 101 netbox \ + && adduser netbox netbox + WORKDIR /opt COPY --from=builder /install /usr/local @@ -68,6 +72,12 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py WORKDIR /opt/netbox/netbox +# Must set permissions for '/opt/netbox/netbox/static' directory +# to a+w so that `./manage.py collectstatic` can be executed during +# container startup. +# Not satisfying +RUN mkdir static && chmod a+w static media + ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..da5fbd5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - redis - netbox-worker env_file: env/netbox.env + user: netbox volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - ./initializers:/opt/netbox/initializers:z,ro From 7942e9edbe89469fd691d1bed634ab84decb00b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 18 Nov 2019 08:23:44 +0100 Subject: [PATCH 11/67] Remove the named user, change permissions to 'g+w' --- Dockerfile | 11 ++++------- docker-compose.yml | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index da15ecb..af70e38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,10 +51,6 @@ RUN apk add --no-cache \ postgresql-libs \ ttf-ubuntu-font-family -RUN addgroup -S -g 101 netbox \ - && adduser -DHS -u 101 netbox \ - && adduser netbox netbox - WORKDIR /opt COPY --from=builder /install /usr/local @@ -73,10 +69,11 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py WORKDIR /opt/netbox/netbox # Must set permissions for '/opt/netbox/netbox/static' directory -# to a+w so that `./manage.py collectstatic` can be executed during +# to g+w so that `./manage.py collectstatic` can be executed during # container startup. -# Not satisfying -RUN mkdir static && chmod a+w static media +# Must set permissions for '/opt/netbox/netbox/media' directory +# to g+w so that pictures can be uploaded to netbox. +RUN mkdir static && chmod g+w static media ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] diff --git a/docker-compose.yml b/docker-compose.yml index da5fbd5..874ff3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - redis - netbox-worker env_file: env/netbox.env - user: netbox + user: '101' volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - ./initializers:/opt/netbox/initializers:z,ro From 8664e42233175ce2a2e9b78163ae01c0de26b789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 11:40:10 +0100 Subject: [PATCH 12/67] Revert "Fix #179 'libc not found'" This reverts commit feb810ab27bf459a54a3e2055e7c399ed478f1f5. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7a9af9..d96fd01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ WORKDIR /install RUN pip install --prefix="/install" --no-warn-script-location \ # gunicorn is used for launching netbox - 'gunicorn<20.0.0' \ + gunicorn \ greenlet \ eventlet \ # napalm is used for gathering information from network devices From 5e92352b0ae83f2c6a065ea31520300d40a3e732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:09:26 +0100 Subject: [PATCH 13/67] set umask in entrypoint --- docker/docker-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e6a8de..7aef836 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +umask 002 # wait shortly and then run db migrations (retry on error) while ! ./manage.py migrate 2>&1; do From 1ce99cf02f2e28bc26f7fb66352714df969d8cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:14:51 +0100 Subject: [PATCH 14/67] update redis --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..e00f8c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: volumes: - netbox-postgres-data:/var/lib/postgresql/data redis: - image: redis:4-alpine + image: redis:5-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env From 150f35ea3bc8e3312d48e9fe97b68e86671dc6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:23:09 +0100 Subject: [PATCH 15/67] update postgres to 12 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..f81f325 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:10.4-alpine + image: postgres:12-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data From 788804627e29a125c9f5668782e3bcf96e6f29ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:27:25 +0100 Subject: [PATCH 16/67] Fix typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c0ff33..a0f1fee 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,12 @@ There is a more complete [Getting Started guide on our Wiki][wiki-getting-starte ## Dependencies -This project relies only on *Docker* and *docker-compose* meeting this requirements: +This project relies only on *Docker* and *docker-compose* meeting these requirements: * The *Docker version* must be at least `17.05`. * The *docker-compose version* must be at least `1.17.0`. -To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. +To check the version installed on your system, compare the output of `docker --version` and `docker-compose --version` with the requirements above. ## Reference Documentation From 8a92640d1001fb074849a98d8d9b725461f322c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:28:43 +0100 Subject: [PATCH 17/67] Re-wording of dependencies check paragraph --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0f1fee..82f01aa 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ This project relies only on *Docker* and *docker-compose* meeting these requirem * The *Docker version* must be at least `17.05`. * The *docker-compose version* must be at least `1.17.0`. -To check the version installed on your system, compare the output of `docker --version` and `docker-compose --version` with the requirements above. +To check the version installed on your system run `docker --version` and `docker-compose --version`. ## Reference Documentation From 809570f4bc5db6b0e54e5dbbada1d6966f44165b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 21:25:36 +0100 Subject: [PATCH 18/67] Fix variables for Dockerhub build --- hooks/common | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/common b/hooks/common index 49cd507..0f0223e 100755 --- a/hooks/common +++ b/hooks/common @@ -14,7 +14,7 @@ ensure_jq() { # Passes args to the scripts run_build() { echo "🐳🐳🐳 Building '${BUILD}' images" - case $BUILD in + case ${BUILD} in release) # build the latest release # shellcheck disable=SC2068 @@ -49,9 +49,9 @@ run_build() { } echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG="index.docker.io/netboxcommunity" +export DOCKER_ORG=netboxcommunity export DOCKER_REPO=netbox -export DOCKERHUB_REPO=netboxcommunity/netbox +export DOCKER_REGISTRY=docker.io # shellcheck disable=SC2153 export BUILD="${DOCKER_TAG}" From a4186c1031e031d89e1d78dcce417b629f084b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 21:44:11 +0100 Subject: [PATCH 19/67] Prepares scripts and documentation for #184 --- DOCKER_HUB.md | 6 +++--- README.md | 4 ++-- build-branches.sh | 2 +- build.sh | 20 ++++++++++---------- hooks/common | 2 +- hooks/push | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index 10c4154..d3b3e68 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -11,21 +11,21 @@ Autotest: Internal and External Pull Requests Repository Links: Enable for Base Image Build Rules: - Source Type: Branch - Source: master + Source: release Docker Tag: branches Dockerfile location: Dockerfile Build Context: / Autobuild: on Build Caching: on - Source Type: Branch - Source: master + Source: release Docker Tag: prerelease Dockerfile location: Dockerfile Build Context: / Autobuild: on Build Caching: on - Source Type: Branch - Source: master + Source: release Docker Tag: release Dockerfile location: Dockerfile Build Context: / diff --git a/README.md b/README.md index 13f7241..00f3aaf 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ Then there is currently one extra tags for each of the above labels: ## Quickstart -To get Netbox up and running: +To get Netbox up and running in Docker: ```bash -git clone -b master https://github.com/netbox-community/netbox-docker.git +git clone -b release https://github.com/netbox-community/netbox-docker.git cd netbox-docker docker-compose pull docker-compose up -d diff --git a/build-branches.sh b/build-branches.sh index a6bc736..483e771 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds develop, develop-* and master branches +# Builds develop, develop-* and master branches of Netbox echo "▶️ $0 $*" diff --git a/build.sh b/build.sh index 14ab409..8ea1476 100755 --- a/build.sh +++ b/build.sh @@ -27,9 +27,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " TAG The version part of the docker tag." echo " Default:" - echo " When \${BRANCH}=master: latest" - echo " When \${BRANCH}=develop: snapshot" - echo " Else: same as \${BRANCH}" + echo " When =master: latest" + echo " When =develop: snapshot" + echo " Else: same as " echo " DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" echo " Used for tagging the image." echo " Default: docker.io" @@ -106,7 +106,7 @@ fi ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" -BRANCH="${1}" +NETBOX_BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" @@ -114,9 +114,9 @@ NETBOX_PATH="${NETBOX_PATH-.netbox}" # fetching the source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then - echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" if [ ! -d "${NETBOX_PATH}" ]; then - $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}" fi ( @@ -127,7 +127,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then fi $DRY git remote set-url origin "${URL}" - $DRY git fetch -qp --depth 10 origin "${BRANCH}" + $DRY git fetch -qp --depth 10 origin "${NETBOX_BRANCH}" $DRY git checkout -qf FETCH_HEAD $DRY git prune ) @@ -174,13 +174,13 @@ fi DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" -case "${BRANCH}" in +case "${NETBOX_BRANCH}" in master) TAG="${TAG-latest}";; develop) TAG="${TAG-snapshot}";; *) - TAG="${TAG-$BRANCH}";; + TAG="${TAG-$NETBOX_BRANCH}";; esac ### @@ -241,7 +241,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if [ "${DOCKER_TARGET}" == "main" ]; then DOCKER_BUILD_ARGS+=( --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}" - + --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.created=${BUILD_DATE}" diff --git a/hooks/common b/hooks/common index 49cd507..c0ad1a8 100755 --- a/hooks/common +++ b/hooks/common @@ -31,7 +31,7 @@ run_build() { ./build-branches.sh $@ ;; this) # Pull Requests - # only build the 'master' branch + # only build the 'master' branch of netbox # (resulting in the 'latest' docker tag) # and the 'main' target. DOCKER_TARGET=main ./build.sh master diff --git a/hooks/push b/hooks/push index 81dc7db..6513b24 100755 --- a/hooks/push +++ b/hooks/push @@ -2,8 +2,8 @@ . hooks/common -if [ "${SOURCE_BRANCH}" == "master" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "master" ]; then +if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then + if [ "${SOURCE_BRANCH}" != "release" ]; then echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". fi From e4e0a63e174e695febae5100ed67e65ffc097072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 22:41:57 +0100 Subject: [PATCH 20/67] Better safe than sorry, keep PostgreSQL at 11 for now. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f81f325..d6a0109 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:12-alpine + image: postgres:11-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data From 7df5da38bf69f6639219918102f14fc2801b4d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 23:09:43 +0100 Subject: [PATCH 21/67] Pull request template suggestion --- .github/ISSUE_TEMPLATE/bug_report.md | 2 ++ .github/ISSUE_TEMPLATE/feature_request.md | 4 ++- .github/pull_request_template.md | 44 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a4771fa..7717abd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -25,6 +25,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Current Behavior diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 65f31dd..6bc6dd4 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -24,6 +24,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Desired Behavior @@ -33,7 +35,7 @@ Please try this means to get help before opening an issue here: ## Contrast to Current Behavior - + ... ## Changes Required diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..9d85010 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,44 @@ + +Related Issue: + +## New Behavior + + + +... + +## Contrast to Current Behavior + + + +... + +## Discussion: Benefits and Drawbacks + + + +... + +## Changes to the Wiki + + + +... From 1064696c96773ac8b1602757bdc24d990b168545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 23:16:11 +0100 Subject: [PATCH 22/67] N/A when there's no issue --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d85010..1c3885f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,7 @@ +But if there is already a related issue, please put it's number here. E.g. #123 or N/A --> Related Issue: ## New Behavior From 93dee744599b520beff6412939216a3b95104aae Mon Sep 17 00:00:00 2001 From: Jeroen Simonetti Date: Mon, 2 Dec 2019 10:23:53 +0100 Subject: [PATCH 23/67] Add rack group initialiser Fixes #192 Signed-off-by: Jeroen Simonetti --- initializers/rack_groups.yml | 3 +++ initializers/racks.yml | 1 + startup_scripts/075_rack_groups.py | 31 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 initializers/rack_groups.yml create mode 100644 startup_scripts/075_rack_groups.py diff --git a/initializers/rack_groups.yml b/initializers/rack_groups.yml new file mode 100644 index 0000000..244fc00 --- /dev/null +++ b/initializers/rack_groups.yml @@ -0,0 +1,3 @@ +# - name: cage 101 +# slug: cage-101 +# site: SING 1 diff --git a/initializers/racks.yml b/initializers/racks.yml index 9a71743..3a31526 100644 --- a/initializers/racks.yml +++ b/initializers/racks.yml @@ -16,6 +16,7 @@ # text_field: Description # - site: SING 1 # name: rack-03 +# group: cage 101 # role: Role 3 # type: 4-post cabinet # width: 19 inches diff --git a/startup_scripts/075_rack_groups.py b/startup_scripts/075_rack_groups.py new file mode 100644 index 0000000..7deaa11 --- /dev/null +++ b/startup_scripts/075_rack_groups.py @@ -0,0 +1,31 @@ +from dcim.models import Site,RackGroup +from ruamel.yaml import YAML + +from pathlib import Path +import sys + +file = Path('/opt/netbox/initializers/rack_groups.yml') +if not file.is_file(): + sys.exit() + +with file.open('r') as stream: + yaml=YAML(typ='safe') + rack_groups= yaml.load(stream) + + required_assocs = { + 'site': (Site, 'name') + } + + if rack_groups is not None: + for params in rack_groups: + + for assoc, details in required_assocs.items(): + model, field = details + query = { field: params.pop(assoc) } + params[assoc] = model.objects.get(**query) + + rack_group, created = RackGroup.objects.get_or_create(**params) + + if created: + print("🎨 Created rack group", rack_group.name) + From daaf72962f8e357272dcca37b2e9219e83b9ae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 11 Dec 2019 11:48:29 +0100 Subject: [PATCH 24/67] Adds a test that validates the initializer yml files --- hooks/test | 13 +++---------- test.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 10 deletions(-) create mode 100755 test.sh diff --git a/hooks/test b/hooks/test index 1dd5538..a66db12 100755 --- a/hooks/test +++ b/hooks/test @@ -2,21 +2,14 @@ . hooks/common -run_test() { - echo "🐳🐳🐳 Testing '${1}'" - VERSION="${1}" docker-compose run netbox ./manage.py test - docker-compose down -v - echo "🐳🐳🐳 Done testing '${1}'" -} - # test on builds of 'branches' if [ "${BUILD}" == "branches" ] \ || [ "${DEBUG}" == "true" ]; then - run_test latest - run_test snapshot + ./test.sh latest + ./test.sh snapshot # test on bulds of 'this' (i.e. pull request) elif [ "${BUILD}" == "this" ]; then - run_test latest + ./test.sh latest else echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." fi diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..d6d784a --- /dev/null +++ b/test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# exit when a command exits with an exit code != 0 +set -e + +# version is used by `docker-compose.yml` do determine the tag +# of the Docker Image that is to be used +export VERSION=${VERSION-latest} + +test_netbox_unit_tests() { + echo "⏱ Running Netbox Unit Tests" + docker-compose run --rm netbox ./manage.py test +} + +test_initializers() { + echo "🏗 Testing Initializers" + + mkdir initializers_test + ( + cd initializers + for script in *.yml; do + sed -E 's/^# //' "${script}" > "../initializers_test/${script}" + done + ) + mv initializers initializers_original + mv initializers_test initializers + + docker-compose run --rm netbox ./manage.py check +} + +test_cleanup() { + echo "💣 Cleaning Up" + docker-compose down -v + rm -rf initializers + mv initializers_original initializers +} + +echo "🐳🐳🐳 Start testing '${VERSION}'" + +# Make sure the cleanup script is executed +trap test_cleanup EXIT ERR + +test_netbox_unit_tests +test_initializers + +echo "🐳🐳🐳 Done testing '${VERSION}'" From 310cda1f185cd9457b2dd7b3e44610d785364bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 11 Dec 2019 13:58:33 +0100 Subject: [PATCH 25/67] Reduce the Docker Context --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.dockerignore b/.dockerignore index 23492a4..bdb18c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,7 @@ env build* docker-compose.override.yml +.netbox/.git* +.netbox/.travis.yml +.netbox/docs +.netbox/scripts From 66e90428b52f9f8dabf35136aa0cbef29f1c0491 Mon Sep 17 00:00:00 2001 From: Matt Olenik Date: Mon, 2 Dec 2019 12:17:40 -0800 Subject: [PATCH 26/67] Remove use of GNU date extensions when labeling images Fixes #197 --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 14ab409..104bb99 100755 --- a/build.sh +++ b/build.sh @@ -152,7 +152,7 @@ fi ### # variables for labelling the docker image ### -BUILD_DATE="$(date --utc --iso-8601=minutes)" +BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" if [ -d ".git" ]; then GIT_REF="$(git rev-parse HEAD)" From 05d32ae70537b590b08cf0c533a1ee20e8032563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 14 Dec 2019 18:16:31 +0100 Subject: [PATCH 27/67] Massive speedup in executing startup_scripts --- docker/docker-entrypoint.sh | 5 +---- startup_scripts/__main__.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 startup_scripts/__main__.py diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e6a8de..0c8d2a1 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -45,10 +45,7 @@ fi if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩️ Skipping startup scripts" else - for script in /opt/netbox/startup_scripts/*.py; do - echo "⚙️ Executing '$script'" - ./manage.py shell --interface python < "${script}" - done + echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi # copy static files diff --git a/startup_scripts/__main__.py b/startup_scripts/__main__.py new file mode 100644 index 0000000..50d2613 --- /dev/null +++ b/startup_scripts/__main__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import runpy +from os import scandir +from os.path import dirname, abspath + +this_dir = dirname(abspath(__file__)) + +def filename(f): + return f.name + +with scandir(dirname(abspath(__file__))) as it: + for f in sorted(it, key = filename): + if f.name.startswith('__') or not f.is_file(): + continue + + print(f"Running {f.path}") + runpy.run_path(f.path) From 7863e5902e55ca06f9a4c6f9e1cfc0c98aee3709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 21 Dec 2019 15:20:33 +0100 Subject: [PATCH 28/67] Preparation for 0.20.0 --- README.md | 6 +++--- VERSION | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed04478..363980c 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.6.7 +export VERSION=v2.6.9 docker-compose pull netbox docker-compose up -d ``` @@ -112,7 +112,7 @@ You can also build a specific version of the Netbox Docker image yourself. `VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=v2.6.7 +export VERSION=v2.6.9 ./build.sh $VERSION docker-compose up -d ``` @@ -126,7 +126,7 @@ From time to time it might become necessary to re-engineer the structure of this Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence. Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label. (The images contained labels since April 2018, although in November 2019 the labels' names changed.) -You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.7 --format "{{json .ContainerConfig.Labels}}"`. +You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.9 --format "{{json .ContainerConfig.Labels}}"`. Please read [the release notes][releases] carefully when updating to a new image version. diff --git a/VERSION b/VERSION index c0b8d59..5a03fb7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.19.4 +0.20.0 From b118cd58127c71c3b2c346926e9728837b98bb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 23 Dec 2019 17:53:19 +0100 Subject: [PATCH 29/67] Building the Docker image with Github Actions --- .github/workflows/push.yml | 31 ++++++++++++++++++ .github/workflows/release.yml | 39 +++++++++++++++++++++++ build.sh | 20 ++++++++---- docker-compose.test.yml | 45 ++++++++++++++++++++++++++ docker-compose.yml | 2 -- docker/docker-entrypoint.sh | 27 ++++++++++++---- hooks/build | 6 ---- hooks/common | 60 ----------------------------------- hooks/push | 14 -------- hooks/test | 15 --------- test.sh | 22 ++++++++----- 11 files changed, 164 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 docker-compose.test.yml delete mode 100755 hooks/build delete mode 100755 hooks/common delete mode 100755 hooks/push delete mode 100755 hooks/test diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..3290e27 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,31 @@ +on: + push: + branches-ignore: + - release + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + docker_from: + - '' # use the default of the DOCKERFILE + - python:3.7-alpine + - python:3.8-alpine + - python:3.9-rc-alpine + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + DOCKER_FROM: ${{ matrix.docker_from }} + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..72a0bdb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +on: + push: + branches: + - release + schedule: + - cron: '45 5 * * *' + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + - name: Login to the Docker Registry + run: | + echo "::add-mask::$DOCKERHUB_USERNAME" + echo "::add-mask::$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}" + env: + DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} + DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} + - name: Push the image + run: ${{ matrix.build_cmd }} --push-only + - name: Logout of the Docker Registry + run: docker logout "${DOCKER_REGISTRY}" diff --git a/build.sh b/build.sh index 01e841a..94cbe6a 100755 --- a/build.sh +++ b/build.sh @@ -63,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " DRY_RUN Prints all build statements instead of running them." echo " Default: undefined" + echo " GH_ACTION If defined, special 'echo' statements are enabled that set the" + echo " following environment variables in Github Actions:" + echo " - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable" + echo " Default: undefined" echo "" echo "Examples:" echo " ${0} master" @@ -92,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi ### -# Determining the build command to use +# Enabling dry-run mode ### if [ -z "${DRY_RUN}" ]; then DRY="" @@ -102,7 +106,7 @@ else fi ### -# variables for fetching the source +# Variables for fetching the source ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" @@ -111,7 +115,7 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" ### -# fetching the source +# Fetching the source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" @@ -150,7 +154,7 @@ if [ ! -f "${DOCKERFILE}" ]; then fi ### -# variables for labelling the docker image +# Variables for labelling the docker image ### BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" @@ -158,7 +162,7 @@ if [ -d ".git" ]; then GIT_REF="$(git rev-parse HEAD)" fi -# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 +# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" # Get the Git information from the netbox directory @@ -169,7 +173,7 @@ if [ -d "${NETBOX_PATH}/.git" ]; then fi ### -# variables for tagging the docker image +# Variables for tagging the docker image ### DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" @@ -193,6 +197,7 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" ### # Build each target ### +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1} for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do echo "🏗 Building the target '${DOCKER_TARGET}'" @@ -203,6 +208,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi + if [ -n "${GH_ACTION}" ]; then + echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + fi ### # composing the additional DOCKER_SHORT_TAG, diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..534a025 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,45 @@ +version: '3.4' +services: + netbox: + image: ${IMAGE-netboxcommunity/netbox:latest} + depends_on: + - postgres + - redis + env_file: env/netbox.env + user: '101' + volumes: + - ./startup_scripts:/opt/netbox/startup_scripts:z,ro + - ./initializers:/opt/netbox/initializers:z,ro + - ./configuration:/etc/netbox/config:z,ro + - ./reports:/etc/netbox/reports:z,ro + - ./scripts:/etc/netbox/scripts:z,ro + - netbox-nginx-config:/etc/netbox-nginx:z + - netbox-static-files:/opt/netbox/netbox/static:z + - netbox-media-files:/opt/netbox/netbox/media:z + nginx: + command: nginx -c /etc/netbox-nginx/nginx.conf + image: nginx:1.17-alpine + depends_on: + - netbox + ports: + - 8080 + volumes: + - netbox-static-files:/opt/netbox/netbox/static:ro + - netbox-nginx-config:/etc/netbox-nginx/:ro + postgres: + image: postgres:11-alpine + env_file: env/postgres.env + redis: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env +volumes: + netbox-static-files: + driver: local + netbox-nginx-config: + driver: local + netbox-media-files: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index e4961d6..10b4ad9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,8 +57,6 @@ volumes: driver: local netbox-media-files: driver: local - netbox-report-files: - driver: local netbox-postgres-data: driver: local netbox-redis-data: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 8b7481e..f555695 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,13 +1,27 @@ #!/bin/bash +# Runs on every start of the Netbox Docker container + +# Stop when an error occures set -e + +# Allows Netbox to be run as non-root users umask 002 -# wait shortly and then run db migrations (retry on error) -while ! ./manage.py migrate 2>&1; do - echo "⏳ Waiting on DB..." - sleep 3 +# Try to connect to the DB +DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} +MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} +CUR_DB_WAIT_TIME=0 +while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do + echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)" + sleep "${DB_WAIT_TIMEOUT}" + CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT )) done +if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then + echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready." + exit 1 +fi +# Create Superuser if required if [ "$SKIP_SUPERUSER" == "true" ]; then echo "↩️ Skip creating the superuser" else @@ -43,18 +57,19 @@ END echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}" fi +# Run the startup scripts (and initializers) if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩️ Skipping startup scripts" else echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi -# copy static files +# Copy static files ./manage.py collectstatic --no-input echo "✅ Initialisation is done." -# launch whatever is passed by docker +# Launch whatever is passed by docker # (i.e. the RUN instruction in the Dockerfile) # # shellcheck disable=SC2068 diff --git a/hooks/build b/hooks/build deleted file mode 100755 index 80c4165..0000000 --- a/hooks/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. hooks/common - -# shellcheck disable=SC2119 -run_build diff --git a/hooks/common b/hooks/common deleted file mode 100755 index ffa1115..0000000 --- a/hooks/common +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -ensure_jq() { - if [ ! -x "$(command -v jq)" ]; then - if [ -x "$(command -v apt-get)" ]; then - echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" - apt-get update && apt-get install -y jq - else - echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'." - fi - fi -} - -# Passes args to the scripts -run_build() { - echo "🐳🐳🐳 Building '${BUILD}' images" - case ${BUILD} in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ - ;; - this) # Pull Requests - # only build the 'master' branch of netbox - # (resulting in the 'latest' docker tag) - # and the 'main' target. - DOCKER_TARGET=main ./build.sh master - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac -} - -echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG=netboxcommunity -export DOCKER_REPO=netbox -export DOCKER_REGISTRY=docker.io -# shellcheck disable=SC2153 -export BUILD="${DOCKER_TAG}" - -unset DOCKER_TAG - -ensure_jq diff --git a/hooks/push b/hooks/push deleted file mode 100755 index 6513b24..0000000 --- a/hooks/push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -. hooks/common - -if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "release" ]; then - echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". - fi - - run_build --push-only -else - echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" - exit 0 -fi diff --git a/hooks/test b/hooks/test deleted file mode 100755 index a66db12..0000000 --- a/hooks/test +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -. hooks/common - -# test on builds of 'branches' -if [ "${BUILD}" == "branches" ] \ - || [ "${DEBUG}" == "true" ]; then - ./test.sh latest - ./test.sh snapshot -# test on bulds of 'this' (i.e. pull request) -elif [ "${BUILD}" == "this" ]; then - ./test.sh latest -else - echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." -fi diff --git a/test.sh b/test.sh index d6d784a..627eef1 100755 --- a/test.sh +++ b/test.sh @@ -5,11 +5,14 @@ set -e # version is used by `docker-compose.yml` do determine the tag # of the Docker Image that is to be used -export VERSION=${VERSION-latest} +export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" + +# The docker compose command to use +doco="docker-compose -f docker-compose.test.yml" test_netbox_unit_tests() { echo "⏱ Running Netbox Unit Tests" - docker-compose run --rm netbox ./manage.py test + $doco run --rm netbox ./manage.py test } test_initializers() { @@ -25,17 +28,20 @@ test_initializers() { mv initializers initializers_original mv initializers_test initializers - docker-compose run --rm netbox ./manage.py check + $doco run --rm netbox ./manage.py check } test_cleanup() { echo "💣 Cleaning Up" - docker-compose down -v - rm -rf initializers - mv initializers_original initializers + $doco down -v + + if [ -d initializers_original ]; then + rm -rf initializers + mv initializers_original initializers + fi } -echo "🐳🐳🐳 Start testing '${VERSION}'" +echo "🐳🐳🐳 Start testing '${IMAGE}'" # Make sure the cleanup script is executed trap test_cleanup EXIT ERR @@ -43,4 +49,4 @@ trap test_cleanup EXIT ERR test_netbox_unit_tests test_initializers -echo "🐳🐳🐳 Done testing '${VERSION}'" +echo "🐳🐳🐳 Done testing '${IMAGE}'" From 57afeec94f9e0a05d95d8af80ecb4902a899f98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 29 Dec 2019 22:09:32 +0100 Subject: [PATCH 30/67] Netbox is not yet compatible with Python 3.9 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3290e27..4a67b5e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,7 @@ jobs: - '' # use the default of the DOCKERFILE - python:3.7-alpine - python:3.8-alpine - - python:3.9-rc-alpine + # - python:3.9-rc-alpine # Netbox does not work with Python 3.9 yet. fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From 79a10dd4456d05a93e39792e3e63ec4c2b02e21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 14:11:28 +0100 Subject: [PATCH 31/67] Preparation for 0.20.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c0b8d59..5a03fb7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.19.4 +0.20.0 From 3b56c827f95bd5cda2a3895f0521c9fd97942f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 17:21:47 +0100 Subject: [PATCH 32/67] Improve test.sh Don't move the original initializers directory, but rather use the one with the modified initializers in it. --- .gitignore | 1 + docker-compose.test.yml | 2 +- test.sh | 39 +++++++++++++++++++++++---------------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 4b029d7..97aa1b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.sql.gz .netbox +.initializers docker-compose.override.yml diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 534a025..9420617 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,7 +9,7 @@ services: user: '101' volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - - ./initializers:/opt/netbox/initializers:z,ro + - ./${INITIALIZERS_DIR-initializers}:/opt/netbox/initializers:z,ro - ./configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro - ./scripts:/etc/netbox/scripts:z,ro diff --git a/test.sh b/test.sh index 627eef1..2b74847 100755 --- a/test.sh +++ b/test.sh @@ -10,24 +10,31 @@ export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" # The docker compose command to use doco="docker-compose -f docker-compose.test.yml" +INITIALIZERS_DIR=".initializers" + +test_setup() { + echo "🏗 Setup up test environment" + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" + fi + + mkdir "${INITIALIZERS_DIR}" + ( + cd initializers + for script in *.yml; do + sed -E 's/^# //' "${script}" > "../${INITIALIZERS_DIR}/${script}" + done + ) +} + test_netbox_unit_tests() { echo "⏱ Running Netbox Unit Tests" $doco run --rm netbox ./manage.py test } test_initializers() { - echo "🏗 Testing Initializers" - - mkdir initializers_test - ( - cd initializers - for script in *.yml; do - sed -E 's/^# //' "${script}" > "../initializers_test/${script}" - done - ) - mv initializers initializers_original - mv initializers_test initializers - + echo "🏭 Testing Initializers" + export INITIALIZERS_DIR $doco run --rm netbox ./manage.py check } @@ -35,9 +42,8 @@ test_cleanup() { echo "💣 Cleaning Up" $doco down -v - if [ -d initializers_original ]; then - rm -rf initializers - mv initializers_original initializers + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" fi } @@ -45,8 +51,9 @@ echo "🐳🐳🐳 Start testing '${IMAGE}'" # Make sure the cleanup script is executed trap test_cleanup EXIT ERR +test_setup -test_netbox_unit_tests +# test_netbox_unit_tests test_initializers echo "🐳🐳🐳 Done testing '${IMAGE}'" From cfbd037f79e58fbd881cc99dacc4adeba9f1d3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:21:53 +0100 Subject: [PATCH 33/67] Enable tests with Python 3.9 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4a67b5e..3290e27 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,7 @@ jobs: - '' # use the default of the DOCKERFILE - python:3.7-alpine - python:3.8-alpine - # - python:3.9-rc-alpine # Netbox does not work with Python 3.9 yet. + - python:3.9-rc-alpine fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From dd0aee081a8e0f818fe8a95a2efd44449838ee5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 18:10:36 +0100 Subject: [PATCH 34/67] Fix failing GH Action --- .github/workflows/push.yml | 10 +++++++--- .github/workflows/release.yml | 22 ++++++++++++++++------ build-latest.sh | 4 ++++ build.sh | 1 + test.sh | 10 ++++++++++ 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3290e27..8f9ca18 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,12 +20,16 @@ jobs: runs-on: ubuntu-latest name: Builds new Netbox Docker Images steps: - - name: Checkout + - id: git-checkout + name: Checkout uses: actions/checkout@v1 - - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + - id: docker-build + name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' run: ${{ matrix.build_cmd }} env: DOCKER_FROM: ${{ matrix.docker_from }} GH_ACTION: enable - - name: Test the image + - id: docker-test + name: Test the image run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + if: steps.docker-build.outputs.skipped != 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 72a0bdb..9a00d3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,15 +17,20 @@ jobs: runs-on: ubuntu-latest name: Builds new Netbox Docker Images steps: - - name: Checkout + - id: git-checkout + name: Checkout uses: actions/checkout@v1 - - name: Build the image with '${{ matrix.build_cmd }}' + - id: docker-build + name: Build the image with '${{ matrix.build_cmd }}' run: ${{ matrix.build_cmd }} env: GH_ACTION: enable - - name: Test the image + - id: docker-test + name: Test the image run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh - - name: Login to the Docker Registry + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-login + name: Login to the Docker Registry run: | echo "::add-mask::$DOCKERHUB_USERNAME" echo "::add-mask::$DOCKERHUB_PASSWORD" @@ -33,7 +38,12 @@ jobs: env: DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} - - name: Push the image + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-push + name: Push the image run: ${{ matrix.build_cmd }} --push-only - - name: Logout of the Docker Registry + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-logout + name: Logout of the Docker Registry run: docker logout "${DOCKER_REGISTRY}" + if: steps.docker-build.outputs.skipped != 'true' diff --git a/build-latest.sh b/build-latest.sh index 31a0f76..468dffe 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -66,6 +66,10 @@ if [ "${PRERELEASE}" == "true" ]; then echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'." if [ -z "$DEBUG" ]; then + if [ -n "${GH_ACTION}" ]; then + echo "::set-output name=skipped::true" + fi + exit 0 else echo "⚠️ Would exit here with code '0', but DEBUG is enabled." diff --git a/build.sh b/build.sh index 94cbe6a..82a6e34 100755 --- a/build.sh +++ b/build.sh @@ -210,6 +210,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do fi if [ -n "${GH_ACTION}" ]; then echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + echo "::set-output name=skipped::false" fi ### diff --git a/test.sh b/test.sh index 627eef1..419589b 100755 --- a/test.sh +++ b/test.sh @@ -7,6 +7,16 @@ set -e # of the Docker Image that is to be used export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" +if [ -z "${IMAGE}" ]; then + echo "⚠️ No image defined" + + if [ -z "${DEBUG}" ]; then + exit 1; + else + echo "⚠️ Would 'exit 1' here, but DEBUG is '${DEBUG}'." + fi +fi + # The docker compose command to use doco="docker-compose -f docker-compose.test.yml" From 25671d42a50c884d0093c0499a632d17e4a9875b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 20 Jan 2020 08:37:18 +0100 Subject: [PATCH 35/67] Re-enable Netbox unit tests --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 2b74847..6684f7c 100755 --- a/test.sh +++ b/test.sh @@ -53,7 +53,7 @@ echo "🐳🐳🐳 Start testing '${IMAGE}'" trap test_cleanup EXIT ERR test_setup -# test_netbox_unit_tests +test_netbox_unit_tests test_initializers echo "🐳🐳🐳 Done testing '${IMAGE}'" From 355f9d4cf7e5fb7b2df412da412da139c3b579a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 16 Dec 2019 12:51:59 +0100 Subject: [PATCH 36/67] Prepare for Netbox 2.7 --- Dockerfile | 4 +++- configuration/configuration.py | 31 ++++++++++++++++++------------- docker-compose.yml | 8 ++++++++ env/netbox.env | 5 ++++- env/redis-cache.env | 1 + 5 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 env/redis-cache.env diff --git a/Dockerfile b/Dockerfile index 82576c1..7764248 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,9 @@ RUN pip install --prefix="/install" --no-warn-script-location \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ # django_auth_ldap is required for ldap - django_auth_ldap + django_auth_ldap \ +# django-storages was introduced in 2.7 and is optional + django-storages ARG NETBOX_PATH COPY ${NETBOX_PATH}/requirements.txt / diff --git a/configuration/configuration.py b/configuration/configuration.py index 4199b9d..af121d9 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -37,9 +37,9 @@ DATABASE = { # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) - 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, # Database connection SSLMODE - 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), + 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), # Database connection persistence } @@ -51,13 +51,22 @@ SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key')) # Redis database settings. The Redis database is used for caching and background processing such as webhooks REDIS = { - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), - 'DATABASE': os.environ.get('REDIS_DATABASE', '0'), - 'CACHE_DATABASE': os.environ.get('REDIS_CACHE_DATABASE', '1'), - 'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + 'webhooks': { + 'HOST': os.environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(os.environ.get('REDIS_PORT', 6379)), + 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), + 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), + 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), + 'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))), + 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), + 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', + }, } ######################### @@ -172,10 +181,6 @@ SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') # Time zone (default: UTC) TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') -# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis -# database be configured and accessible by NetBox (see `REDIS` below). -WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true' - # Date/time formatting. See the following link for supported formats: # https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y') diff --git a/docker-compose.yml b/docker-compose.yml index 10b4ad9..3806480 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: depends_on: - postgres - redis + - redis-cache - netbox-worker env_file: env/netbox.env user: '101' @@ -50,6 +51,13 @@ services: env_file: env/redis.env volumes: - netbox-redis-data:/data + redis-cache: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env volumes: netbox-static-files: driver: local diff --git a/env/netbox.env b/env/netbox.env index 4a75905..aaa7482 100644 --- a/env/netbox.env +++ b/env/netbox.env @@ -17,8 +17,11 @@ MAX_PAGE_SIZE=1000 REDIS_HOST=redis REDIS_PASSWORD=H733Kdjndks81 REDIS_DATABASE=0 -REDIS_CACHE_DATABASE=1 REDIS_SSL=false +REDIS_CACHE_HOST=redis-cache +REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36 +REDIS_CACHE_DATABASE=0 +REDIS_CACHE_SSL=false SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj SKIP_STARTUP_SCRIPTS=false SKIP_SUPERUSER=false diff --git a/env/redis-cache.env b/env/redis-cache.env new file mode 100644 index 0000000..6285c33 --- /dev/null +++ b/env/redis-cache.env @@ -0,0 +1 @@ +REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 From c001626b85ba07bf2203faee16b09372fbe42ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:22:16 +0100 Subject: [PATCH 37/67] Update Custom Fields Initializer for Netbox 2.7 The custom field database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your custom_fields.yml file as follows: - type must be lowercase - the `selection` type was changed to `select` - the filter_logic must be lower case This is to achieve compatibility with the naming schema that Netbox uses internally. It allows us to become forward compatible in case Netbox ever introduces a new type for custom fields. See the diff of this commit for further information how this is meant. --- initializers/custom_fields.yml | 23 +++++++++++++++++++---- startup_scripts/020_custom_fields.py | 18 +----------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/initializers/custom_fields.yml b/initializers/custom_fields.yml index 0b6472a..4085ab0 100644 --- a/initializers/custom_fields.yml +++ b/initializers/custom_fields.yml @@ -1,3 +1,18 @@ +## Possible Choices: +## type: +## - text +## - integer +## - boolean +## - date +## - url +## - select +## filter_logic: +## - disabled +## - loose +## - exact +## +## Examples: + # text_field: # type: text # label: Custom Text @@ -22,8 +37,8 @@ # weight: 10 # on_objects: # - tenancy.models.Tenant -# selection_field: -# type: selection +# select_field: +# type: select # label: Choose between items # required: false # filter_logic: exact @@ -41,8 +56,8 @@ # weight: 50 # - value: Fourth Item # weight: 40 -# selection_field_auto_weight: -# type: selection +# select_field_auto_weight: +# type: select # label: Choose between items # required: false # filter_logic: loose diff --git a/startup_scripts/020_custom_fields.py b/startup_scripts/020_custom_fields.py index 76a32bb..2f6ba72 100644 --- a/startup_scripts/020_custom_fields.py +++ b/startup_scripts/020_custom_fields.py @@ -1,19 +1,9 @@ -from extras.constants import CF_TYPE_TEXT, CF_TYPE_INTEGER, CF_TYPE_BOOLEAN, CF_TYPE_DATE, CF_TYPE_URL, CF_TYPE_SELECT, CF_FILTER_CHOICES from extras.models import CustomField, CustomFieldChoice from ruamel.yaml import YAML from pathlib import Path import sys -text_to_fields = { - 'boolean': CF_TYPE_BOOLEAN, - 'date': CF_TYPE_DATE, - 'integer': CF_TYPE_INTEGER, - 'selection': CF_TYPE_SELECT, - 'text': CF_TYPE_TEXT, - 'url': CF_TYPE_URL, -} - def get_class_for_class_path(class_path): import importlib from django.contrib.contenttypes.models import ContentType @@ -42,12 +32,6 @@ with file.open('r') as stream: if cf_details.get('description', 0): custom_field.description = cf_details['description'] - # If no filter_logic is specified then it will default to 'Loose' - if cf_details.get('filter_logic', 0): - for choice_id, choice_text in CF_FILTER_CHOICES: - if choice_text.lower() == cf_details['filter_logic']: - custom_field.filter_logic = choice_id - if cf_details.get('label', 0): custom_field.label = cf_details['label'] @@ -58,7 +42,7 @@ with file.open('r') as stream: custom_field.required = cf_details['required'] if cf_details.get('type', 0): - custom_field.type = text_to_fields[cf_details['type']] + custom_field.type = cf_details['type'] if cf_details.get('weight', 0): custom_field.weight = cf_details['weight'] From 7b914d31d66f56bb9cf6ab8fbba131a23a6127a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:44:19 +0100 Subject: [PATCH 38/67] Update Rack Initializer for Netbox 2.7 The rack database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your racks.yml file as follows: - Rack types must match one of the 5 rack types given, e.g. '4-post-cabinet'. - Rack width must match one of the 2 rack widths given, i.e. '19' or '23'. See the diff of this commit for further information how this is meant. --- initializers/racks.yml | 28 ++++++++++++++++++++++------ startup_scripts/080_racks.py | 9 --------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/initializers/racks.yml b/initializers/racks.yml index 3a31526..51502de 100644 --- a/initializers/racks.yml +++ b/initializers/racks.yml @@ -1,16 +1,32 @@ +## Possible Choices: +## width: +## - 19 +## - 23 +## types: +## - 2-post-frame +## - 4-post-frame +## - 4-post-cabinet +## - wall-frame +## - wall-cabinet +## outer_unit: +## - mm +## - in +## +## Examples: + # - site: AMS 1 # name: rack-01 # role: Role 1 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description # - site: AMS 2 # name: rack-02 # role: Role 2 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description @@ -18,8 +34,8 @@ # name: rack-03 # group: cage 101 # role: Role 3 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description diff --git a/startup_scripts/080_racks.py b/startup_scripts/080_racks.py index 05bca10..ed7713d 100644 --- a/startup_scripts/080_racks.py +++ b/startup_scripts/080_racks.py @@ -1,7 +1,6 @@ from dcim.models import Site, RackRole, Rack, RackGroup from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue -from dcim.constants import RACK_TYPE_CHOICES, RACK_WIDTH_CHOICES from ruamel.yaml import YAML from pathlib import Path import sys @@ -41,14 +40,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - for rack_type in RACK_TYPE_CHOICES: - if params['type'] in rack_type: - params['type'] = rack_type[0] - - for rack_width in RACK_WIDTH_CHOICES: - if params['width'] in rack_width: - params['width'] = rack_width[0] - rack, created = Rack.objects.get_or_create(**params) if created: From 4a58676647e41115d5ecc10c4e9035ca73db53ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:52:17 +0100 Subject: [PATCH 39/67] Update Device Initializer for Netbox 2.7 The device database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your devices.yml file as follows: - Make sure the rack face is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/devices.yml | 21 ++++++++++++++++++--- startup_scripts/130_devices.py | 7 ------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/initializers/devices.yml b/initializers/devices.yml index 0beb6f2..708b68c 100644 --- a/initializers/devices.yml +++ b/initializers/devices.yml @@ -1,9 +1,24 @@ +## Possible Choices: +## face: +## - front +## - rear +## status: +## - offline +## - active +## - planned +## - staged +## - failed +## - inventory +## - decommissioning +## +## Examples: + # - name: server01 # device_role: server # device_type: Other # site: AMS 1 # rack: rack-01 -# face: Front +# face: front # position: 1 # custom_fields: # text_field: Description @@ -12,7 +27,7 @@ # device_type: Other # site: AMS 2 # rack: rack-02 -# face: Front +# face: front # position: 2 # custom_fields: # text_field: Description @@ -21,7 +36,7 @@ # device_type: Other # site: SING 1 # rack: rack-03 -# face: Front +# face: front # position: 3 # custom_fields: # text_field: Description diff --git a/startup_scripts/130_devices.py b/startup_scripts/130_devices.py index 2d8d3ca..4217549 100644 --- a/startup_scripts/130_devices.py +++ b/startup_scripts/130_devices.py @@ -1,5 +1,4 @@ from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform -from dcim.constants import RACK_FACE_CHOICES from ipam.models import IPAddress from virtualization.models import Cluster from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'face' in params: - for rack_face in RACK_FACE_CHOICES: - if params['face'] in rack_face: - params['face'] = rack_face[0] - break - device, created = Device.objects.get_or_create(**params) if created: From 8d8b9a157e219ce3ed5944f8fd19ca668b4c6cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:59:21 +0100 Subject: [PATCH 40/67] Update VLAN Initializer for Netbox 2.7 The VLAN database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your vlans.yml file as follows: - Make sure the status is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/vlans.yml | 12 ++++++++++-- startup_scripts/210_vlans.py | 7 ------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/initializers/vlans.yml b/initializers/vlans.yml index 26532c7..a8cd521 100644 --- a/initializers/vlans.yml +++ b/initializers/vlans.yml @@ -1,11 +1,19 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## +## Examples: + # - name: vlan1 # site: AMS 1 -# status: Active +# status: active # vid: 5 # role: Main Management # description: VLAN 5 for MGMT # - group: VLAN group 2 # name: vlan2 # site: AMS 1 -# status: Active +# status: active # vid: 1300 diff --git a/startup_scripts/210_vlans.py b/startup_scripts/210_vlans.py index e3a0ef1..ab6bd2b 100644 --- a/startup_scripts/210_vlans.py +++ b/startup_scripts/210_vlans.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import VLAN, VLANGroup, Role -from ipam.constants import VLAN_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -35,12 +34,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vlan_status in VLAN_STATUS_CHOICES: - if params['status'] in vlan_status: - params['status'] = vlan_status[0] - break - vlan, created = VLAN.objects.get_or_create(**params) if created: From f3403cd0f5a5c26e5aea9afed2e4d3a5a129a1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:03:45 +0100 Subject: [PATCH 41/67] Update VM Initializer for Netbox 2.7 The vm database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your virtual_machines.yml file as follows: - Make sure the status is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/prefixes.yml | 15 ++++++++++++--- initializers/virtual_machines.yml | 12 ++++++++++-- startup_scripts/220_prefixes.py | 7 ------- startup_scripts/230_virtual_machines.py | 7 ------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/initializers/prefixes.yml b/initializers/prefixes.yml index a7e6815..fbf3eee 100644 --- a/initializers/prefixes.yml +++ b/initializers/prefixes.yml @@ -1,13 +1,22 @@ +## Possible Choices: +## status: +## - container +## - active +## - reserved +## - deprecated +## +## Examples: + # - description: prefix1 # prefix: 10.1.1.0/24 # site: AMS 1 -# status: Active +# status: active # tenant: tenant1 # vlan: vlan1 # - description: prefix2 # prefix: 10.1.2.0/24 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 # is_pool: true @@ -15,6 +24,6 @@ # - description: ipv6 prefix1 # prefix: 2001:db8:a000:1::/64 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 diff --git a/initializers/virtual_machines.yml b/initializers/virtual_machines.yml index 5da75a2..2122920 100644 --- a/initializers/virtual_machines.yml +++ b/initializers/virtual_machines.yml @@ -1,10 +1,18 @@ +## Possible Choices: +## status: +## - active +## - offline +## - staged +## +## Examples: + # - cluster: cluster1 # comments: VM1 # disk: 200 # memory: 4096 # name: virtual machine 1 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 # - cluster: cluster1 @@ -13,6 +21,6 @@ # memory: 2048 # name: virtual machine 2 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 diff --git a/startup_scripts/220_prefixes.py b/startup_scripts/220_prefixes.py index a832c88..d13578a 100644 --- a/startup_scripts/220_prefixes.py +++ b/startup_scripts/220_prefixes.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import Prefix, VLAN, Role, VRF -from ipam.constants import PREFIX_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -38,12 +37,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for prefix_status in PREFIX_STATUS_CHOICES: - if params['status'] in prefix_status: - params['status'] = prefix_status[0] - break - prefix, created = Prefix.objects.get_or_create(**params) if created: diff --git a/startup_scripts/230_virtual_machines.py b/startup_scripts/230_virtual_machines.py index 065b600..449df8a 100644 --- a/startup_scripts/230_virtual_machines.py +++ b/startup_scripts/230_virtual_machines.py @@ -1,6 +1,5 @@ from dcim.models import Site, Platform, DeviceRole from virtualization.models import Cluster, VirtualMachine -from virtualization.constants import VM_STATUS_CHOICES from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -43,12 +42,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vm_status in VM_STATUS_CHOICES: - if params['status'] in vm_status: - params['status'] = vm_status[0] - break - virtual_machine, created = VirtualMachine.objects.get_or_create(**params) if created: From 74a0e2cf6ee38d28d312386514a9e3ce8116b015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:14:09 +0100 Subject: [PATCH 42/67] Update Interfaces Initializer for Netbox 2.7 The interface database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your dcim_interfaces.yml file as follows: - Make sure the type is a value out of the possible choices. See the diff of this commit for further information how this is meant. --- initializers/dcim_interfaces.yml | 14 ++++++++++++-- startup_scripts/250_dcim_interfaces.py | 11 ----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/initializers/dcim_interfaces.yml b/initializers/dcim_interfaces.yml index 18920fe..4030530 100644 --- a/initializers/dcim_interfaces.yml +++ b/initializers/dcim_interfaces.yml @@ -1,8 +1,18 @@ +## Possible Choices: +## type: +## - virtual +## - lag +## - 1000base-t +## - ... and many more. See for yourself: +## https://github.com/netbox-community/netbox/blob/295d4f0394b431351c0cb2c3ecc791df68c6c2fb/netbox/dcim/choices.py#L510 +## +## Examples: + # - device: server01 # enabled: true -# type: Virtual +# type: virtual # name: to-server02 # - device: server02 # enabled: true -# type: Virtual +# type: virtual # name: to-server01 diff --git a/startup_scripts/250_dcim_interfaces.py b/startup_scripts/250_dcim_interfaces.py index ce7e7bd..ec30b5c 100644 --- a/startup_scripts/250_dcim_interfaces.py +++ b/startup_scripts/250_dcim_interfaces.py @@ -1,5 +1,4 @@ from dcim.models import Interface, Device -from dcim.constants import IFACE_TYPE_CHOICES from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -28,16 +27,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'type' in params: - for outer_list in IFACE_TYPE_CHOICES: - for type_choices in outer_list[1]: - if params['type'] in type_choices: - params['type'] = type_choices[0] - break - else: - continue - break - interface, created = Interface.objects.get_or_create(**params) if created: From 27f671e41ad561ad3ecf3493e410624c2c503c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:18:52 +0100 Subject: [PATCH 43/67] Update IP Initializer for Netbox 2.7 The ip address database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your ip_addresses.yml file as follows: - Make sure the status is written in lower case. See the diff of this commit for further information how this is meant. --- initializers/ip_addresses.yml | 30 +++++++++++++++++++++++------ startup_scripts/260_ip_addresses.py | 7 ------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/initializers/ip_addresses.yml b/initializers/ip_addresses.yml index 5738749..6ac38e9 100644 --- a/initializers/ip_addresses.yml +++ b/initializers/ip_addresses.yml @@ -1,26 +1,44 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## - dhcp +## role: +## - loopback +## - secondary +## - anycast +## - vip +## - vrrp +## - hsrp +## - glbp +## - carp +## +## Examples: + # - address: 10.1.1.1/24 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 2001:db8:a000:1::1/64 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 10.1.1.2/24 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 2001:db8:a000:1::2/64 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 10.1.1.10/24 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 # - address: 2001:db8:a000:1::10/64 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 diff --git a/startup_scripts/260_ip_addresses.py b/startup_scripts/260_ip_addresses.py index f7f2bc7..d109a36 100644 --- a/startup_scripts/260_ip_addresses.py +++ b/startup_scripts/260_ip_addresses.py @@ -1,5 +1,4 @@ from ipam.models import IPAddress, VRF -from ipam.constants import IPADDRESS_STATUS_CHOICES from dcim.models import Device, Interface from virtualization.models import VirtualMachine from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: query = { field: params.pop(assoc) } params[assoc] = model.objects.get(**query) - if 'status' in params: - for ip_status in IPADDRESS_STATUS_CHOICES: - if params['status'] in ip_status: - params['status'] = ip_status[0] - break - ip_address, created = IPAddress.objects.get_or_create(**params) if created: From 2b21e14c2c69d1c747d1a349c365521d2f877054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 13 Jan 2020 18:14:23 +0100 Subject: [PATCH 44/67] Check that `develop` is target in PR This is an update to the PR template. It mostly adds text and some checks related to how good the template is filled in and whether the `develop` branch was selected as base. --- .github/pull_request_template.md | 46 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1c3885f..792055a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,13 +1,31 @@ + + -But if there is already a related issue, please put it's number here. E.g. #123 or N/A --> Related Issue: ## New Behavior ... @@ -15,7 +33,8 @@ please describe in a few words the behavior your PR implements ## Contrast to Current Behavior ... @@ -24,12 +43,15 @@ please describe in a few words how the new behavior is different from the curren ... @@ -38,7 +60,17 @@ Please make your case here: ... + +## Double Check + + + +* [ ] I have read the comments and followed the PR template. +* [ ] I have provided a explained my PR according to the information in the comments. +* [ ] My PR targets the `develop` branch. From 1bad9b4fa89639e205472f49d5a0e8f7e41be8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 13 Jan 2020 18:24:51 +0100 Subject: [PATCH 45/67] Fix typo in pull request template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 792055a..6f755c3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -72,5 +72,5 @@ Please put an x into the brackets (like `[x]`) if you've completed that task. --> * [ ] I have read the comments and followed the PR template. -* [ ] I have provided a explained my PR according to the information in the comments. +* [ ] I have provided and explained my PR according to the information in the comments. * [ ] My PR targets the `develop` branch. From 8892ea99369ae6115d47588886373179b51232a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 18 Jan 2020 11:48:08 +0100 Subject: [PATCH 46/67] Ask for a release note entry --- .github/pull_request_template.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6f755c3..32b34f0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -65,6 +65,15 @@ If the Wiki must be updated, please make a suggestion below. ... +## Proposed Release Note Entry + + + +... + ## Double Check * [ ] I have read the comments and followed the PR template. -* [ ] I have provided and explained my PR according to the information in the comments. +* [ ] I have explained my PR according to the information in the comments. * [ ] My PR targets the `develop` branch. From 653321994a4431f69c2c0501ce72c141c00d759e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 23 Jan 2020 10:26:21 +0100 Subject: [PATCH 51/67] Fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db6e867..2f48b4e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Do you have any questions? Before opening an issue on Github, please join the [N [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/ [ntc-slack]: http://slack.networktocode.com/ [netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E -[netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/master/LICENSE +[netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/release/LICENSE ## Docker Tags From 1c8d695fc2dcc53e2be0684f428f7c014798495f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 23 Jan 2020 10:36:13 +0100 Subject: [PATCH 52/67] Remove references to the old build system --- DOCKER_HUB.md | 63 --------------------------------------------------- README.md | 19 +++++++++------- 2 files changed, 11 insertions(+), 71 deletions(-) delete mode 100644 DOCKER_HUB.md diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md deleted file mode 100644 index d3b3e68..0000000 --- a/DOCKER_HUB.md +++ /dev/null @@ -1,63 +0,0 @@ -# cloud.docker.com Configuration - -The automatic build is configured in cloud.docker.com. - -The following build configuration is expected: - -```yaml -Source Repository: github.com/netbox-community/netbox-docker -Build Location: Build on Docker Hub's infrastructure -Autotest: Internal and External Pull Requests -Repository Links: Enable for Base Image -Build Rules: -- Source Type: Branch - Source: release - Docker Tag: branches - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -- Source Type: Branch - Source: release - Docker Tag: prerelease - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -- Source Type: Branch - Source: release - Docker Tag: release - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -Build Environment Variables: -# Create an app on Github and use it's OATH credentials here -- Key: GITHUB_OAUTH_CLIENT_ID - Value: -- Key: GITHUB_OAUTH_CLIENT_SECRET - Value: -Build Triggers: -- Name: Cron Trigger - Trigger URL: -# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds -``` - -## Background Knowledge - -The build system of cloud.docker.com is not made for this kind of project. -But we found a way to make it work, and this is how: - -1. The docker hub build system [allows to overwrite the scripts that get executed - for `build`, `test` and `push`](overwrite). See `/hooks/*`. -2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`. -3. The `build` script runs `run_build()` from `/hooks/common`. - This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`. -4. The `test` script just invokes `docker-compose` commands. -5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. - This causes the `build.sh` script to not re-build the Docker image, but just the just built image. - -The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`. -Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose. - -[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands diff --git a/README.md b/README.md index 2f48b4e..5f12bfc 100644 --- a/README.md +++ b/README.md @@ -124,31 +124,34 @@ docker-compose up -d From time to time it might become necessary to re-engineer the structure of this setup. Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence. + Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label. (The images contained labels since April 2018, although in November 2019 the labels' names changed.) -You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.7.1 --format "{{json .ContainerConfig.Labels}}"`. +You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.7.1 --format "{{json .Config.Labels}}"`. Please read [the release notes][releases] carefully when updating to a new image version. [releases]: https://github.com/netbox-community/netbox-docker/releases -## Rebuilding & Publishing images +## Rebuilding the Image `./build.sh` can be used to rebuild the Docker image. See `./build.sh --help` for more information. -### Publishing Docker Images +For more details on custom builds [consult our wiki][netbox-docker-wiki-build]. -New Docker images are built and published every 24h on the [Docker Build Infrastructure][docker-build-infra]. -`DOCKER_HUB.md` contains more information about the build infrastructure. +[netbox-docker-wiki-build]: https://github.com/netbox-community/netbox-docker/wiki/Build -[docker-build-infra]: https://hub.docker.com/r/netboxcommunity/netbox/builds/ +### Pre-made Docker Images + +New Docker images are built and published every 24h. ## Tests -To run the tests coming with Netbox, use the `docker-compose.yml` file as such: +We have a test script. +It runs Netbox's own unit tests and ensures that all initializers work: ```bash -docker-compose run netbox ./manage.py test +IMAGE=netboxcommunity/netbox:latest ./test.sh ``` ## About From f0b00ee104646af78140d2963608ae576b46742e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 23 Jan 2020 10:40:28 +0100 Subject: [PATCH 53/67] Update docker inspect command in bug_report template --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7717abd..82f22b1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -48,7 +48,7 @@ The output of `git rev-parse HEAD`: `XXXXX` The command you used to start the project: `XXXXX` -The output of `docker inspect netboxcommunity/netbox:latest --format "{{json .ContainerConfig.Labels}}"`: +The output of `docker inspect netboxcommunity/netbox:latest --format "{{json .Config.Labels}}"`: ```json { From 0e625a3b5c1780532e8e091bfd670abce185cf9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 23 Jan 2020 10:44:02 +0100 Subject: [PATCH 54/67] Disable python 3.9 tests once more --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8f9ca18..66062e6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,7 @@ jobs: - '' # use the default of the DOCKERFILE - python:3.7-alpine - python:3.8-alpine - - python:3.9-rc-alpine + # - python:3.9-rc-alpine # disable until Netbox's unit tests work fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From 1eb40d1774988e7c7b4703afc563b1e513d26c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 23 Jan 2020 10:02:10 +0100 Subject: [PATCH 55/67] Preparation for 0.21.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8854156..a67ceba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0 +0.21.1 From 00986573d907ea6e79fdf144b01992d56fe43d51 Mon Sep 17 00:00:00 2001 From: Kevin Newland Date: Tue, 28 Jan 2020 17:36:45 -0600 Subject: [PATCH 56/67] Update LDAP Caching Options Update LDAP caching configuration to match changes made to django-auth-ldap in 1.6.0 Django social auth now uses different cache configuration options: https://github.com/django-auth-ldap/django-auth-ldap/blob/master/django_auth_ldap/backend.py#L1041-L1056 NetBox settings.py reference: https://github.com/netbox-community/netbox/blob/master/netbox/netbox/settings.py#L360 --- configuration/ldap_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configuration/ldap_config.py b/configuration/ldap_config.py index 39fc894..2ec75cd 100644 --- a/configuration/ldap_config.py +++ b/configuration/ldap_config.py @@ -70,8 +70,7 @@ AUTH_LDAP_USER_FLAGS_BY_GROUP = { AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' # Cache groups for one hour to reduce LDAP traffic -AUTH_LDAP_CACHE_GROUPS = os.environ.get('AUTH_LDAP_CACHE_GROUPS', 'True').lower() == 'true' -AUTH_LDAP_GROUP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_GROUP_CACHE_TIMEOUT', 3600)) +AUTH_LDAP_CACHE_TIMEOUT = = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) # Populate the Django user from the LDAP directory. AUTH_LDAP_USER_ATTR_MAP = { From 74eaae6bc82a959a752751e905c39255adc0c307 Mon Sep 17 00:00:00 2001 From: Kevin Newland Date: Tue, 28 Jan 2020 17:43:35 -0600 Subject: [PATCH 57/67] Update ldap_config.py --- configuration/ldap_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/ldap_config.py b/configuration/ldap_config.py index 2ec75cd..ba2067c 100644 --- a/configuration/ldap_config.py +++ b/configuration/ldap_config.py @@ -70,7 +70,7 @@ AUTH_LDAP_USER_FLAGS_BY_GROUP = { AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' # Cache groups for one hour to reduce LDAP traffic -AUTH_LDAP_CACHE_TIMEOUT = = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) +AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) # Populate the Django user from the LDAP directory. AUTH_LDAP_USER_ATTR_MAP = { From 778f7546b8e4d5f20fbc9b2cb7000bf0495749fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 30 Jan 2020 15:48:01 +0100 Subject: [PATCH 58/67] Enable push workflow for PRs --- .github/workflows/push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 569b333..4932ae8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -4,6 +4,9 @@ on: push: branches-ignore: - release + pull_request: + branches-ignore: + - release jobs: build: From 927a545f41b6ca8490e12de63ca0d346043884e9 Mon Sep 17 00:00:00 2001 From: Les Begnaud Date: Thu, 5 Dec 2019 09:41:11 -0600 Subject: [PATCH 59/67] adjust groups and users startup scripts to allow custom codename filter --- initializers/groups.yml | 14 ++++++++++---- initializers/users.yml | 8 ++++---- startup_scripts/000_users.py | 27 ++++++++++++++++++++------- startup_scripts/010_groups.py | 24 ++++++++++++++++++------ 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/initializers/groups.yml b/initializers/groups.yml index 7bdd0a7..fc9ff28 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -7,10 +7,16 @@ # writers: # users: # - writer +## specify explicit permission codenames or codename filter functions and filters to match on # permissions: -# - add_device -# - change_device # - delete_device -# - add_virtualmachine -# - change_virtualmachine # - delete_virtualmachine +# - codename__startswith: +# - add_ +# - change_ +# vm_managers: +# - codename__endswith: +# - _virtualmachine +# creators: +# - codename__startswith: +# - add_ diff --git a/initializers/users.yml b/initializers/users.yml index 64c005c..a7a57a3 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -4,10 +4,10 @@ # password: reader # writer: # password: writer +## specify explicit permission codenames or codename filter functions and filters to match on # permissions: -# - add_device -# - change_device # - delete_device -# - add_virtualmachine -# - change_virtualmachine # - delete_virtualmachine +# - codename__startswith: +# - add_ +# - change_ diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index a1340a1..0037ee1 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -25,10 +25,23 @@ with file.open('r') as stream: if user_details.get('api_token', 0): Token.objects.create(user=user, key=user_details['api_token']) - user_permissions = user_details.get('permissions', []) - if user_permissions: - user.user_permissions.clear() - for permission_codename in user_details.get('permissions', []): - for permission in Permission.objects.filter(codename=permission_codename): - user.user_permissions.add(permission) - user.save() + yaml_permissions = user_details.get('permissions', []) + permission_object = user + if yaml_permissions: + permission_object.permissions.clear() + for yaml_permission in yaml_permissions: + if isinstance(yaml_permission,dict): + # assume this is the specific codename filter function instead of an exact codename + permission_codename_function = list(yaml_permission.keys())[0] + permission_codenames = yaml_permission[permission_codename_function] + else: + permission_codename_function = 'codename' + permission_codenames = list({yaml_permission}) + + # supports either one codename from the permissions list, or multiple codenames in a codename_function dict + for permission_codename in permission_codenames: + # supports non-unique permission codenames + for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): + permission_object.permissions.add(permission) + + permission_object.save() diff --git a/startup_scripts/010_groups.py b/startup_scripts/010_groups.py index e68a5f0..49b6678 100644 --- a/startup_scripts/010_groups.py +++ b/startup_scripts/010_groups.py @@ -24,9 +24,21 @@ with file.open('r') as stream: if user: user.groups.add(group) - group_permissions = group_details.get('permissions', []) - if group_permissions: - group.permissions.clear() - for permission_codename in group_details.get('permissions', []): - for permission in Permission.objects.filter(codename=permission_codename): - group.permissions.add(permission) + yaml_permissions = group_details.get('permissions', []) + permission_object = group + if yaml_permissions: + permission_object.permissions.clear() + for yaml_permission in yaml_permissions: + if isinstance(yaml_permission,dict): + # assume this is the specific codename filter function instead of an exact codename + permission_codename_function = list(yaml_permission.keys())[0] + permission_codenames = yaml_permission[permission_codename_function] + else: + permission_codename_function = 'codename' + permission_codenames = list({yaml_permission}) + + # supports either one codename from the permissions list, or multiple codenames in a codename_function dict + for permission_codename in permission_codenames: + # supports non-unique permission codenames + for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): + permission_object.permissions.add(permission) From cce4370d4141db79b1df37fdcf905b4e6b6d1189 Mon Sep 17 00:00:00 2001 From: Les Begnaud Date: Thu, 5 Dec 2019 09:44:40 -0600 Subject: [PATCH 60/67] add permission example --- initializers/groups.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/initializers/groups.yml b/initializers/groups.yml index fc9ff28..6d0665f 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -17,6 +17,9 @@ # vm_managers: # - codename__endswith: # - _virtualmachine +# device_managers: +# - codename__contains: +# - device # creators: # - codename__startswith: # - add_ From aa0d2a6e01b7ace9e81f91a56b09eae844352278 Mon Sep 17 00:00:00 2001 From: Les Begnaud Date: Wed, 11 Dec 2019 10:14:38 -0600 Subject: [PATCH 61/67] simplify yml definitions to use wildcard syntax --- initializers/groups.yml | 16 ++++++---------- initializers/users.yml | 7 +++---- startup_scripts/000_users.py | 19 ++++++++----------- startup_scripts/010_groups.py | 19 ++++++++----------- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/initializers/groups.yml b/initializers/groups.yml index 6d0665f..d471fe3 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -7,19 +7,15 @@ # writers: # users: # - writer -## specify explicit permission codenames or codename filter functions and filters to match on +## specify explicit permission codenames or include wildcard to match multiple permissions # permissions: # - delete_device # - delete_virtualmachine -# - codename__startswith: -# - add_ -# - change_ +# - add_* +# - change_* # vm_managers: -# - codename__endswith: -# - _virtualmachine +# - *_virtualmachine # device_managers: -# - codename__contains: -# - device +# - *device* # creators: -# - codename__startswith: -# - add_ +# - add_* diff --git a/initializers/users.yml b/initializers/users.yml index a7a57a3..ad654b7 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -4,10 +4,9 @@ # password: reader # writer: # password: writer -## specify explicit permission codenames or codename filter functions and filters to match on +## specify explicit permission codenames or include wildcard to match multiple permissions # permissions: # - delete_device # - delete_virtualmachine -# - codename__startswith: -# - add_ -# - change_ +# - add_* +# - change_* diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index 0037ee1..cb04a16 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -30,18 +30,15 @@ with file.open('r') as stream: if yaml_permissions: permission_object.permissions.clear() for yaml_permission in yaml_permissions: - if isinstance(yaml_permission,dict): - # assume this is the specific codename filter function instead of an exact codename - permission_codename_function = list(yaml_permission.keys())[0] - permission_codenames = yaml_permission[permission_codename_function] + if '*' in yaml_permission: + permission_codename_function = 'codename__iregex' + permission_codename = '^' + yaml_permission.replace('*','.*') + '$' else: permission_codename_function = 'codename' - permission_codenames = list({yaml_permission}) - - # supports either one codename from the permissions list, or multiple codenames in a codename_function dict - for permission_codename in permission_codenames: - # supports non-unique permission codenames - for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): - permission_object.permissions.add(permission) + permission_codename = yaml_permission + + # supports non-unique permission codenames + for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): + permission_object.permissions.add(permission) permission_object.save() diff --git a/startup_scripts/010_groups.py b/startup_scripts/010_groups.py index 49b6678..990e065 100644 --- a/startup_scripts/010_groups.py +++ b/startup_scripts/010_groups.py @@ -29,16 +29,13 @@ with file.open('r') as stream: if yaml_permissions: permission_object.permissions.clear() for yaml_permission in yaml_permissions: - if isinstance(yaml_permission,dict): - # assume this is the specific codename filter function instead of an exact codename - permission_codename_function = list(yaml_permission.keys())[0] - permission_codenames = yaml_permission[permission_codename_function] + if '*' in yaml_permission: + permission_codename_function = 'codename__iregex' + permission_codename = '^' + yaml_permission.replace('*','.*') + '$' else: permission_codename_function = 'codename' - permission_codenames = list({yaml_permission}) - - # supports either one codename from the permissions list, or multiple codenames in a codename_function dict - for permission_codename in permission_codenames: - # supports non-unique permission codenames - for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): - permission_object.permissions.add(permission) + permission_codename = yaml_permission + + # supports non-unique permission codenames + for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): + permission_object.permissions.add(permission) From f4e243d5ad55c56090164e14fc68173b002eaa78 Mon Sep 17 00:00:00 2001 From: Les Begnaud Date: Wed, 11 Dec 2019 10:21:28 -0600 Subject: [PATCH 62/67] update example to note yaml restriction --- initializers/groups.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/initializers/groups.yml b/initializers/groups.yml index d471fe3..67a9290 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -14,8 +14,9 @@ # - add_* # - change_* # vm_managers: -# - *_virtualmachine +## yaml doesn't allow starting a key with an asterisk without explicit use of single quote +# - '*_virtualmachine' # device_managers: -# - *device* +# - '*device*' # creators: # - add_* From a2c06026d57ff2a5160df9eed116ba713e019e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 31 Jan 2020 11:35:25 +0100 Subject: [PATCH 63/67] Ajdust indents in __main__.py ... so that the match the style of the other python code in this project --- startup_scripts/__main__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/startup_scripts/__main__.py b/startup_scripts/__main__.py index 50d2613..b13b8dd 100644 --- a/startup_scripts/__main__.py +++ b/startup_scripts/__main__.py @@ -7,12 +7,12 @@ from os.path import dirname, abspath this_dir = dirname(abspath(__file__)) def filename(f): - return f.name + return f.name with scandir(dirname(abspath(__file__))) as it: - for f in sorted(it, key = filename): - if f.name.startswith('__') or not f.is_file(): - continue - - print(f"Running {f.path}") - runpy.run_path(f.path) + for f in sorted(it, key = filename): + if f.name.startswith('__') or not f.is_file(): + continue + + print(f"Running {f.path}") + runpy.run_path(f.path) From ba3176f140973b6bad219269a24467e6aa1b3a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 31 Jan 2020 11:37:05 +0100 Subject: [PATCH 64/67] Added missing keywords to the yaml ... and moved some documentatory comments to the beginning of the file. --- initializers/groups.yml | 17 +++++++++++++++-- initializers/users.yml | 13 ++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/initializers/groups.yml b/initializers/groups.yml index 67a9290..b91ef39 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -1,3 +1,15 @@ +## To list all permissions, run: +## +## docker-compose run --rm --entrypoint /bin/bash netbox +## $ ./manage.py migrate +## $ ./manage.py shell +## > from django.contrib.auth.models import Permission +## > print('\n'.join([p.codename for p in Permission.objects.all()])) +## +## Permission lists support wildcards. See the examples below. +## +## Examples: + # applications: # users: # - technical_user @@ -7,16 +19,17 @@ # writers: # users: # - writer -## specify explicit permission codenames or include wildcard to match multiple permissions # permissions: # - delete_device # - delete_virtualmachine # - add_* # - change_* # vm_managers: -## yaml doesn't allow starting a key with an asterisk without explicit use of single quote +# permissions: # - '*_virtualmachine' # device_managers: +# permissions: # - '*device*' # creators: +# permissions: # - add_* diff --git a/initializers/users.yml b/initializers/users.yml index ad654b7..2aea62e 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -1,10 +1,21 @@ +## To list all permissions, run: +## +## docker-compose run --rm --entrypoint /bin/bash netbox +## $ ./manage.py migrate +## $ ./manage.py shell +## > from django.contrib.auth.models import Permission +## > print('\n'.join([p.codename for p in Permission.objects.all()])) +## +## Permission lists support wildcards. See the examples below. +## +## Examples: + # technical_user: # api_token: 0123456789technicaluser789abcdef01234567 # must be looooong! # reader: # password: reader # writer: # password: writer -## specify explicit permission codenames or include wildcard to match multiple permissions # permissions: # - delete_device # - delete_virtualmachine From 69ef7b78272654ea8e3610da8b30eb981e391275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 31 Jan 2020 11:39:05 +0100 Subject: [PATCH 65/67] Removed the eval from the code ... and changed it to make it work with the latest Netbox version. --- startup_scripts/000_users.py | 22 ++++++++++------------ startup_scripts/010_groups.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index cb04a16..446aaeb 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -20,25 +20,23 @@ with file.open('r') as stream: username = username, password = user_details.get('password', 0) or User.objects.make_random_password) - print("👤 Created user ",username) + print("👤 Created user",username) if user_details.get('api_token', 0): Token.objects.create(user=user, key=user_details['api_token']) yaml_permissions = user_details.get('permissions', []) - permission_object = user + subject = user.user_permissions if yaml_permissions: - permission_object.permissions.clear() + subject.clear() for yaml_permission in yaml_permissions: if '*' in yaml_permission: - permission_codename_function = 'codename__iregex' - permission_codename = '^' + yaml_permission.replace('*','.*') + '$' + permission_filter = '^' + yaml_permission.replace('*','.*') + '$' + permissions = Permission.objects.filter(codename__iregex=permission_filter) + print(" ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'") else: - permission_codename_function = 'codename' - permission_codename = yaml_permission + permissions = Permission.objects.filter(codename=yaml_permission) + print(" ⚿ Granting permission", yaml_permission) - # supports non-unique permission codenames - for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): - permission_object.permissions.add(permission) - - permission_object.save() + for permission in permissions: + subject.add(permission) diff --git a/startup_scripts/010_groups.py b/startup_scripts/010_groups.py index 990e065..0a35387 100644 --- a/startup_scripts/010_groups.py +++ b/startup_scripts/010_groups.py @@ -25,17 +25,17 @@ with file.open('r') as stream: user.groups.add(group) yaml_permissions = group_details.get('permissions', []) - permission_object = group + subject = group.permissions if yaml_permissions: - permission_object.permissions.clear() + subject.clear() for yaml_permission in yaml_permissions: if '*' in yaml_permission: - permission_codename_function = 'codename__iregex' - permission_codename = '^' + yaml_permission.replace('*','.*') + '$' + permission_filter = '^' + yaml_permission.replace('*','.*') + '$' + permissions = Permission.objects.filter(codename__iregex=permission_filter) + print(" ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'") else: - permission_codename_function = 'codename' - permission_codename = yaml_permission - - # supports non-unique permission codenames - for permission in eval('Permission.objects.filter(' + permission_codename_function + '=permission_codename)'): - permission_object.permissions.add(permission) + permissions = Permission.objects.filter(codename=yaml_permission) + print(" ⚿ Granting permission", yaml_permission) + + for permission in permissions: + subject.add(permission) From 3d80cc5a722c80285be4b803ecbdde60e0199f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 2 Feb 2020 09:48:02 +0100 Subject: [PATCH 66/67] Tiny code refactoring --- startup_scripts/000_users.py | 2 +- startup_scripts/010_groups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index 446aaeb..6962df0 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -26,8 +26,8 @@ with file.open('r') as stream: Token.objects.create(user=user, key=user_details['api_token']) yaml_permissions = user_details.get('permissions', []) - subject = user.user_permissions if yaml_permissions: + subject = user.user_permissions subject.clear() for yaml_permission in yaml_permissions: if '*' in yaml_permission: diff --git a/startup_scripts/010_groups.py b/startup_scripts/010_groups.py index 0a35387..08fb4bf 100644 --- a/startup_scripts/010_groups.py +++ b/startup_scripts/010_groups.py @@ -25,8 +25,8 @@ with file.open('r') as stream: user.groups.add(group) yaml_permissions = group_details.get('permissions', []) - subject = group.permissions if yaml_permissions: + subject = group.permissions subject.clear() for yaml_permission in yaml_permissions: if '*' in yaml_permission: From e99a222a70fe3ae27762a3ced8882f71ca614ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 31 Jan 2020 09:19:53 +0100 Subject: [PATCH 67/67] Prepare v0.22.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a67ceba..2157409 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.1 +0.22.0