Merge pull request #226 from netbox-community/develop

Release 0.21.0

.dockerignore

@@ -5,3 +5,7 @@
 env
 build*
 docker-compose.override.yml
+.netbox/.git*
+.netbox/.travis.yml
+.netbox/docs
+.netbox/scripts

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,6 +25,8 @@ Please try this means to get help before opening an issue here:
 * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/
 * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss
 
+Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough.
+
 -->
 
 ## Current Behavior

.github/ISSUE_TEMPLATE/feature_request.md

@@ -24,6 +24,8 @@ Please try this means to get help before opening an issue here:
 * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/
 * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss
 
+Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough.
+
 -->
 
 ## Desired Behavior
@@ -33,7 +35,7 @@ Please try this means to get help before opening an issue here:
 
 ## Contrast to Current Behavior
 
-<!-- please describe how the desired behavior is different to the current behavior -->
+<!-- please describe how the desired behavior is different from the current behavior -->
 ...
 
 ## Changes Required

.github/pull_request_template.md

@@ -0,0 +1,85 @@
+<!--
+###############################################################################
+
+Thank you for sharing your work and for opening a PR.
+
+(!) IMPORTANT (!):
+First make sure that you point your PR to the `develop` branch!
+
+Now please read the comments carefully and try to provide information
+on all relevant titles.
+
+###############################################################################
+-->
+
+<!--
+Please don't open an extra issue when submiting a PR.
+
+But if there is already a related issue, please put it's number here.
+
+E.g. #123 or N/A
+-->
+
+Related Issue:
+
+## New Behavior
+
+<!--
+Please describe in a few words the intentions of your PR.
+-->
+
+...
+
+## Contrast to Current Behavior
+
+<!--
+Please describe in a few words how the new behavior is different
+from the current behavior.
+-->
+
+...
+
+## Discussion: Benefits and Drawbacks
+
+<!--
+Please make your case here:
+
+- Why do you think this project and the community will benefit from your
+  proposed change?
+- What are the drawbacks of this change?
+- Is it backwards-compatible?
+- Anything else that you think is relevant to the discussion of this PR.
+
+(No need to write a huge article here. Just a few sentences that give some
+additional context about the motivations for the change.)
+-->
+
+...
+
+## Changes to the Wiki
+
+<!--
+If the README.md must be updated, please include the changes in the PR.
+If the Wiki must be updated, please make a suggestion below.
+-->
+
+...
+
+## Proposed Release Note Entry
+
+<!--
+Please provide a short summary of your PR that we can copy & paste
+into the release notes.
+-->
+
+...
+
+## Double Check
+
+<!--
+Please put an x into the brackets (like `[x]`) if you've completed that task.
+-->
+
+* [ ] I have read the comments and followed the PR template.
+* [ ] I have provided and explained my PR according to the information in the comments.
+* [ ] My PR targets the `develop` branch.

.github/workflows/push.yml

@@ -0,0 +1,35 @@
+on:
+  push:
+    branches-ignore:
+      - release
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        build_cmd:
+        - ./build-latest.sh
+        - PRERELEASE=true ./build-latest.sh
+        - ./build-branches.sh
+        docker_from:
+        - '' # use the default of the DOCKERFILE
+        - python:3.7-alpine
+        - python:3.8-alpine
+        - python:3.9-rc-alpine
+      fail-fast: false
+    runs-on: ubuntu-latest
+    name: Builds new Netbox Docker Images
+    steps:
+    - id: git-checkout
+      name: Checkout
+      uses: actions/checkout@v1
+    - id: docker-build
+      name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        DOCKER_FROM: ${{ matrix.docker_from }}
+        GH_ACTION: enable
+    - id: docker-test
+      name: Test the image
+      run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh
+      if: steps.docker-build.outputs.skipped != 'true'

.github/workflows/release.yml

@@ -0,0 +1,49 @@
+on:
+  push:
+    branches:
+      - release
+  schedule:
+    - cron: '45 5 * * *'
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        build_cmd:
+        - ./build-latest.sh
+        - PRERELEASE=true ./build-latest.sh
+        - ./build-branches.sh
+      fail-fast: false
+    runs-on: ubuntu-latest
+    name: Builds new Netbox Docker Images
+    steps:
+    - id: git-checkout
+      name: Checkout
+      uses: actions/checkout@v1
+    - id: docker-build
+      name: Build the image with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        GH_ACTION: enable
+    - id: docker-test
+      name: Test the image
+      run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: registry-login
+      name: Login to the Docker Registry
+      run: |
+        echo "::add-mask::$DOCKERHUB_USERNAME"
+        echo "::add-mask::$DOCKERHUB_PASSWORD"
+        docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}"
+      env:
+        DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }}
+        DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }}
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: registry-push
+      name: Push the image
+      run: ${{ matrix.build_cmd }} --push-only
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: registry-logout
+      name: Logout of the Docker Registry
+      run: docker logout "${DOCKER_REGISTRY}"
+      if: steps.docker-build.outputs.skipped != 'true'

.gitignore

@@ -1,3 +1,4 @@
 *.sql.gz
 .netbox
+.initializers
 docker-compose.override.yml

DOCKER_HUB.md

@@ -11,21 +11,21 @@ Autotest: Internal and External Pull Requests
 Repository Links: Enable for Base Image
 Build Rules:
 - Source Type: Branch
-  Source: master
+  Source: release
   Docker Tag: branches
   Dockerfile location: Dockerfile
   Build Context: /
   Autobuild: on
   Build Caching: on
 - Source Type: Branch
-  Source: master
+  Source: release
   Docker Tag: prerelease
   Dockerfile location: Dockerfile
   Build Context: /
   Autobuild: on
   Build Caching: on
 - Source Type: Branch
-  Source: master
+  Source: release
   Docker Tag: release
   Dockerfile location: Dockerfile
   Build Context: /

Dockerfile

@@ -18,7 +18,7 @@ WORKDIR /install
 
 RUN pip install --prefix="/install" --no-warn-script-location \
 # gunicorn is used for launching netbox
-      'gunicorn<20.0.0' \
+      gunicorn \
       greenlet \
       eventlet \
 # napalm is used for gathering information from network devices
@@ -26,7 +26,9 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 # ruamel is used in startup_scripts
       'ruamel.yaml>=0.15,<0.16' \
 # django_auth_ldap is required for ldap
-      django_auth_ldap
+      django_auth_ldap \
+# django-storages was introduced in 2.7 and is optional
+      django-storages
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH}/requirements.txt /
@@ -68,6 +70,13 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py
 
 WORKDIR /opt/netbox/netbox
 
+# Must set permissions for '/opt/netbox/netbox/static' directory
+# to g+w so that `./manage.py collectstatic` can be executed during
+# container startup.
+# Must set permissions for '/opt/netbox/netbox/media' directory
+# to g+w so that pictures can be uploaded to netbox.
+RUN mkdir static && chmod g+w static media
+
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
 CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]

README.md

@@ -43,10 +43,10 @@ Then there is currently one extra tags for each of the above labels:
 
 ## Quickstart
 
-To get Netbox up and running:
+To get Netbox up and running in Docker:
 
 ```bash
-git clone -b master https://github.com/netbox-community/netbox-docker.git
+git clone -b release https://github.com/netbox-community/netbox-docker.git
 cd netbox-docker
 docker-compose pull
 docker-compose up -d
@@ -68,22 +68,25 @@ $ xdg-open "http://$(docker-compose port nginx 8080)/" &>/dev/null &
 
 Alternatively, use something like [Reception][docker-reception] to connect to _docker-compose_ projects.
 
-Default credentials:
+The default credentials are:
 
 * Username: **admin**
 * Password: **admin**
 * API Token: **0123456789abcdef0123456789abcdef01234567**
 
+There is a more complete [Getting Started guide on our Wiki][wiki-getting-started].
+
+[wiki-getting-started]: https://github.com/netbox-community/netbox-docker/wiki/Getting-Started
 [docker-reception]: https://github.com/nxt-engineering/reception
 
 ## Dependencies
 
-This project relies only on *Docker* and *docker-compose* meeting this requirements:
+This project relies only on *Docker* and *docker-compose* meeting these requirements:
 
 * The *Docker version* must be at least `17.05`.
 * The *docker-compose version* must be at least `1.17.0`.
 
-To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above.
+To check the version installed on your system run `docker --version` and `docker-compose --version`.
 
 ## Documentation
 
@@ -100,7 +103,7 @@ To use this feature, set the environment-variable `VERSION` before launching `do
 [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
 
 ```bash
-export VERSION=v2.6.7
+export VERSION=v2.7.1
 docker-compose pull netbox
 docker-compose up -d
 ```
@@ -109,7 +112,7 @@ You can also build a specific version of the Netbox Docker image yourself.
 `VERSION` can be any valid [git ref][git-ref] in that case.
 
 ```bash
-export VERSION=v2.6.7
+export VERSION=v2.7.1
 ./build.sh $VERSION
 docker-compose up -d
 ```
@@ -123,7 +126,7 @@ From time to time it might become necessary to re-engineer the structure of this
 Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence.
 Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label.
 (The images contained labels since April 2018, although in November 2019 the labels' names changed.)
-You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.7 --format "{{json .ContainerConfig.Labels}}"`.
+You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.7.1 --format "{{json .ContainerConfig.Labels}}"`.
 
 Please read [the release notes][releases] carefully when updating to a new image version.
 

VERSION

@@ -1 +1 @@
-0.20.0
+0.21.0

build-branches.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Builds develop, develop-* and master branches
+# Builds develop, develop-* and master branches of Netbox
 
 echo "▶️ $0 $*"
 

build-latest.sh

@@ -66,6 +66,10 @@ if [ "${PRERELEASE}" == "true" ]; then
 
     echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'."
     if [ -z "$DEBUG" ]; then
+      if [ -n "${GH_ACTION}" ]; then
+        echo "::set-output name=skipped::true"
+      fi
+
       exit 0
     else
       echo "⚠️ Would exit here with code '0', but DEBUG is enabled."

build.sh

@@ -27,9 +27,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "              Default: undefined"
   echo "  TAG         The version part of the docker tag."
   echo "              Default:"
-  echo "                When \${BRANCH}=master:  latest"
+  echo "                When <branch>=master:  latest"
-  echo "                When \${BRANCH}=develop: snapshot"
+  echo "                When <branch>=develop: snapshot"
-  echo "                Else:          same as \${BRANCH}"
+  echo "                Else:                  same as <branch>"
   echo "  DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')"
   echo "              Used for tagging the image."
   echo "              Default: docker.io"
@@ -63,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "              Default: undefined"
   echo "  DRY_RUN     Prints all build statements instead of running them."
   echo "              Default: undefined"
+  echo "  GH_ACTION   If defined, special 'echo' statements are enabled that set the"
+  echo "              following environment variables in Github Actions:"
+  echo "              - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable"
+  echo "              Default: undefined"
   echo ""
   echo "Examples:"
   echo "  ${0} master"
@@ -92,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
 fi
 
 ###
-# Determining the build command to use
+# Enabling dry-run mode
 ###
 if [ -z "${DRY_RUN}" ]; then
   DRY=""
@@ -102,21 +106,21 @@ else
 fi
 
 ###
-# variables for fetching the source
+# Variables for fetching the source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
-BRANCH="${1}"
+NETBOX_BRANCH="${1}"
 URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# fetching the source
+# Fetching the source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
-  echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
-    $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}"
+    $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
   fi
 
   (
@@ -127,7 +131,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
     fi
 
     $DRY git remote set-url origin "${URL}"
-    $DRY git fetch -qp --depth 10 origin "${BRANCH}"
+    $DRY git fetch -qp --depth 10 origin "${NETBOX_BRANCH}"
     $DRY git checkout -qf FETCH_HEAD
     $DRY git prune
   )
@@ -150,15 +154,15 @@ if [ ! -f "${DOCKERFILE}" ]; then
 fi
 
 ###
-# variables for labelling the docker image
+# Variables for labelling the docker image
 ###
-BUILD_DATE="$(date --utc --iso-8601=minutes)"
+BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')"
 
 if [ -d ".git" ]; then
   GIT_REF="$(git rev-parse HEAD)"
 fi
 
-# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132
+# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132
 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
 
 # Get the Git information from the netbox directory
@@ -169,18 +173,18 @@ if [ -d "${NETBOX_PATH}/.git" ]; then
 fi
 
 ###
-# variables for tagging the docker image
+# Variables for tagging the docker image
 ###
 DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}"
 DOCKER_ORG="${DOCKER_ORG-netboxcommunity}"
 DOCKER_REPO="${DOCKER_REPO-netbox}"
-case "${BRANCH}" in
+case "${NETBOX_BRANCH}" in
   master)
     TAG="${TAG-latest}";;
   develop)
     TAG="${TAG-snapshot}";;
   *)
-    TAG="${TAG-$BRANCH}";;
+    TAG="${TAG-$NETBOX_BRANCH}";;
 esac
 
 ###
@@ -193,6 +197,7 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
 ###
 # Build each target
 ###
+export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1}
 for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   echo "🏗 Building the target '${DOCKER_TARGET}'"
 
@@ -203,6 +208,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   if [ "${DOCKER_TARGET}" != "main" ]; then
     TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
   fi
+  if [ -n "${GH_ACTION}" ]; then
+    echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}"
+    echo "::set-output name=skipped::false"
+  fi
 
   ###
   # composing the additional DOCKER_SHORT_TAG,

configuration/configuration.py

@@ -37,6 +37,8 @@ DATABASE = {
                                                          # PostgreSQL password
     'HOST': os.environ.get('DB_HOST', 'localhost'),      # Database server
     'PORT': os.environ.get('DB_PORT', ''),               # Database port (leave blank for default)
+    'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')},
+                                                         # Database connection SSLMODE
     'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')),
                                                          # Database connection persistence
 }
@@ -49,13 +51,22 @@ SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key'))
 
 # Redis database settings. The Redis database is used for caching and background processing such as webhooks
 REDIS = {
+    'webhooks': {
         'HOST': os.environ.get('REDIS_HOST', 'localhost'),
         'PORT': int(os.environ.get('REDIS_PORT', 6379)),
         'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
-    'DATABASE': os.environ.get('REDIS_DATABASE', '0'),
+        'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)),
-    'CACHE_DATABASE': os.environ.get('REDIS_CACHE_DATABASE', '1'),
+        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)),
-    'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'),
         'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true',
+    },
+    'caching': {
+        'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')),
+        'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))),
+        'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))),
+        'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)),
+        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))),
+        'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true',
+    },
 }
 
 #########################
@@ -170,10 +181,6 @@ SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
 # Time zone (default: UTC)
 TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
 
-# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis
-# database be configured and accessible by NetBox (see `REDIS` below).
-WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true'
-
 # Date/time formatting. See the following link for supported formats:
 # https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
 DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y')

docker-compose.test.yml

@@ -0,0 +1,45 @@
+version: '3.4'
+services:
+  netbox:
+    image: ${IMAGE-netboxcommunity/netbox:latest}
+    depends_on:
+    - postgres
+    - redis
+    env_file: env/netbox.env
+    user: '101'
+    volumes:
+    - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
+    - ./${INITIALIZERS_DIR-initializers}:/opt/netbox/initializers:z,ro
+    - ./configuration:/etc/netbox/config:z,ro
+    - ./reports:/etc/netbox/reports:z,ro
+    - ./scripts:/etc/netbox/scripts:z,ro
+    - netbox-nginx-config:/etc/netbox-nginx:z
+    - netbox-static-files:/opt/netbox/netbox/static:z
+    - netbox-media-files:/opt/netbox/netbox/media:z
+  nginx:
+    command: nginx -c /etc/netbox-nginx/nginx.conf
+    image: nginx:1.17-alpine
+    depends_on:
+    - netbox
+    ports:
+    - 8080
+    volumes:
+    - netbox-static-files:/opt/netbox/netbox/static:ro
+    - netbox-nginx-config:/etc/netbox-nginx/:ro
+  postgres:
+    image: postgres:11-alpine
+    env_file: env/postgres.env
+  redis:
+    image: redis:5-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: env/redis.env
+volumes:
+  netbox-static-files:
+    driver: local
+  netbox-nginx-config:
+    driver: local
+  netbox-media-files:
+    driver: local

docker-compose.yml

@@ -5,8 +5,10 @@ services:
     depends_on:
     - postgres
     - redis
+    - redis-cache
     - netbox-worker
     env_file: env/netbox.env
+    user: '101'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
     - ./initializers:/opt/netbox/initializers:z,ro
@@ -36,12 +38,12 @@ services:
     - netbox-static-files:/opt/netbox/netbox/static:ro
     - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
-    image: postgres:10.4-alpine
+    image: postgres:11-alpine
     env_file: env/postgres.env
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data
   redis:
-    image: redis:4-alpine
+    image: redis:5-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
@@ -49,6 +51,13 @@ services:
     env_file: env/redis.env
     volumes:
     - netbox-redis-data:/data
+  redis-cache:
+    image: redis:5-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: env/redis.env
 volumes:
   netbox-static-files:
     driver: local
@@ -56,8 +65,6 @@ volumes:
     driver: local
   netbox-media-files:
     driver: local
-  netbox-report-files:
-    driver: local
   netbox-postgres-data:
     driver: local
   netbox-redis-data:

docker/docker-entrypoint.sh

@@ -1,12 +1,27 @@
 #!/bin/bash
+# Runs on every start of the Netbox Docker container
+
+# Stop when an error occures
 set -e
 
-# wait shortly and then run db migrations (retry on error)
+# Allows Netbox to be run as non-root users
-while ! ./manage.py migrate 2>&1; do
+umask 002
-  echo "⏳ Waiting on DB..."
-  sleep 3
-done
 
+# Try to connect to the DB
+DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
+MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
+CUR_DB_WAIT_TIME=0
+while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+  echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)"
+  sleep "${DB_WAIT_TIMEOUT}"
+  CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT ))
+done
+if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then
+  echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready."
+  exit 1
+fi
+
+# Create Superuser if required
 if [ "$SKIP_SUPERUSER" == "true" ]; then
   echo "↩️ Skip creating the superuser"
 else
@@ -42,21 +57,19 @@ END
   echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}"
 fi
 
+# Run the startup scripts (and initializers)
 if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then
   echo "↩️ Skipping startup scripts"
 else
-  for script in /opt/netbox/startup_scripts/*.py; do
+  echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
-    echo "⚙️ Executing '$script'"
-    ./manage.py shell --interface python < "${script}"
-  done
 fi
 
-# copy static files
+# Copy static files
 ./manage.py collectstatic --no-input
 
 echo "✅ Initialisation is done."
 
-# launch whatever is passed by docker
+# Launch whatever is passed by docker
 # (i.e. the RUN instruction in the Dockerfile)
 #
 # shellcheck disable=SC2068

env/netbox.env

@@ -17,8 +17,11 @@ MAX_PAGE_SIZE=1000
 REDIS_HOST=redis
 REDIS_PASSWORD=H733Kdjndks81
 REDIS_DATABASE=0
-REDIS_CACHE_DATABASE=1
 REDIS_SSL=false
+REDIS_CACHE_HOST=redis-cache
+REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36
+REDIS_CACHE_DATABASE=0
+REDIS_CACHE_SSL=false
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 SKIP_STARTUP_SCRIPTS=false
 SKIP_SUPERUSER=false

env/redis-cache.env

@@ -0,0 +1 @@
+REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36

hooks/build

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-# shellcheck disable=SC2119
-run_build

hooks/common

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-ensure_jq() {
-  if [ ! -x "$(command -v jq)" ]; then
-    if [ -x "$(command -v apt-get)" ]; then
-      echo "🛠🛠🛠 Installing 'jq' via 'apt-get'"
-      apt-get update && apt-get install -y jq
-    else
-      echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'."
-    fi
-  fi
-}
-
-# Passes args to the scripts
-run_build() {
-  echo "🐳🐳🐳 Building '${BUILD}' images"
-  case $BUILD in
-    release)
-      # build the latest release
-      # shellcheck disable=SC2068
-      ./build-latest.sh $@
-      ;;
-    prerelease)
-      # build the latest pre-release
-      # shellcheck disable=SC2068
-      PRERELEASE=true ./build-latest.sh $@
-      ;;
-    branches)
-      # build all branches
-      # shellcheck disable=SC2068
-      ./build-branches.sh $@
-      ;;
-    this) # Pull Requests
-      # only build the 'master' branch
-      # (resulting in the 'latest' docker tag)
-      # and the 'main' target.
-      DOCKER_TARGET=main ./build.sh master
-      ;;
-    *)
-      echo "🚨 Unrecognized build '$BUILD'."
-
-      if [ -z "$DEBUG" ]; then
-        exit 1
-      else
-        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-      fi
-      ;;
-  esac
-}
-
-echo "🤖🤖🤖 Preparing build"
-export DOCKER_ORG="index.docker.io/netboxcommunity"
-export DOCKER_REPO=netbox
-export DOCKERHUB_REPO=netboxcommunity/netbox
-# shellcheck disable=SC2153
-export BUILD="${DOCKER_TAG}"
-
-unset DOCKER_TAG
-
-ensure_jq

hooks/push

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-if [ "${SOURCE_BRANCH}" == "master" ] || [ "${DEBUG}" == "true" ]; then
-  if [ "${SOURCE_BRANCH}" != "master" ]; then
-    echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'".
-  fi
-
-  run_build --push-only
-else
-  echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'"
-  exit 0
-fi

hooks/test

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-run_test() {
-  echo "🐳🐳🐳 Testing '${1}'"
-  VERSION="${1}" docker-compose run netbox ./manage.py test
-  docker-compose down -v
-  echo "🐳🐳🐳 Done testing '${1}'"
-}
-
-# test on builds of 'branches'
-if [ "${BUILD}" == "branches" ] \
-   || [ "${DEBUG}" == "true" ]; then
-  run_test latest
-  run_test snapshot
-# test on bulds of 'this' (i.e. pull request)
-elif [ "${BUILD}" == "this" ]; then
-  run_test latest
-else
-  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'."
-fi

initializers/custom_fields.yml

@@ -1,3 +1,18 @@
+## Possible Choices:
+##   type:
+##   - text
+##   - integer
+##   - boolean
+##   - date
+##   - url
+##   - select
+##   filter_logic:
+##   - disabled
+##   - loose
+##   - exact
+##
+## Examples:
+
 # text_field:
 #   type: text
 #   label: Custom Text
@@ -22,8 +37,8 @@
 #   weight: 10
 #   on_objects:
 #   - tenancy.models.Tenant
-# selection_field:
+# select_field:
-#   type: selection
+#   type: select
 #   label: Choose between items
 #   required: false
 #   filter_logic: exact
@@ -41,8 +56,8 @@
 #     weight: 50
 #   - value: Fourth Item
 #     weight: 40
-# selection_field_auto_weight:
+# select_field_auto_weight:
-#    type: selection
+#    type: select
 #    label: Choose between items
 #    required: false
 #    filter_logic: loose

initializers/dcim_interfaces.yml

@@ -1,8 +1,18 @@
+## Possible Choices:
+##   type:
+##   - virtual
+##   - lag
+##   - 1000base-t
+##   - ... and many more. See for yourself:
+##     https://github.com/netbox-community/netbox/blob/295d4f0394b431351c0cb2c3ecc791df68c6c2fb/netbox/dcim/choices.py#L510
+##
+## Examples:
+
 # - device: server01
 #   enabled: true
-#   type: Virtual
+#   type: virtual
 #   name: to-server02
 # - device: server02
 #   enabled: true
-#   type: Virtual
+#   type: virtual
 #   name: to-server01

initializers/devices.yml

@@ -1,9 +1,24 @@
+## Possible Choices:
+##   face:
+##   - front
+##   - rear
+##   status:
+##   - offline
+##   - active
+##   - planned
+##   - staged
+##   - failed
+##   - inventory
+##   - decommissioning
+##
+## Examples:
+
 # - name: server01
 #   device_role: server
 #   device_type: Other
 #   site: AMS 1
 #   rack: rack-01
-#   face: Front
+#   face: front
 #   position: 1
 #   custom_fields:
 #     text_field: Description
@@ -12,7 +27,7 @@
 #   device_type: Other
 #   site: AMS 2
 #   rack: rack-02
-#   face: Front
+#   face: front
 #   position: 2
 #   custom_fields:
 #     text_field: Description
@@ -21,7 +36,7 @@
 #   device_type: Other
 #   site: SING 1
 #   rack: rack-03
-#   face: Front
+#   face: front
 #   position: 3
 #   custom_fields:
 #     text_field: Description

initializers/ip_addresses.yml

@@ -1,26 +1,44 @@
+## Possible Choices:
+##   status:
+##   - active
+##   - reserved
+##   - deprecated
+##   - dhcp
+##   role:
+##   - loopback
+##   - secondary
+##   - anycast
+##   - vip
+##   - vrrp
+##   - hsrp
+##   - glbp
+##   - carp
+##
+## Examples:
+
 # - address: 10.1.1.1/24
 #   device: server01
 #   interface: to-server02
-#   status: Active
+#   status: active
 #   vrf: vrf1
 # - address: 2001:db8:a000:1::1/64
 #   device: server01
 #   interface: to-server02
-#   status: Active
+#   status: active
 #   vrf: vrf1
 # - address: 10.1.1.2/24
 #   device: server02
 #   interface: to-server01
-#   status: Active
+#   status: active
 # - address: 2001:db8:a000:1::2/64
 #   device: server02
 #   interface: to-server01
-#   status: Active
+#   status: active
 # - address: 10.1.1.10/24
 #   description: reserved IP
-#   status: Reserved
+#   status: reserved
 #   tenant: tenant1
 # - address: 2001:db8:a000:1::10/64
 #   description: reserved IP
-#   status: Reserved
+#   status: reserved
 #   tenant: tenant1

initializers/prefixes.yml

@@ -1,13 +1,22 @@
+## Possible Choices:
+##   status:
+##   - container
+##   - active
+##   - reserved
+##   - deprecated
+##
+## Examples:
+
 # - description: prefix1
 #   prefix: 10.1.1.0/24
 #   site: AMS 1
-#   status: Active
+#   status: active
 #   tenant: tenant1
 #   vlan: vlan1
 # - description: prefix2
 #   prefix: 10.1.2.0/24
 #   site: AMS 2
-#   status: Active
+#   status: active
 #   tenant: tenant2
 #   vlan: vlan2
 #   is_pool: true
@@ -15,6 +24,6 @@
 # - description: ipv6 prefix1
 #   prefix: 2001:db8:a000:1::/64
 #   site: AMS 2
-#   status: Active
+#   status: active
 #   tenant: tenant2
 #   vlan: vlan2

initializers/rack_groups.yml

@@ -0,0 +1,3 @@
+# - name: cage 101
+#   slug: cage-101
+#   site: SING 1

initializers/racks.yml

@@ -1,24 +1,41 @@
+## Possible Choices:
+##   width:
+##   - 19
+##   - 23
+##   types:
+##   - 2-post-frame
+##   - 4-post-frame
+##   - 4-post-cabinet
+##   - wall-frame
+##   - wall-cabinet
+##   outer_unit:
+##   - mm
+##   - in
+##
+## Examples:
+
 # - site: AMS 1
 #   name: rack-01
 #   role: Role 1
-#   type: 4-post cabinet
+#   type: 4-post-cabinet
-#   width: 19 inches
+#   width: 19
 #   u_height: 47
 #   custom_fields:
 #     text_field: Description
 # - site: AMS 2
 #   name: rack-02
 #   role: Role 2
-#   type: 4-post cabinet
+#   type: 4-post-cabinet
-#   width: 19 inches
+#   width: 19
 #   u_height: 47
 #   custom_fields:
 #     text_field: Description
 # - site: SING 1
 #   name: rack-03
+#   group: cage 101
 #   role: Role 3
-#   type: 4-post cabinet
+#   type: 4-post-cabinet
-#   width: 19 inches
+#   width: 19
 #   u_height: 47
 #   custom_fields:
 #     text_field: Description

initializers/virtual_machines.yml

@@ -1,10 +1,18 @@
+## Possible Choices:
+##   status:
+##   - active
+##   - offline
+##   - staged
+##
+## Examples:
+
 # - cluster: cluster1
 #   comments: VM1
 #   disk: 200
 #   memory: 4096
 #   name: virtual machine 1
 #   platform: Platform 2
-#   status: Active
+#   status: active
 #   tenant: tenant1
 #   vcpus: 8
 # - cluster: cluster1
@@ -13,6 +21,6 @@
 #   memory: 2048
 #   name: virtual machine 2
 #   platform: Platform 2
-#   status: Active
+#   status: active
 #   tenant: tenant1
 #   vcpus: 8

initializers/vlans.yml

@@ -1,11 +1,19 @@
+## Possible Choices:
+##   status:
+##   - active
+##   - reserved
+##   - deprecated
+##
+## Examples:
+
 # - name: vlan1
 #   site: AMS 1
-#   status: Active
+#   status: active
 #   vid: 5
 #   role: Main Management
 #   description: VLAN 5 for MGMT
 # - group: VLAN group 2
 #   name: vlan2
 #   site: AMS 1
-#   status: Active
+#   status: active
 #   vid: 1300

startup_scripts/020_custom_fields.py

@@ -1,19 +1,9 @@
-from extras.constants import CF_TYPE_TEXT, CF_TYPE_INTEGER, CF_TYPE_BOOLEAN, CF_TYPE_DATE, CF_TYPE_URL, CF_TYPE_SELECT, CF_FILTER_CHOICES
 from extras.models import CustomField, CustomFieldChoice
 
 from ruamel.yaml import YAML
 from pathlib import Path
 import sys
 
-text_to_fields = {
-  'boolean': CF_TYPE_BOOLEAN,
-  'date': CF_TYPE_DATE,
-  'integer': CF_TYPE_INTEGER,
-  'selection': CF_TYPE_SELECT,
-  'text': CF_TYPE_TEXT,
-  'url': CF_TYPE_URL,
-}
-
 def get_class_for_class_path(class_path):
   import importlib
   from django.contrib.contenttypes.models import ContentType
@@ -42,12 +32,6 @@ with file.open('r') as stream:
         if cf_details.get('description', 0):
           custom_field.description = cf_details['description']
 
-        # If no filter_logic is specified then it will default to 'Loose'
-        if cf_details.get('filter_logic', 0):
-          for choice_id, choice_text in CF_FILTER_CHOICES:
-            if choice_text.lower() == cf_details['filter_logic']:
-              custom_field.filter_logic = choice_id
-
         if cf_details.get('label', 0):
           custom_field.label = cf_details['label']
 
@@ -58,7 +42,7 @@ with file.open('r') as stream:
           custom_field.required = cf_details['required']
 
         if cf_details.get('type', 0):
-          custom_field.type = text_to_fields[cf_details['type']]
+          custom_field.type = cf_details['type']
 
         if cf_details.get('weight', 0):
           custom_field.weight = cf_details['weight']

startup_scripts/075_rack_groups.py

@@ -0,0 +1,31 @@
+from dcim.models import Site,RackGroup
+from ruamel.yaml import YAML
+
+from pathlib import Path
+import sys
+
+file = Path('/opt/netbox/initializers/rack_groups.yml')
+if not file.is_file():
+  sys.exit()
+
+with file.open('r') as stream:
+  yaml=YAML(typ='safe')
+  rack_groups= yaml.load(stream)
+
+  required_assocs = {
+    'site': (Site, 'name')
+  }
+
+  if rack_groups is not None:
+    for params in rack_groups:
+
+      for assoc, details in required_assocs.items():
+        model, field = details
+        query = { field: params.pop(assoc) }
+        params[assoc] = model.objects.get(**query)
+
+      rack_group, created = RackGroup.objects.get_or_create(**params)
+
+      if created:
+        print("🎨 Created rack group", rack_group.name)
+

startup_scripts/080_racks.py

@@ -1,7 +1,6 @@
 from dcim.models import Site, RackRole, Rack, RackGroup
 from tenancy.models import Tenant
 from extras.models import CustomField, CustomFieldValue
-from dcim.constants import RACK_TYPE_CHOICES, RACK_WIDTH_CHOICES
 from ruamel.yaml import YAML
 from pathlib import Path
 import sys
@@ -41,14 +40,6 @@ with file.open('r') as stream:
 
           params[assoc] = model.objects.get(**query)
 
-      for rack_type in RACK_TYPE_CHOICES:
-        if params['type'] in rack_type:
-          params['type'] = rack_type[0]
-
-      for rack_width in RACK_WIDTH_CHOICES:
-        if params['width'] in rack_width:
-          params['width'] = rack_width[0]
-
       rack, created = Rack.objects.get_or_create(**params)
 
       if created:

startup_scripts/130_devices.py

@@ -1,5 +1,4 @@
 from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
-from dcim.constants import RACK_FACE_CHOICES
 from ipam.models import IPAddress
 from virtualization.models import Cluster
 from tenancy.models import Tenant
@@ -49,12 +48,6 @@ with file.open('r') as stream:
 
           params[assoc] = model.objects.get(**query)
 
-      if 'face' in params:
-        for rack_face in RACK_FACE_CHOICES:
-          if params['face'] in rack_face:
-            params['face'] = rack_face[0]
-            break
-
       device, created = Device.objects.get_or_create(**params)
 
       if created:

startup_scripts/210_vlans.py

@@ -1,6 +1,5 @@
 from dcim.models import Site
 from ipam.models import VLAN, VLANGroup, Role
-from ipam.constants import VLAN_STATUS_CHOICES
 from tenancy.models import Tenant, TenantGroup
 from extras.models import CustomField, CustomFieldValue
 from ruamel.yaml import YAML
@@ -35,12 +34,6 @@ with file.open('r') as stream:
 
           params[assoc] = model.objects.get(**query)
 
-      if 'status' in params:
-        for vlan_status in VLAN_STATUS_CHOICES:
-          if params['status'] in vlan_status:
-            params['status'] = vlan_status[0]
-            break
-
       vlan, created = VLAN.objects.get_or_create(**params)
 
       if created:

startup_scripts/220_prefixes.py

@@ -1,6 +1,5 @@
 from dcim.models import Site
 from ipam.models import Prefix, VLAN, Role, VRF
-from ipam.constants import PREFIX_STATUS_CHOICES
 from tenancy.models import Tenant, TenantGroup
 from extras.models import CustomField, CustomFieldValue
 from ruamel.yaml import YAML
@@ -38,12 +37,6 @@ with file.open('r') as stream:
 
           params[assoc] = model.objects.get(**query)
 
-      if 'status' in params:
-        for prefix_status in PREFIX_STATUS_CHOICES:
-          if params['status'] in prefix_status:
-            params['status'] = prefix_status[0]
-            break
-
       prefix, created = Prefix.objects.get_or_create(**params)
 
       if created:

startup_scripts/230_virtual_machines.py

@@ -1,6 +1,5 @@
 from dcim.models import Site, Platform, DeviceRole
 from virtualization.models import Cluster, VirtualMachine
-from virtualization.constants import VM_STATUS_CHOICES
 from tenancy.models import Tenant
 from extras.models import CustomField, CustomFieldValue
 from ruamel.yaml import YAML
@@ -43,12 +42,6 @@ with file.open('r') as stream:
 
           params[assoc] = model.objects.get(**query)
 
-      if 'status' in params:
-        for vm_status in VM_STATUS_CHOICES:
-          if params['status'] in vm_status:
-            params['status'] = vm_status[0]
-            break
-
       virtual_machine, created = VirtualMachine.objects.get_or_create(**params)
 
       if created:

startup_scripts/250_dcim_interfaces.py

@@ -1,5 +1,4 @@
 from dcim.models import Interface, Device
-from dcim.constants import IFACE_TYPE_CHOICES
 from extras.models import CustomField, CustomFieldValue
 from ruamel.yaml import YAML
 
@@ -28,16 +27,6 @@ with file.open('r') as stream:
 
         params[assoc] = model.objects.get(**query)
 
-      if 'type' in params:
-        for outer_list in IFACE_TYPE_CHOICES:
-          for type_choices in outer_list[1]:
-            if params['type'] in type_choices:
-              params['type'] = type_choices[0]
-              break
-          else:
-            continue
-          break
-
       interface, created = Interface.objects.get_or_create(**params)
 
       if created:

startup_scripts/260_ip_addresses.py

@@ -1,5 +1,4 @@
 from ipam.models import IPAddress, VRF
-from ipam.constants import IPADDRESS_STATUS_CHOICES
 from dcim.models import Device, Interface
 from virtualization.models import VirtualMachine
 from tenancy.models import Tenant
@@ -49,12 +48,6 @@ with file.open('r') as stream:
               query = { field: params.pop(assoc) }
           params[assoc] = model.objects.get(**query)
 
-      if 'status' in params:
-        for ip_status in IPADDRESS_STATUS_CHOICES:
-          if params['status'] in ip_status:
-            params['status'] = ip_status[0]
-            break
-
       ip_address, created = IPAddress.objects.get_or_create(**params)
 
       if created:

startup_scripts/__main__.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import runpy
+from os import scandir
+from os.path import dirname, abspath
+
+this_dir = dirname(abspath(__file__))
+
+def filename(f):
+    return f.name
+
+with scandir(dirname(abspath(__file__))) as it:
+    for f in sorted(it, key = filename):
+        if f.name.startswith('__') or not f.is_file():
+            continue
+        
+        print(f"Running {f.path}")
+        runpy.run_path(f.path)

test.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# exit when a command exits with an exit code != 0
+set -e
+
+# version is used by `docker-compose.yml` do determine the tag
+# of the Docker Image that is to be used
+export IMAGE="${IMAGE-netboxcommunity/netbox:latest}"
+
+if [ -z "${IMAGE}" ]; then
+  echo "⚠️ No image defined"
+
+  if [ -z "${DEBUG}" ]; then
+    exit 1;
+  else
+    echo "⚠️ Would 'exit 1' here, but DEBUG is '${DEBUG}'."
+  fi
+fi
+
+# The docker compose command to use
+doco="docker-compose -f docker-compose.test.yml"
+
+INITIALIZERS_DIR=".initializers"
+
+test_setup() {
+  echo "🏗 Setup up test environment"
+  if [ -d "${INITIALIZERS_DIR}" ]; then
+    rm -rf "${INITIALIZERS_DIR}"
+  fi
+
+  mkdir "${INITIALIZERS_DIR}"
+  (
+    cd initializers
+    for script in *.yml; do
+      sed -E 's/^# //' "${script}" > "../${INITIALIZERS_DIR}/${script}"
+    done
+  )
+}
+
+test_netbox_unit_tests() {
+  echo "⏱ Running Netbox Unit Tests"
+  $doco run --rm netbox ./manage.py test
+}
+
+test_initializers() {
+  echo "🏭 Testing Initializers"
+  export INITIALIZERS_DIR
+  $doco run --rm netbox ./manage.py check
+}
+
+test_cleanup() {
+  echo "💣 Cleaning Up"
+  $doco down -v
+
+  if [ -d "${INITIALIZERS_DIR}" ]; then
+    rm -rf "${INITIALIZERS_DIR}"
+  fi
+}
+
+echo "🐳🐳🐳 Start testing '${IMAGE}'"
+
+# Make sure the cleanup script is executed
+trap test_cleanup EXIT ERR
+test_setup
+
+test_netbox_unit_tests
+test_initializers
+
+echo "🐳🐳🐳 Done testing '${IMAGE}'"