Merge pull request #394 from netbox-community/develop

Prepare version 1.0.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -65,13 +65,3 @@ If your log is very long, create a Gist instead (and post the link to it): https
 ```text
 LOG LOG LOG
 ```
-
-The output of `docker-compose logs nginx`:
-<!--
-Only if you have gotten a 5xx http error, else delete this section.
-If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com
--->
-
-```text
-LOG LOG LOG
-```

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,13 @@
+blank_issues_enabled: false
+contact_links:
+  - name: The \#netbox-docker Slack channel
+    url: http://slack.networktocode.com/
+    about: It's usually the quickest way to seek help when you're in trouble with regards to Netbox Docker.
+    
+  - name: Github Discussions
+    url: https://github.com/netbox-community/netbox-docker/discussions
+    about: This is the right place to ask questions about how to use or do certain things with Netbox Docker.
+
+  - name: Have you had a look at our Wiki?
+    url: https://github.com/netbox-community/netbox-docker/wiki
+    about: Our wiki contains information for common problems and tips for operating Netbox Docker in production.

.github/workflows/push.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   build:
+    continue-on-error: ${{ matrix.docker_from == 'alpine:edge' }}
     strategy:
       matrix:
         build_cmd:
@@ -19,7 +20,7 @@ jobs:
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script
-        # - python:3.10-rc-alpine # disable until dependencies work
+        - alpine:edge
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new Netbox Docker Images

Dockerfile

@@ -12,27 +12,15 @@ RUN apk add --no-cache \
       libffi-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev
-
-WORKDIR /install
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-# gunicorn is used for launching netbox
-      gunicorn \
-      greenlet \
-      eventlet \
-# napalm is used for gathering information from network devices
-      napalm \
-# ruamel is used in startup_scripts
-      'ruamel.yaml>=0.15,<0.16' \
-# django_auth_ldap is required for ldap
-      django_auth_ldap \
-# django-storages was introduced in 2.7 and is optional
-      django-storages
+      postgresql-dev \
+      py3-pip \
+      python3-dev \
+      && python3 -m venv /opt/netbox/venv \
+      && /opt/netbox/venv/bin/python3 -m pip install --upgrade pip setuptools
 
 ARG NETBOX_PATH
-COPY ${NETBOX_PATH}/requirements.txt /
-RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt
+COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
+RUN /opt/netbox/venv/bin/pip install -r /requirements.txt -r /requirements-container.txt
 
 ###
 # Main stage
@@ -44,6 +32,7 @@ FROM ${FROM} as main
 RUN apk add --no-cache \
       bash \
       ca-certificates \
+      curl \
       graphviz \
       libevent \
       libffi \
@@ -51,35 +40,38 @@ RUN apk add --no-cache \
       libressl \
       libxslt \
       postgresql-libs \
-      ttf-ubuntu-font-family
+      python3 \
+      py3-pip \
+      ttf-ubuntu-font-family \
+      unit \
+      unit-python3
 
 WORKDIR /opt
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
-COPY docker/gunicorn_config.py /etc/netbox/
-COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
 COPY configuration/ /etc/netbox/config/
+COPY docker/nginx-unit.json /etc/unit/
 
 WORKDIR /opt/netbox/netbox
 
-# Must set permissions for '/opt/netbox/netbox/static' directory
-# to g+w so that `./manage.py collectstatic` can be executed during
-# container startup.
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
-RUN mkdir static && chmod -R g+w static media
+RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chmod -R g+w media /opt/unit/ \
+      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-CMD ["gunicorn", "-c /etc/netbox/gunicorn_config.py", "netbox.wsgi"]
+CMD [ "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \

README.md

@@ -56,7 +56,7 @@ cd netbox-docker
 tee docker-compose.override.yml <<EOF
 version: '3.4'
 services:
-  nginx:
+  netbox:
     ports:
       - 8000:8080
 EOF

VERSION

@@ -1 +1 @@
-0.27.0
+1.0.0

build.sh

@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'python:3.9-alpine'"
+  echo "              Default: 'alpine:3.13'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -106,7 +106,7 @@ else
 fi
 
 ###
-# Variables for fetching the source
+# Variables for fetching the Netbox source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
@@ -115,10 +115,10 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# Fetching the source
+# Fetching the Netbox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
-  echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  echo "🌐 Checking out '${NETBOX_BRANCH}' of Netbox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
   fi
@@ -135,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
     $DRY git checkout -qf FETCH_HEAD
     $DRY git prune
   )
-  echo "✅ Checked out netbox"
+  echo "✅ Checked out Netbox"
 fi
 
 ###
@@ -157,7 +157,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="python:3.9-alpine"
+  DOCKER_FROM="alpine:3.13"
 fi
 
 ###
@@ -271,7 +271,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
 
       if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
         SHOULD_BUILD="true"
-        BUILD_REASON="${BUILD_REASON} python"
+        BUILD_REASON="${BUILD_REASON} alpine"
       fi
       if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
         SHOULD_BUILD="true"

docker-compose.test.yml

@@ -7,6 +7,8 @@ services:
     - redis
     - redis-cache
     env_file: env/netbox.env
+    environment:
+      SKIP_STARTUP_SCRIPTS: ${SKIP_STARTUP_SCRIPTS-false}
     user: '101'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
@@ -14,19 +16,9 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
     ports:
     - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
     image: postgres:12-alpine
     env_file: env/postgres.env
@@ -45,9 +37,5 @@ services:
     - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
     env_file: env/redis-cache.env
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local

docker-compose.yml

@@ -15,30 +15,19 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
+    ports:
+    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
     entrypoint:
-    - python3
+    - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
     command:
     - rqworker
-
-  # nginx
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
-    ports:
-    - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
+    ports: []
 
   # postgres
   postgres:
@@ -66,10 +55,6 @@ services:
     env_file: env/redis-cache.env
 
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
   netbox-postgres-data:

docker/configuration.docker.py

@@ -9,6 +9,7 @@ from os import scandir
 import importlib.util
 import sys
 
+
 def _filename(f):
   return f.name
 

docker/docker-entrypoint.sh

@@ -7,6 +7,9 @@ set -e
 # Allows Netbox to be run as non-root users
 umask 002
 
+# Load correct Python3 env
+source /opt/netbox/venv/bin/activate
+
 # Try to connect to the DB
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
@@ -60,9 +63,6 @@ else
   echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-# Copy static files
-./manage.py collectstatic --no-input
-
 echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker

docker/gunicorn_config.py

@@ -1,8 +0,0 @@
-command = '/usr/bin/gunicorn'
-pythonpath = '/opt/netbox/netbox'
-bind = '0.0.0.0:8001'
-workers = 3
-errorlog = '-'
-accesslog = '-'
-capture_output = False
-loglevel = 'info'

docker/launch-netbox.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}"
+UNIT_SOCKET="/opt/unit/unit.sock"
+
+load_configuration() {
+  MAX_WAIT=10
+  WAIT_COUNT=0
+  while [ ! -S $UNIT_SOCKET ]; do
+    if [ $WAIT_COUNT -gte $MAX_WAIT ]; then
+      echo "⚠️ No control socket found; configuration will not be loaded."
+      return 1
+    fi
+
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    echo "⏳ Waiting for control socket to be created... (${WAIT_COUNT}/${MAX_WAIT})"
+
+    sleep 1
+  done
+
+  # even when the control socket exists, it does not mean unit has finished initialisation
+  # this curl call will get a reply once unit is fully launched
+  curl --silent --output /dev/null --request GET --unix-socket $UNIT_SOCKET http://localhost/
+
+  echo "⚙️ Applying configuration from $UNIT_CONFIG";
+
+  RESP_CODE=$(curl \
+                  --silent \
+                  --output /dev/null \
+                  --write-out '%{http_code}' \
+                  --request PUT \
+                  --data-binary "@${UNIT_CONFIG}" \
+                  --unix-socket $UNIT_SOCKET \
+                  http://localhost/config
+            )
+  if [ "$RESP_CODE" != "200" ]; then
+    echo "⚠️ Could no load Unit configuration"
+    kill "$(cat /opt/unit/unit.pid)"
+    return 1
+  fi
+
+  echo "✅ Unit configuration loaded successfully"
+}
+
+load_configuration &
+
+exec unitd \
+    --no-daemon \
+    --control unix:$UNIT_SOCKET \
+    --pid /opt/unit/unit.pid \
+    --log /dev/stdout \
+    --state /opt/unit/state/ \
+    --tmp /opt/unit/tmp/

docker/nginx-unit.json

@@ -0,0 +1,40 @@
+{
+    "listeners": {
+        "*:8080": {
+            "pass": "routes"
+        }
+    },
+
+    "routes": [
+        {
+            "match": {
+                "uri": "/static/*"
+            },
+            "action": {
+                "share": "/opt/netbox/netbox"
+            }
+        },
+
+        {
+            "action": {
+                "pass": "applications/netbox"
+            }
+        }
+    ],
+
+    "applications": {
+        "netbox": {
+            "type": "python 3",
+            "path": "/opt/netbox/netbox/",
+            "module": "netbox.wsgi",
+            "home": "/opt/netbox/venv",
+            "processes": {
+                "max": 4,
+                "spare": 1,
+                "idle_timeout": 120
+            }
+        }
+    },
+
+    "access_log": "/dev/stdout"
+}

docker/nginx.conf

@@ -1,44 +0,0 @@
-daemon off;
-worker_processes 1;
-
-error_log /dev/stderr info;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include              /etc/nginx/mime.types;
-    default_type         application/octet-stream;
-    sendfile             on;
-    tcp_nopush           on;
-    keepalive_timeout    65;
-    gzip                 on;
-    server_tokens        off;
-    client_max_body_size 10M;
-
-    server {
-        listen      8080;
-        access_log  off;
-
-        location /static/ {
-            alias /opt/netbox/netbox/static/;
-        }
-
-        location / {
-            proxy_pass http://netbox:8001;
-            proxy_set_header X-Forwarded-Host $http_host;
-            proxy_set_header X-Real-IP $remote_addr;
-            add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
-        }
-    }
-
-    server {
-        listen      8081;
-        access_log  off;
-
-        location = /stub_status {
-            stub_status;
-        }
-    }
-}

initializers/aggregates.yml

@@ -1,5 +1,6 @@
 # - prefix: 10.0.0.0/16
 #   rir: RFC1918
+#   tenant: tenant1
 # - prefix: fd00:ccdd::/32
 #   rir: RFC4193 ULA
 # - prefix: 2001:db8::/32

initializers/circuit_types.yml

@@ -0,0 +1,6 @@
+# - name: VPLS
+#   slug: vpls
+# - name: MPLS
+#   slug: mpls
+# - name: Internet
+#   slug: internet

initializers/circuits.yml

@@ -0,0 +1,7 @@
+# - cid: Circuit_ID-1
+#   provider: Provider1
+#   type: Internet
+#   tenant: tenant1
+# - cid: Circuit_ID-2
+#   provider: Provider2
+#   type: MPLS

initializers/cluster_groups.yml

@@ -0,0 +1,4 @@
+# - name: Group 1
+#   slug: group-1
+# - name: Group 2
+#   slug: group-2

initializers/clusters.yml

@@ -1,5 +1,7 @@
 # - name: cluster1
 #   type: Hyper-V
+#   group: Group 1
+#   tenant: tenant1
 # - name: cluster2
 #   type: Hyper-V
 #   site: SING 1

initializers/power_feeds.yml

@@ -0,0 +1,14 @@
+# - name: power feed 1
+#   power_panel: power panel AMS 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Single phase
+#   rack: rack-01
+# - name: power feed 2
+#   power_panel: power panel SING 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Three-phase
+#   rack: rack-03

initializers/power_panels.yml

@@ -0,0 +1,5 @@
+# - name: power panel AMS 1
+#   site: AMS 1   
+# - name: power panel SING 1
+#   site: SING 1 
+#   rack_group: cage 101

initializers/providers.yml

@@ -0,0 +1,6 @@
+# - name: Provider1
+#   slug: provider1
+#   asn: 121
+# - name: Provider2
+#   slug: provider2
+#   asn: 122

initializers/route_targets.yml

@@ -0,0 +1,3 @@
+# - name: 65000:1001
+#   tenant: tenant1
+# - name: 65000:1002

initializers/secret_roles.yml

@@ -0,0 +1,4 @@
+# - name: Super Secret Passwords
+#   slug: super-secret
+# - name: SNMP Communities
+#   slug: snmp

initializers/services.yml

@@ -0,0 +1,15 @@
+# - name: DNS
+#   protocol: TCP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: DNS
+#   protocol: UDP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: MISC
+#   protocol: UDP
+#   ports: 
+#     - 4000
+#   device: server01

initializers/tags.yml

@@ -0,0 +1,12 @@
+# - name: Tag 1
+#   slug: tag-1
+#   color: Pink
+# - name: Tag 2
+#   slug: tag-2
+#   color: Cyan
+# - name: Tag 3
+#   slug: tag-3
+#   color: Grey
+# - name: Tag 4
+#   slug: tag-4
+#   color: Teal

renovate.json

@@ -0,0 +1,13 @@
+{
+  "extends": [
+    "config:base"
+  ],
+  "enabled": true,
+  "labels": ["maintenance"],
+  "baseBranches": ["develop"],
+  "pip_requirements": {
+    "fileMatch": [
+      "requirements-container.txt"
+    ]
+  }
+}

requirements-container.txt

@@ -0,0 +1,4 @@
+napalm==3.2.0
+ruamel.yaml==0.16.12
+django-auth-ldap==2.2.0
+django-storages==1.10.1

startup_scripts/020_tags.py

@@ -0,0 +1,23 @@
+from extras.models import Tag
+from utilities.choices import ColorChoices
+
+from startup_script_utils import load_yaml
+import sys
+
+tags = load_yaml('/opt/netbox/initializers/tags.yml')
+
+if tags is None:
+  sys.exit()
+
+for params in tags:
+  if 'color' in params:
+    color = params.pop('color')
+
+    for color_tpl in ColorChoices:
+      if color in color_tpl:
+        params['color'] = color_tpl[0]
+
+  tag, created = Tag.objects.get_or_create(**params)
+
+  if created:
+    print("🎨 Created Tag", tag.name)

startup_scripts/135_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/135_clusters.py

@@ -3,6 +3,7 @@ import sys
 from dcim.models import Site
 from startup_script_utils import *
 from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
 
 clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
 
@@ -15,7 +16,8 @@ required_assocs = {
 
 optional_assocs = {
   'site': (Site, 'name'),
-  'group': (ClusterGroup, 'name')
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
 }
 
 for params in clusters:

startup_scripts/140_clusters.py

@@ -0,0 +1,44 @@
+import sys
+
+from dcim.models import Site
+from startup_script_utils import *
+from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
+
+clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
+
+if clusters is None:
+  sys.exit()
+
+required_assocs = {
+  'type': (ClusterType, 'name')
+}
+
+optional_assocs = {
+  'site': (Site, 'name'),
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
+}
+
+for params in clusters:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  cluster, created = Cluster.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(cluster, custom_field_data)
+
+    print("🗄️ Created cluster", cluster.name)

startup_scripts/145_devices.py

@@ -0,0 +1,51 @@
+import sys
+
+from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster
+
+devices = load_yaml('/opt/netbox/initializers/devices.yml')
+
+if devices is None:
+  sys.exit()
+
+required_assocs = {
+  'device_role': (DeviceRole, 'name'),
+  'device_type': (DeviceType, 'model'),
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'platform': (Platform, 'name'),
+  'rack': (Rack, 'name'),
+  'cluster': (Cluster, 'name')
+}
+
+for params in devices:
+  custom_field_data = pop_custom_fields(params)
+
+  # primary ips are handled later in `270_primary_ips.py`
+  params.pop('primary_ip4', None)
+  params.pop('primary_ip6', None)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  device, created = Device.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(device, custom_field_data)
+
+    print("🖥️  Created device", device.name)

startup_scripts/160_aggregates.py

@@ -3,6 +3,7 @@ import sys
 from ipam.models import Aggregate, RIR
 from netaddr import IPNetwork
 from startup_script_utils import *
+from tenancy.models import Tenant
 
 aggregates = load_yaml('/opt/netbox/initializers/aggregates.yml')
 
@@ -13,6 +14,10 @@ required_assocs = {
   'rir': (RIR, 'name')
 }
 
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+}
+
 for params in aggregates:
   custom_field_data = pop_custom_fields(params)
 
@@ -24,6 +29,13 @@ for params in aggregates:
 
     params[assoc] = model.objects.get(**query)
 
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
   aggregate, created = Aggregate.objects.get_or_create(**params)
 
   if created:

startup_scripts/165_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/175_route_targets.py

@@ -0,0 +1,31 @@
+import sys
+
+from ipam.models import RouteTarget
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+route_targets = load_yaml('/opt/netbox/initializers/route_targets.yml')
+
+if route_targets is None:
+  sys.exit()
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in route_targets:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  route_target, created = RouteTarget.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(route_target, custom_field_data)
+
+    print("🎯 Created Route Target", route_target.name)

startup_scripts/270_primary_ips.py

@@ -31,13 +31,14 @@ def link_primary_ip(assets, asset_model):
 devices = load_yaml('/opt/netbox/initializers/devices.yml')
 virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml')
 
-if devices is None and virtual_machines is None:
-    sys.exit()
-
 optional_assocs = {
     'primary_ip4': (IPAddress, 'address'),
     'primary_ip6': (IPAddress, 'address')
 }
 
-link_primary_ip(devices, Device)
-link_primary_ip(virtual_machines, VirtualMachine)
+if devices is None and virtual_machines is None:
+    sys.exit()
+if devices is not None:
+  link_primary_ip(devices, Device)
+if virtual_machines is not None:
+  link_primary_ip(virtual_machines, VirtualMachine)

startup_scripts/280_providers.py

@@ -0,0 +1,18 @@
+from circuits.models import Provider
+from startup_script_utils import *
+import sys
+
+providers = load_yaml('/opt/netbox/initializers/providers.yml')
+
+if providers is None:
+  sys.exit()
+
+for params in providers:
+  custom_field_data = pop_custom_fields(params)
+
+  provider, created = Provider.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(provider, custom_field_data)
+
+    print("📡 Created provider", provider.name)

startup_scripts/290_circuit_types.py

@@ -0,0 +1,18 @@
+from circuits.models import CircuitType
+from startup_script_utils import *
+import sys
+
+circuit_types = load_yaml('/opt/netbox/initializers/circuit_types.yml')
+
+if circuit_types is None:
+  sys.exit()
+
+for params in circuit_types:
+  custom_field_data = pop_custom_fields(params)
+
+  circuit_type, created = CircuitType.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit_type, custom_field_data)
+
+    print("⚡ Created Circuit Type", circuit_type.name)

startup_scripts/300_circuits.py

@@ -0,0 +1,41 @@
+from circuits.models import Circuit, Provider, CircuitType
+from tenancy.models import Tenant
+from startup_script_utils import *
+import sys
+
+circuits = load_yaml('/opt/netbox/initializers/circuits.yml')
+
+if circuits is None:
+  sys.exit()
+
+required_assocs = {
+  'provider': (Provider, 'name'),
+  'type': (CircuitType, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in circuits:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  circuit, created = Circuit.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit, custom_field_data)
+
+    print("⚡ Created Circuit", circuit.cid)

startup_scripts/310_secret_roles.py

@@ -0,0 +1,14 @@
+from secrets.models import SecretRole
+from startup_script_utils import load_yaml
+import sys
+
+secret_roles = load_yaml('/opt/netbox/initializers/secret_roles.yml')
+
+if secret_roles is None:
+  sys.exit()
+
+for params in secret_roles:
+  secret_role, created = SecretRole.objects.get_or_create(**params)
+
+  if created:
+    print("🔑 Created Secret Role", secret_role.name)

startup_scripts/320_services.py

@@ -0,0 +1,29 @@
+from ipam.models import Service
+from dcim.models import Device
+from virtualization.models import VirtualMachine
+from startup_script_utils import load_yaml
+import sys
+
+services = load_yaml('/opt/netbox/initializers/services.yml')
+
+if services is None:
+  sys.exit()
+
+optional_assocs = {
+  'device': (Device, 'name'),
+  'virtual_machine': (VirtualMachine, 'name')
+}
+
+for params in services:
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  service, created = Service.objects.get_or_create(**params)
+
+  if created:
+    print("🧰 Created Service", service.name)

startup_scripts/330_power_panels.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Site, RackGroup, PowerPanel
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_panels = load_yaml('/opt/netbox/initializers/power_panels.yml')
+
+if power_panels is None:
+  sys.exit()
+
+required_assocs = {
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'rack_group': (RackGroup, 'name')
+}
+
+for params in power_panels:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_panel, created = PowerPanel.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_panel, custom_field_data)
+
+    print("⚡ Created Power Panel", power_panel.site, power_panel.name)

startup_scripts/340_power_feeds.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Rack, PowerPanel, PowerFeed
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_feeds = load_yaml('/opt/netbox/initializers/power_feeds.yml')
+
+if power_feeds is None:
+  sys.exit()
+
+required_assocs = {
+  'power_panel': (PowerPanel, 'name')
+}
+
+optional_assocs = {
+  'rack': (Rack, 'name')
+}
+
+for params in power_feeds:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_feed, created = PowerFeed.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_feed, custom_field_data)
+
+    print("⚡ Created Power Feed", power_feed.name)

startup_scripts/__main__.py

@@ -20,7 +20,7 @@ with scandir(this_dir) as it:
     if not f.name.endswith('.py'):
       continue
 
-    print(f"▶️ Running the startup script {f.path}")
+    print(f"▶️  Running the startup script {f.path}")
     try:
       runpy.run_path(f.path)
     except SystemExit as e:

test.sh

@@ -35,7 +35,7 @@ if [ -z "${IMAGE}" ]; then
 fi
 
 # The docker compose command to use
-doco="docker-compose -f docker-compose.test.yml"
+doco="docker-compose --file docker-compose.test.yml --project-name netbox_docker_test_${1}"
 
 INITIALIZERS_DIR=".initializers"
 
@@ -56,7 +56,7 @@ test_setup() {
 
 test_netbox_unit_tests() {
   echo "⏱ Running Netbox Unit Tests"
-  $doco run --rm netbox ./manage.py test
+  SKIP_STARTUP_SCRIPTS=true $doco run --rm netbox ./manage.py test
 }
 
 test_initializers() {