From 35a94cb7e58e9876badbdd19d7ec6236c76d35f6 Mon Sep 17 00:00:00 2001 From: Ryan Merolle Date: Sat, 22 Oct 2022 09:40:03 -0400 Subject: [PATCH 01/11] Update docker-compose.yml Address housekeeping tracebacks & remove whitespace --- docker-compose.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c20f7e9..780c732 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,9 +6,13 @@ services: - postgres - redis - redis-cache - - netbox-worker env_file: env/netbox.env user: 'unit:root' + healthcheck: + start_period: 60s + timeout: 3s + interval: 15s + test: "curl -f http://localhost:8080/api/ || exit 1" volumes: - ./configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro @@ -17,19 +21,29 @@ services: netbox-worker: <<: *netbox depends_on: - - redis - - postgres + netbox: + condition: service_healthy command: - /opt/netbox/venv/bin/python - /opt/netbox/netbox/manage.py - rqworker + healthcheck: + start_period: 20s + timeout: 3s + interval: 15s + test: "ps -aux | grep -v grep | grep -q rqworker || exit 1" netbox-housekeeping: <<: *netbox depends_on: - - redis - - postgres + netbox: + condition: service_healthy command: - /opt/netbox/housekeeping.sh + healthcheck: + start_period: 20s + timeout: 3s + interval: 15s + test: "ps -aux | grep -v grep | grep -q housekeeping || exit 1" # postgres postgres: @@ -56,7 +70,7 @@ services: - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis-cache.env volumes: - - netbox-redis-cache-data:/data + - netbox-redis-cache-data:/data volumes: netbox-media-files: @@ -66,4 +80,4 @@ volumes: netbox-redis-data: driver: local netbox-redis-cache-data: - driver: local + driver: local From 7d871778ebf2f38f02ee7df29844bfbdbdc7beb0 Mon Sep 17 00:00:00 2001 From: movelg <43134135+movelg@users.noreply.github.com> Date: Fri, 11 Nov 2022 08:40:29 +0100 Subject: [PATCH 02/11] Don't use bash internal variable name --- docker/housekeeping.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/housekeeping.sh b/docker/housekeeping.sh index cfe06b1..47a4a3d 100755 --- a/docker/housekeeping.sh +++ b/docker/housekeeping.sh @@ -1,8 +1,8 @@ #!/bin/bash -SECONDS=${HOUSEKEEPING_INTERVAL:=86400} -echo "Interval set to ${SECONDS} seconds" +SLEEP_SECONDS=${HOUSEKEEPING_INTERVAL:=86400} +echo "Interval set to ${SLEEP_SECONDS} seconds" while true; do date /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping - sleep "${SECONDS}s" + sleep "${SLEEP_SECONDS}s" done From dd01e3c2275dfa0f8d85c14a777606d176028bb3 Mon Sep 17 00:00:00 2001 From: Wellyas Date: Mon, 31 Oct 2022 16:00:18 +0100 Subject: [PATCH 03/11] Enable SAML & OPENIDCONNECT for social-auth-core --- Dockerfile | 8 ++++++++ requirements-container.txt | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2bc5dff..21c58dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,13 @@ RUN export DEBIAN_FRONTEND=noninteractive \ libpq-dev \ libsasl2-dev \ libssl-dev \ + libxml2-dev \ + libxml2-dev \ + libxmlsec1 \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + libxslt-dev \ + pkg-config \ python3-dev \ python3-pip \ python3-venv \ @@ -46,6 +53,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ curl \ libldap-common \ libpq5 \ + libxmlsec1-openssl \ openssl \ python3 \ python3-distutils \ diff --git a/requirements-container.txt b/requirements-container.txt index f8b720e..0d16b26 100644 --- a/requirements-container.txt +++ b/requirements-container.txt @@ -2,4 +2,5 @@ django-auth-ldap==4.1.0 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.13.1 napalm==4.0.0 psycopg2==2.9.4 -social-auth-core[openidconnect]==4.3.0 +python3-saml==1.14.0 +social-auth-core[all]==4.3.0 From a4f494db14edc792712fb473efd80480d5fce5e5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 11 Nov 2022 14:24:44 +0000 Subject: [PATCH 04/11] Update postgres Docker tag to v15 --- docker-compose.test.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 3e4686e..2f6b24c 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -17,7 +17,7 @@ services: - ./scripts:/etc/netbox/scripts:z,ro - netbox-media-files:/opt/netbox/netbox/media:z postgres: - image: postgres:14-alpine + image: postgres:15-alpine env_file: env/postgres.env redis: image: redis:7-alpine diff --git a/docker-compose.yml b/docker-compose.yml index 3d50c77..043705d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: # postgres postgres: - image: postgres:14-alpine + image: postgres:15-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data From 6020f4503add335a709a4a702006b8b801debb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 23 Nov 2022 14:40:03 +0100 Subject: [PATCH 05/11] Make nginx-unit listen on IPv4 and IPv6 --- docker/nginx-unit.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/nginx-unit.json b/docker/nginx-unit.json index 87a186b..d6ff9cc 100644 --- a/docker/nginx-unit.json +++ b/docker/nginx-unit.json @@ -1,6 +1,9 @@ { "listeners": { - "*:8080": { + "0.0.0.0:8080": { + "pass": "routes" + }, + "[::]:8080": { "pass": "routes" } }, From 1779ba790d3a3af2d716a61e90b02c5aed26bb46 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Wed, 30 Nov 2022 09:51:36 +0100 Subject: [PATCH 06/11] Removed warning for initializer scripts --- docker/docker-entrypoint.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6afa1ea..ff495e6 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -80,15 +80,6 @@ END echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}" fi -# Print warning if startup scripts (and initializers) would've been run # Remove for next release -if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then - # Nothing to do - echo "" # Empty block not allowed -else - echo "⚠️⚠️⚠️ WARNING: The initializers have been moved to a plugin. See release notes." - echo "⚠️⚠️⚠️ Set environment variable 'SKIP_STARTUP_SCRIPTS' to 'true' to remove this warning." -fi - echo "✅ Initialisation is done." # Launch whatever is passed by docker From 9441be459c0fe4c84aa12e38acb14333989f7ba6 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Wed, 30 Nov 2022 09:52:07 +0100 Subject: [PATCH 07/11] Improved testing After the initializer scripts were removed, we didn't test the actual compose setup anymore. This adds new tests to run the database migrations. --- docker-compose.test.yml | 16 +++++++++++----- test.sh | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 2f6b24c..b8444f0 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -3,12 +3,13 @@ services: netbox: image: ${IMAGE-netboxcommunity/netbox:latest} depends_on: - - postgres - - redis - - redis-cache + postgres: + condition: service_healthy + redis: + condition: service_started + redis-cache: + condition: service_started env_file: env/netbox.env - environment: - SKIP_STARTUP_SCRIPTS: ${SKIP_STARTUP_SCRIPTS-false} user: 'unit:root' volumes: - ./configuration:/etc/netbox/config:z,ro @@ -19,6 +20,11 @@ services: postgres: image: postgres:15-alpine env_file: env/postgres.env + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 redis: image: redis:7-alpine command: diff --git a/test.sh b/test.sh index ca36a8f..31b4d87 100755 --- a/test.sh +++ b/test.sh @@ -35,10 +35,14 @@ if [ -z "${IMAGE}" ]; then fi # The docker compose command to use -doco="docker-compose --file docker-compose.test.yml --project-name netbox_docker_test_${1}" +doco="docker compose --file docker-compose.test.yml --project-name netbox_docker_test" test_setup() { echo "🏗 Setup up test environment" + $doco up --detach --quiet-pull --wait --force-recreate --renew-anon-volumes --no-start + $doco start postgres + $doco start redis + $doco start redis-cache } test_netbox_unit_tests() { @@ -46,9 +50,14 @@ test_netbox_unit_tests() { $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py test } +test_compose_db_setup() { + echo "⏱ Running NetBox DB migrations" + $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py migrate +} + test_cleanup() { echo "💣 Cleaning Up" - $doco down -v + $doco down --volumes } echo "🐳🐳🐳 Start testing '${IMAGE}'" @@ -58,5 +67,6 @@ trap test_cleanup EXIT ERR test_setup test_netbox_unit_tests +test_compose_db_setup echo "🐳🐳🐳 Done testing '${IMAGE}'" From aa3357817a765d09c22799bb972a408ae5586425 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Wed, 30 Nov 2022 10:03:27 +0100 Subject: [PATCH 08/11] Disable Gitleaks --- .github/workflows/push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3dc25ba..acd08d4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -31,6 +31,7 @@ jobs: LINTER_RULES_PATH: / VALIDATE_ALL_CODEBASE: false VALIDATE_DOCKERFILE: false + VALIDATE_GITLEAKS: false FILTER_REGEX_EXCLUDE: (.*/)?(LICENSE|configuration/.*) EDITORCONFIG_FILE_NAME: .ecrc DOCKERFILE_HADOLINT_FILE_NAME: .hadolint.yaml From 6d465e6f813bc8e29c846f313487a086cfa50428 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Wed, 30 Nov 2022 14:36:53 +0100 Subject: [PATCH 09/11] Optimize psycopg2 dependency We have beeing installing psycopg2 for a while now. This updates to the latest version. Because psycopg2-binary is a direct dependency of Netbox both versions were installled. Now we remove the pre-compiled version from the dependency file. --- Dockerfile | 3 ++- requirements-container.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21c58dd..fcf01b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,8 @@ RUN export DEBIAN_FRONTEND=noninteractive \ ARG NETBOX_PATH COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt / -RUN /opt/netbox/venv/bin/pip install \ +RUN sed -i -e '/psycopg2-binary/d' requirements.txt && \ + /opt/netbox/venv/bin/pip install \ -r /requirements.txt \ -r /requirements-container.txt diff --git a/requirements-container.txt b/requirements-container.txt index 0d16b26..96f3aa3 100644 --- a/requirements-container.txt +++ b/requirements-container.txt @@ -1,6 +1,6 @@ django-auth-ldap==4.1.0 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.13.1 napalm==4.0.0 -psycopg2==2.9.4 +psycopg2==2.9.5 python3-saml==1.14.0 social-auth-core[all]==4.3.0 From bdb4396275f3f622f7f3de3a9e0506f9705e7075 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Thu, 1 Dec 2022 08:17:25 +0100 Subject: [PATCH 10/11] Added settings for CA certificates for LDAP --- configuration/ldap/ldap_config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/configuration/ldap/ldap_config.py b/configuration/ldap/ldap_config.py index 7fb62d7..8aa3ecb 100644 --- a/configuration/ldap/ldap_config.py +++ b/configuration/ldap/ldap_config.py @@ -49,6 +49,16 @@ AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'tr # ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' +# Include this setting if you want to validate the LDAP server certificates against a CA certificate directory on your server +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, LDAP_CA_CERT_DIR) +LDAP_CA_CERT_DIR = environ.get('LDAP_CA_CERT_DIR', None) + +# Include this setting if you want to validate the LDAP server certificates against your own CA. +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, LDAP_CA_CERT_FILE) +LDAP_CA_CERT_FILE = environ.get('LDAP_CA_CERT_FILE', None) + AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') AUTH_LDAP_USER_SEARCH = LDAPSearch( From 238f95c5ce7f21ca764b08052babe8cd629eec1f Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Thu, 15 Dec 2022 12:57:51 +0100 Subject: [PATCH 11/11] Preparation for 2.4.0 --- VERSION | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 276cbf9..197c4d5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.0 +2.4.0 diff --git a/docker-compose.yml b/docker-compose.yml index 6619283..d3e8234 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.4' services: netbox: &netbox - image: netboxcommunity/netbox:${VERSION-v3.3-2.3.0} + image: netboxcommunity/netbox:${VERSION-v3.4-2.4.0} depends_on: - postgres - redis