diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fc00ee8..414f3fe 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -17,7 +17,9 @@ jobs: with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Lint Code Base uses: github/super-linter@v4 env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93f6868..7f448eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,7 +42,7 @@ jobs: # docker.io - name: Login to docker.io - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: docker.io username: ${{ secrets.dockerhub_username }} @@ -56,7 +56,7 @@ jobs: # quay.io - name: Login to Quay.io - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: quay.io username: ${{ secrets.quayio_username }} @@ -72,7 +72,7 @@ jobs: # ghcr.io - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.hadolint.yaml b/.hadolint.yaml index 610e431..4442bd4 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,3 +1,4 @@ ignored: - DL3006 -- DL3018 +- DL3008 +- DL3003 diff --git a/Dockerfile b/Dockerfile index 51cd359..6bec8a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,27 @@ ARG FROM FROM ${FROM} as builder -RUN apk add --no-cache \ - bash \ - build-base \ - cargo \ +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -qq \ + && apt-get upgrade \ + --yes -qq --no-install-recommends \ + && apt-get install \ + --yes -qq --no-install-recommends \ + build-essential \ ca-certificates \ - cmake \ - cyrus-sasl-dev \ - git \ - graphviz \ - jpeg-dev \ - libevent-dev \ - libffi-dev \ - libxslt-dev \ - make \ - musl-dev \ - openldap-dev \ - postgresql-dev \ - py3-pip \ + libldap-dev \ + libpq-dev \ + libsasl2-dev \ + libssl-dev \ python3-dev \ - && python3 -m venv /opt/netbox/venv \ - && /opt/netbox/venv/bin/python3 -m pip install --upgrade \ + python3-pip \ + python3-venv \ + && python3 -m venv /opt/netbox/venv \ + && /opt/netbox/venv/bin/python3 -m pip install --upgrade \ pip \ setuptools \ wheel -# Build libcrc32c for google-crc32c python module -RUN git clone https://github.com/google/crc32c \ - && cd crc32c \ - && git submodule update --init --recursive \ - && mkdir build \ - && cd build \ - && cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCRC32C_BUILD_TESTS=no \ - -DCRC32C_BUILD_BENCHMARKS=no \ - -DBUILD_SHARED_LIBS=yes \ - .. \ - && make all install - ARG NETBOX_PATH COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt / RUN /opt/netbox/venv/bin/pip install \ @@ -53,35 +35,37 @@ RUN /opt/netbox/venv/bin/pip install \ ARG FROM FROM ${FROM} as main -RUN apk add --no-cache \ - bash \ +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -qq \ + && apt-get upgrade \ + --yes -qq --no-install-recommends \ + && apt-get install \ + --yes -qq --no-install-recommends \ ca-certificates \ curl \ - graphviz \ - libevent \ - libffi \ - libjpeg-turbo \ - libxslt \ + libpq5 \ openssl \ - postgresql-client \ - postgresql-libs \ - py3-pip \ python3 \ + python3-distutils \ tini \ - unit \ - unit-python3 + && curl -sL https://nginx.org/keys/nginx_signing.key \ + > /etc/apt/trusted.gpg.d/nginx.asc && \ + echo "deb https://packages.nginx.org/unit/debian/ bullseye unit" \ + > /etc/apt/sources.list.d/unit.list \ + && apt-get update -qq \ + && apt-get install \ + --yes -qq --no-install-recommends \ + unit=1.27.0-1~bullseye \ + unit-python3.9=1.27.0-1~bullseye \ + && rm -rf /var/lib/apt/lists/* -WORKDIR /opt - -COPY --from=builder /usr/local/lib/libcrc32c.* /usr/local/lib/ -COPY --from=builder /usr/local/include/crc32c /usr/local/include -COPY --from=builder /usr/local/lib/cmake/Crc32c /usr/local/lib/cmake/ COPY --from=builder /opt/netbox/venv /opt/netbox/venv ARG NETBOX_PATH COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py +COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh COPY docker/housekeeping.sh /opt/netbox/housekeeping.sh COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh @@ -101,26 +85,15 @@ RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \ --config-file /opt/netbox/mkdocs.yml --site-dir /opt/netbox/netbox/project-static/docs/ \ && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input -ENTRYPOINT [ "/sbin/tini", "--" ] +ENV LANG=C.UTF-8 +ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/opt/netbox/docker-entrypoint.sh", "/opt/netbox/launch-netbox.sh" ] -LABEL ORIGINAL_TAG="" \ - NETBOX_GIT_BRANCH="" \ - NETBOX_GIT_REF="" \ - NETBOX_GIT_URL="" \ -# See http://label-schema.org/rc1/#build-time-labels -# Also https://microbadger.com/labels - org.label-schema.schema-version="1.0" \ - org.label-schema.build-date="" \ - org.label-schema.name="NetBox Docker" \ - org.label-schema.description="A container based distribution of NetBox, the free and open IPAM and DCIM solution." \ - org.label-schema.vendor="The netbox-docker contributors." \ - org.label-schema.url="https://github.com/netbox-community/netbox-docker" \ - org.label-schema.usage="https://github.com/netbox-community/netbox-docker/wiki" \ - org.label-schema.vcs-url="https://github.com/netbox-community/netbox-docker.git" \ - org.label-schema.vcs-ref="" \ - org.label-schema.version="snapshot" \ +LABEL netbox.original-tag="" \ + netbox.git-branch="" \ + netbox.git-ref="" \ + netbox.git-url="" \ # See https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys org.opencontainers.image.created="" \ org.opencontainers.image.title="NetBox Docker" \ @@ -132,17 +105,4 @@ LABEL ORIGINAL_TAG="" \ org.opencontainers.image.documentation="https://github.com/netbox-community/netbox-docker/wiki" \ org.opencontainers.image.source="https://github.com/netbox-community/netbox-docker.git" \ org.opencontainers.image.revision="" \ - org.opencontainers.image.version="snapshot" - -##### -## LDAP specific configuration -##### - -FROM main as ldap - -RUN apk add --no-cache \ - libsasl \ - libldap \ - util-linux - -COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py + org.opencontainers.image.version="" diff --git a/VERSION b/VERSION index 9c6d629..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6.1 +2.0.0 diff --git a/build.sh b/build.sh index 1cf6cd1..fbf102d 100755 --- a/build.sh +++ b/build.sh @@ -49,10 +49,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" echo " DOCKER_FROM The base image to use." - echo " Default: 'alpine:3.14'" + echo " Default: 'debian:11-slim'" echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." - echo " Default: main ldap" + echo " Default: main" echo " HTTP_PROXY The proxy to use for http requests." echo " Example: http://proxy.domain.tld:3128" echo " Default: undefined" @@ -170,7 +170,7 @@ fi # Determining the value for DOCKER_FROM ### if [ -z "$DOCKER_FROM" ]; then - DOCKER_FROM="alpine:3.14" + DOCKER_FROM="debian:11-slim" fi ### @@ -178,7 +178,7 @@ fi ### BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" -if [ -d ".git" ]; then +if [ -d ".git" ] && [ -z "${SKIP_GIT}" ]; then GIT_REF="$(git rev-parse HEAD)" fi @@ -186,7 +186,7 @@ fi PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" # Get the Git information from the netbox directory -if [ -d "${NETBOX_PATH}/.git" ]; then +if [ -d "${NETBOX_PATH}/.git" ] && [ -z "${SKIP_GIT}" ]; then NETBOX_GIT_REF=$( cd "${NETBOX_PATH}" git rev-parse HEAD @@ -222,7 +222,7 @@ esac ### # Determine targets to build ### -DEFAULT_DOCKER_TARGETS=("main" "ldap") +DEFAULT_DOCKER_TARGETS=("main") DOCKER_TARGETS=("${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}") echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" @@ -277,9 +277,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### # Checking if the build is necessary, # meaning build only if one of those values changed: - # - Python base image digest (Label: PYTHON_BASE_DIGEST) - # - netbox git ref (Label: NETBOX_GIT_REF) - # - netbox-docker git ref (Label: org.label-schema.vcs-ref) + # - base image digest + # - netbox git ref (Label: netbox.git-ref) + # - netbox-docker git ref (Label: org.opencontainers.image.revision) ### # Load information from registry (only for docker.io) SHOULD_BUILD="false" @@ -295,14 +295,14 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # Need to use "library/..." for images the have no two part name DOCKER_FROM_SPLIT[0]="library/${DOCKER_FROM_SPLIT[0]}" fi - PYTHON_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}") + BASE_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}") mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") - NETBOX_GIT_REF_OLD=$(get_image_label NETBOX_GIT_REF "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") - GIT_REF_OLD=$(get_image_label org.label-schema.vcs-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") + NETBOX_GIT_REF_OLD=$(get_image_label netbox.git-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") + GIT_REF_OLD=$(get_image_label org.opencontainers.image.revision "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}") - if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then + if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${BASE_LAST_LAYER}\$"; then SHOULD_BUILD="true" - BUILD_REASON="${BUILD_REASON} alpine" + BUILD_REASON="${BUILD_REASON} debian" fi if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then SHOULD_BUILD="true" @@ -335,30 +335,25 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do # --label DOCKER_BUILD_ARGS+=( - --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG_PROJECT}" - - --label "org.label-schema.build-date=${BUILD_DATE}" + --label "netbox.original-tag=${TARGET_DOCKER_TAG_PROJECT}" --label "org.opencontainers.image.created=${BUILD_DATE}" - - --label "org.label-schema.version=${PROJECT_VERSION}" --label "org.opencontainers.image.version=${PROJECT_VERSION}" ) if [ -d ".git" ]; then DOCKER_BUILD_ARGS+=( - --label "org.label-schema.vcs-ref=${GIT_REF}" --label "org.opencontainers.image.revision=${GIT_REF}" ) fi if [ -d "${NETBOX_PATH}/.git" ]; then DOCKER_BUILD_ARGS+=( - --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}" - --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}" - --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}" + --label "netbox.git-branch=${NETBOX_GIT_BRANCH}" + --label "netbox.git-ref=${NETBOX_GIT_REF}" + --label "netbox.git-url=${NETBOX_GIT_URL}" ) fi if [ -n "${BUILD_REASON}" ]; then BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<"$BUILD_REASON") - DOCKER_BUILD_ARGS+=(--label "BUILD_REASON=${BUILD_REASON}") + DOCKER_BUILD_ARGS+=(--label "netbox.build-reason=${BUILD_REASON}") fi # --build-arg @@ -385,7 +380,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . echo "βœ… Finished building the Docker images '${TARGET_DOCKER_TAG_PROJECT}'" echo "πŸ”Ž Inspecting labels on '${TARGET_DOCKER_TAG_PROJECT}'" - $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}" + $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}" | jq else echo "Build skipped because sources didn't change" echo "::set-output name=skipped::true" diff --git a/configuration/configuration.py b/configuration/configuration.py index 1db6051..c8ddd14 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -120,6 +120,11 @@ CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] +# Cross-Site-Request-Forgery-Attack settings. If Netbox is sitting behind a reverse proxy, you might need to set the CSRF_TRUSTED_ORIGINS flag. +# Django 4.0 requires to specify the URL Scheme in this setting. An example environment variable could be specified like: +# CSRF_TRUSTED_ORIGINS=https://demo.netbox.dev http://demo.netbox.dev +CSRF_TRUSTED_ORIGINS = list(filter(None, environ.get('CSRF_TRUSTED_ORIGINS', '').split(' '))) + # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. @@ -165,6 +170,9 @@ LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600)) # Setting this to True will display a "maintenance mode" banner at the top of every page. MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' +# Maps provider +MAPS_URL = environ.get('MAPS_URL', None) + # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request # all objects by specifying "?limit=0". diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 33c0606..04b5939 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -21,14 +21,14 @@ services: image: postgres:14-alpine env_file: env/postgres.env redis: - image: redis:6-alpine + image: redis:7-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis.env redis-cache: - image: redis:6-alpine + image: redis:7-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env diff --git a/docker-compose.yml b/docker-compose.yml index c7373be..31d053e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.4' services: netbox: &netbox - image: netboxcommunity/netbox:${VERSION-v3.2-1.6.1} + image: netboxcommunity/netbox:${VERSION-v3.2-2.0.0} depends_on: - postgres - redis @@ -42,7 +42,7 @@ services: # redis redis: - image: redis:6-alpine + image: redis:7-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env @@ -51,7 +51,7 @@ services: volumes: - netbox-redis-data:/data redis-cache: - image: redis:6-alpine + image: redis:7-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env diff --git a/docker/nginx-unit.json b/docker/nginx-unit.json index fbe8c2b..87a186b 100644 --- a/docker/nginx-unit.json +++ b/docker/nginx-unit.json @@ -11,7 +11,7 @@ "uri": "/static/*" }, "action": { - "share": "/opt/netbox/netbox" + "share": "/opt/netbox/netbox${uri}" } }, diff --git a/initializers/cables.yml b/initializers/cables.yml new file mode 100644 index 0000000..3257643 --- /dev/null +++ b/initializers/cables.yml @@ -0,0 +1,71 @@ +# # Required parameters for termination X ('a' or 'b'): +# # +# # ``` +# # termination_x_name -> name of interface +# # termination_x_device -> name of the device interface belongs to +# # termination_x_class -> required if different than 'Interface' which is the default +# # ``` +# # +# # Supported termination classes: Interface, ConsolePort, ConsoleServerPort, FrontPort, RearPort, PowerPort, PowerOutlet +# # +# # +# # If a termination is a circuit then the required parameter is termination_x_circuit. +# # Required parameters for a circuit termination: +# # +# # ``` +# # termination_x_circuit: +# # term_side -> termination side of a circuit. Must be A or B +# # cid -> circuit ID value +# # site OR provider_network -> name of Site or ProviderNetwork respectively. If both provided, Site takes precedence +# # ``` +# # +# # If a termination is a power feed then the required parameter is termination_x_feed. +# # +# # ``` +# # termination_x_feed: +# # name -> name of the PowerFeed object +# # power_panel: +# # name -> name of the PowerPanel the PowerFeed is attached to +# # site -> name of the Site in which the PowerPanel is present +# # ``` +# # +# # Any other Cable parameters supported by Netbox are supported as the top level keys, e.g. 'type', 'status', etc. +# # +# # - termination_a_name: console +# # termination_a_device: spine +# # termination_a_class: ConsolePort +# # termination_b_name: tty9 +# # termination_b_device: console-server +# # termination_b_class: ConsoleServerPort +# # type: cat6 +# # +# - termination_a_name: to-server02 +# termination_a_device: server01 +# termination_b_name: to-server01 +# termination_b_device: server02 +# status: planned +# type: mmf + +# - termination_a_name: eth0 +# termination_a_device: server02 +# termination_b_circuit: +# term_side: A +# cid: Circuit_ID-1 +# site: AMS 1 +# type: cat6 + +# - termination_a_name: psu0 +# termination_a_device: server04 +# termination_a_class: PowerPort +# termination_b_feed: +# name: power feed 1 +# power_panel: +# name: power panel AMS 1 +# site: AMS 1 + +# - termination_a_name: outlet1 +# termination_a_device: server04 +# termination_a_class: PowerOutlet +# termination_b_name: psu1 +# termination_b_device: server04 +# termination_b_class: PowerPort diff --git a/initializers/contact_groups.yml b/initializers/contact_groups.yml new file mode 100644 index 0000000..a3ead09 --- /dev/null +++ b/initializers/contact_groups.yml @@ -0,0 +1,7 @@ +# - name: Network-Team +# slug: network-team +# description: This is a new contact group for the Network-Team +# - name: New Contact Group +# slug: new-contact-group +# description: This is a new contact group sub under of Network-Team +# parent: Network-Team diff --git a/initializers/contact_roles.yml b/initializers/contact_roles.yml new file mode 100644 index 0000000..1a72e43 --- /dev/null +++ b/initializers/contact_roles.yml @@ -0,0 +1,3 @@ +# - name: New Contact Role +# slug: new-contact-role +# description: This is a new contact role description diff --git a/initializers/contacts.yml b/initializers/contacts.yml new file mode 100644 index 0000000..3ab41a2 --- /dev/null +++ b/initializers/contacts.yml @@ -0,0 +1,20 @@ +# - name: Lee Widget +# title: CEO of Widget Corp +# phone: 221-555-1212 +# email: widgetCEO@widgetcorp.com +# address: 1200 Nowhere Blvd, Scranton NJ, 555111 +# comments: This is a very important contact +# - name: Ali Gator +# group: Network-Team +# title: Consultant for Widget Corp +# phone: 221-555-1213 +# email: Consultant@widgetcorp.com +# address: 1200 Nowhere Blvd, Scranton NJ, 555111 +# comments: This is a very important contact +# - name: Karlchen Maier +# group: New Contact Group +# title: COO of Widget Corp +# phone: 221-555-1214 +# email: Karlchen@widgetcorp.com +# address: 1200 Nowhere Blvd, Scranton NJ, 555111 +# comments: This is a very important contact diff --git a/initializers/dcim_interfaces.yml b/initializers/dcim_interfaces.yml index 4030530..a889d46 100644 --- a/initializers/dcim_interfaces.yml +++ b/initializers/dcim_interfaces.yml @@ -8,11 +8,28 @@ ## ## Examples: +# - device: server01 +# name: ath0 +# type: 1000base-t +# lag: ae0 +# bridge: br0 +# - device: server01 +# name: ath1 +# type: 1000base-t +# parent: ath0 # - device: server01 # enabled: true -# type: virtual +# type: 1000base-x-sfp # name: to-server02 # - device: server02 # enabled: true -# type: virtual +# type: 1000base-x-sfp # name: to-server01 +# - device: server02 +# enabled: true +# type: 1000base-t +# name: eth0 +# - device: server02 +# enabled: true +# type: virtual +# name: loopback diff --git a/initializers/device_types.yml b/initializers/device_types.yml index 88798b5..b1ee130 100644 --- a/initializers/device_types.yml +++ b/initializers/device_types.yml @@ -21,3 +21,37 @@ # slug: other # custom_field_data: # text_field: Description +# interfaces: +# - name: eth0 +# type: 1000base-t +# mgmt_only: True +# - name: eth1 +# type: 1000base-t +# console_server_ports: +# - name_template: ttyS[1-48] +# type: rj-45 +# power_ports: +# - name_template: psu[0,1] +# type: iec-60320-c14 +# maximum_draw: 35 +# allocated_draw: 35 +# front_ports: +# - name_template: front[1,2] +# type: 8p8c +# rear_port_template: rear[0,1] +# rear_port_position_template: "[1,2]" +# rear_ports: +# - name_template: rear[0,1] +# type: 8p8c +# positions_template: "[3,2]" +# device_bays: +# - name: bay0 # both non-template and template field specified; non-template field takes precedence +# name_template: bay[0-9] +# label: test0 +# label_template: test[0-5,9,6-8] +# description: Test description +# power_outlets: +# - name_template: outlet[0,1] +# type: iec-60320-c5 +# power_port: psu0 +# feed_leg: B diff --git a/initializers/users.yml b/initializers/users.yml index c163d50..d6484c2 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -4,6 +4,7 @@ # password: reader # writer: # password: writer +# api_token: "" # a token is generated automatically unless the value is explicity set to empty # jdoe: # first_name: John # last_name: Doe diff --git a/requirements-container.txt b/requirements-container.txt index 15b728e..b80f0eb 100644 --- a/requirements-container.txt +++ b/requirements-container.txt @@ -1,5 +1,5 @@ -django-auth-ldap==4.0.0 +django-auth-ldap==4.1.0 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.12.3 -google-crc32c==1.3.0 -napalm==3.4.0 +napalm==4.0.0 +psycopg2==2.9.3 ruamel.yaml==0.17.21 diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index 1435d81..7ca6305 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -9,13 +9,17 @@ if users is None: sys.exit() for username, user_details in users.items(): - if not User.objects.filter(username=username): - user = User.objects.create_user( - username=username, - password=user_details.get("password", 0) or User.objects.make_random_password(), - ) + + api_token = user_details.pop("api_token", Token.generate_key()) + password = user_details.pop("password", User.objects.make_random_password()) + + user, created = User.objects.get_or_create(username=username, defaults=user_details) + + if created: + user.set_password(password) + user.save() + + if api_token: + Token.objects.get_or_create(user=user, key=api_token) print("πŸ‘€ Created user", username) - - if user_details.get("api_token", 0): - Token.objects.create(user=user, key=user_details["api_token"]) diff --git a/startup_scripts/020_object_permissions.py b/startup_scripts/020_object_permissions.py index 8a5ecd4..562a09c 100644 --- a/startup_scripts/020_object_permissions.py +++ b/startup_scripts/020_object_permissions.py @@ -14,9 +14,11 @@ for permission_name, permission_details in object_permissions.items(): object_permission, created = ObjectPermission.objects.get_or_create( name=permission_name, - description=permission_details["description"], - enabled=permission_details["enabled"], - actions=permission_details["actions"], + defaults={ + "description": permission_details["description"], + "enabled": permission_details["enabled"], + "actions": permission_details["actions"], + }, ) if permission_details.get("object_types", 0): diff --git a/startup_scripts/040_custom_links.py b/startup_scripts/040_custom_links.py index 40144bd..d8c0bba 100644 --- a/startup_scripts/040_custom_links.py +++ b/startup_scripts/040_custom_links.py @@ -2,7 +2,7 @@ import sys from django.contrib.contenttypes.models import ContentType from extras.models import CustomLink -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params custom_links = load_yaml("/opt/netbox/initializers/custom_links.yml") @@ -28,6 +28,8 @@ for link in custom_links: ) continue - custom_link, created = CustomLink.objects.get_or_create(**link) + matching_params, defaults = split_params(link) + custom_link, created = CustomLink.objects.get_or_create(**matching_params, defaults=defaults) + if created: print("πŸ”— Created Custom Link '{0}'".format(custom_link.name)) diff --git a/startup_scripts/050_tags.py b/startup_scripts/050_tags.py index e50a000..06a18cb 100644 --- a/startup_scripts/050_tags.py +++ b/startup_scripts/050_tags.py @@ -1,7 +1,7 @@ import sys from extras.models import Tag -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices tags = load_yaml("/opt/netbox/initializers/tags.yml") @@ -17,7 +17,8 @@ for params in tags: if color in color_tpl: params["color"] = color_tpl[0] - tag, created = Tag.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tag, created = Tag.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created Tag", tag.name) diff --git a/startup_scripts/060_webhooks.py b/startup_scripts/060_webhooks.py index 8787f5f..9de6e7e 100644 --- a/startup_scripts/060_webhooks.py +++ b/startup_scripts/060_webhooks.py @@ -2,7 +2,7 @@ import sys from django.contrib.contenttypes.models import ContentType from extras.models import Webhook -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params webhooks = load_yaml("/opt/netbox/initializers/webhooks.yml") @@ -26,7 +26,9 @@ for hook in webhooks: except ContentType.DoesNotExist: continue - webhook, created = Webhook.objects.get_or_create(**hook) + matching_params, defaults = split_params(hook) + webhook, created = Webhook.objects.get_or_create(**matching_params, defaults=defaults) + if created: webhook.content_types.set(obj_type_ids) webhook.save() diff --git a/startup_scripts/070_tenant_groups.py b/startup_scripts/070_tenant_groups.py index 65cf155..0bfebad 100644 --- a/startup_scripts/070_tenant_groups.py +++ b/startup_scripts/070_tenant_groups.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from tenancy.models import TenantGroup tenant_groups = load_yaml("/opt/netbox/initializers/tenant_groups.yml") @@ -9,7 +9,8 @@ if tenant_groups is None: sys.exit() for params in tenant_groups: - tenant_group, created = TenantGroup.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tenant_group, created = TenantGroup.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ”³ Created Tenant Group", tenant_group.name) diff --git a/startup_scripts/080_tenants.py b/startup_scripts/080_tenants.py index 7b1a629..88d68b7 100644 --- a/startup_scripts/080_tenants.py +++ b/startup_scripts/080_tenants.py @@ -1,6 +1,11 @@ import sys -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup tenants = load_yaml("/opt/netbox/initializers/tenants.yml") @@ -20,9 +25,10 @@ for params in tenants: params[assoc] = model.objects.get(**query) - tenant, created = Tenant.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tenant, created = Tenant.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(tenant, custom_field_data) - print("πŸ‘©β€πŸ’» Created Tenant", tenant.name) + + set_custom_fields_values(tenant, custom_field_data) diff --git a/startup_scripts/090_regions.py b/startup_scripts/090_regions.py index 9d5c91f..592c4c5 100644 --- a/startup_scripts/090_regions.py +++ b/startup_scripts/090_regions.py @@ -1,7 +1,7 @@ import sys from dcim.models import Region -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params regions = load_yaml("/opt/netbox/initializers/regions.yml") @@ -19,7 +19,8 @@ for params in regions: params[assoc] = model.objects.get(**query) - region, created = Region.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + region, created = Region.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🌐 Created region", region.name) diff --git a/startup_scripts/110_sites.py b/startup_scripts/110_sites.py index f785139..c310e14 100644 --- a/startup_scripts/110_sites.py +++ b/startup_scripts/110_sites.py @@ -1,7 +1,12 @@ import sys from dcim.models import Region, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant sites = load_yaml("/opt/netbox/initializers/sites.yml") @@ -21,9 +26,10 @@ for params in sites: params[assoc] = model.objects.get(**query) - site, created = Site.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + site, created = Site.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(site, custom_field_data) - print("πŸ“ Created site", site.name) + + set_custom_fields_values(site, custom_field_data) diff --git a/startup_scripts/120_locations.py b/startup_scripts/120_locations.py index d8a2c5a..6a269b2 100644 --- a/startup_scripts/120_locations.py +++ b/startup_scripts/120_locations.py @@ -1,13 +1,14 @@ import sys from dcim.models import Location, Site -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params rack_groups = load_yaml("/opt/netbox/initializers/locations.yml") if rack_groups is None: sys.exit() +match_params = ["name", "slug", "site"] required_assocs = {"site": (Site, "name")} for params in rack_groups: @@ -17,7 +18,8 @@ for params in rack_groups: query = {field: params.pop(assoc)} params[assoc] = model.objects.get(**query) - location, created = Location.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + location, created = Location.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created location", location.name) diff --git a/startup_scripts/130_rack_roles.py b/startup_scripts/130_rack_roles.py index 5850405..8d43237 100644 --- a/startup_scripts/130_rack_roles.py +++ b/startup_scripts/130_rack_roles.py @@ -1,7 +1,7 @@ import sys from dcim.models import RackRole -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices rack_roles = load_yaml("/opt/netbox/initializers/rack_roles.yml") @@ -17,7 +17,8 @@ for params in rack_roles: if color in color_tpl: params["color"] = color_tpl[0] - rack_role, created = RackRole.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + rack_role, created = RackRole.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created rack role", rack_role.name) diff --git a/startup_scripts/140_racks.py b/startup_scripts/140_racks.py index b2cfc80..f3dc145 100644 --- a/startup_scripts/140_racks.py +++ b/startup_scripts/140_racks.py @@ -1,7 +1,12 @@ import sys from dcim.models import Location, Rack, RackRole, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant racks = load_yaml("/opt/netbox/initializers/racks.yml") @@ -9,8 +14,8 @@ racks = load_yaml("/opt/netbox/initializers/racks.yml") if racks is None: sys.exit() +match_params = ["name", "site"] required_assocs = {"site": (Site, "name")} - optional_assocs = { "role": (RackRole, "name"), "tenant": (Tenant, "name"), @@ -33,9 +38,10 @@ for params in racks: params[assoc] = model.objects.get(**query) - rack, created = Rack.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + rack, created = Rack.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(rack, custom_field_data) - print("πŸ”³ Created rack", rack.site, rack.name) + + set_custom_fields_values(rack, custom_field_data) diff --git a/startup_scripts/150_power_panels.py b/startup_scripts/150_power_panels.py index 8542435..1397a93 100644 --- a/startup_scripts/150_power_panels.py +++ b/startup_scripts/150_power_panels.py @@ -1,15 +1,20 @@ import sys from dcim.models import Location, PowerPanel, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) power_panels = load_yaml("/opt/netbox/initializers/power_panels.yml") if power_panels is None: sys.exit() +match_params = ["name", "site"] required_assocs = {"site": (Site, "name")} - optional_assocs = {"location": (Location, "name")} for params in power_panels: @@ -28,9 +33,10 @@ for params in power_panels: params[assoc] = model.objects.get(**query) - power_panel, created = PowerPanel.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + power_panel, created = PowerPanel.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(power_panel, custom_field_data) - print("⚑ Created Power Panel", power_panel.site, power_panel.name) + + set_custom_fields_values(power_panel, custom_field_data) diff --git a/startup_scripts/160_power_feeds.py b/startup_scripts/160_power_feeds.py index f5aa5b5..7b3e6ac 100644 --- a/startup_scripts/160_power_feeds.py +++ b/startup_scripts/160_power_feeds.py @@ -1,15 +1,20 @@ import sys from dcim.models import PowerFeed, PowerPanel, Rack -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) power_feeds = load_yaml("/opt/netbox/initializers/power_feeds.yml") if power_feeds is None: sys.exit() +match_params = ["name", "power_panel"] required_assocs = {"power_panel": (PowerPanel, "name")} - optional_assocs = {"rack": (Rack, "name")} for params in power_feeds: @@ -28,9 +33,10 @@ for params in power_feeds: params[assoc] = model.objects.get(**query) - power_feed, created = PowerFeed.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + power_feed, created = PowerFeed.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(power_feed, custom_field_data) - print("⚑ Created Power Feed", power_feed.name) + + set_custom_fields_values(power_feed, custom_field_data) diff --git a/startup_scripts/170_manufacturers.py b/startup_scripts/170_manufacturers.py index d11b440..c105a46 100644 --- a/startup_scripts/170_manufacturers.py +++ b/startup_scripts/170_manufacturers.py @@ -1,7 +1,7 @@ import sys from dcim.models import Manufacturer -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params manufacturers = load_yaml("/opt/netbox/initializers/manufacturers.yml") @@ -9,7 +9,8 @@ if manufacturers is None: sys.exit() for params in manufacturers: - manufacturer, created = Manufacturer.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + manufacturer, created = Manufacturer.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🏭 Created Manufacturer", manufacturer.name) diff --git a/startup_scripts/180_device_roles.py b/startup_scripts/180_device_roles.py index 635acff..3cea0f1 100644 --- a/startup_scripts/180_device_roles.py +++ b/startup_scripts/180_device_roles.py @@ -1,7 +1,7 @@ import sys from dcim.models import DeviceRole -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices device_roles = load_yaml("/opt/netbox/initializers/device_roles.yml") @@ -18,7 +18,8 @@ for params in device_roles: if color in color_tpl: params["color"] = color_tpl[0] - device_role, created = DeviceRole.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + device_role, created = DeviceRole.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created device role", device_role.name) diff --git a/startup_scripts/190_device_types.py b/startup_scripts/190_device_types.py index 0d3050b..d20c9e6 100644 --- a/startup_scripts/190_device_types.py +++ b/startup_scripts/190_device_types.py @@ -1,20 +1,95 @@ import sys +from typing import List from dcim.models import DeviceType, Manufacturer, Region -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from dcim.models.device_component_templates import ( + ConsolePortTemplate, + ConsoleServerPortTemplate, + DeviceBayTemplate, + FrontPortTemplate, + InterfaceTemplate, + PowerOutletTemplate, + PowerPortTemplate, + RearPortTemplate, +) +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant +from utilities.forms.utils import expand_alphanumeric_pattern + + +def expand_templates(params: List[dict], device_type: DeviceType) -> List[dict]: + templateable_fields = ["name", "label", "positions", "rear_port", "rear_port_position"] + + expanded = [] + for param in params: + param["device_type"] = device_type + expanded_fields = {} + has_plain_fields = False + + for field in templateable_fields: + template_value = param.pop(f"{field}_template", None) + + if field in param: + has_plain_fields = True + elif template_value: + expanded_fields[field] = list(expand_alphanumeric_pattern(template_value)) + + if expanded_fields and has_plain_fields: + raise ValueError(f"Mix of plain and template keys provided for {templateable_fields}") + elif not expanded_fields: + expanded.append(param) + continue + + elements = list(expanded_fields.values()) + master_len = len(elements[0]) + if not all([len(elem) == master_len for elem in elements]): + raise ValueError( + f"Number of elements in template fields " + f"{list(expanded_fields.keys())} must be equal" + ) + + for idx in range(master_len): + tmp = param.copy() + for field, value in expanded_fields.items(): + if field in nested_assocs: + model, match_key = nested_assocs[field] + query = {match_key: value[idx], "device_type": device_type} + tmp[field] = model.objects.get(**query) + else: + tmp[field] = value[idx] + expanded.append(tmp) + return expanded + device_types = load_yaml("/opt/netbox/initializers/device_types.yml") if device_types is None: sys.exit() +match_params = ["manufacturer", "model", "slug"] required_assocs = {"manufacturer": (Manufacturer, "name")} - optional_assocs = {"region": (Region, "name"), "tenant": (Tenant, "name")} +nested_assocs = {"rear_port": (RearPortTemplate, "name"), "power_port": (PowerPortTemplate, "name")} + +supported_components = { + "interfaces": (InterfaceTemplate, ["name"]), + "console_ports": (ConsolePortTemplate, ["name"]), + "console_server_ports": (ConsoleServerPortTemplate, ["name"]), + "power_ports": (PowerPortTemplate, ["name"]), + "power_outlets": (PowerOutletTemplate, ["name"]), + "rear_ports": (RearPortTemplate, ["name"]), + "front_ports": (FrontPortTemplate, ["name"]), + "device_bays": (DeviceBayTemplate, ["name"]), +} for params in device_types: custom_field_data = pop_custom_fields(params) + components = [(v[0], v[1], params.pop(k, [])) for k, v in supported_components.items()] for assoc, details in required_assocs.items(): model, field = details @@ -29,9 +104,36 @@ for params in device_types: params[assoc] = model.objects.get(**query) - device_type, created = DeviceType.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + device_type, created = DeviceType.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(device_type, custom_field_data) - print("πŸ”‘ Created device type", device_type.manufacturer, device_type.model) + + set_custom_fields_values(device_type, custom_field_data) + + for component in components: + c_model, c_match_params, c_params = component + c_match_params.append("device_type") + + if not c_params: + continue + + expanded_c_params = expand_templates(c_params, device_type) + + for n_assoc, n_details in nested_assocs.items(): + n_model, n_field = n_details + for c_param in expanded_c_params: + if n_assoc in c_param: + n_query = {n_field: c_param[n_assoc], "device_type": device_type} + c_param[n_assoc] = n_model.objects.get(**n_query) + + for new_param in expanded_c_params: + new_matching_params, new_defaults = split_params(new_param, c_match_params) + new_obj, new_obj_created = c_model.objects.get_or_create( + **new_matching_params, defaults=new_defaults + ) + if new_obj_created: + print( + f"🧷 Created {c_model._meta} {new_obj} component for device type {device_type}" + ) diff --git a/startup_scripts/200_devices.py b/startup_scripts/200_devices.py index 423b7c9..189471c 100644 --- a/startup_scripts/200_devices.py +++ b/startup_scripts/200_devices.py @@ -1,7 +1,12 @@ import sys from dcim.models import Device, DeviceRole, DeviceType, Location, Platform, Rack, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster @@ -10,12 +15,12 @@ devices = load_yaml("/opt/netbox/initializers/devices.yml") if devices is None: sys.exit() +match_params = ["device_type", "name", "site"] required_assocs = { "device_role": (DeviceRole, "name"), "device_type": (DeviceType, "model"), "site": (Site, "name"), } - optional_assocs = { "tenant": (Tenant, "name"), "platform": (Platform, "name"), @@ -27,7 +32,7 @@ optional_assocs = { for params in devices: custom_field_data = pop_custom_fields(params) - # primary ips are handled later in `270_primary_ips.py` + # primary ips are handled later in `380_primary_ips.py` params.pop("primary_ip4", None) params.pop("primary_ip6", None) @@ -44,9 +49,10 @@ for params in devices: params[assoc] = model.objects.get(**query) - device, created = Device.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + device, created = Device.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(device, custom_field_data) - print("πŸ–₯️ Created device", device.name) + + set_custom_fields_values(device, custom_field_data) diff --git a/startup_scripts/210_dcim_interfaces.py b/startup_scripts/210_dcim_interfaces.py index a802628..6e57e73 100644 --- a/startup_scripts/210_dcim_interfaces.py +++ b/startup_scripts/210_dcim_interfaces.py @@ -1,27 +1,70 @@ import sys from dcim.models import Device, Interface -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) interfaces = load_yaml("/opt/netbox/initializers/dcim_interfaces.yml") if interfaces is None: sys.exit() +match_params = ["device", "name"] required_assocs = {"device": (Device, "name")} +related_assocs = { + "bridge": (Interface, "name"), + "lag": (Interface, "name"), + "parent": (Interface, "name"), +} for params in interfaces: custom_field_data = pop_custom_fields(params) + related_interfaces = {k: params.pop(k, None) for k in related_assocs} + for assoc, details in required_assocs.items(): model, field = details query = {field: params.pop(assoc)} params[assoc] = model.objects.get(**query) - interface, created = Interface.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + interface, created = Interface.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(interface, custom_field_data) + print(f"🧷 Created interface {interface} on {interface.device}") - print("🧷 Created interface", interface.name, interface.device.name) + set_custom_fields_values(interface, custom_field_data) + + for related_field, related_value in related_interfaces.items(): + if not related_value: + continue + + r_model, r_field = related_assocs[related_field] + + if related_field == "parent" and not interface.parent_id: + query = {r_field: related_value, "device": interface.device} + try: + related_obj = r_model.objects.get(**query) + except Interface.DoesNotExist: + print(f"⚠️ Could not find parent interface with: {query} for interface {interface}") + raise + + interface.parent_id = related_obj.id + interface.save() + print( + f"🧷 Attached interface {interface} on {interface.device} " + f"to parent {related_obj}" + ) + else: + query = {r_field: related_value, "device": interface.device, "type": related_field} + related_obj, rel_obj_created = r_model.objects.get_or_create(**query) + + if rel_obj_created: + setattr(interface, f"{related_field}_id", related_obj.id) + interface.save() + print(f"🧷 Created {related_field} interface {interface} on {interface.device}") diff --git a/startup_scripts/220_platforms.py b/startup_scripts/220_platforms.py index 633b89f..73905d4 100644 --- a/startup_scripts/220_platforms.py +++ b/startup_scripts/220_platforms.py @@ -1,7 +1,7 @@ import sys from dcim.models import Manufacturer, Platform -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params platforms = load_yaml("/opt/netbox/initializers/platforms.yml") @@ -21,7 +21,8 @@ for params in platforms: params[assoc] = model.objects.get(**query) - platform, created = Platform.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + platform, created = Platform.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ’Ύ Created platform", platform.name) diff --git a/startup_scripts/230_route_targets.py b/startup_scripts/230_route_targets.py index e1c8221..12ca9a5 100644 --- a/startup_scripts/230_route_targets.py +++ b/startup_scripts/230_route_targets.py @@ -1,7 +1,12 @@ import sys from ipam.models import RouteTarget -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant route_targets = load_yaml("/opt/netbox/initializers/route_targets.yml") @@ -21,9 +26,10 @@ for params in route_targets: params[assoc] = model.objects.get(**query) - route_target, created = RouteTarget.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + route_target, created = RouteTarget.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(route_target, custom_field_data) - print("🎯 Created Route Target", route_target.name) + + set_custom_fields_values(route_target, custom_field_data) diff --git a/startup_scripts/240_vrfs.py b/startup_scripts/240_vrfs.py index a67c8c8..1b4cd1a 100644 --- a/startup_scripts/240_vrfs.py +++ b/startup_scripts/240_vrfs.py @@ -1,7 +1,12 @@ import sys from ipam.models import VRF -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant vrfs = load_yaml("/opt/netbox/initializers/vrfs.yml") @@ -9,6 +14,7 @@ vrfs = load_yaml("/opt/netbox/initializers/vrfs.yml") if vrfs is None: sys.exit() +match_params = ["name", "rd"] optional_assocs = {"tenant": (Tenant, "name")} for params in vrfs: @@ -21,9 +27,10 @@ for params in vrfs: params[assoc] = model.objects.get(**query) - vrf, created = VRF.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + vrf, created = VRF.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(vrf, custom_field_data) - print("πŸ“¦ Created VRF", vrf.name) + + set_custom_fields_values(vrf, custom_field_data) diff --git a/startup_scripts/250_rirs.py b/startup_scripts/250_rirs.py index 0e0df20..08d1703 100644 --- a/startup_scripts/250_rirs.py +++ b/startup_scripts/250_rirs.py @@ -1,7 +1,7 @@ import sys from ipam.models import RIR -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params rirs = load_yaml("/opt/netbox/initializers/rirs.yml") @@ -9,7 +9,8 @@ if rirs is None: sys.exit() for params in rirs: - rir, created = RIR.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + rir, created = RIR.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ—ΊοΈ Created RIR", rir.name) diff --git a/startup_scripts/260_asns.py b/startup_scripts/260_asns.py index 893f3ba..93776d2 100644 --- a/startup_scripts/260_asns.py +++ b/startup_scripts/260_asns.py @@ -1,7 +1,7 @@ import sys from ipam.models import ASN, RIR -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from tenancy.models import Tenant asns = load_yaml("/opt/netbox/initializers/asns.yml") @@ -9,8 +9,8 @@ asns = load_yaml("/opt/netbox/initializers/asns.yml") if asns is None: sys.exit() +match_params = ["asn", "rir"] required_assocs = {"rir": (RIR, "name")} - optional_assocs = {"tenant": (Tenant, "name")} for params in asns: @@ -27,7 +27,8 @@ for params in asns: params[assoc] = model.objects.get(**query) - asn, created = ASN.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + asn, created = ASN.objects.get_or_create(**matching_params, defaults=defaults) if created: print(f"πŸ”‘ Created ASN {asn.asn}") diff --git a/startup_scripts/270_aggregates.py b/startup_scripts/270_aggregates.py index c638e6f..d37c4cc 100644 --- a/startup_scripts/270_aggregates.py +++ b/startup_scripts/270_aggregates.py @@ -2,7 +2,12 @@ import sys from ipam.models import RIR, Aggregate from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant aggregates = load_yaml("/opt/netbox/initializers/aggregates.yml") @@ -10,8 +15,8 @@ aggregates = load_yaml("/opt/netbox/initializers/aggregates.yml") if aggregates is None: sys.exit() +match_params = ["prefix", "rir"] required_assocs = {"rir": (RIR, "name")} - optional_assocs = { "tenant": (Tenant, "name"), } @@ -34,9 +39,10 @@ for params in aggregates: params[assoc] = model.objects.get(**query) - aggregate, created = Aggregate.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + aggregate, created = Aggregate.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(aggregate, custom_field_data) - print("πŸ—žοΈ Created Aggregate", aggregate.prefix) + + set_custom_fields_values(aggregate, custom_field_data) diff --git a/startup_scripts/280_prefix_vlan_roles.py b/startup_scripts/280_prefix_vlan_roles.py index ec359fb..c17feb8 100644 --- a/startup_scripts/280_prefix_vlan_roles.py +++ b/startup_scripts/280_prefix_vlan_roles.py @@ -1,7 +1,7 @@ import sys from ipam.models import Role -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params roles = load_yaml("/opt/netbox/initializers/prefix_vlan_roles.yml") @@ -9,7 +9,8 @@ if roles is None: sys.exit() for params in roles: - role, created = Role.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + role, created = Role.objects.get_or_create(**matching_params, defaults=defaults) if created: print("⛹️‍ Created Prefix/VLAN Role", role.name) diff --git a/startup_scripts/290_cluster_types.py b/startup_scripts/290_cluster_types.py index 9f361b1..642bc7a 100644 --- a/startup_scripts/290_cluster_types.py +++ b/startup_scripts/290_cluster_types.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import ClusterType cluster_types = load_yaml("/opt/netbox/initializers/cluster_types.yml") @@ -9,7 +9,8 @@ if cluster_types is None: sys.exit() for params in cluster_types: - cluster_type, created = ClusterType.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + cluster_type, created = ClusterType.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🧰 Created Cluster Type", cluster_type.name) diff --git a/startup_scripts/300_cluster_groups.py b/startup_scripts/300_cluster_groups.py index fedd292..a8e3573 100644 --- a/startup_scripts/300_cluster_groups.py +++ b/startup_scripts/300_cluster_groups.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import ClusterGroup cluster_groups = load_yaml("/opt/netbox/initializers/cluster_groups.yml") @@ -9,7 +9,10 @@ if cluster_groups is None: sys.exit() for params in cluster_groups: - cluster_group, created = ClusterGroup.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + cluster_group, created = ClusterGroup.objects.get_or_create( + **matching_params, defaults=defaults + ) if created: print("πŸ—„οΈ Created Cluster Group", cluster_group.name) diff --git a/startup_scripts/310_clusters.py b/startup_scripts/310_clusters.py index 2748f20..e36a5f9 100644 --- a/startup_scripts/310_clusters.py +++ b/startup_scripts/310_clusters.py @@ -1,7 +1,12 @@ import sys from dcim.models import Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster, ClusterGroup, ClusterType @@ -10,8 +15,8 @@ clusters = load_yaml("/opt/netbox/initializers/clusters.yml") if clusters is None: sys.exit() +match_params = ["name", "type"] required_assocs = {"type": (ClusterType, "name")} - optional_assocs = { "site": (Site, "name"), "group": (ClusterGroup, "name"), @@ -34,9 +39,10 @@ for params in clusters: params[assoc] = model.objects.get(**query) - cluster, created = Cluster.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + cluster, created = Cluster.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(cluster, custom_field_data) - print("πŸ—„οΈ Created cluster", cluster.name) + + set_custom_fields_values(cluster, custom_field_data) diff --git a/startup_scripts/320_vlan_groups.py b/startup_scripts/320_vlan_groups.py index 2a4a33d..a81e695 100644 --- a/startup_scripts/320_vlan_groups.py +++ b/startup_scripts/320_vlan_groups.py @@ -2,7 +2,12 @@ import sys from django.contrib.contenttypes.models import ContentType from ipam.models import VLANGroup -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) vlan_groups = load_yaml("/opt/netbox/initializers/vlan_groups.yml") @@ -32,9 +37,11 @@ for params in vlan_groups: ) continue params["scope_id"] = ct.model_class().objects.get(**query).id - vlan_group, created = VLANGroup.objects.get_or_create(**params) + + matching_params, defaults = split_params(params) + vlan_group, created = VLANGroup.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(vlan_group, custom_field_data) - print("🏘️ Created VLAN Group", vlan_group.name) + + set_custom_fields_values(vlan_group, custom_field_data) diff --git a/startup_scripts/330_vlans.py b/startup_scripts/330_vlans.py index e8ebb94..e378881 100644 --- a/startup_scripts/330_vlans.py +++ b/startup_scripts/330_vlans.py @@ -2,7 +2,12 @@ import sys from dcim.models import Site from ipam.models import VLAN, Role, VLANGroup -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup vlans = load_yaml("/opt/netbox/initializers/vlans.yml") @@ -10,6 +15,7 @@ vlans = load_yaml("/opt/netbox/initializers/vlans.yml") if vlans is None: sys.exit() +match_params = ["name", "vid"] optional_assocs = { "site": (Site, "name"), "tenant": (Tenant, "name"), @@ -28,9 +34,10 @@ for params in vlans: params[assoc] = model.objects.get(**query) - vlan, created = VLAN.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + vlan, created = VLAN.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(vlan, custom_field_data) - print("🏠 Created VLAN", vlan.name) + + set_custom_fields_values(vlan, custom_field_data) diff --git a/startup_scripts/340_virtual_machines.py b/startup_scripts/340_virtual_machines.py index 2e3f428..51d3bc6 100644 --- a/startup_scripts/340_virtual_machines.py +++ b/startup_scripts/340_virtual_machines.py @@ -1,7 +1,12 @@ import sys from dcim.models import DeviceRole, Platform -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster, VirtualMachine @@ -10,8 +15,8 @@ virtual_machines = load_yaml("/opt/netbox/initializers/virtual_machines.yml") if virtual_machines is None: sys.exit() +match_params = ["cluster", "name"] required_assocs = {"cluster": (Cluster, "name")} - optional_assocs = { "tenant": (Tenant, "name"), "platform": (Platform, "name"), @@ -38,9 +43,12 @@ for params in virtual_machines: params[assoc] = model.objects.get(**query) - virtual_machine, created = VirtualMachine.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + virtual_machine, created = VirtualMachine.objects.get_or_create( + **matching_params, defaults=defaults + ) if created: - set_custom_fields_values(virtual_machine, custom_field_data) - print("πŸ–₯️ Created virtual machine", virtual_machine.name) + + set_custom_fields_values(virtual_machine, custom_field_data) diff --git a/startup_scripts/350_virtualization_interfaces.py b/startup_scripts/350_virtualization_interfaces.py index 6ee6347..c6bbfa0 100644 --- a/startup_scripts/350_virtualization_interfaces.py +++ b/startup_scripts/350_virtualization_interfaces.py @@ -1,6 +1,11 @@ import sys -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from virtualization.models import VirtualMachine, VMInterface interfaces = load_yaml("/opt/netbox/initializers/virtualization_interfaces.yml") @@ -8,6 +13,7 @@ interfaces = load_yaml("/opt/netbox/initializers/virtualization_interfaces.yml") if interfaces is None: sys.exit() +match_params = ["name", "virtual_machine"] required_assocs = {"virtual_machine": (VirtualMachine, "name")} for params in interfaces: @@ -19,9 +25,10 @@ for params in interfaces: params[assoc] = model.objects.get(**query) - interface, created = VMInterface.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + interface, created = VMInterface.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(interface, custom_field_data) - print("🧷 Created interface", interface.name, interface.virtual_machine.name) + + set_custom_fields_values(interface, custom_field_data) diff --git a/startup_scripts/360_prefixes.py b/startup_scripts/360_prefixes.py index 4e2b0d0..ffb774b 100644 --- a/startup_scripts/360_prefixes.py +++ b/startup_scripts/360_prefixes.py @@ -3,7 +3,12 @@ import sys from dcim.models import Site from ipam.models import VLAN, VRF, Prefix, Role from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup prefixes = load_yaml("/opt/netbox/initializers/prefixes.yml") @@ -11,6 +16,7 @@ prefixes = load_yaml("/opt/netbox/initializers/prefixes.yml") if prefixes is None: sys.exit() +match_params = ["prefix", "site", "vrf", "vlan"] optional_assocs = { "site": (Site, "name"), "tenant": (Tenant, "name"), @@ -31,9 +37,10 @@ for params in prefixes: query = {field: params.pop(assoc)} params[assoc] = model.objects.get(**query) - prefix, created = Prefix.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + prefix, created = Prefix.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(prefix, custom_field_data) - print("πŸ“Œ Created Prefix", prefix.prefix) + + set_custom_fields_values(prefix, custom_field_data) diff --git a/startup_scripts/370_ip_addresses.py b/startup_scripts/370_ip_addresses.py index 7f166f0..b35cb49 100644 --- a/startup_scripts/370_ip_addresses.py +++ b/startup_scripts/370_ip_addresses.py @@ -5,7 +5,12 @@ from django.contrib.contenttypes.models import ContentType from django.db.models import Q from ipam.models import VRF, IPAddress from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import VirtualMachine, VMInterface @@ -14,10 +19,11 @@ ip_addresses = load_yaml("/opt/netbox/initializers/ip_addresses.yml") if ip_addresses is None: sys.exit() +match_params = ["address", "vrf"] optional_assocs = { "tenant": (Tenant, "name"), "vrf": (VRF, "name"), - "interface": (None, None), + "interface": (Interface, "name"), } vm_interface_ct = ContentType.objects.filter( @@ -55,9 +61,10 @@ for params in ip_addresses: params[assoc] = model.objects.get(**query) - ip_address, created = IPAddress.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + ip_address, created = IPAddress.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(ip_address, custom_field_data) - print("🧬 Created IP Address", ip_address.address) + + set_custom_fields_values(ip_address, custom_field_data) diff --git a/startup_scripts/400_services.py b/startup_scripts/400_services.py index a28eb09..a92d82b 100644 --- a/startup_scripts/400_services.py +++ b/startup_scripts/400_services.py @@ -2,7 +2,7 @@ import sys from dcim.models import Device from ipam.models import Service -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import VirtualMachine services = load_yaml("/opt/netbox/initializers/services.yml") @@ -10,6 +10,7 @@ services = load_yaml("/opt/netbox/initializers/services.yml") if services is None: sys.exit() +match_params = ["name", "device", "virtual_machine"] optional_assocs = { "device": (Device, "name"), "virtual_machine": (VirtualMachine, "name"), @@ -24,6 +25,7 @@ for params in services: params[assoc] = model.objects.get(**query) + matching_params, defaults = split_params(params, match_params) service, created = Service.objects.get_or_create(**params) if created: diff --git a/startup_scripts/420_providers.py b/startup_scripts/420_providers.py index 5c4330a..e68e1f3 100644 --- a/startup_scripts/420_providers.py +++ b/startup_scripts/420_providers.py @@ -1,7 +1,12 @@ import sys from circuits.models import Provider -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) providers = load_yaml("/opt/netbox/initializers/providers.yml") @@ -11,9 +16,10 @@ if providers is None: for params in providers: custom_field_data = pop_custom_fields(params) - provider, created = Provider.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + provider, created = Provider.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(provider, custom_field_data) - print("πŸ“‘ Created provider", provider.name) + + set_custom_fields_values(provider, custom_field_data) diff --git a/startup_scripts/440_circuit_types.py b/startup_scripts/440_circuit_types.py index 071793c..6d3f7fa 100644 --- a/startup_scripts/440_circuit_types.py +++ b/startup_scripts/440_circuit_types.py @@ -1,7 +1,12 @@ import sys from circuits.models import CircuitType -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) circuit_types = load_yaml("/opt/netbox/initializers/circuit_types.yml") @@ -11,9 +16,10 @@ if circuit_types is None: for params in circuit_types: custom_field_data = pop_custom_fields(params) - circuit_type, created = CircuitType.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + circuit_type, created = CircuitType.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(circuit_type, custom_field_data) - print("⚑ Created Circuit Type", circuit_type.name) + + set_custom_fields_values(circuit_type, custom_field_data) diff --git a/startup_scripts/450_circuits.py b/startup_scripts/450_circuits.py index f82d3b7..68f159b 100644 --- a/startup_scripts/450_circuits.py +++ b/startup_scripts/450_circuits.py @@ -1,7 +1,12 @@ import sys from circuits.models import Circuit, CircuitType, Provider -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant circuits = load_yaml("/opt/netbox/initializers/circuits.yml") @@ -9,8 +14,8 @@ circuits = load_yaml("/opt/netbox/initializers/circuits.yml") if circuits is None: sys.exit() +match_params = ["cid", "provider", "type"] required_assocs = {"provider": (Provider, "name"), "type": (CircuitType, "name")} - optional_assocs = {"tenant": (Tenant, "name")} for params in circuits: @@ -29,9 +34,10 @@ for params in circuits: params[assoc] = model.objects.get(**query) - circuit, created = Circuit.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + circuit, created = Circuit.objects.get_or_create(**matching_params, defaults=defaults) if created: - set_custom_fields_values(circuit, custom_field_data) - print("⚑ Created Circuit", circuit.cid) + + set_custom_fields_values(circuit, custom_field_data) diff --git a/startup_scripts/460_cables.py b/startup_scripts/460_cables.py new file mode 100644 index 0000000..6e4cec2 --- /dev/null +++ b/startup_scripts/460_cables.py @@ -0,0 +1,227 @@ +import sys +from typing import Tuple + +from circuits.models import Circuit, CircuitTermination, ProviderNetwork +from dcim.models import ( + Cable, + ConsolePort, + ConsoleServerPort, + FrontPort, + Interface, + PowerFeed, + PowerOutlet, + PowerPanel, + PowerPort, + RearPort, + Site, +) +from django.contrib.contenttypes.models import ContentType +from django.db.models import Q +from startup_script_utils import load_yaml + +CONSOLE_PORT_TERMINATION = ContentType.objects.get_for_model(ConsolePort) +CONSOLE_SERVER_PORT_TERMINATION = ContentType.objects.get_for_model(ConsoleServerPort) +FRONT_PORT_TERMINATION = ContentType.objects.get_for_model(FrontPort) +REAR_PORT_TERMINATION = ContentType.objects.get_for_model(RearPort) +FRONT_AND_REAR = [FRONT_PORT_TERMINATION, REAR_PORT_TERMINATION] +POWER_PORT_TERMINATION = ContentType.objects.get_for_model(PowerPort) +POWER_OUTLET_TERMINATION = ContentType.objects.get_for_model(PowerOutlet) +POWER_FEED_TERMINATION = ContentType.objects.get_for_model(PowerFeed) +POWER_TERMINATIONS = [POWER_PORT_TERMINATION, POWER_OUTLET_TERMINATION, POWER_FEED_TERMINATION] + +VIRTUAL_INTERFACES = ["bridge", "lag", "virtual"] + + +def get_termination_object(params: dict, side: str): + klass = params.pop(f"termination_{side}_class") + name = params.pop(f"termination_{side}_name", None) + device = params.pop(f"termination_{side}_device", None) + feed_params = params.pop(f"termination_{side}_feed", None) + circuit_params = params.pop(f"termination_{side}_circuit", {}) + + if name and device: + termination = klass.objects.get(name=name, device__name=device) + return termination + elif feed_params: + q = { + "name": feed_params["power_panel"]["name"], + "site__name": feed_params["power_panel"]["site"], + } + power_panel = PowerPanel.objects.get(**q) + termination = PowerFeed.objects.get(name=feed_params["name"], power_panel=power_panel) + return termination + elif circuit_params: + circuit = Circuit.objects.get(cid=circuit_params.pop("cid")) + term_side = circuit_params.pop("term_side").upper() + + site_name = circuit_params.pop("site", None) + provider_network = circuit_params.pop("provider_network", None) + + if site_name: + circuit_params["site"] = Site.objects.get(name=site_name) + elif provider_network: + circuit_params["provider_network"] = ProviderNetwork.objects.get(name=provider_network) + else: + raise ValueError( + f"⚠️ Missing one of required parameters: 'site' or 'provider_network' " + f"for side {term_side} of circuit {circuit}" + ) + + termination, created = CircuitTermination.objects.get_or_create( + circuit=circuit, term_side=term_side, defaults=circuit_params + ) + if created: + print(f"⚑ Created new CircuitTermination {termination}") + + return termination + + raise ValueError( + f"⚠️ Missing parameters for termination_{side}. " + "Need termination_{side}_name AND termination_{side}_device OR termination_{side}_circuit" + ) + + +def get_termination_class_by_name(port_class: str): + if not port_class: + return Interface + + return globals()[port_class] + + +def cable_in_cables(term_a: tuple, term_b: tuple) -> bool: + """Check if cable exist for given terminations. + Each tuple should consist termination object and termination type + """ + + cable = Cable.objects.filter( + Q( + termination_a_id=term_a[0].id, + termination_a_type=term_a[1], + termination_b_id=term_b[0].id, + termination_b_type=term_b[1], + ) + | Q( + termination_a_id=term_b[0].id, + termination_a_type=term_b[1], + termination_b_id=term_a[0].id, + termination_b_type=term_a[1], + ) + ) + return cable.exists() + + +def check_termination_types(type_a, type_b) -> Tuple[bool, str]: + if type_a in POWER_TERMINATIONS and type_b in POWER_TERMINATIONS: + if type_a == type_b: + return False, "Can't connect the same power terminations together" + elif ( + type_a == POWER_OUTLET_TERMINATION + and type_b == POWER_FEED_TERMINATION + or type_a == POWER_FEED_TERMINATION + and type_b == POWER_OUTLET_TERMINATION + ): + return False, "PowerOutlet can't be connected with PowerFeed" + elif type_a in POWER_TERMINATIONS or type_b in POWER_TERMINATIONS: + return False, "Can't mix power terminations with port terminations" + elif type_a in FRONT_AND_REAR or type_b in FRONT_AND_REAR: + return True, "" + elif ( + type_a == CONSOLE_PORT_TERMINATION + and type_b != CONSOLE_SERVER_PORT_TERMINATION + or type_b == CONSOLE_PORT_TERMINATION + and type_a != CONSOLE_SERVER_PORT_TERMINATION + ): + return False, "ConsolePorts can only be connected to ConsoleServerPorts or Front/Rear ports" + return True, "" + + +def get_cable_name(termination_a: tuple, termination_b: tuple) -> str: + """Returns name of a cable in format: + device_a interface_a <---> interface_b device_b + or for circuits: + circuit_a termination_a <---> termination_b circuit_b + """ + cable_name = [] + + for is_side_b, termination in enumerate([termination_a, termination_b]): + try: + power_panel_id = getattr(termination[0], "power_panel_id", None) + if power_panel_id: + power_feed = PowerPanel.objects.get(id=power_panel_id) + segment = [f"{power_feed}", f"{termination[0]}"] + else: + segment = [f"{termination[0].device}", f"{termination[0]}"] + except AttributeError: + segment = [f"{termination[0].circuit.cid}", f"{termination[0]}"] + + if is_side_b: + segment.reverse() + + cable_name.append(" ".join(segment)) + + return " <---> ".join(cable_name) + + +def check_interface_types(*args): + for termination in args: + try: + if termination.type in VIRTUAL_INTERFACES: + raise Exception( + f"⚠️ Virtual interfaces are not supported for cabling. " + f"Termination {termination.device} {termination} {termination.type}" + ) + except AttributeError: + # CircuitTermination doesn't have a type field + pass + + +def check_terminations_are_free(*args): + any_failed = False + for termination in args: + if termination.cable_id: + any_failed = True + print( + f"⚠️ Termination {termination} is already occupied " + f"with cable #{termination.cable_id}" + ) + if any_failed: + raise Exception("⚠️ At least one end of the cable is already occupied.") + + +cables = load_yaml("/opt/netbox/initializers/cables.yml") + +if cables is None: + sys.exit() + +for params in cables: + params["termination_a_class"] = get_termination_class_by_name(params.get("termination_a_class")) + params["termination_b_class"] = get_termination_class_by_name(params.get("termination_b_class")) + + term_a = get_termination_object(params, side="a") + term_b = get_termination_object(params, side="b") + + check_interface_types(term_a, term_b) + + term_a_ct = ContentType.objects.get_for_model(term_a) + term_b_ct = ContentType.objects.get_for_model(term_b) + + types_ok, msg = check_termination_types(term_a_ct, term_b_ct) + cable_name = get_cable_name((term_a, term_a_ct), (term_b, term_b_ct)) + + if not types_ok: + print(f"⚠️ Invalid termination types for {cable_name}. {msg}") + continue + + if cable_in_cables((term_a, term_a_ct), (term_b, term_b_ct)): + continue + + check_terminations_are_free(term_a, term_b) + + params["termination_a_id"] = term_a.id + params["termination_b_id"] = term_b.id + params["termination_a_type"] = term_a_ct + params["termination_b_type"] = term_b_ct + + cable = Cable.objects.create(**params) + + print(f"🧷 Created cable {cable} {cable_name}") diff --git a/startup_scripts/470_contact_groups.py b/startup_scripts/470_contact_groups.py new file mode 100644 index 0000000..f5fc4fd --- /dev/null +++ b/startup_scripts/470_contact_groups.py @@ -0,0 +1,36 @@ +import sys + +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) +from tenancy.models import ContactGroup + +contact_groups = load_yaml("/opt/netbox/initializers/contact_groups.yml") + +if contact_groups is None: + sys.exit() + +optional_assocs = {"parent": (ContactGroup, "name")} + +for params in contact_groups: + custom_field_data = pop_custom_fields(params) + + for assoc, details in optional_assocs.items(): + if assoc in params: + model, field = details + query = {field: params.pop(assoc)} + + params[assoc] = model.objects.get(**query) + + matching_params, defaults = split_params(params) + contact_group, created = ContactGroup.objects.get_or_create( + **matching_params, defaults=defaults + ) + + if created: + print("πŸ”³ Created Contact Group", contact_group.name) + + set_custom_fields_values(contact_group, custom_field_data) diff --git a/startup_scripts/480_contact_roles.py b/startup_scripts/480_contact_roles.py new file mode 100644 index 0000000..1b3d8bc --- /dev/null +++ b/startup_scripts/480_contact_roles.py @@ -0,0 +1,26 @@ +import sys + +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) +from tenancy.models import ContactRole + +contact_roles = load_yaml("/opt/netbox/initializers/contact_roles.yml") + +if contact_roles is None: + sys.exit() + + +for params in contact_roles: + custom_field_data = pop_custom_fields(params) + + matching_params, defaults = split_params(params) + contact_role, created = ContactRole.objects.get_or_create(**matching_params, defaults=defaults) + + if created: + print("πŸ”³ Created Contact Role", contact_role.name) + + set_custom_fields_values(contact_role, custom_field_data) diff --git a/startup_scripts/490_contacts.py b/startup_scripts/490_contacts.py new file mode 100644 index 0000000..80d75d6 --- /dev/null +++ b/startup_scripts/490_contacts.py @@ -0,0 +1,34 @@ +import sys + +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) +from tenancy.models import Contact, ContactGroup + +contacts = load_yaml("/opt/netbox/initializers/contacts.yml") + +if contacts is None: + sys.exit() + +optional_assocs = {"group": (ContactGroup, "name")} + +for params in contacts: + custom_field_data = pop_custom_fields(params) + + for assoc, details in optional_assocs.items(): + if assoc in params: + model, field = details + query = {field: params.pop(assoc)} + + params[assoc] = model.objects.get(**query) + + matching_params, defaults = split_params(params) + contact, created = Contact.objects.get_or_create(**matching_params, defaults=defaults) + + if created: + print("πŸ‘©β€πŸ’» Created Contact", contact.name) + + set_custom_fields_values(contact, custom_field_data) diff --git a/startup_scripts/startup_script_utils/__init__.py b/startup_scripts/startup_script_utils/__init__.py index 290b87b..4c25c6b 100644 --- a/startup_scripts/startup_script_utils/__init__.py +++ b/startup_scripts/startup_script_utils/__init__.py @@ -1,2 +1,3 @@ from .custom_fields import pop_custom_fields, set_custom_fields_values from .load_yaml import load_yaml +from .utils import split_params diff --git a/startup_scripts/startup_script_utils/custom_fields.py b/startup_scripts/startup_script_utils/custom_fields.py index c000970..74e8a25 100644 --- a/startup_scripts/startup_script_utils/custom_fields.py +++ b/startup_scripts/startup_script_utils/custom_fields.py @@ -1,9 +1,38 @@ +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from extras.models import CustomField + + def set_custom_fields_values(entity, custom_field_data): if not custom_field_data: return - entity.custom_field_data = custom_field_data - return entity.save() + missing_cfs = [] + save = False + for key, value in custom_field_data.items(): + try: + cf = CustomField.objects.get(name=key) + except ObjectDoesNotExist: + missing_cfs.append(key) + else: + ct = ContentType.objects.get_for_model(entity) + if ct not in cf.content_types.all(): + print( + f"⚠️ Custom field {key} is not enabled for {entity}'s model!" + "Please check the 'on_objects' for that custom field in custom_fields.yml" + ) + elif key not in entity.custom_field_data: + entity.custom_field_data[key] = value + save = True + + if missing_cfs: + raise Exception( + f"⚠️ Custom field(s) '{missing_cfs}' requested for {entity} but not found in Netbox!" + "Please chceck the custom_fields.yml" + ) + + if save: + entity.save() def pop_custom_fields(params): diff --git a/startup_scripts/startup_script_utils/utils.py b/startup_scripts/startup_script_utils/utils.py new file mode 100644 index 0000000..e31f796 --- /dev/null +++ b/startup_scripts/startup_script_utils/utils.py @@ -0,0 +1,15 @@ +from typing import Tuple + + +def split_params(params: dict, unique_params: list = None) -> Tuple[dict, dict]: + """Split params dict into dict with matching params and a dict with default values""" + + if unique_params is None: + unique_params = ["name", "slug"] + + matching_params = {} + for unique_param in unique_params: + param = params.pop(unique_param, None) + if param: + matching_params[unique_param] = param + return matching_params, params