diff --git a/.dockerignore b/.dockerignore index 1b2bacc..bdb18c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,11 @@ +.git .github .travis.yml +*.md +env build* -*.env -.git +docker-compose.override.yml +.netbox/.git* +.netbox/.travis.yml +.netbox/docs +.netbox/scripts diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 88437e0..82f22b1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -25,6 +25,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Current Behavior @@ -45,12 +47,22 @@ The output of `docker version`: `XXXXX` The output of `git rev-parse HEAD`: `XXXXX` The command you used to start the project: `XXXXX` + +The output of `docker inspect netboxcommunity/netbox:latest --format "{{json .Config.Labels}}"`: + +```json +{ + "JSON JSON JSON": + "--> Please paste formatted json. (Use e.g. `jq` or https://jsonformatter.curiousconcept.com/)" +} +``` + The output of `docker-compose logs netbox`: -``` +```text LOG LOG LOG ``` @@ -60,6 +72,6 @@ Only if you have gotten a 5xx http error, else delete this section. If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com --> -``` +```text LOG LOG LOG ``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 65f31dd..6bc6dd4 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -24,6 +24,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Desired Behavior @@ -33,7 +35,7 @@ Please try this means to get help before opening an issue here: ## Contrast to Current Behavior - + ... ## Changes Required diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..139ff96 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,85 @@ + + + + +Related Issue: + +## New Behavior + + + +... + +## Contrast to Current Behavior + + + +... + +## Discussion: Benefits and Drawbacks + + + +... + +## Changes to the Wiki + + + +... + +## Proposed Release Note Entry + + + +... + +## Double Check + + + +* [ ] I have read the comments and followed the PR template. +* [ ] I have explained my PR according to the information in the comments. +* [ ] My PR targets the `develop` branch. diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..4932ae8 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,40 @@ +name: push + +on: + push: + branches-ignore: + - release + pull_request: + branches-ignore: + - release + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + docker_from: + - '' # use the default of the DOCKERFILE + - python:3.7-alpine + - python:3.8-alpine + # - python:3.9-rc-alpine # disable until Netbox's unit tests work + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - id: git-checkout + name: Checkout + uses: actions/checkout@v1 + - id: docker-build + name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + DOCKER_FROM: ${{ matrix.docker_from }} + GH_ACTION: enable + - id: docker-test + name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + if: steps.docker-build.outputs.skipped != 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1796097 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: release + +on: + push: + branches: + - release + schedule: + - cron: '45 5 * * *' + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - id: git-checkout + name: Checkout + uses: actions/checkout@v1 + - id: docker-build + name: Build the image with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + GH_ACTION: enable + - id: docker-test + name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-login + name: Login to the Docker Registry + run: | + echo "::add-mask::$DOCKERHUB_USERNAME" + echo "::add-mask::$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}" + env: + DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} + DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-push + name: Push the image + run: ${{ matrix.build_cmd }} --push-only + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-logout + name: Logout of the Docker Registry + run: docker logout "${DOCKER_REGISTRY}" + if: steps.docker-build.outputs.skipped != 'true' diff --git a/.gitignore b/.gitignore index cbaffa8..97aa1b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.sql.gz .netbox +.initializers +docker-compose.override.yml diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md deleted file mode 100644 index 10c4154..0000000 --- a/DOCKER_HUB.md +++ /dev/null @@ -1,63 +0,0 @@ -# cloud.docker.com Configuration - -The automatic build is configured in cloud.docker.com. - -The following build configuration is expected: - -```yaml -Source Repository: github.com/netbox-community/netbox-docker -Build Location: Build on Docker Hub's infrastructure -Autotest: Internal and External Pull Requests -Repository Links: Enable for Base Image -Build Rules: -- Source Type: Branch - Source: master - Docker Tag: branches - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -- Source Type: Branch - Source: master - Docker Tag: prerelease - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -- Source Type: Branch - Source: master - Docker Tag: release - Dockerfile location: Dockerfile - Build Context: / - Autobuild: on - Build Caching: on -Build Environment Variables: -# Create an app on Github and use it's OATH credentials here -- Key: GITHUB_OAUTH_CLIENT_ID - Value: -- Key: GITHUB_OAUTH_CLIENT_SECRET - Value: -Build Triggers: -- Name: Cron Trigger - Trigger URL: -# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds -``` - -## Background Knowledge - -The build system of cloud.docker.com is not made for this kind of project. -But we found a way to make it work, and this is how: - -1. The docker hub build system [allows to overwrite the scripts that get executed - for `build`, `test` and `push`](overwrite). See `/hooks/*`. -2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`. -3. The `build` script runs `run_build()` from `/hooks/common`. - This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`. -4. The `test` script just invokes `docker-compose` commands. -5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. - This causes the `build.sh` script to not re-build the Docker image, but just the just built image. - -The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`. -Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose. - -[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands diff --git a/Dockerfile b/Dockerfile index 7ee1216..7764248 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ WORKDIR /install RUN pip install --prefix="/install" --no-warn-script-location \ # gunicorn is used for launching netbox - 'gunicorn<20.0.0' \ + gunicorn \ greenlet \ eventlet \ # napalm is used for gathering information from network devices @@ -26,7 +26,9 @@ RUN pip install --prefix="/install" --no-warn-script-location \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ # django_auth_ldap is required for ldap - django_auth_ldap + django_auth_ldap \ +# django-storages was introduced in 2.7 and is optional + django-storages ARG NETBOX_PATH COPY ${NETBOX_PATH}/requirements.txt / @@ -68,15 +70,45 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py WORKDIR /opt/netbox/netbox +# Must set permissions for '/opt/netbox/netbox/static' directory +# to g+w so that `./manage.py collectstatic` can be executed during +# container startup. +# Must set permissions for '/opt/netbox/netbox/media' directory +# to g+w so that pictures can be uploaded to netbox. +RUN mkdir static && chmod g+w static media + ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] -LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \ - NETBOX_BRANCH="custom build" \ - ORIGINAL_DOCKER_TAG="custom build" \ - NETBOX_GIT_COMMIT="not built from git" \ - NETBOX_GIT_URL="not built from git" +LABEL ORIGINAL_TAG="" \ + NETBOX_GIT_BRANCH="" \ + NETBOX_GIT_REF="" \ + NETBOX_GIT_URL="" \ +# See http://label-schema.org/rc1/#build-time-labels +# Also https://microbadger.com/labels + org.label-schema.schema-version="1.0" \ + org.label-schema.build-date="" \ + org.label-schema.name="Netbox Docker" \ + org.label-schema.description="A container based distribution of Netbox, the free and open IPAM and DCIM solution." \ + org.label-schema.vendor="The netbox-docker contributors." \ + org.label-schema.url="https://github.com/netbox-community/netbox-docker" \ + org.label-schema.usage="https://github.com/netbox-community/netbox-docker/wiki" \ + org.label-schema.vcs-url="https://github.com/netbox-community/netbox-docker.git" \ + org.label-schema.vcs-ref="" \ + org.label-schema.version="snapshot" \ +# See https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys + org.opencontainers.image.created="" \ + org.opencontainers.image.title="Netbox Docker" \ + org.opencontainers.image.description="A container based distribution of Netbox, the free and open IPAM and DCIM solution." \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.authors="The netbox-docker contributors." \ + org.opencontainers.image.vendor="The netbox-docker contributors." \ + org.opencontainers.image.url="https://github.com/netbox-community/netbox-docker" \ + org.opencontainers.image.documentation="https://github.com/netbox-community/netbox-docker/wiki" \ + org.opencontainers.image.source="https://github.com/netbox-community/netbox-docker.git" \ + org.opencontainers.image.revision="" \ + org.opencontainers.image.version="snapshot" ##### ## LDAP specific configuration diff --git a/README.md b/README.md index e375bea..5f12bfc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,52 @@ # netbox-docker +[![GitHub release (latest by date)](https://img.shields.io/github/v/release/netbox-community/netbox-docker)][github-release] +[![GitHub stars](https://img.shields.io/github/stars/netbox-community/netbox-docker)][github-stargazers] +![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/netbox-community/netbox-docker) +![Github release workflow](https://img.shields.io/github/workflow/status/netbox-community/netbox-docker/release) +![Docker Pulls](https://img.shields.io/docker/pulls/netboxcommunity/netbox) +[![MicroBadger Layers](https://img.shields.io/microbadger/layers/netboxcommunity/netbox)][netbox-docker-microbadger] +[![MicroBadger Size](https://img.shields.io/microbadger/image-size/netboxcommunity/netbox)][netbox-docker-microbadger] +[![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license] + [The Github repository](netbox-docker-github) houses the components needed to build Netbox as a Docker container. Images are built using this code and are released to [Docker Hub][netbox-dockerhub] once a day. Do you have any questions? Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel. +[github-stargazers]: https://github.com/netbox-community/netbox-docker/stargazers +[github-release]: https://github.com/netbox-community/netbox-docker/releases +[netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox [netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/tags/ [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/ [ntc-slack]: http://slack.networktocode.com/ [netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E +[netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/release/LICENSE + +## Docker Tags + +* `vX.Y.Z`: Release builds, built from [releases of Netbox][netbox-releases]. +* `latest`: Release builds, built from [`master` branch of Netbox][netbox-master]. +* `snapshot`: Pre-release builds, built from the [`develop` branch of Netbox][netbox-develop]. +* `develop-X.Y`: Pre-release builds, built from the corresponding [branch of Netbox][netbox-branches]. + +Then there is currently one extra tags for each of the above labels: + +* `-ldap`: Contains additional dependencies and configurations for connecting Netbox to an LDAP directroy. + [Learn more about that in our wiki][netbox-docker-ldap]. + +[netbox-releases]: https://github.com/netbox-community/netbox/releases +[netbox-master]: https://github.com/netbox-community/netbox/tree/master +[netbox-develop]: https://github.com/netbox-community/netbox/tree/develop +[netbox-branches]: https://github.com/netbox-community/netbox/branches +[netbox-docker-ldap]: https://github.com/netbox-community/netbox-docker/wiki/LDAP ## Quickstart -To get Netbox up and running: +To get Netbox up and running in Docker: ```bash -git clone -b master https://github.com/netbox-community/netbox-docker.git +git clone -b release https://github.com/netbox-community/netbox-docker.git cd netbox-docker docker-compose pull docker-compose up -d @@ -37,29 +68,32 @@ $ xdg-open "http://$(docker-compose port nginx 8080)/" &>/dev/null & Alternatively, use something like [Reception][docker-reception] to connect to _docker-compose_ projects. -Default credentials: +The default credentials are: * Username: **admin** * Password: **admin** * API Token: **0123456789abcdef0123456789abcdef01234567** +There is a more complete [Getting Started guide on our Wiki][wiki-getting-started]. + +[wiki-getting-started]: https://github.com/netbox-community/netbox-docker/wiki/Getting-Started [docker-reception]: https://github.com/nxt-engineering/reception ## Dependencies -This project relies only on *Docker* and *docker-compose* meeting this requirements: +This project relies only on *Docker* and *docker-compose* meeting these requirements: * The *Docker version* must be at least `17.05`. * The *docker-compose version* must be at least `1.17.0`. -To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. +To check the version installed on your system run `docker --version` and `docker-compose --version`. -## Reference Documentation +## Documentation -Please refer [to the wiki][wiki] for further information on how to use this Netbox Docker image properly. +Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly. It covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration. -[wiki]: https://github.com/netbox-community/netbox-docker/wiki/ +[netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/ ## Netbox Version @@ -69,7 +103,7 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.6.6 +export VERSION=v2.7.1 docker-compose pull netbox docker-compose up -d ``` @@ -78,7 +112,7 @@ You can also build a specific version of the Netbox Docker image yourself. `VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=v2.6.6 +export VERSION=v2.7.1 ./build.sh $VERSION docker-compose up -d ``` @@ -90,30 +124,34 @@ docker-compose up -d From time to time it might become necessary to re-engineer the structure of this setup. Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence. -Since April 2018 each image built from this repo contains a `NETBOX_DOCKER_PROJECT_VERSION` label. -You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.3.1 --format "{{json .ContainerConfig.Labels}}"`. + +Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label. +(The images contained labels since April 2018, although in November 2019 the labels' names changed.) +You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.7.1 --format "{{json .Config.Labels}}"`. Please read [the release notes][releases] carefully when updating to a new image version. [releases]: https://github.com/netbox-community/netbox-docker/releases -## Rebuilding & Publishing images +## Rebuilding the Image `./build.sh` can be used to rebuild the Docker image. See `./build.sh --help` for more information. -### Publishing Docker Images +For more details on custom builds [consult our wiki][netbox-docker-wiki-build]. -New Docker images are built and published every 24h on the [Docker Build Infrastructure][docker-build-infra]. -`DOCKER_HUB.md` contains more information about the build infrastructure. +[netbox-docker-wiki-build]: https://github.com/netbox-community/netbox-docker/wiki/Build -[docker-build-infra]: https://hub.docker.com/r/netboxcommunity/netbox/builds/ +### Pre-made Docker Images + +New Docker images are built and published every 24h. ## Tests -To run the tests coming with Netbox, use the `docker-compose.yml` file as such: +We have a test script. +It runs Netbox's own unit tests and ensures that all initializers work: ```bash -docker-compose run netbox ./manage.py test +IMAGE=netboxcommunity/netbox:latest ./test.sh ``` ## About diff --git a/VERSION b/VERSION index b72b05e..2157409 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.19.3 +0.22.0 diff --git a/build-branches.sh b/build-branches.sh index a6bc736..483e771 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds develop, develop-* and master branches +# Builds develop, develop-* and master branches of Netbox echo "▶️ $0 $*" diff --git a/build-latest.sh b/build-latest.sh index 31a0f76..468dffe 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -66,6 +66,10 @@ if [ "${PRERELEASE}" == "true" ]; then echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'." if [ -z "$DEBUG" ]; then + if [ -n "${GH_ACTION}" ]; then + echo "::set-output name=skipped::true" + fi + exit 0 else echo "⚠️ Would exit here with code '0', but DEBUG is enabled." diff --git a/build.sh b/build.sh index 123cd17..82a6e34 100755 --- a/build.sh +++ b/build.sh @@ -27,26 +27,29 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " TAG The version part of the docker tag." echo " Default:" - echo " When \${BRANCH}=master: latest" - echo " When \${BRANCH}=develop: snapshot" - echo " Else: same as \${BRANCH}" - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" - echo " Also used for tagging the image." + echo " When =master: latest" + echo " When =develop: snapshot" + echo " Else: same as " + echo " DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." + echo " Default: docker.io" + echo " DOCKER_ORG The Docker repository's organisation (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" - echo " Also used for tagging the image." + echo " DOCKER_REPO The Docker repository's name (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" + echo " Used for tagging the image." echo " Default: netbox" - echo " DOCKER_FROM The base image to use." - echo " Default: Whatever is defined as default in the Dockerfile." echo " DOCKER_TAG The name of the tag which is applied to the image." echo " Useful for pushing into another registry than hub.docker.com." - echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" + echo " Default: \${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" echo " image. This is used to tag all patch releases to their" echo " containing version e.g. v2.5.1 -> v2.5" - echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:." + echo " Default: \${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}:." echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" + echo " DOCKER_FROM The base image to use." + echo " Default: Whatever is defined as default in the Dockerfile." echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." echo " Default: main ldap" @@ -60,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " DRY_RUN Prints all build statements instead of running them." echo " Default: undefined" + echo " GH_ACTION If defined, special 'echo' statements are enabled that set the" + echo " following environment variables in Github Actions:" + echo " - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable" + echo " Default: undefined" echo "" echo "Examples:" echo " ${0} master" @@ -89,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi ### -# Determining the build command to use +# Enabling dry-run mode ### if [ -z "${DRY_RUN}" ]; then DRY="" @@ -99,27 +106,21 @@ else fi ### -# read the project version from the `VERSION` file and trim it -# see https://stackoverflow.com/a/3232433/172132 -### -NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" - -### -# variables for fetching the source +# Variables for fetching the source ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" -BRANCH="${1}" +NETBOX_BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" +NETBOX_PATH="${NETBOX_PATH-.netbox}" ### -# fetching the source +# Fetching the source ### -if [ "${2}" != "--push-only" ] ; then - NETBOX_PATH="${NETBOX_PATH-.netbox}" - echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" +if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then + echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" if [ ! -d "${NETBOX_PATH}" ]; then - $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}" fi ( @@ -130,7 +131,7 @@ if [ "${2}" != "--push-only" ] ; then fi $DRY git remote set-url origin "${URL}" - $DRY git fetch -qp --depth 10 origin "${BRANCH}" + $DRY git fetch -qp --depth 10 origin "${NETBOX_BRANCH}" $DRY git checkout -qf FETCH_HEAD $DRY git prune ) @@ -153,17 +154,37 @@ if [ ! -f "${DOCKERFILE}" ]; then fi ### -# variables for tagging the docker image +# Variables for labelling the docker image ### +BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" + +if [ -d ".git" ]; then + GIT_REF="$(git rev-parse HEAD)" +fi + +# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 +PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" + +# Get the Git information from the netbox directory +if [ -d "${NETBOX_PATH}/.git" ]; then + NETBOX_GIT_REF=$(cd ${NETBOX_PATH}; git rev-parse HEAD) + NETBOX_GIT_BRANCH=$(cd ${NETBOX_PATH}; git rev-parse --abbrev-ref HEAD) + NETBOX_GIT_URL=$(cd ${NETBOX_PATH}; git remote get-url origin) +fi + +### +# Variables for tagging the docker image +### +DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" -case "${BRANCH}" in +case "${NETBOX_BRANCH}" in master) TAG="${TAG-latest}";; develop) TAG="${TAG-snapshot}";; *) - TAG="${TAG-$BRANCH}";; + TAG="${TAG-$NETBOX_BRANCH}";; esac ### @@ -176,16 +197,21 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" ### # Build each target ### +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1} for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do echo "🏗 Building the target '${DOCKER_TARGET}'" ### # composing the final TARGET_DOCKER_TAG ### - TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" + TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi + if [ -n "${GH_ACTION}" ]; then + echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + echo "::set-output name=skipped::false" + fi ### # composing the additional DOCKER_SHORT_TAG, @@ -196,10 +222,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do MAJOR=${BASH_REMATCH[1]} MINOR=${BASH_REMATCH[2]} - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" + TARGET_DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" if [ "${DOCKER_TARGET}" != "main" ]; then - DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" + TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}" fi fi @@ -216,25 +242,39 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do -f "${DOCKERFILE}" -t "${TARGET_DOCKER_TAG}" ) - if [ -n "${DOCKER_SHORT_TAG}" ]; then - DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" ) + if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then + DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_SHORT_TAG}" ) fi # --label - DOCKER_BUILD_ARGS+=( - --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --label "NETBOX_BRANCH=${BRANCH}" - --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" - ) - if [ -d "${NETBOX_PATH}/.git" ]; then + if [ "${DOCKER_TARGET}" == "main" ]; then DOCKER_BUILD_ARGS+=( - --label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)" - --label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)" + --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}" + + --label "org.label-schema.build-date=${BUILD_DATE}" + --label "org.opencontainers.image.created=${BUILD_DATE}" + + --label "org.label-schema.version=${PROJECT_VERSION}" + --label "org.opencontainers.image.version=${PROJECT_VERSION}" ) + if [ -d ".git" ]; then + DOCKER_BUILD_ARGS+=( + --label "org.label-schema.vcs-ref=${GIT_REF}" + --label "org.opencontainers.image.revision=${GIT_REF}" + ) + fi + if [ -d "${NETBOX_PATH}/.git" ]; then + DOCKER_BUILD_ARGS+=( + --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}" + --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}" + --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}" + ) + fi fi # --build-arg - DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) + DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" ) + if [ -n "${DOCKER_FROM}" ]; then DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) fi @@ -262,10 +302,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do $DRY docker push "${TARGET_DOCKER_TAG}" echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DRY docker push "${DOCKER_SHORT_TAG}" - echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." + if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then + echo "⏫ Pushing '${TARGET_DOCKER_SHORT_TAG}'" + $DRY docker push "${TARGET_DOCKER_SHORT_TAG}" + echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_SHORT_TAG}'." fi fi done diff --git a/configuration/configuration.py b/configuration/configuration.py index bf941a4..af121d9 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -37,7 +37,9 @@ DATABASE = { # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) - 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), + 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE + 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), # Database connection persistence } @@ -49,13 +51,22 @@ SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key')) # Redis database settings. The Redis database is used for caching and background processing such as webhooks REDIS = { - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), - 'DATABASE': os.environ.get('REDIS_DATABASE', '0'), - 'CACHE_DATABASE': os.environ.get('REDIS_CACHE_DATABASE', '1'), - 'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + 'webhooks': { + 'HOST': os.environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(os.environ.get('REDIS_PORT', 6379)), + 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), + 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), + 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), + 'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))), + 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), + 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', + }, } ######################### @@ -170,10 +181,6 @@ SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') # Time zone (default: UTC) TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') -# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis -# database be configured and accessible by NetBox (see `REDIS` below). -WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true' - # Date/time formatting. See the following link for supported formats: # https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y') diff --git a/configuration/ldap_config.py b/configuration/ldap_config.py index 39fc894..ba2067c 100644 --- a/configuration/ldap_config.py +++ b/configuration/ldap_config.py @@ -70,8 +70,7 @@ AUTH_LDAP_USER_FLAGS_BY_GROUP = { AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' # Cache groups for one hour to reduce LDAP traffic -AUTH_LDAP_CACHE_GROUPS = os.environ.get('AUTH_LDAP_CACHE_GROUPS', 'True').lower() == 'true' -AUTH_LDAP_GROUP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_GROUP_CACHE_TIMEOUT', 3600)) +AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) # Populate the Django user from the LDAP directory. AUTH_LDAP_USER_ATTR_MAP = { diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..9420617 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,45 @@ +version: '3.4' +services: + netbox: + image: ${IMAGE-netboxcommunity/netbox:latest} + depends_on: + - postgres + - redis + env_file: env/netbox.env + user: '101' + volumes: + - ./startup_scripts:/opt/netbox/startup_scripts:z,ro + - ./${INITIALIZERS_DIR-initializers}:/opt/netbox/initializers:z,ro + - ./configuration:/etc/netbox/config:z,ro + - ./reports:/etc/netbox/reports:z,ro + - ./scripts:/etc/netbox/scripts:z,ro + - netbox-nginx-config:/etc/netbox-nginx:z + - netbox-static-files:/opt/netbox/netbox/static:z + - netbox-media-files:/opt/netbox/netbox/media:z + nginx: + command: nginx -c /etc/netbox-nginx/nginx.conf + image: nginx:1.17-alpine + depends_on: + - netbox + ports: + - 8080 + volumes: + - netbox-static-files:/opt/netbox/netbox/static:ro + - netbox-nginx-config:/etc/netbox-nginx/:ro + postgres: + image: postgres:11-alpine + env_file: env/postgres.env + redis: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env +volumes: + netbox-static-files: + driver: local + netbox-nginx-config: + driver: local + netbox-media-files: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 7a8e6f9..469202c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,10 @@ services: depends_on: - postgres - redis + - redis-cache - netbox-worker env_file: env/netbox.env + user: '101' volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - ./initializers:/opt/netbox/initializers:z,ro @@ -27,7 +29,7 @@ services: - rqworker nginx: command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.15-alpine + image: nginx:1.17-alpine depends_on: - netbox ports: @@ -37,12 +39,12 @@ services: - netbox-nginx-config:/etc/netbox-nginx/:ro - /etc/cert-client:/etc/cert-client:ro postgres: - image: postgres:10.4-alpine + image: postgres:11-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data redis: - image: redis:4-alpine + image: redis:5-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env @@ -50,6 +52,13 @@ services: env_file: env/redis.env volumes: - netbox-redis-data:/data + redis-cache: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env volumes: netbox-static-files: driver: local @@ -57,8 +66,6 @@ volumes: driver: local netbox-media-files: driver: local - netbox-report-files: - driver: local netbox-postgres-data: driver: local netbox-redis-data: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e6a8de..f555695 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,12 +1,27 @@ #!/bin/bash +# Runs on every start of the Netbox Docker container + +# Stop when an error occures set -e -# wait shortly and then run db migrations (retry on error) -while ! ./manage.py migrate 2>&1; do - echo "⏳ Waiting on DB..." - sleep 3 -done +# Allows Netbox to be run as non-root users +umask 002 +# Try to connect to the DB +DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} +MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} +CUR_DB_WAIT_TIME=0 +while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do + echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)" + sleep "${DB_WAIT_TIMEOUT}" + CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT )) +done +if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then + echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready." + exit 1 +fi + +# Create Superuser if required if [ "$SKIP_SUPERUSER" == "true" ]; then echo "↩️ Skip creating the superuser" else @@ -42,21 +57,19 @@ END echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}" fi +# Run the startup scripts (and initializers) if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩️ Skipping startup scripts" else - for script in /opt/netbox/startup_scripts/*.py; do - echo "⚙️ Executing '$script'" - ./manage.py shell --interface python < "${script}" - done + echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi -# copy static files +# Copy static files ./manage.py collectstatic --no-input echo "✅ Initialisation is done." -# launch whatever is passed by docker +# Launch whatever is passed by docker # (i.e. the RUN instruction in the Dockerfile) # # shellcheck disable=SC2068 diff --git a/env/netbox.env b/env/netbox.env index 023e0ed..155ce52 100644 --- a/env/netbox.env +++ b/env/netbox.env @@ -17,8 +17,11 @@ MAX_PAGE_SIZE=1000 REDIS_HOST=redis REDIS_PASSWORD=H733Kdjndks81 REDIS_DATABASE=0 -REDIS_CACHE_DATABASE=1 REDIS_SSL=false +REDIS_CACHE_HOST=redis-cache +REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36 +REDIS_CACHE_DATABASE=0 +REDIS_CACHE_SSL=false SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj SKIP_STARTUP_SCRIPTS=false SKIP_SUPERUSER=false diff --git a/env/redis-cache.env b/env/redis-cache.env new file mode 100644 index 0000000..6285c33 --- /dev/null +++ b/env/redis-cache.env @@ -0,0 +1 @@ +REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 diff --git a/hooks/build b/hooks/build deleted file mode 100755 index 80c4165..0000000 --- a/hooks/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. hooks/common - -# shellcheck disable=SC2119 -run_build diff --git a/hooks/common b/hooks/common deleted file mode 100755 index 49cd507..0000000 --- a/hooks/common +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -ensure_jq() { - if [ ! -x "$(command -v jq)" ]; then - if [ -x "$(command -v apt-get)" ]; then - echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" - apt-get update && apt-get install -y jq - else - echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'." - fi - fi -} - -# Passes args to the scripts -run_build() { - echo "🐳🐳🐳 Building '${BUILD}' images" - case $BUILD in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ - ;; - this) # Pull Requests - # only build the 'master' branch - # (resulting in the 'latest' docker tag) - # and the 'main' target. - DOCKER_TARGET=main ./build.sh master - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac -} - -echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG="index.docker.io/netboxcommunity" -export DOCKER_REPO=netbox -export DOCKERHUB_REPO=netboxcommunity/netbox -# shellcheck disable=SC2153 -export BUILD="${DOCKER_TAG}" - -unset DOCKER_TAG - -ensure_jq diff --git a/hooks/push b/hooks/push deleted file mode 100755 index 81dc7db..0000000 --- a/hooks/push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -. hooks/common - -if [ "${SOURCE_BRANCH}" == "master" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "master" ]; then - echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". - fi - - run_build --push-only -else - echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" - exit 0 -fi diff --git a/hooks/test b/hooks/test deleted file mode 100755 index 1dd5538..0000000 --- a/hooks/test +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -. hooks/common - -run_test() { - echo "🐳🐳🐳 Testing '${1}'" - VERSION="${1}" docker-compose run netbox ./manage.py test - docker-compose down -v - echo "🐳🐳🐳 Done testing '${1}'" -} - -# test on builds of 'branches' -if [ "${BUILD}" == "branches" ] \ - || [ "${DEBUG}" == "true" ]; then - run_test latest - run_test snapshot -# test on bulds of 'this' (i.e. pull request) -elif [ "${BUILD}" == "this" ]; then - run_test latest -else - echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'." -fi diff --git a/initializers/custom_fields.yml b/initializers/custom_fields.yml index 0b6472a..4085ab0 100644 --- a/initializers/custom_fields.yml +++ b/initializers/custom_fields.yml @@ -1,3 +1,18 @@ +## Possible Choices: +## type: +## - text +## - integer +## - boolean +## - date +## - url +## - select +## filter_logic: +## - disabled +## - loose +## - exact +## +## Examples: + # text_field: # type: text # label: Custom Text @@ -22,8 +37,8 @@ # weight: 10 # on_objects: # - tenancy.models.Tenant -# selection_field: -# type: selection +# select_field: +# type: select # label: Choose between items # required: false # filter_logic: exact @@ -41,8 +56,8 @@ # weight: 50 # - value: Fourth Item # weight: 40 -# selection_field_auto_weight: -# type: selection +# select_field_auto_weight: +# type: select # label: Choose between items # required: false # filter_logic: loose diff --git a/initializers/dcim_interfaces.yml b/initializers/dcim_interfaces.yml index 18920fe..4030530 100644 --- a/initializers/dcim_interfaces.yml +++ b/initializers/dcim_interfaces.yml @@ -1,8 +1,18 @@ +## Possible Choices: +## type: +## - virtual +## - lag +## - 1000base-t +## - ... and many more. See for yourself: +## https://github.com/netbox-community/netbox/blob/295d4f0394b431351c0cb2c3ecc791df68c6c2fb/netbox/dcim/choices.py#L510 +## +## Examples: + # - device: server01 # enabled: true -# type: Virtual +# type: virtual # name: to-server02 # - device: server02 # enabled: true -# type: Virtual +# type: virtual # name: to-server01 diff --git a/initializers/devices.yml b/initializers/devices.yml index 0beb6f2..708b68c 100644 --- a/initializers/devices.yml +++ b/initializers/devices.yml @@ -1,9 +1,24 @@ +## Possible Choices: +## face: +## - front +## - rear +## status: +## - offline +## - active +## - planned +## - staged +## - failed +## - inventory +## - decommissioning +## +## Examples: + # - name: server01 # device_role: server # device_type: Other # site: AMS 1 # rack: rack-01 -# face: Front +# face: front # position: 1 # custom_fields: # text_field: Description @@ -12,7 +27,7 @@ # device_type: Other # site: AMS 2 # rack: rack-02 -# face: Front +# face: front # position: 2 # custom_fields: # text_field: Description @@ -21,7 +36,7 @@ # device_type: Other # site: SING 1 # rack: rack-03 -# face: Front +# face: front # position: 3 # custom_fields: # text_field: Description diff --git a/initializers/groups.yml b/initializers/groups.yml index 7bdd0a7..b91ef39 100644 --- a/initializers/groups.yml +++ b/initializers/groups.yml @@ -1,3 +1,15 @@ +## To list all permissions, run: +## +## docker-compose run --rm --entrypoint /bin/bash netbox +## $ ./manage.py migrate +## $ ./manage.py shell +## > from django.contrib.auth.models import Permission +## > print('\n'.join([p.codename for p in Permission.objects.all()])) +## +## Permission lists support wildcards. See the examples below. +## +## Examples: + # applications: # users: # - technical_user @@ -8,9 +20,16 @@ # users: # - writer # permissions: -# - add_device -# - change_device # - delete_device -# - add_virtualmachine -# - change_virtualmachine # - delete_virtualmachine +# - add_* +# - change_* +# vm_managers: +# permissions: +# - '*_virtualmachine' +# device_managers: +# permissions: +# - '*device*' +# creators: +# permissions: +# - add_* diff --git a/initializers/ip_addresses.yml b/initializers/ip_addresses.yml index 5738749..6ac38e9 100644 --- a/initializers/ip_addresses.yml +++ b/initializers/ip_addresses.yml @@ -1,26 +1,44 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## - dhcp +## role: +## - loopback +## - secondary +## - anycast +## - vip +## - vrrp +## - hsrp +## - glbp +## - carp +## +## Examples: + # - address: 10.1.1.1/24 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 2001:db8:a000:1::1/64 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 10.1.1.2/24 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 2001:db8:a000:1::2/64 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 10.1.1.10/24 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 # - address: 2001:db8:a000:1::10/64 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 diff --git a/initializers/prefixes.yml b/initializers/prefixes.yml index a7e6815..fbf3eee 100644 --- a/initializers/prefixes.yml +++ b/initializers/prefixes.yml @@ -1,13 +1,22 @@ +## Possible Choices: +## status: +## - container +## - active +## - reserved +## - deprecated +## +## Examples: + # - description: prefix1 # prefix: 10.1.1.0/24 # site: AMS 1 -# status: Active +# status: active # tenant: tenant1 # vlan: vlan1 # - description: prefix2 # prefix: 10.1.2.0/24 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 # is_pool: true @@ -15,6 +24,6 @@ # - description: ipv6 prefix1 # prefix: 2001:db8:a000:1::/64 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 diff --git a/initializers/rack_groups.yml b/initializers/rack_groups.yml new file mode 100644 index 0000000..244fc00 --- /dev/null +++ b/initializers/rack_groups.yml @@ -0,0 +1,3 @@ +# - name: cage 101 +# slug: cage-101 +# site: SING 1 diff --git a/initializers/racks.yml b/initializers/racks.yml index 9a71743..51502de 100644 --- a/initializers/racks.yml +++ b/initializers/racks.yml @@ -1,24 +1,41 @@ +## Possible Choices: +## width: +## - 19 +## - 23 +## types: +## - 2-post-frame +## - 4-post-frame +## - 4-post-cabinet +## - wall-frame +## - wall-cabinet +## outer_unit: +## - mm +## - in +## +## Examples: + # - site: AMS 1 # name: rack-01 # role: Role 1 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description # - site: AMS 2 # name: rack-02 # role: Role 2 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description # - site: SING 1 # name: rack-03 +# group: cage 101 # role: Role 3 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description diff --git a/initializers/users.yml b/initializers/users.yml index 64c005c..2aea62e 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -1,3 +1,15 @@ +## To list all permissions, run: +## +## docker-compose run --rm --entrypoint /bin/bash netbox +## $ ./manage.py migrate +## $ ./manage.py shell +## > from django.contrib.auth.models import Permission +## > print('\n'.join([p.codename for p in Permission.objects.all()])) +## +## Permission lists support wildcards. See the examples below. +## +## Examples: + # technical_user: # api_token: 0123456789technicaluser789abcdef01234567 # must be looooong! # reader: @@ -5,9 +17,7 @@ # writer: # password: writer # permissions: -# - add_device -# - change_device # - delete_device -# - add_virtualmachine -# - change_virtualmachine # - delete_virtualmachine +# - add_* +# - change_* diff --git a/initializers/virtual_machines.yml b/initializers/virtual_machines.yml index 5da75a2..2122920 100644 --- a/initializers/virtual_machines.yml +++ b/initializers/virtual_machines.yml @@ -1,10 +1,18 @@ +## Possible Choices: +## status: +## - active +## - offline +## - staged +## +## Examples: + # - cluster: cluster1 # comments: VM1 # disk: 200 # memory: 4096 # name: virtual machine 1 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 # - cluster: cluster1 @@ -13,6 +21,6 @@ # memory: 2048 # name: virtual machine 2 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 diff --git a/initializers/vlans.yml b/initializers/vlans.yml index 26532c7..a8cd521 100644 --- a/initializers/vlans.yml +++ b/initializers/vlans.yml @@ -1,11 +1,19 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## +## Examples: + # - name: vlan1 # site: AMS 1 -# status: Active +# status: active # vid: 5 # role: Main Management # description: VLAN 5 for MGMT # - group: VLAN group 2 # name: vlan2 # site: AMS 1 -# status: Active +# status: active # vid: 1300 diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index a1340a1..6962df0 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -20,15 +20,23 @@ with file.open('r') as stream: username = username, password = user_details.get('password', 0) or User.objects.make_random_password) - print("👤 Created user ",username) + print("👤 Created user",username) if user_details.get('api_token', 0): Token.objects.create(user=user, key=user_details['api_token']) - user_permissions = user_details.get('permissions', []) - if user_permissions: - user.user_permissions.clear() - for permission_codename in user_details.get('permissions', []): - for permission in Permission.objects.filter(codename=permission_codename): - user.user_permissions.add(permission) - user.save() + yaml_permissions = user_details.get('permissions', []) + if yaml_permissions: + subject = user.user_permissions + subject.clear() + for yaml_permission in yaml_permissions: + if '*' in yaml_permission: + permission_filter = '^' + yaml_permission.replace('*','.*') + '$' + permissions = Permission.objects.filter(codename__iregex=permission_filter) + print(" ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'") + else: + permissions = Permission.objects.filter(codename=yaml_permission) + print(" ⚿ Granting permission", yaml_permission) + + for permission in permissions: + subject.add(permission) diff --git a/startup_scripts/010_groups.py b/startup_scripts/010_groups.py index e68a5f0..08fb4bf 100644 --- a/startup_scripts/010_groups.py +++ b/startup_scripts/010_groups.py @@ -24,9 +24,18 @@ with file.open('r') as stream: if user: user.groups.add(group) - group_permissions = group_details.get('permissions', []) - if group_permissions: - group.permissions.clear() - for permission_codename in group_details.get('permissions', []): - for permission in Permission.objects.filter(codename=permission_codename): - group.permissions.add(permission) + yaml_permissions = group_details.get('permissions', []) + if yaml_permissions: + subject = group.permissions + subject.clear() + for yaml_permission in yaml_permissions: + if '*' in yaml_permission: + permission_filter = '^' + yaml_permission.replace('*','.*') + '$' + permissions = Permission.objects.filter(codename__iregex=permission_filter) + print(" ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'") + else: + permissions = Permission.objects.filter(codename=yaml_permission) + print(" ⚿ Granting permission", yaml_permission) + + for permission in permissions: + subject.add(permission) diff --git a/startup_scripts/020_custom_fields.py b/startup_scripts/020_custom_fields.py index 76a32bb..2f6ba72 100644 --- a/startup_scripts/020_custom_fields.py +++ b/startup_scripts/020_custom_fields.py @@ -1,19 +1,9 @@ -from extras.constants import CF_TYPE_TEXT, CF_TYPE_INTEGER, CF_TYPE_BOOLEAN, CF_TYPE_DATE, CF_TYPE_URL, CF_TYPE_SELECT, CF_FILTER_CHOICES from extras.models import CustomField, CustomFieldChoice from ruamel.yaml import YAML from pathlib import Path import sys -text_to_fields = { - 'boolean': CF_TYPE_BOOLEAN, - 'date': CF_TYPE_DATE, - 'integer': CF_TYPE_INTEGER, - 'selection': CF_TYPE_SELECT, - 'text': CF_TYPE_TEXT, - 'url': CF_TYPE_URL, -} - def get_class_for_class_path(class_path): import importlib from django.contrib.contenttypes.models import ContentType @@ -42,12 +32,6 @@ with file.open('r') as stream: if cf_details.get('description', 0): custom_field.description = cf_details['description'] - # If no filter_logic is specified then it will default to 'Loose' - if cf_details.get('filter_logic', 0): - for choice_id, choice_text in CF_FILTER_CHOICES: - if choice_text.lower() == cf_details['filter_logic']: - custom_field.filter_logic = choice_id - if cf_details.get('label', 0): custom_field.label = cf_details['label'] @@ -58,7 +42,7 @@ with file.open('r') as stream: custom_field.required = cf_details['required'] if cf_details.get('type', 0): - custom_field.type = text_to_fields[cf_details['type']] + custom_field.type = cf_details['type'] if cf_details.get('weight', 0): custom_field.weight = cf_details['weight'] diff --git a/startup_scripts/075_rack_groups.py b/startup_scripts/075_rack_groups.py new file mode 100644 index 0000000..7deaa11 --- /dev/null +++ b/startup_scripts/075_rack_groups.py @@ -0,0 +1,31 @@ +from dcim.models import Site,RackGroup +from ruamel.yaml import YAML + +from pathlib import Path +import sys + +file = Path('/opt/netbox/initializers/rack_groups.yml') +if not file.is_file(): + sys.exit() + +with file.open('r') as stream: + yaml=YAML(typ='safe') + rack_groups= yaml.load(stream) + + required_assocs = { + 'site': (Site, 'name') + } + + if rack_groups is not None: + for params in rack_groups: + + for assoc, details in required_assocs.items(): + model, field = details + query = { field: params.pop(assoc) } + params[assoc] = model.objects.get(**query) + + rack_group, created = RackGroup.objects.get_or_create(**params) + + if created: + print("🎨 Created rack group", rack_group.name) + diff --git a/startup_scripts/080_racks.py b/startup_scripts/080_racks.py index 05bca10..ed7713d 100644 --- a/startup_scripts/080_racks.py +++ b/startup_scripts/080_racks.py @@ -1,7 +1,6 @@ from dcim.models import Site, RackRole, Rack, RackGroup from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue -from dcim.constants import RACK_TYPE_CHOICES, RACK_WIDTH_CHOICES from ruamel.yaml import YAML from pathlib import Path import sys @@ -41,14 +40,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - for rack_type in RACK_TYPE_CHOICES: - if params['type'] in rack_type: - params['type'] = rack_type[0] - - for rack_width in RACK_WIDTH_CHOICES: - if params['width'] in rack_width: - params['width'] = rack_width[0] - rack, created = Rack.objects.get_or_create(**params) if created: diff --git a/startup_scripts/130_devices.py b/startup_scripts/130_devices.py index 2d8d3ca..4217549 100644 --- a/startup_scripts/130_devices.py +++ b/startup_scripts/130_devices.py @@ -1,5 +1,4 @@ from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform -from dcim.constants import RACK_FACE_CHOICES from ipam.models import IPAddress from virtualization.models import Cluster from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'face' in params: - for rack_face in RACK_FACE_CHOICES: - if params['face'] in rack_face: - params['face'] = rack_face[0] - break - device, created = Device.objects.get_or_create(**params) if created: diff --git a/startup_scripts/210_vlans.py b/startup_scripts/210_vlans.py index e3a0ef1..ab6bd2b 100644 --- a/startup_scripts/210_vlans.py +++ b/startup_scripts/210_vlans.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import VLAN, VLANGroup, Role -from ipam.constants import VLAN_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -35,12 +34,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vlan_status in VLAN_STATUS_CHOICES: - if params['status'] in vlan_status: - params['status'] = vlan_status[0] - break - vlan, created = VLAN.objects.get_or_create(**params) if created: diff --git a/startup_scripts/220_prefixes.py b/startup_scripts/220_prefixes.py index a832c88..d13578a 100644 --- a/startup_scripts/220_prefixes.py +++ b/startup_scripts/220_prefixes.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import Prefix, VLAN, Role, VRF -from ipam.constants import PREFIX_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -38,12 +37,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for prefix_status in PREFIX_STATUS_CHOICES: - if params['status'] in prefix_status: - params['status'] = prefix_status[0] - break - prefix, created = Prefix.objects.get_or_create(**params) if created: diff --git a/startup_scripts/230_virtual_machines.py b/startup_scripts/230_virtual_machines.py index 065b600..449df8a 100644 --- a/startup_scripts/230_virtual_machines.py +++ b/startup_scripts/230_virtual_machines.py @@ -1,6 +1,5 @@ from dcim.models import Site, Platform, DeviceRole from virtualization.models import Cluster, VirtualMachine -from virtualization.constants import VM_STATUS_CHOICES from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -43,12 +42,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vm_status in VM_STATUS_CHOICES: - if params['status'] in vm_status: - params['status'] = vm_status[0] - break - virtual_machine, created = VirtualMachine.objects.get_or_create(**params) if created: diff --git a/startup_scripts/250_dcim_interfaces.py b/startup_scripts/250_dcim_interfaces.py index ce7e7bd..ec30b5c 100644 --- a/startup_scripts/250_dcim_interfaces.py +++ b/startup_scripts/250_dcim_interfaces.py @@ -1,5 +1,4 @@ from dcim.models import Interface, Device -from dcim.constants import IFACE_TYPE_CHOICES from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -28,16 +27,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'type' in params: - for outer_list in IFACE_TYPE_CHOICES: - for type_choices in outer_list[1]: - if params['type'] in type_choices: - params['type'] = type_choices[0] - break - else: - continue - break - interface, created = Interface.objects.get_or_create(**params) if created: diff --git a/startup_scripts/260_ip_addresses.py b/startup_scripts/260_ip_addresses.py index f7f2bc7..d109a36 100644 --- a/startup_scripts/260_ip_addresses.py +++ b/startup_scripts/260_ip_addresses.py @@ -1,5 +1,4 @@ from ipam.models import IPAddress, VRF -from ipam.constants import IPADDRESS_STATUS_CHOICES from dcim.models import Device, Interface from virtualization.models import VirtualMachine from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: query = { field: params.pop(assoc) } params[assoc] = model.objects.get(**query) - if 'status' in params: - for ip_status in IPADDRESS_STATUS_CHOICES: - if params['status'] in ip_status: - params['status'] = ip_status[0] - break - ip_address, created = IPAddress.objects.get_or_create(**params) if created: diff --git a/startup_scripts/__main__.py b/startup_scripts/__main__.py new file mode 100644 index 0000000..b13b8dd --- /dev/null +++ b/startup_scripts/__main__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import runpy +from os import scandir +from os.path import dirname, abspath + +this_dir = dirname(abspath(__file__)) + +def filename(f): + return f.name + +with scandir(dirname(abspath(__file__))) as it: + for f in sorted(it, key = filename): + if f.name.startswith('__') or not f.is_file(): + continue + + print(f"Running {f.path}") + runpy.run_path(f.path) diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..26de625 --- /dev/null +++ b/test.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# exit when a command exits with an exit code != 0 +set -e + +# version is used by `docker-compose.yml` do determine the tag +# of the Docker Image that is to be used +export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" + +if [ -z "${IMAGE}" ]; then + echo "⚠️ No image defined" + + if [ -z "${DEBUG}" ]; then + exit 1; + else + echo "⚠️ Would 'exit 1' here, but DEBUG is '${DEBUG}'." + fi +fi + +# The docker compose command to use +doco="docker-compose -f docker-compose.test.yml" + +INITIALIZERS_DIR=".initializers" + +test_setup() { + echo "🏗 Setup up test environment" + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" + fi + + mkdir "${INITIALIZERS_DIR}" + ( + cd initializers + for script in *.yml; do + sed -E 's/^# //' "${script}" > "../${INITIALIZERS_DIR}/${script}" + done + ) +} + +test_netbox_unit_tests() { + echo "⏱ Running Netbox Unit Tests" + $doco run --rm netbox ./manage.py test +} + +test_initializers() { + echo "🏭 Testing Initializers" + export INITIALIZERS_DIR + $doco run --rm netbox ./manage.py check +} + +test_cleanup() { + echo "💣 Cleaning Up" + $doco down -v + + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" + fi +} + +echo "🐳🐳🐳 Start testing '${IMAGE}'" + +# Make sure the cleanup script is executed +trap test_cleanup EXIT ERR +test_setup + +test_netbox_unit_tests +test_initializers + +echo "🐳🐳🐳 Done testing '${IMAGE}'"