From a9232e4f305fd4273a37cff615a5e4e47a9ad006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= Date: Fri, 22 May 2020 17:37:14 +0200 Subject: [PATCH 01/36] Add optional primary_ip on virtual machines initializer --- startup_scripts/230_virtual_machines.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/startup_scripts/230_virtual_machines.py b/startup_scripts/230_virtual_machines.py index 6776c4d..4236fa4 100644 --- a/startup_scripts/230_virtual_machines.py +++ b/startup_scripts/230_virtual_machines.py @@ -1,4 +1,5 @@ from dcim.models import Site, Platform, DeviceRole +from ipam.models import IPAddress from virtualization.models import Cluster, VirtualMachine from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue @@ -17,7 +18,9 @@ required_assocs = { optional_assocs = { 'tenant': (Tenant, 'name'), 'platform': (Platform, 'name'), - 'role': (DeviceRole, 'name') + 'role': (DeviceRole, 'name'), + 'primary_ip4': (IPAddress, 'address'), + 'primary_ip6': (IPAddress, 'address') } for params in virtual_machines: From df72d6bbb56a19807b14adce56b7e0e9ca0073ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= Date: Thu, 4 Jun 2020 20:46:40 +0200 Subject: [PATCH 02/36] Handle primary IPs on devices and VMs --- initializers/devices.yml | 2 ++ initializers/virtual_machines.yml | 2 ++ startup_scripts/130_devices.py | 8 ++--- startup_scripts/230_virtual_machines.py | 8 ++--- startup_scripts/270_primary_ips.py | 43 +++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 startup_scripts/270_primary_ips.py diff --git a/initializers/devices.yml b/initializers/devices.yml index 708b68c..e968503 100644 --- a/initializers/devices.yml +++ b/initializers/devices.yml @@ -29,6 +29,8 @@ # rack: rack-02 # face: front # position: 2 +# primary_ip4: 10.1.1.2/24 +# primary_ip6: 2001:db8:a000:1::2/64 # custom_fields: # text_field: Description # - name: server03 diff --git a/initializers/virtual_machines.yml b/initializers/virtual_machines.yml index 2122920..918df93 100644 --- a/initializers/virtual_machines.yml +++ b/initializers/virtual_machines.yml @@ -21,6 +21,8 @@ # memory: 2048 # name: virtual machine 2 # platform: Platform 2 +# primary_ip4: 10.1.1.10/24 +# primary_ip6: 2001:db8:a000:1::10/64 # status: active # tenant: tenant1 # vcpus: 8 diff --git a/startup_scripts/130_devices.py b/startup_scripts/130_devices.py index 1e0d894..7233dd0 100644 --- a/startup_scripts/130_devices.py +++ b/startup_scripts/130_devices.py @@ -1,5 +1,4 @@ from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform -from ipam.models import IPAddress from virtualization.models import Cluster from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue @@ -21,13 +20,14 @@ optional_assocs = { 'tenant': (Tenant, 'name'), 'platform': (Platform, 'name'), 'rack': (Rack, 'name'), - 'cluster': (Cluster, 'name'), - 'primary_ip4': (IPAddress, 'address'), - 'primary_ip6': (IPAddress, 'address') + 'cluster': (Cluster, 'name') } for params in devices: custom_fields = params.pop('custom_fields', None) + # primary ips are handled later in `270_primary_ips.py` + params.pop('primary_ip4', None) + params.pop('primary_ip6', None) for assoc, details in required_assocs.items(): model, field = details diff --git a/startup_scripts/230_virtual_machines.py b/startup_scripts/230_virtual_machines.py index 4236fa4..f138886 100644 --- a/startup_scripts/230_virtual_machines.py +++ b/startup_scripts/230_virtual_machines.py @@ -1,5 +1,4 @@ from dcim.models import Site, Platform, DeviceRole -from ipam.models import IPAddress from virtualization.models import Cluster, VirtualMachine from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue @@ -18,13 +17,14 @@ required_assocs = { optional_assocs = { 'tenant': (Tenant, 'name'), 'platform': (Platform, 'name'), - 'role': (DeviceRole, 'name'), - 'primary_ip4': (IPAddress, 'address'), - 'primary_ip6': (IPAddress, 'address') + 'role': (DeviceRole, 'name') } for params in virtual_machines: custom_fields = params.pop('custom_fields', None) + # primary ips are handled later in `270_primary_ips.py` + params.pop('primary_ip4', None) + params.pop('primary_ip6', None) for assoc, details in required_assocs.items(): model, field = details diff --git a/startup_scripts/270_primary_ips.py b/startup_scripts/270_primary_ips.py new file mode 100644 index 0000000..2ddab81 --- /dev/null +++ b/startup_scripts/270_primary_ips.py @@ -0,0 +1,43 @@ +from dcim.models import Device +from ipam.models import IPAddress +from virtualization.models import VirtualMachine +from startup_script_utils import load_yaml +import sys + +def link_primary_ip(assets, asset_model): + for params in assets: + primary_ip_fields = set(params) & {'primary_ip4', 'primary_ip6'} + if not primary_ip_fields: + continue + + for assoc, details in optional_assocs.items(): + if assoc in params: + model, field = details + query = { field: params.pop(assoc) } + + try: + params[assoc] = model.objects.get(**query) + except model.DoesNotExist: + primary_ip_fields -= {assoc} + print(f"⚠️ IP Address {query[field]} not found") + + asset = asset_model.objects.get(name=params['name']) + for field in primary_ip_fields: + if getattr(asset, field) != params[field]: + setattr(asset, field, params[field]) + print(f"🧬 Linked primary IP {params[field].address} on {asset.name}") + asset.save() + +devices = load_yaml('/opt/netbox/initializers/devices.yml') +virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml') + +if devices is None and virtual_machines is None: + sys.exit() + +optional_assocs = { + 'primary_ip4': (IPAddress, 'address'), + 'primary_ip6': (IPAddress, 'address') +} + +link_primary_ip(devices, Device) +link_primary_ip(virtual_machines, VirtualMachine) From a68f315acb31e738e17d08c745dfa0575cf92ca9 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Mon, 24 Aug 2020 14:39:50 +0200 Subject: [PATCH 03/36] Move default source image to Python 3.8 --- .github/workflows/push.yml | 3 +-- build.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d803cbc..da397de 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,8 +18,7 @@ jobs: - ./build-next.sh - ./build.sh develop docker_from: - - '' # use the default of the DOCKERFILE - - python:3.7-alpine + - '' # use the default of the build script - python:3.8-alpine # - python:3.9-rc-alpine # disable until Netbox's unit tests work fail-fast: false diff --git a/build.sh b/build.sh index 7489ebb..be65bd4 100755 --- a/build.sh +++ b/build.sh @@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" echo " DOCKER_FROM The base image to use." - echo " Default: 'python:3.7-alpine'" + echo " Default: 'python:3.8-alpine'" echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." echo " Default: main ldap" @@ -157,7 +157,7 @@ fi # Determining the value for DOCKER_FROM ### if [ -z "$DOCKER_FROM" ]; then - DOCKER_FROM="python:3.7-alpine" + DOCKER_FROM="python:3.8-alpine" fi ### From 74833a9b2137f46d9dfe50bb5def641118ba6add Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Tue, 1 Sep 2020 13:34:42 +0200 Subject: [PATCH 04/36] Update images in docker-compose --- docker-compose.test.yml | 8 ++++---- docker-compose.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 5ccd92a..23f74ca 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -19,7 +19,7 @@ services: - netbox-media-files:/opt/netbox/netbox/media:z nginx: command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.17-alpine + image: nginx:1.19-alpine depends_on: - netbox ports: @@ -28,17 +28,17 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:11-alpine + image: postgres:12-alpine env_file: env/postgres.env redis: - image: redis:5-alpine + image: redis:6-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis.env redis-cache: - image: redis:5-alpine + image: redis:6-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env diff --git a/docker-compose.yml b/docker-compose.yml index 9272b91..001694c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: - rqworker nginx: command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.17-alpine + image: nginx:1.19-alpine depends_on: - netbox ports: @@ -38,12 +38,12 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:11-alpine + image: postgres:12-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data redis: - image: redis:5-alpine + image: redis:6-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env @@ -52,7 +52,7 @@ services: volumes: - netbox-redis-data:/data redis-cache: - image: redis:5-alpine + image: redis:6-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env From 32cef1dcf8495f7460f12b3a67ae889e5dde7e4c Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Tue, 1 Sep 2020 13:52:01 +0200 Subject: [PATCH 05/36] Enable test on Python 3.9 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index da397de..243aa68 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,7 +20,7 @@ jobs: docker_from: - '' # use the default of the build script - python:3.8-alpine - # - python:3.9-rc-alpine # disable until Netbox's unit tests work + - python:3.9-rc-alpine fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From 1133ef50debd8e02db7e42137f44d512ca799d6a Mon Sep 17 00:00:00 2001 From: Eric Geldmacher Date: Tue, 15 Sep 2020 15:38:24 -0500 Subject: [PATCH 06/36] Fix sites.yml initializer --- initializers/sites.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/initializers/sites.yml b/initializers/sites.yml index 2381f99..f3e05ba 100644 --- a/initializers/sites.yml +++ b/initializers/sites.yml @@ -1,7 +1,7 @@ # - name: AMS 1 # slug: ams1 # region: Downtown -# status: 1 +# status: active # facility: Amsterdam 1 # asn: 12345 # custom_fields: @@ -9,7 +9,7 @@ # - name: AMS 2 # slug: ams2 # region: Downtown -# status: 1 +# status: active # facility: Amsterdam 2 # asn: 54321 # custom_fields: @@ -17,7 +17,7 @@ # - name: AMS 3 # slug: ams3 # region: Suburbs -# status: 1 +# status: active # facility: Amsterdam 3 # asn: 67890 # custom_fields: @@ -25,7 +25,7 @@ # - name: SING 1 # slug: sing1 # region: Singapore -# status: 1 +# status: active # facility: Singapore 1 # asn: 09876 # custom_fields: From f553d176552341f6c3d5684293a73ae2320f18ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 11:31:48 +0200 Subject: [PATCH 07/36] Change log message --- startup_scripts/270_primary_ips.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/startup_scripts/270_primary_ips.py b/startup_scripts/270_primary_ips.py index 2ddab81..74acece 100644 --- a/startup_scripts/270_primary_ips.py +++ b/startup_scripts/270_primary_ips.py @@ -19,13 +19,13 @@ def link_primary_ip(assets, asset_model): params[assoc] = model.objects.get(**query) except model.DoesNotExist: primary_ip_fields -= {assoc} - print(f"⚠️ IP Address {query[field]} not found") + print(f"⚠️ IP Address '{query[field]}' not found") asset = asset_model.objects.get(name=params['name']) for field in primary_ip_fields: if getattr(asset, field) != params[field]: setattr(asset, field, params[field]) - print(f"🧬 Linked primary IP {params[field].address} on {asset.name}") + print(f"🔗 Define primary IP '{params[field].address}' on '{asset.name}'") asset.save() devices = load_yaml('/opt/netbox/initializers/devices.yml') From cad803ed31ba9f482aff9072a67c14706ca3ed8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 19 Oct 2020 16:50:04 +0200 Subject: [PATCH 08/36] Create FUNDING.yml to enable sponsoring --- .github/FUNDING.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..acdb4e0 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,14 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +- cimnine +- tobiasge +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 9f4a9f528ce7163e8ebaa73963d6e5308a7aad9a Mon Sep 17 00:00:00 2001 From: Alex Houlton Date: Mon, 19 Oct 2020 07:50:33 -0700 Subject: [PATCH 09/36] add option to mirror ldap groups into netbox this commit allows ldap assigned groups to be mirrored into netbox users. The default is None as this is not the primary way to do this change. --- configuration/ldap_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/ldap_config.py b/configuration/ldap_config.py index 19277e1..23a1e6d 100644 --- a/configuration/ldap_config.py +++ b/configuration/ldap_config.py @@ -71,6 +71,7 @@ AUTH_LDAP_USER_FLAGS_BY_GROUP = { # For more granular permissions, we can map LDAP groups to Django groups. AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' +AUTH_LDAP_MIRROR_GROUPS = os.environ.get('AUTH_LDAP_MIRROR_GROUPS', None).lower() == 'true' # Cache groups for one hour to reduce LDAP traffic AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) From 38f4474b14f25c790d8d1928cbcde93b966df8ef Mon Sep 17 00:00:00 2001 From: Shuichiro MAKIGAKI Date: Mon, 8 Jun 2020 15:02:39 +0900 Subject: [PATCH 10/36] Add REMOTE_AUTH_* support --- configuration/configuration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/configuration/configuration.py b/configuration/configuration.py index 404b6a0..6e5ff19 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -203,6 +203,13 @@ REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports') # this setting is derived from the installed location. SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') +# Remote authentication support +REMOTE_AUTH_ENABLED = os.environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' +REMOTE_AUTH_BACKEND = os.environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') +REMOTE_AUTH_HEADER = os.environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = os.environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' +REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, os.environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) + # Time zone (default: UTC) TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') From 6bada6660a499a81086de8413239dee31bf9c47d Mon Sep 17 00:00:00 2001 From: eemil Date: Wed, 13 May 2020 15:44:41 +0300 Subject: [PATCH 11/36] Prefer secret to env variable if both are configured --- configuration/configuration.py | 16 ++++++++-------- configuration/ldap_config.py | 6 +++--- docker/docker-entrypoint.sh | 20 ++++++++------------ 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index 6e5ff19..0081072 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -6,11 +6,11 @@ import socket # Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py # Read secret from file -def read_secret(secret_name): +def read_secret(secret_name, default=''): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: - return '' + return default else: with f: return f.readline().strip() @@ -33,7 +33,7 @@ ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(' ') DATABASE = { 'NAME': os.environ.get('DB_NAME', 'netbox'), # Database name 'USER': os.environ.get('DB_USER', ''), # PostgreSQL username - 'PASSWORD': os.environ.get('DB_PASSWORD', read_secret('db_password')), + 'PASSWORD': read_secret('db_password', os.environ.get('DB_PASSWORD', '')), # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) @@ -47,7 +47,7 @@ DATABASE = { # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key')) +SECRET_KEY = read_secret('secret_key', os.environ.get('SECRET_KEY', '')) # Redis database settings. The Redis database is used for caching and background processing such as webhooks REDIS = { @@ -62,7 +62,7 @@ REDIS = { 'webhooks': { # legacy setting, can be removed after Netbox seizes support for it 'HOST': os.environ.get('REDIS_HOST', 'localhost'), 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), + 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')), 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', @@ -70,7 +70,7 @@ REDIS = { 'caching': { 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), - 'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))), + 'PASSWORD': read_secret('redis_cache_password', os.environ.get('REDIS_CACHE_PASSWORD', read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')))), 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', @@ -124,7 +124,7 @@ EMAIL = { 'SERVER': os.environ.get('EMAIL_SERVER', 'localhost'), 'PORT': int(os.environ.get('EMAIL_PORT', 25)), 'USERNAME': os.environ.get('EMAIL_USERNAME', ''), - 'PASSWORD': os.environ.get('EMAIL_PASSWORD', read_secret('email_password')), + 'PASSWORD': read_secret('email_password', os.environ.get('EMAIL_PASSWORD', '')), 'TIMEOUT': int(os.environ.get('EMAIL_TIMEOUT', 10)), # seconds 'FROM_EMAIL': os.environ.get('EMAIL_FROM', ''), 'USE_SSL': os.environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', @@ -171,7 +171,7 @@ METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true' # Credentials that NetBox will use to access live devices. NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '') -NAPALM_PASSWORD = os.environ.get('NAPALM_PASSWORD', read_secret('napalm_password')) +NAPALM_PASSWORD = read_secret('napalm_password', os.environ.get('NAPALM_PASSWORD', '')) # NAPALM timeout (in seconds). (Default: 30) NAPALM_TIMEOUT = int(os.environ.get('NAPALM_TIMEOUT', 30)) diff --git a/configuration/ldap_config.py b/configuration/ldap_config.py index 23a1e6d..da6c6fe 100644 --- a/configuration/ldap_config.py +++ b/configuration/ldap_config.py @@ -5,11 +5,11 @@ from django_auth_ldap.config import LDAPSearch from importlib import import_module # Read secret from file -def read_secret(secret_name): +def read_secret(secret_name, default=''): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: - return '' + return default else: with f: return f.readline().strip() @@ -32,7 +32,7 @@ AUTH_LDAP_CONNECTION_OPTIONS = { # Set the DN and password for the NetBox service account. AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '') -AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', read_secret('auth_ldap_bind_password')) +AUTH_LDAP_BIND_PASSWORD = read_secret('auth_ldap_bind_password', os.environ.get('AUTH_LDAP_BIND_PASSWORD', '')) # Set a string template that describes any user’s distinguished name based on the username. AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index f555695..4887d96 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -31,19 +31,15 @@ else if [ -z ${SUPERUSER_EMAIL+x} ]; then SUPERUSER_EMAIL='admin@example.com' fi - if [ -z ${SUPERUSER_PASSWORD+x} ]; then - if [ -f "/run/secrets/superuser_password" ]; then - SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)" - else - SUPERUSER_PASSWORD='admin' - fi + if [ -f "/run/secrets/superuser_password" ]; then + SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)" + elif [ -z ${SUPERUSER_PASSWORD+x} ]; then + SUPERUSER_PASSWORD='admin' fi - if [ -z ${SUPERUSER_API_TOKEN+x} ]; then - if [ -f "/run/secrets/superuser_api_token" ]; then - SUPERUSER_API_TOKEN="$(< /run/secrets/superuser_api_token)" - else - SUPERUSER_API_TOKEN='0123456789abcdef0123456789abcdef01234567' - fi + if [ -f "/run/secrets/superuser_api_token" ]; then + SUPERUSER_API_TOKEN="$(< /run/secrets/superuser_api_token)" + elif [ -z ${SUPERUSER_API_TOKEN+x} ]; then + SUPERUSER_API_TOKEN='0123456789abcdef0123456789abcdef01234567' fi ./manage.py shell --interface python << END From 85fbb0af7046a8f3784c192eae0b716b4e11fe23 Mon Sep 17 00:00:00 2001 From: eemil Date: Thu, 14 May 2020 17:46:26 +0000 Subject: [PATCH 12/36] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christian Mäder --- configuration/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index 0081072..ae777e4 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -62,7 +62,7 @@ REDIS = { 'webhooks': { # legacy setting, can be removed after Netbox seizes support for it 'HOST': os.environ.get('REDIS_HOST', 'localhost'), 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')), + 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')), 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', From 9287995df4ad5f126514c274895935c9c785d267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 19:22:52 +0200 Subject: [PATCH 13/36] Update to latest configuration --- configuration/configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index ae777e4..f7c8ed5 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -6,7 +6,7 @@ import socket # Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py # Read secret from file -def read_secret(secret_name, default=''): +def read_secret(secret_name, default=None): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: @@ -54,7 +54,7 @@ REDIS = { 'tasks': { 'HOST': os.environ.get('REDIS_HOST', 'localhost'), 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), + 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', ''), 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', @@ -62,7 +62,7 @@ REDIS = { 'webhooks': { # legacy setting, can be removed after Netbox seizes support for it 'HOST': os.environ.get('REDIS_HOST', 'localhost'), 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')), + 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', ''), 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', From fd3d3d11d1f00dc28bc136afa1cccfd94419f4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 17 Oct 2020 23:09:26 +0200 Subject: [PATCH 14/36] Tiny refactoring --- configuration/configuration.py | 1 - startup_scripts/__main__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index f7c8ed5..7230927 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -1,6 +1,5 @@ import os import re -import socket # For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/ # Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py diff --git a/startup_scripts/__main__.py b/startup_scripts/__main__.py index c41bba6..dc4cdc9 100644 --- a/startup_scripts/__main__.py +++ b/startup_scripts/__main__.py @@ -9,7 +9,7 @@ this_dir = dirname(abspath(__file__)) def filename(f): return f.name -with scandir(dirname(abspath(__file__))) as it: +with scandir(this_dir) as it: for f in sorted(it, key = filename): if not f.is_file(): continue From 43c05d816df73a9a4268543aa5691432379cd91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:34:45 +0200 Subject: [PATCH 15/36] Dynamically load configuration files --- Dockerfile | 7 +-- configuration/{ => ldap}/ldap_config.py | 0 docker/configuration.docker.py | 62 ++++++++++++++++++-- {configuration => docker}/gunicorn_config.py | 0 docker/ldap_config.docker.py | 62 ++++++++++++++++++-- 5 files changed, 117 insertions(+), 14 deletions(-) rename configuration/{ => ldap}/ldap_config.py (100%) rename {configuration => docker}/gunicorn_config.py (100%) diff --git a/Dockerfile b/Dockerfile index 4347aa8..8db4cac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,12 +61,12 @@ ARG NETBOX_PATH COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py -COPY configuration/gunicorn_config.py /etc/netbox/config/ +COPY docker/gunicorn_config.py /etc/netbox/ COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh COPY startup_scripts/ /opt/netbox/startup_scripts/ COPY initializers/ /opt/netbox/initializers/ -COPY configuration/configuration.py /etc/netbox/config/configuration.py +COPY configuration/ /etc/netbox/config/ WORKDIR /opt/netbox/netbox @@ -79,7 +79,7 @@ RUN mkdir static && chmod -R g+w static media ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] -CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] +CMD ["gunicorn", "-c /etc/netbox/gunicorn_config.py", "netbox.wsgi"] LABEL ORIGINAL_TAG="" \ NETBOX_GIT_BRANCH="" \ @@ -122,4 +122,3 @@ RUN apk add --no-cache \ util-linux COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py -COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/configuration/ldap_config.py b/configuration/ldap/ldap_config.py similarity index 100% rename from configuration/ldap_config.py rename to configuration/ldap/ldap_config.py diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index 733887f..3a4d2a2 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -1,10 +1,62 @@ +from os.path import abspath, isfile +from os import scandir import importlib.util import sys -try: - spec = importlib.util.spec_from_file_location('configuration', '/etc/netbox/config/configuration.py') +_CONFIG_DIR = '/etc/netbox/config/' +_MAIN_CONFIG = 'configuration' +_MODULE = 'netbox.configuration' +_loaded_configurations = [] + + +def __getattr__(name): + for config in _loaded_configurations: + try: + return getattr(config, name) + except: + pass + raise AttributeError + + +def _filename(f): + return f.name + + +def _import(module_name, path): + spec = importlib.util.spec_from_file_location('', path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - sys.modules['netbox.configuration'] = module -except: - raise ImportError('') + sys.modules[module_name] = module + + _loaded_configurations.insert(0, module) + + print(f"🧬 loaded config '{path}'") + + +_main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') +if isfile(_main_config_path): + _import(f'{_MODULE}.configuration', _main_config_path) +else: + print(f"⚠️ Main configuration '{_main_config_path}' not found.") + +with scandir(_CONFIG_DIR) as it: + for f in sorted(it, key=_filename): + if not f.is_file(): + continue + + if f.name.startswith('__'): + continue + + if not f.name.endswith('.py'): + continue + + if f.name == f'{_MAIN_CONFIG}.py': + continue + + module_name = f"{_MODULE}.{f.name[:-len('.py')]}" + + _import(module_name, f.path) + +if len(_loaded_configurations) == 0: + print(f"‼️ No configuration files found in '{_CONFIG_DIR}'.") + raise ImportError(f"No configuration files found in '{_CONFIG_DIR}'.") diff --git a/configuration/gunicorn_config.py b/docker/gunicorn_config.py similarity index 100% rename from configuration/gunicorn_config.py rename to docker/gunicorn_config.py diff --git a/docker/ldap_config.docker.py b/docker/ldap_config.docker.py index 8d82173..601ba71 100644 --- a/docker/ldap_config.docker.py +++ b/docker/ldap_config.docker.py @@ -1,10 +1,62 @@ +from os.path import abspath, isfile +from os import scandir import importlib.util import sys -try: - spec = importlib.util.spec_from_file_location('ldap_config', '/etc/netbox/config/ldap_config.py') +_CONFIG_DIR = '/etc/netbox/config/ldap/' +_MAIN_CONFIG = 'ldap_config' +_MODULE = 'netbox.configuration.ldap' +_loaded_configurations = [] + + +def __getattr__(name): + for config in _loaded_configurations: + try: + return getattr(config, name) + except: + pass + raise AttributeError + + +def _filename(f): + return f.name + + +def _import(module_name, path): + spec = importlib.util.spec_from_file_location('', path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - sys.modules['netbox.ldap_config'] = module -except: - raise ImportError('') + sys.modules[module_name] = module + + _loaded_configurations.insert(0, module) + + print(f"🧬 loaded config '{path}'") + + +_main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') +if isfile(_main_config_path): + _import(f'{_MODULE}.configuration', _main_config_path) +else: + print(f"⚠️ Main configuration '{_main_config_path}' not found.") + +with scandir(_CONFIG_DIR) as it: + for f in sorted(it, key=_filename): + if not f.is_file(): + continue + + if f.name.startswith('__'): + continue + + if not f.name.endswith('.py'): + continue + + if f.name == f'{_MAIN_CONFIG}.py': + continue + + module_name = f"{_MODULE}.{f.name[:-len('.py')]}" + + _import(module_name, f.path) + +if len(_loaded_configurations) == 0: + print(f"‼️ No configuration files found in '{_CONFIG_DIR}'.") + raise ImportError(f"No configuration files found in '{_CONFIG_DIR}'.") From 7429835970773416d6a0841261e65c5ddc85f9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:34:57 +0200 Subject: [PATCH 16/36] Sample dynamic configuration file --- configuration/example.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 configuration/example.py diff --git a/configuration/example.py b/configuration/example.py new file mode 100644 index 0000000..df74559 --- /dev/null +++ b/configuration/example.py @@ -0,0 +1,49 @@ +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + + +# URL schemes that are allowed within links in NetBox +# ALLOWED_URL_SCHEMES = ( +# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +# ) + + +# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +# NAPALM_ARGS = {} + + +# Enable installed plugins. Add the name of each plugin to the list. +# from netbox.configuration.configuration import PLUGINS +# PLUGINS.append('my_plugin') + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# from netbox.configuration.configuration import PLUGINS_CONFIG +# PLUGINS_CONFIG['my_plugin'] = { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } + + +# Remote authentication support +# REMOTE_AUTH_ENABLED = False +# REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +# REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' +# REMOTE_AUTH_AUTO_CREATE_USER = True +# REMOTE_AUTH_DEFAULT_GROUPS = [] +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + + +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } From 811618b3591c7fac0f6441eff3319e6ecdd1c480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:35:09 +0200 Subject: [PATCH 17/36] Lower gunicorn default loglevel --- docker/gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/gunicorn_config.py b/docker/gunicorn_config.py index 063d822..4ce7763 100644 --- a/docker/gunicorn_config.py +++ b/docker/gunicorn_config.py @@ -5,4 +5,4 @@ workers = 3 errorlog = '-' accesslog = '-' capture_output = False -loglevel = 'debug' +loglevel = 'info' From 553c8ea47026bb8aa8dfa9797aa3cfd1fd02d3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:35:22 +0200 Subject: [PATCH 18/36] Update configuration.py --- configuration/configuration.py | 228 ++++++++++++++++++--------------- 1 file changed, 128 insertions(+), 100 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index 7230927..5a9da1e 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -1,11 +1,13 @@ -import os import re -# For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/ -# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py +from os.path import dirname, abspath, join +from os import environ + +# For reference see https://netbox.readthedocs.io/en/stable/configuration/ +# Based on https://github.com/netbox-community/netbox/blob/master/netbox/netbox/configuration.example.py # Read secret from file -def read_secret(secret_name, default=None): +def _read_secret(secret_name, default = None): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: @@ -14,7 +16,7 @@ def read_secret(secret_name, default=None): with f: return f.readline().strip() -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_BASE_DIR = dirname(dirname(abspath(__file__))) ######################### # # @@ -26,55 +28,49 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(' ') +ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ') -# PostgreSQL database configuration. +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases DATABASE = { - 'NAME': os.environ.get('DB_NAME', 'netbox'), # Database name - 'USER': os.environ.get('DB_USER', ''), # PostgreSQL username - 'PASSWORD': read_secret('db_password', os.environ.get('DB_PASSWORD', '')), + 'NAME': environ.get('DB_NAME', 'netbox'), # Database name + 'USER': environ.get('DB_USER', ''), # PostgreSQL username + 'PASSWORD': environ.get('DB_PASSWORD', _read_secret('db_password')), # PostgreSQL password - 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server - 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) - 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + 'HOST': environ.get('DB_HOST', 'localhost'), # Database server + 'PORT': environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')}, # Database connection SSLMODE - 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), - # Database connection persistence + 'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')), + # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(environ.get('REDIS_PORT', 6379)), + 'PASSWORD': environ.get('REDIS_PASSWORD', _read_secret('redis_password')), + 'DATABASE': int(environ.get('REDIS_DATABASE', 0)), + 'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))), + 'PASSWORD': environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', _read_secret('redis_cache_password'))), + 'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)), + 'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true', + }, } # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see -# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = read_secret('secret_key', os.environ.get('SECRET_KEY', '')) +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = environ.get('SECRET_KEY', _read_secret('secret_key')) -# Redis database settings. The Redis database is used for caching and background processing such as webhooks -REDIS = { - 'tasks': { - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', ''), - 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), - 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', - }, - 'webhooks': { # legacy setting, can be removed after Netbox seizes support for it - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': read_secret('redis_password', os.environ.get('REDIS_PASSWORD', ''), - 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), - 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', - }, - 'caching': { - 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), - 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), - 'PASSWORD': read_secret('redis_cache_password', os.environ.get('REDIS_CACHE_PASSWORD', read_secret('redis_password', os.environ.get('REDIS_PASSWORD', '')))), - 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), - 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), - 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', - }, -} ######################### # # @@ -88,135 +84,167 @@ ADMINS = [ # ['John Doe', 'jdoe@example.com'], ] +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. -BANNER_TOP = os.environ.get('BANNER_TOP', '') -BANNER_BOTTOM = os.environ.get('BANNER_BOTTOM', '') +BANNER_TOP = environ.get('BANNER_TOP', '') +BANNER_BOTTOM = environ.get('BANNER_BOTTOM', '') # Text to include on the login page above the login form. HTML is allowed. -BANNER_LOGIN = os.environ.get('BANNER_LOGIN', '') +BANNER_LOGIN = environ.get('BANNER_LOGIN', '') # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = os.environ.get('BASE_PATH', '') +BASE_PATH = environ.get('BASE_PATH', '') # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) -CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT', 900)) +CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900)) # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) -CHANGELOG_RETENTION = int(os.environ.get('CHANGELOG_RETENTION', 90)) +CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90)) # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers -CORS_ORIGIN_ALLOW_ALL = os.environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true' -CORS_ORIGIN_WHITELIST = list(filter(None, os.environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) -CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, os.environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] +CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true' +CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) +CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. -DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true' +DEBUG = environ.get('DEBUG', 'False').lower() == 'true' # Email settings EMAIL = { - 'SERVER': os.environ.get('EMAIL_SERVER', 'localhost'), - 'PORT': int(os.environ.get('EMAIL_PORT', 25)), - 'USERNAME': os.environ.get('EMAIL_USERNAME', ''), - 'PASSWORD': read_secret('email_password', os.environ.get('EMAIL_PASSWORD', '')), - 'TIMEOUT': int(os.environ.get('EMAIL_TIMEOUT', 10)), # seconds - 'FROM_EMAIL': os.environ.get('EMAIL_FROM', ''), - 'USE_SSL': os.environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', - 'USE_TLS': os.environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', - 'SSL_CERTFILE': os.environ.get('EMAIL_SSL_CERTFILE', ''), - 'SSL_KEYFILE': os.environ.get('EMAIL_SSL_KEYFILE', ''), + 'SERVER': environ.get('EMAIL_SERVER', 'localhost'), + 'PORT': int(environ.get('EMAIL_PORT', 25)), + 'USERNAME': environ.get('EMAIL_USERNAME', ''), + 'PASSWORD': environ.get('EMAIL_PASSWORD', _read_secret('email_password')), + 'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', + 'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', + 'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''), + 'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''), + 'TIMEOUT': int(environ.get('EMAIL_TIMEOUT', 10)), # seconds + 'FROM_EMAIL': environ.get('EMAIL_FROM', ''), } -# Enforcement of unique IP space can be toggled on a per-VRF basis. -# To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF), -# set ENFORCE_GLOBAL_UNIQUE to True. -ENFORCE_GLOBAL_UNIQUE = os.environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true' +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true' # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. -EXEMPT_VIEW_PERMISSIONS = list(filter(None, os.environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' '))) +EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' '))) # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: -# https://docs.djangoproject.com/en/1.11/topics/logging/ +# https://docs.djangoproject.com/en/stable/topics/logging/ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = os.environ.get('LOGIN_REQUIRED', 'False').lower() == 'true' +LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true' # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to # re-authenticate. (Default: 1209600 [14 days]) -LOGIN_TIMEOUT = os.environ.get('LOGIN_TIMEOUT', None) +LOGIN_TIMEOUT = environ.get('LOGIN_TIMEOUT', None) # Setting this to True will display a "maintenance mode" banner at the top of every page. -MAINTENANCE_MODE = os.environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' +MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request # all objects by specifying "?limit=0". -MAX_PAGE_SIZE = int(os.environ.get('MAX_PAGE_SIZE', 1000)) +MAX_PAGE_SIZE = int(environ.get('MAX_PAGE_SIZE', 1000)) # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media')) # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' -METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true' +METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true' -# Credentials that NetBox will use to access live devices. -NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '') -NAPALM_PASSWORD = read_secret('napalm_password', os.environ.get('NAPALM_PASSWORD', '')) +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '') +NAPALM_PASSWORD = environ.get('NAPALM_PASSWORD', _read_secret('napalm_password')) # NAPALM timeout (in seconds). (Default: 30) -NAPALM_TIMEOUT = int(os.environ.get('NAPALM_TIMEOUT', 30)) +NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30)) # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must # be provided as a dictionary. NAPALM_ARGS = {} # Determine how many objects to display per page within a list. (Default: 50) -PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50)) +PAGINATE_COUNT = int(environ.get('PAGINATE_COUNT', 50)) + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = [] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +PLUGINS_CONFIG = { +} # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to # prefer IPv4 instead. -PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true' +PREFER_IPV4 = environ.get('PREFER_IPV4', 'False').lower() == 'true' -# This determines how often the GitHub API is called to check the latest release of NetBox in seconds. Must be at least 1 hour. -RELEASE_CHECK_TIMEOUT = os.environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600) +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', 22)) +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220)) + +# Remote authentication support +REMOTE_AUTH_ENABLED = False +# REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +# REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' +# REMOTE_AUTH_AUTO_CREATE_USER = True +# REMOTE_AUTH_DEFAULT_GROUPS = [] +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600)) # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. # https://api.github.com/repos/netbox-community/netbox/releases -RELEASE_CHECK_URL = os.environ.get('RELEASE_CHECK_URL', None) +RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None) # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports') +REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports') + +# Maximum execution time for background tasks, in seconds. +RQ_DEFAULT_TIMEOUT = int(environ.get('RQ_DEFAULT_TIMEOUT', 300)) # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') +SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = environ.get('REPORTS_ROOT', None) # Remote authentication support -REMOTE_AUTH_ENABLED = os.environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' -REMOTE_AUTH_BACKEND = os.environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') -REMOTE_AUTH_HEADER = os.environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') -REMOTE_AUTH_AUTO_CREATE_USER = os.environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' -REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, os.environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) +REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' +REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') +REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' +REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) # Time zone (default: UTC) -TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') +TIME_ZONE = environ.get('TIME_ZONE', 'UTC') # Date/time formatting. See the following link for supported formats: -# https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date -DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y') -SHORT_DATE_FORMAT = os.environ.get('SHORT_DATE_FORMAT', 'Y-m-d') -TIME_FORMAT = os.environ.get('TIME_FORMAT', 'g:i a') -SHORT_TIME_FORMAT = os.environ.get('SHORT_TIME_FORMAT', 'H:i:s') -DATETIME_FORMAT = os.environ.get('DATETIME_FORMAT', 'N j, Y g:i a') -SHORT_DATETIME_FORMAT = os.environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i') +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = environ.get('DATE_FORMAT', 'N j, Y') +SHORT_DATE_FORMAT = environ.get('SHORT_DATE_FORMAT', 'Y-m-d') +TIME_FORMAT = environ.get('TIME_FORMAT', 'g:i a') +SHORT_TIME_FORMAT = environ.get('SHORT_TIME_FORMAT', 'H:i:s') +DATETIME_FORMAT = environ.get('DATETIME_FORMAT', 'N j, Y g:i a') +SHORT_DATETIME_FORMAT = environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i') From 20820e10c25a833c9d9d71cb66472d85e35bf5c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:38:06 +0200 Subject: [PATCH 19/36] Optimize Imports in ldap_config.py --- configuration/ldap/ldap_config.py | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/configuration/ldap/ldap_config.py b/configuration/ldap/ldap_config.py index da6c6fe..b92a79a 100644 --- a/configuration/ldap/ldap_config.py +++ b/configuration/ldap/ldap_config.py @@ -1,11 +1,11 @@ import ldap -import os from django_auth_ldap.config import LDAPSearch from importlib import import_module +from os import environ # Read secret from file -def read_secret(secret_name, default=''): +def _read_secret(secret_name, default=None): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: @@ -15,7 +15,7 @@ def read_secret(secret_name, default=''): return f.readline().strip() # Import and return the group type based on string name -def import_group_type(group_type_name): +def _import_group_type(group_type_name): mod = import_module('django_auth_ldap.config') try: return getattr(mod, group_type_name)() @@ -23,7 +23,7 @@ def import_group_type(group_type_name): return None # Server URI -AUTH_LDAP_SERVER_URI = os.environ.get('AUTH_LDAP_SERVER_URI', '') +AUTH_LDAP_SERVER_URI = environ.get('AUTH_LDAP_SERVER_URI', '') # The following may be needed if you are binding to Active Directory. AUTH_LDAP_CONNECTION_OPTIONS = { @@ -31,54 +31,54 @@ AUTH_LDAP_CONNECTION_OPTIONS = { } # Set the DN and password for the NetBox service account. -AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '') -AUTH_LDAP_BIND_PASSWORD = read_secret('auth_ldap_bind_password', os.environ.get('AUTH_LDAP_BIND_PASSWORD', '')) +AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '') +AUTH_LDAP_BIND_PASSWORD = environ.get('AUTH_LDAP_BIND_PASSWORD', _read_secret('auth_ldap_bind_password')) # Set a string template that describes any user’s distinguished name based on the username. -AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) +AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) # Enable STARTTLS for ldap authentication. -AUTH_LDAP_START_TLS = os.environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true' +AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true' # Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert. # Note that this is a NetBox-specific setting which sets: # ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) -LDAP_IGNORE_CERT_ERRORS = os.environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' +LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' -AUTH_LDAP_USER_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') -AUTH_LDAP_USER_SEARCH_ATTR = os.environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') +AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') +AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)") # This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group # heirarchy. -AUTH_LDAP_GROUP_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '') -AUTH_LDAP_GROUP_SEARCH_CLASS = os.environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group') +AUTH_LDAP_GROUP_SEARCH_BASEDN = environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '') +AUTH_LDAP_GROUP_SEARCH_CLASS = environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group') AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")") -AUTH_LDAP_GROUP_TYPE = import_group_type(os.environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType')) +AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType')) # Define a group required to login. -AUTH_LDAP_REQUIRE_GROUP = os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '') +AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '') # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_active": os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''), - "is_staff": os.environ.get('AUTH_LDAP_IS_ADMIN_DN', ''), - "is_superuser": os.environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '') + "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''), + "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''), + "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '') } # For more granular permissions, we can map LDAP groups to Django groups. -AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' -AUTH_LDAP_MIRROR_GROUPS = os.environ.get('AUTH_LDAP_MIRROR_GROUPS', None).lower() == 'true' +AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' +AUTH_LDAP_MIRROR_GROUPS = environ.get('AUTH_LDAP_MIRROR_GROUPS', None).lower() == 'true' # Cache groups for one hour to reduce LDAP traffic -AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) +AUTH_LDAP_CACHE_TIMEOUT = int(environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) # Populate the Django user from the LDAP directory. AUTH_LDAP_USER_ATTR_MAP = { - "first_name": os.environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'), - "last_name": os.environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'), - "email": os.environ.get('AUTH_LDAP_ATTR_MAIL', 'mail') + "first_name": environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'), + "last_name": environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'), + "email": environ.get('AUTH_LDAP_ATTR_MAIL', 'mail') } From 16b567939c2742f9b2e8ed194c3944f6f3a2af5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 02:50:51 +0200 Subject: [PATCH 20/36] Dot character has special meaning in module names Therefore we esacpe it using the underscore character. --- docker/configuration.docker.py | 2 +- docker/ldap_config.docker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index 3a4d2a2..a47c353 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -53,7 +53,7 @@ with scandir(_CONFIG_DIR) as it: if f.name == f'{_MAIN_CONFIG}.py': continue - module_name = f"{_MODULE}.{f.name[:-len('.py')]}" + module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") _import(module_name, f.path) diff --git a/docker/ldap_config.docker.py b/docker/ldap_config.docker.py index 601ba71..7248195 100644 --- a/docker/ldap_config.docker.py +++ b/docker/ldap_config.docker.py @@ -53,7 +53,7 @@ with scandir(_CONFIG_DIR) as it: if f.name == f'{_MAIN_CONFIG}.py': continue - module_name = f"{_MODULE}.{f.name[:-len('.py')]}" + module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") _import(module_name, f.path) From dc77f1a0b170e336328d45960a79ffd553fc2a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 20:03:35 +0200 Subject: [PATCH 21/36] Ignore non-default configuration files --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 3389e7a..22b3cee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ .initializers docker-compose.override.yml *.pem +configuration/* +!configuration/configuration.py +!configuration/example.py +configuration/ldap/* +!configuration/ldap/ldap_config.py From dad06088348c1a061331eedd000489d3496f2f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 20:11:10 +0200 Subject: [PATCH 22/36] Rename example.py to extra.py And add some comments to the the file that hopefully guide the user. --- .gitignore | 2 +- configuration/configuration.py | 6 ++++ configuration/example.py | 49 ------------------------------ configuration/extra.py | 55 ++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 50 deletions(-) delete mode 100644 configuration/example.py create mode 100644 configuration/extra.py diff --git a/.gitignore b/.gitignore index 22b3cee..8e90907 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,6 @@ docker-compose.override.yml *.pem configuration/* !configuration/configuration.py -!configuration/example.py +!configuration/extra.py configuration/ldap/* !configuration/ldap/ldap_config.py diff --git a/configuration/configuration.py b/configuration/configuration.py index 5a9da1e..4d1a146 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -1,3 +1,9 @@ +#### +## We recommend to not edit this file. +## Create separate files to overwrite the settings. +## See `extra.py` as an example. +#### + import re from os.path import dirname, abspath, join diff --git a/configuration/example.py b/configuration/example.py deleted file mode 100644 index df74559..0000000 --- a/configuration/example.py +++ /dev/null @@ -1,49 +0,0 @@ -# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of -# application errors (assuming correct email settings are provided). -# ADMINS = [ -# # ['John Doe', 'jdoe@example.com'], -# ] - - -# URL schemes that are allowed within links in NetBox -# ALLOWED_URL_SCHEMES = ( -# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', -# ) - - -# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must -# be provided as a dictionary. -# NAPALM_ARGS = {} - - -# Enable installed plugins. Add the name of each plugin to the list. -# from netbox.configuration.configuration import PLUGINS -# PLUGINS.append('my_plugin') - -# Plugins configuration settings. These settings are used by various plugins that the user may have installed. -# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. -# from netbox.configuration.configuration import PLUGINS_CONFIG -# PLUGINS_CONFIG['my_plugin'] = { -# 'foo': 'bar', -# 'buzz': 'bazz' -# } - - -# Remote authentication support -# REMOTE_AUTH_ENABLED = False -# REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' -# REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' -# REMOTE_AUTH_AUTO_CREATE_USER = True -# REMOTE_AUTH_DEFAULT_GROUPS = [] -# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} - - -# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the -# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: -# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_CONFIG = { -# 'AWS_ACCESS_KEY_ID': 'Key ID', -# 'AWS_SECRET_ACCESS_KEY': 'Secret', -# 'AWS_STORAGE_BUCKET_NAME': 'netbox', -# 'AWS_S3_REGION_NAME': 'eu-west-1', -# } diff --git a/configuration/extra.py b/configuration/extra.py new file mode 100644 index 0000000..46f1877 --- /dev/null +++ b/configuration/extra.py @@ -0,0 +1,55 @@ +#### +## This file contains extra configuration options that can't be configured +## directly through environment variables. +#### + +## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +## application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + + +## URL schemes that are allowed within links in NetBox +# ALLOWED_URL_SCHEMES = ( +# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +# ) + + +## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +## be provided as a dictionary. +# NAPALM_ARGS = {} + + +## Enable installed plugins. Add the name of each plugin to the list. +# from netbox.configuration.configuration import PLUGINS +# PLUGINS.append('my_plugin') + +## Plugins configuration settings. These settings are used by various plugins that the user may have installed. +## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# from netbox.configuration.configuration import PLUGINS_CONFIG +# PLUGINS_CONFIG['my_plugin'] = { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } + + +## Remote authentication support +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + + +## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + + +## This file can contain arbitrary Python code, e.g.: +# from datetime import datetime +# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") +# BANNER_TOP = f'This instance started on {now}.' From 75554ef5b48dbb67df5eacd76bb24f47c8ac7617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 21:29:08 +0200 Subject: [PATCH 23/36] Consistency in namespace name --- docker/configuration.docker.py | 2 +- docker/ldap_config.docker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index a47c353..68a12e4 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -35,7 +35,7 @@ def _import(module_name, path): _main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') if isfile(_main_config_path): - _import(f'{_MODULE}.configuration', _main_config_path) + _import(f'{_MODULE}.{_MAIN_CONFIG}', _main_config_path) else: print(f"⚠️ Main configuration '{_main_config_path}' not found.") diff --git a/docker/ldap_config.docker.py b/docker/ldap_config.docker.py index 7248195..ca8329d 100644 --- a/docker/ldap_config.docker.py +++ b/docker/ldap_config.docker.py @@ -35,7 +35,7 @@ def _import(module_name, path): _main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') if isfile(_main_config_path): - _import(f'{_MODULE}.configuration', _main_config_path) + _import(f'{_MODULE}.{_MAIN_CONFIG}', _main_config_path) else: print(f"⚠️ Main configuration '{_main_config_path}' not found.") From ec3fd8a7907f19d406c87d7e6ef954a39e8cd972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 21:36:40 +0200 Subject: [PATCH 24/36] Revert accidential change to _read_secret order --- configuration/configuration.py | 12 ++++++------ configuration/ldap/ldap_config.py | 2 +- docker/configuration.docker.py | 1 - docker/ldap_config.docker.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index 4d1a146..f111c19 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -41,7 +41,7 @@ ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ') DATABASE = { 'NAME': environ.get('DB_NAME', 'netbox'), # Database name 'USER': environ.get('DB_USER', ''), # PostgreSQL username - 'PASSWORD': environ.get('DB_PASSWORD', _read_secret('db_password')), + 'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')), # PostgreSQL password 'HOST': environ.get('DB_HOST', 'localhost'), # Database server 'PORT': environ.get('DB_PORT', ''), # Database port (leave blank for default) @@ -58,14 +58,14 @@ REDIS = { 'tasks': { 'HOST': environ.get('REDIS_HOST', 'localhost'), 'PORT': int(environ.get('REDIS_PORT', 6379)), - 'PASSWORD': environ.get('REDIS_PASSWORD', _read_secret('redis_password')), + 'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')), 'DATABASE': int(environ.get('REDIS_DATABASE', 0)), 'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true', }, 'caching': { 'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')), 'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))), - 'PASSWORD': environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', _read_secret('redis_cache_password'))), + 'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))), 'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)), 'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true', }, @@ -75,7 +75,7 @@ REDIS = { # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = environ.get('SECRET_KEY', _read_secret('secret_key')) +SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY')) ######################### @@ -130,7 +130,7 @@ EMAIL = { 'SERVER': environ.get('EMAIL_SERVER', 'localhost'), 'PORT': int(environ.get('EMAIL_PORT', 25)), 'USERNAME': environ.get('EMAIL_USERNAME', ''), - 'PASSWORD': environ.get('EMAIL_PASSWORD', _read_secret('email_password')), + 'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')), 'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', 'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', 'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''), @@ -176,7 +176,7 @@ METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true' # Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '') -NAPALM_PASSWORD = environ.get('NAPALM_PASSWORD', _read_secret('napalm_password')) +NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', '')) # NAPALM timeout (in seconds). (Default: 30) NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30)) diff --git a/configuration/ldap/ldap_config.py b/configuration/ldap/ldap_config.py index b92a79a..d1e4444 100644 --- a/configuration/ldap/ldap_config.py +++ b/configuration/ldap/ldap_config.py @@ -32,7 +32,7 @@ AUTH_LDAP_CONNECTION_OPTIONS = { # Set the DN and password for the NetBox service account. AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '') -AUTH_LDAP_BIND_PASSWORD = environ.get('AUTH_LDAP_BIND_PASSWORD', _read_secret('auth_ldap_bind_password')) +AUTH_LDAP_BIND_PASSWORD = _read_secret('auth_ldap_bind_password', environ.get('AUTH_LDAP_BIND_PASSWORD', '')) # Set a string template that describes any user’s distinguished name based on the username. AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index 68a12e4..1a3590d 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -54,7 +54,6 @@ with scandir(_CONFIG_DIR) as it: continue module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") - _import(module_name, f.path) if len(_loaded_configurations) == 0: diff --git a/docker/ldap_config.docker.py b/docker/ldap_config.docker.py index ca8329d..45ffe7c 100644 --- a/docker/ldap_config.docker.py +++ b/docker/ldap_config.docker.py @@ -54,7 +54,6 @@ with scandir(_CONFIG_DIR) as it: continue module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") - _import(module_name, f.path) if len(_loaded_configurations) == 0: From d0c7b87925b5d876e0f7a8f1609686ffa9e061f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 21:40:14 +0200 Subject: [PATCH 25/36] Improve order in configuration.py --- configuration/configuration.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index f111c19..7d5477c 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -205,12 +205,11 @@ RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNI RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220)) # Remote authentication support -REMOTE_AUTH_ENABLED = False -# REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' -# REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' -# REMOTE_AUTH_AUTO_CREATE_USER = True -# REMOTE_AUTH_DEFAULT_GROUPS = [] -# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} +REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' +REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') +REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' +REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600)) @@ -236,13 +235,6 @@ SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. SESSION_FILE_PATH = environ.get('REPORTS_ROOT', None) -# Remote authentication support -REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' -REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') -REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') -REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' -REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) - # Time zone (default: UTC) TIME_ZONE = environ.get('TIME_ZONE', 'UTC') From 2dba2b881131bc020724a9d95a84370bb94e9073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 20 Oct 2020 21:45:37 +0200 Subject: [PATCH 26/36] Fix default for secret_key in configuration.py --- configuration/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index 7d5477c..639e1b6 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -75,7 +75,7 @@ REDIS = { # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY')) +SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', '')) ######################### From 58050e5287f92ad8c6d2c3ce2126c047678063d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 26 Oct 2020 14:43:11 +0100 Subject: [PATCH 27/36] Merge core functionality into `configuration.py` Which is the file `docker/configuration.docker.py` in our repo. The common code is then imported by `docker/ldap_config.docker.py`. --- docker/configuration.docker.py | 112 +++++++++++++++++++-------------- docker/ldap_config.docker.py | 56 ++--------------- 2 files changed, 70 insertions(+), 98 deletions(-) diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index 1a3590d..e7a7c55 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -1,12 +1,73 @@ +## Generic Parts +# These functions are providing the functionality to load +# arbitrary configuration files. +# +# They can be imported by other code (see `ldap_config.py` for an example). + from os.path import abspath, isfile from os import scandir import importlib.util import sys -_CONFIG_DIR = '/etc/netbox/config/' -_MAIN_CONFIG = 'configuration' -_MODULE = 'netbox.configuration' -_loaded_configurations = [] +def _filename(f): + return f.name + + +def _import(module_name, path, loaded_configurations): + spec = importlib.util.spec_from_file_location('', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[module_name] = module + + loaded_configurations.insert(0, module) + + print(f"🧬 loaded config '{path}'") + + +def read_configurations(config_module, config_dir, main_config): + loaded_configurations = [] + + main_config_path = abspath(f'{config_dir}/{main_config}.py') + if isfile(main_config_path): + _import(f'{config_module}.{main_config}', main_config_path, loaded_configurations) + else: + print(f"⚠️ Main configuration '{main_config_path}' not found.") + + with scandir(config_dir) as it: + for f in sorted(it, key=_filename): + if not f.is_file(): + continue + + if f.name.startswith('__'): + continue + + if not f.name.endswith('.py'): + continue + + if f.name == f'{config_dir}.py': + continue + + module_name = f"{config_module}.{f.name[:-len('.py')]}".replace(".", "_") + _import(module_name, f.path, loaded_configurations) + + if len(loaded_configurations) == 0: + print(f"‼️ No configuration files found in '{config_dir}'.") + raise ImportError(f"No configuration files found in '{config_dir}'.") + + return loaded_configurations + + +## Specific Parts +# This section's code actually loads the various configuration files +# into the module with the given name. +# It contains the logic to resolve arbitrary configuration options by +# levaraging dynamic programming using `__getattr__`. + + +_loaded_configurations = read_configurations( + config_dir = '/etc/netbox/config/', + config_module = 'netbox.configuration', + main_config = 'configuration') def __getattr__(name): @@ -16,46 +77,3 @@ def __getattr__(name): except: pass raise AttributeError - - -def _filename(f): - return f.name - - -def _import(module_name, path): - spec = importlib.util.spec_from_file_location('', path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules[module_name] = module - - _loaded_configurations.insert(0, module) - - print(f"🧬 loaded config '{path}'") - - -_main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') -if isfile(_main_config_path): - _import(f'{_MODULE}.{_MAIN_CONFIG}', _main_config_path) -else: - print(f"⚠️ Main configuration '{_main_config_path}' not found.") - -with scandir(_CONFIG_DIR) as it: - for f in sorted(it, key=_filename): - if not f.is_file(): - continue - - if f.name.startswith('__'): - continue - - if not f.name.endswith('.py'): - continue - - if f.name == f'{_MAIN_CONFIG}.py': - continue - - module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") - _import(module_name, f.path) - -if len(_loaded_configurations) == 0: - print(f"‼️ No configuration files found in '{_CONFIG_DIR}'.") - raise ImportError(f"No configuration files found in '{_CONFIG_DIR}'.") diff --git a/docker/ldap_config.docker.py b/docker/ldap_config.docker.py index 45ffe7c..bcaf7e7 100644 --- a/docker/ldap_config.docker.py +++ b/docker/ldap_config.docker.py @@ -1,12 +1,9 @@ -from os.path import abspath, isfile -from os import scandir -import importlib.util -import sys +from .configuration import read_configurations -_CONFIG_DIR = '/etc/netbox/config/ldap/' -_MAIN_CONFIG = 'ldap_config' -_MODULE = 'netbox.configuration.ldap' -_loaded_configurations = [] +_loaded_configurations = read_configurations( + config_dir = '/etc/netbox/config/ldap/', + config_module = 'netbox.configuration.ldap', + main_config = 'ldap_config') def __getattr__(name): @@ -16,46 +13,3 @@ def __getattr__(name): except: pass raise AttributeError - - -def _filename(f): - return f.name - - -def _import(module_name, path): - spec = importlib.util.spec_from_file_location('', path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules[module_name] = module - - _loaded_configurations.insert(0, module) - - print(f"🧬 loaded config '{path}'") - - -_main_config_path = abspath(f'{_CONFIG_DIR}/{_MAIN_CONFIG}.py') -if isfile(_main_config_path): - _import(f'{_MODULE}.{_MAIN_CONFIG}', _main_config_path) -else: - print(f"⚠️ Main configuration '{_main_config_path}' not found.") - -with scandir(_CONFIG_DIR) as it: - for f in sorted(it, key=_filename): - if not f.is_file(): - continue - - if f.name.startswith('__'): - continue - - if not f.name.endswith('.py'): - continue - - if f.name == f'{_MAIN_CONFIG}.py': - continue - - module_name = f"{_MODULE}.{f.name[:-len('.py')]}".replace(".", "_") - _import(module_name, f.path) - -if len(_loaded_configurations) == 0: - print(f"‼️ No configuration files found in '{_CONFIG_DIR}'.") - raise ImportError(f"No configuration files found in '{_CONFIG_DIR}'.") From f46d8a77820cc2e99ada9475dd15ce8aa49f3633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 15:16:16 +0200 Subject: [PATCH 28/36] Adds Prometheus/Grafana monitoring infrastructure --- README.md | 5 +- docker-compose.yml | 73 +- docker/gunicorn_config.py | 1 + docker/nginx.conf | 9 + env/grafana.env | 3 + env/netbox.env | 5 +- env/postgres-exporter.env | 4 + env/redis-cache-exporter.env | 2 + env/redis-worker-exporter.env | 2 + env/{redis.env => redis-worker.env} | 0 monitoring/grafana/dashboards/netbox/.keep | 0 .../dashboards/nginx/nginx_1d070f2.json | 579 +++ .../postgres/postgresql-database_rev4.json | 3395 +++++++++++++++++ .../prometheus/prometheus-stats_rev2.json | 894 +++++ ...is-exporter-helm-stable-redis-ha_rev1.json | 1411 +++++++ monitoring/grafana/plugins/.keep | 0 .../provisioning/dashboards/dashboards.yml | 9 + .../grafana/provisioning/datasources/prom.yml | 8 + .../grafana/provisioning/notifiers/.keep | 0 monitoring/grafana/provisioning/plugins/.keep | 0 monitoring/prometheus/prometheus.yml | 20 + 21 files changed, 6414 insertions(+), 6 deletions(-) create mode 100644 env/grafana.env create mode 100644 env/postgres-exporter.env create mode 100644 env/redis-cache-exporter.env create mode 100644 env/redis-worker-exporter.env rename env/{redis.env => redis-worker.env} (100%) create mode 100644 monitoring/grafana/dashboards/netbox/.keep create mode 100644 monitoring/grafana/dashboards/nginx/nginx_1d070f2.json create mode 100644 monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json create mode 100644 monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json create mode 100644 monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json create mode 100644 monitoring/grafana/plugins/.keep create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prom.yml create mode 100644 monitoring/grafana/provisioning/notifiers/.keep create mode 100644 monitoring/grafana/provisioning/plugins/.keep create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/README.md b/README.md index 25c1286..f41aafb 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,11 @@ The default credentials are: ## Documentation Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly. -It covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration. +You'll learn how to read the integrated metrics using [Prometheus][prometheus] and [Grafana][grafana]. +It also covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration. +[prometheus]: https://prometheus.io/ +[grafana]: https://grafana.com/grafana/ [netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/ ## Getting Help diff --git a/docker-compose.yml b/docker-compose.yml index 001694c..3c0af7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,9 @@ services: - netbox-nginx-config:/etc/netbox-nginx:z - netbox-static-files:/opt/netbox/netbox/static:z - netbox-media-files:/opt/netbox/netbox/media:z + - type: tmpfs + target: /tmp/metrics + read_only: false netbox-worker: <<: *netbox depends_on: @@ -27,6 +30,8 @@ services: - /opt/netbox/netbox/manage.py command: - rqworker + + # nginx nginx: command: nginx -c /etc/netbox-nginx/nginx.conf image: nginx:1.19-alpine @@ -37,18 +42,36 @@ services: volumes: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro + + nginx-exporter: + image: nginx/nginx-prometheus-exporter + depends_on: + - nginx + command: + - -nginx.scrape-uri + - http://nginx:8081/stub_status + + # postgres postgres: image: postgres:12-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data - redis: + + postgres-exporter: + image: wrouesnel/postgres_exporter:v0.8.0 + depends_on: + - postgres + env_file: env/postgres-exporter.env + + # redis + redis: &redis image: redis:6-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis.env + env_file: env/redis-worker.env volumes: - netbox-redis-data:/data redis-cache: @@ -56,8 +79,48 @@ services: command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis-cache.env + + redis-worker-exporter: + image: oliver006/redis_exporter + depends_on: + - redis + env_file: env/redis-worker-exporter.env + redis-cache-exporter: + image: oliver006/redis_exporter + depends_on: + - redis-cache + env_file: env/redis-cache-exporter.env + + # prometheus + prometheus: + image: prom/prometheus:v2.22.0 + depends_on: + - postgres-exporter + - redis-cache-exporter + - redis-worker-exporter + - nginx-exporter + - netbox + ports: + - 9090 + volumes: + - ./monitoring/prometheus/:/etc/prometheus/ + - prometheus-data:/prometheus/data + + # grafana + grafana: + image: grafana/grafana:7.2.1 + depends_on: + - prometheus + ports: + - 3000 + volumes: + - ./monitoring/grafana/plugins/:/var/lib/grafana/plugins/:z,ro + - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:z,ro + - ./monitoring/grafana/dashboards/:/etc/grafana/dashboards/:z,ro + - grafana-data:/var/lib/grafana + volumes: netbox-static-files: driver: local @@ -69,3 +132,7 @@ volumes: driver: local netbox-redis-data: driver: local + prometheus-data: + driver: local + grafana-data: + driver: local diff --git a/docker/gunicorn_config.py b/docker/gunicorn_config.py index 4ce7763..9e22310 100644 --- a/docker/gunicorn_config.py +++ b/docker/gunicorn_config.py @@ -6,3 +6,4 @@ errorlog = '-' accesslog = '-' capture_output = False loglevel = 'info' +raw_env = 'prometheus_multiproc_dir=/tmp/metrics' diff --git a/docker/nginx.conf b/docker/nginx.conf index edbd927..75e3bfd 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -32,4 +32,13 @@ http { add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"'; } } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } } diff --git a/env/grafana.env b/env/grafana.env new file mode 100644 index 0000000..c0c22bf --- /dev/null +++ b/env/grafana.env @@ -0,0 +1,3 @@ +GF_SECURITY_ADMIN_USER=admin +GF_SECURITY_ADMIN_PASSWORD=admin +GF_SECURITY_SECRET_KEY=oew5RCBGGBba0MdsEKrj diff --git a/env/netbox.env b/env/netbox.env index 9849ce6..eec999c 100644 --- a/env/netbox.env +++ b/env/netbox.env @@ -14,11 +14,12 @@ EMAIL_USE_SSL=false EMAIL_USE_TLS=false EMAIL_SSL_CERTFILE= EMAIL_SSL_KEYFILE= +MAX_PAGE_SIZE=1000 MEDIA_ROOT=/opt/netbox/netbox/media +METRICS_ENABLED=true NAPALM_USERNAME= NAPALM_PASSWORD= NAPALM_TIMEOUT=10 -MAX_PAGE_SIZE=1000 REDIS_HOST=redis REDIS_PASSWORD=H733Kdjndks81 REDIS_DATABASE=0 @@ -27,6 +28,7 @@ REDIS_CACHE_HOST=redis-cache REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36 REDIS_CACHE_DATABASE=1 REDIS_CACHE_SSL=false +RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj SKIP_STARTUP_SCRIPTS=false SKIP_SUPERUSER=false @@ -35,4 +37,3 @@ SUPERUSER_EMAIL=admin@example.com SUPERUSER_PASSWORD=admin SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 WEBHOOKS_ENABLED=true -RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases diff --git a/env/postgres-exporter.env b/env/postgres-exporter.env new file mode 100644 index 0000000..ab0e652 --- /dev/null +++ b/env/postgres-exporter.env @@ -0,0 +1,4 @@ +DATA_SOURCE_URI=postgres?sslmode=disable +DATA_SOURCE_USER=netbox +DATA_SOURCE_PASS=J5brHrAXFLQSif0K +PG_EXPORTER_AUTO_DISCOVER_DATABASES=true diff --git a/env/redis-cache-exporter.env b/env/redis-cache-exporter.env new file mode 100644 index 0000000..2894b8d --- /dev/null +++ b/env/redis-cache-exporter.env @@ -0,0 +1,2 @@ +REDIS_ADDR=redis://redis-cache:6379 +REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 diff --git a/env/redis-worker-exporter.env b/env/redis-worker-exporter.env new file mode 100644 index 0000000..5d937c9 --- /dev/null +++ b/env/redis-worker-exporter.env @@ -0,0 +1,2 @@ +REDIS_ADDR=redis://redis:6379 +REDIS_PASSWORD=H733Kdjndks81 diff --git a/env/redis.env b/env/redis-worker.env similarity index 100% rename from env/redis.env rename to env/redis-worker.env diff --git a/monitoring/grafana/dashboards/netbox/.keep b/monitoring/grafana/dashboards/netbox/.keep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json b/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json new file mode 100644 index 0000000..905ac25 --- /dev/null +++ b/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json @@ -0,0 +1,579 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Official dashboard for NGINX Prometheus exporter", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 8, + "iteration": 1603022185715, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "DS_PROMETHEUS", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "title": "Status", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorPostfix": false, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#E02F44", + "#FF9830", + "#299c46" + ], + "datasource": "DS_PROMETHEUS", + "decimals": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": "instance", + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "nginx-exporter:9113", + "value": "nginx-exporter:9113" + } + }, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "nginx_up{instance=~\"$instance\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,1", + "timeFrom": null, + "timeShift": null, + "title": "NGINX Status for $instance", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "Down", + "value": "0" + }, + { + "op": "=", + "text": "Up", + "value": "1" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": "DS_PROMETHEUS", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 6, + "panels": [], + "title": "Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "decimals": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(nginx_connections_accepted{instance=~\"$instance\"}[5m])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{instance}} accepted", + "refId": "A" + }, + { + "expr": "irate(nginx_connections_handled{instance=~\"$instance\"}[5m])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{instance}} handled", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Processed connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": "Connections (rate)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nginx_connections_active{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} active", + "refId": "A" + }, + { + "expr": "nginx_connections_reading{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} reading", + "refId": "B" + }, + { + "expr": "nginx_connections_waiting{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} waiting", + "refId": "C" + }, + { + "expr": "nginx_connections_writing{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} writing", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "DS_PROMETHEUS", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(nginx_http_requests_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}} total requests", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "nginx", + "prometheus", + "nginx prometheus exporter" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "DS_PROMETHEUS", + "value": "DS_PROMETHEUS" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "DS_PROMETHEUS", + "definition": "label_values(nginx_up, instance)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(nginx_up, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "NGINX Dashboard", + "uid": "MsjffzSZz", + "version": 1 +} diff --git a/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json b/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json new file mode 100644 index 0000000..9e3c0cd --- /dev/null +++ b/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json @@ -0,0 +1,3395 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This dashboard works with postgres_exporter for prometheus", + "editable": true, + "gnetId": 9628, + "graphTooltip": 0, + "iteration": 1603018468350, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "title": "General Counters, CPU, Memory and File Descriptor Stats", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "#7eb26d", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 36, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "start time of the process", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "dateTimeFromNow", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "110%", + "prefix": "", + "prefixFontSize": "110%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "process_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Start Time", + "type": "singlestat", + "valueFontSize": "110%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 1 + }, + "height": "200px", + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Current fetch data", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "height": "200px", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Current insert data", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 1 + }, + "height": "200px", + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Current update data", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 38, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Average user and system CPU time spent in seconds.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_cpu_seconds_total{release=\"$release\", instance=\"$instance\"}[5m]) * 1000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CPU Time", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(rate(process_resident_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Mem", + "refId": "A" + }, + { + "expr": "avg(rate(process_virtual_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Virtual Mem", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Number of open file descriptors", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open FD", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 32, + "panels": [], + "title": "Settings", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 11 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Shared Buffers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 11 + }, + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Effective Cache", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 11 + }, + "id": 44, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Maintenance Work Mem", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 11 + }, + "id": 46, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Work Mem", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 1, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 11 + }, + "id": 48, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max WAL Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 11 + }, + "id": 50, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Random Page Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 11 + }, + "id": 52, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_seq_page_cost", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Seq Page Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 20, + "y": 11 + }, + "id": 54, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Worker Processes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 22, + "y": 11 + }, + "id": 56, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Parallel Workers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 30, + "panels": [], + "title": "Database Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=\"active\"} !=0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} commits", + "refId": "A" + }, + { + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} rollbacks", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Update data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Fetch data (SELECT)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Insert data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_locks_count{datname=~\"$datname\", instance=~\"$instance\", mode=~\"$mode\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}},{{mode}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lock tables", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Return data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Idle sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 29 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delete data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=~\"$datname\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ datname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache Hit Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 36 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffers (bgwriter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 36 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} conflicts", + "refId": "B" + }, + { + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} deadlocks", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Temp File (Bytes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 43 + }, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Checkpoint Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "postgres", + "db", + "stats" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 200, + "auto_min": "1s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,5s,1m,5m,1h,6h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds)", + "refresh": 2, + "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Release", + "multi": false, + "name": "release", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*release=\"([^\"]+)/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "localhost:9090", + "value": "localhost:9090" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "query_result(up{release=\"$release\"})", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Database", + "multi": true, + "name": "datname", + "options": [], + "query": "label_values(datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Lock table", + "multi": true, + "name": "mode", + "options": [], + "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "DS_PROMETHEUS", + "value": "DS_PROMETHEUS" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "PostgreSQL Database", + "uid": "000000039", + "version": 1 +} diff --git a/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json b/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json new file mode 100644 index 0000000..c2f9b73 --- /dev/null +++ b/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json @@ -0,0 +1,894 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "The official, pre-built Prometheus Stats Dashboard.", + "editable": true, + "gnetId": 2, + "graphTooltip": 0, + "links": [ + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Grafana Docs", + "tooltip": "", + "type": "link", + "url": "http://www.grafana.org/docs" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Prometheus Docs", + "type": "link", + "url": "http://prometheus.io/docs/introduction/overview/" + } + ], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 1, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_local_storage_memory_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "1,5", + "title": "Local Storage Memory Series", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_local_storage_indexing_queue_length", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": "500,4000", + "title": "Interal Storage Queue Length", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "Empty", + "value": "0" + } + ], + "valueName": "current" + }, + { + "content": "\"Prometheus\nPrometheus\n\n

You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the Grafana and Prometheus projects.

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 9, + "links": [], + "mode": "html", + "options": { + "content": "\"Prometheus\nPrometheus\n\n

You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the Grafana and Prometheus projects.

", + "mode": "html" + }, + "pluginVersion": "7.1.0", + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "prometheus": "#C15C17", + "{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 18, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_local_storage_ingested_samples_total[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Samples ingested (rate-5m)", + "tooltip": { + "msResolution": false, + "ordering": "alphabetical", + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 18, + "y": 5 + }, + "id": 8, + "links": [], + "mode": "markdown", + "options": { + "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": { + "prometheus": "#F9BA8F", + "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_count[5m])", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Target Scrapes (last 5m)", + "tooltip": { + "msResolution": false, + "ordering": "alphabetical", + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 10, + "y": 11 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}} ({{interval}})", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "msResolution": false, + "ordering": "alphabetical", + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 11, + "links": [], + "mode": "markdown", + "options": { + "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_evaluator_duration_milliseconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule Eval Duration", + "tooltip": { + "msResolution": false, + "ordering": "alphabetical", + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 15, + "links": [], + "mode": "markdown", + "options": { + "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "DS_PROMETHEUS", + "value": "DS_PROMETHEUS" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Prometheus Stats", + "uid": "HplbpgtGz", + "version": 1 +} diff --git a/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json b/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json new file mode 100644 index 0000000..79c822e --- /dev/null +++ b/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json @@ -0,0 +1,1411 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Redis Dashboard for Prometheus Redis Exporter 1.x, it works with helm stable/redis-ha exporter.\r\n\r\nIf you missing redis memory utilization, please modify \"maxmemory\" value in values.yaml", + "editable": true, + "gnetId": 11835, + "graphTooltip": 0, + "iteration": 1603019449855, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(max_over_time(redis_uptime_in_seconds{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 2, + "y": 0 + }, + "hideTimeOverride": true, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "redis_connected_clients{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": "", + "timeFrom": "1m", + "timeShift": null, + "title": "Clients", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "hideTimeOverride": true, + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 * (redis_memory_used_bytes{instance=~\"$instance\"} / redis_memory_max_bytes{instance=~\"$instance\"} )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": "80,95", + "timeFrom": "1m", + "timeShift": null, + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(redis_commands_processed_total{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "A", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Commands Executed / sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(redis_keyspace_hits_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hits", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "irate(redis_keyspace_misses_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "misses", + "metric": "", + "refId": "B", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hits / Misses per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_memory_used_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "redis_memory_max_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "max", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(redis_net_input_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ input }}", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(redis_net_output_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ output }}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ db }} ", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Items per DB", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "not expiring", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "expiring", + "metric": "", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expiring vs Not-Expiring Keys", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(redis_expired_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "expired", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(rate(redis_evicted_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "evicted", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expired / Evicted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 8, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 14, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, irate(redis_commands_total{instance=~\"$instance\"} [1m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ cmd }}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Command Calls / sec", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_connected_clients{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Redis connected clients", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "prometheus", + "redis" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "DS_PROMETHEUS", + "value": "DS_PROMETHEUS" + }, + "hide": 2, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "redis-worker-exporter:9121", + "value": "redis-worker-exporter:9121" + }, + "datasource": "$DS_PROMETHEUS", + "definition": "label_values(redis_up, instance)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(redis_up, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Redis Dashboard for Prometheus Redis Exporter (helm stable/redis-ha)", + "uid": "xDLNRKUWz", + "version": 1 +} diff --git a/monitoring/grafana/plugins/.keep b/monitoring/grafana/plugins/.keep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..0198297 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: +- name: dashboards + type: file + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true diff --git a/monitoring/grafana/provisioning/datasources/prom.yml b/monitoring/grafana/provisioning/datasources/prom.yml new file mode 100644 index 0000000..9d7d545 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prom.yml @@ -0,0 +1,8 @@ +# config file version +apiVersion: 1 + +datasources: + - name: DS_PROMETHEUS + type: prometheus + default: true + url: http://prometheus:9090 diff --git a/monitoring/grafana/provisioning/notifiers/.keep b/monitoring/grafana/provisioning/notifiers/.keep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/grafana/provisioning/plugins/.keep b/monitoring/grafana/provisioning/plugins/.keep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..2a88357 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,20 @@ +scrape_configs: +- job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + +- job_name: netbox + static_configs: + - targets: ['netbox:8001'] + +- job_name: nginx + static_configs: + - targets: ['nginx-exporter:9113'] + +- job_name: postgresql + static_configs: + - targets: ['postgres-exporter:9187'] + +- job_name: redis + static_configs: + - targets: ['redis-worker-exporter:9121', 'redis-cache-exporter:9121'] From 6ac65a64c9ca483c235ed16a2095884711f5356c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 18 Oct 2020 15:36:50 +0200 Subject: [PATCH 29/36] Fix test --- docker-compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 23f74ca..b04d9c2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -36,7 +36,7 @@ services: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis.env + env_file: env/redis-worker.env redis-cache: image: redis:6-alpine command: From 48decdeb0e650a89c9f5d498cd56c236d9b8327a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Tue, 20 Oct 2020 09:07:19 +0200 Subject: [PATCH 30/36] Monitoring parts to docker-compose.monitorin.yml --- docker-compose.monitoring.yml | 63 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 59 +------------------------------- 2 files changed, 64 insertions(+), 58 deletions(-) create mode 100644 docker-compose.monitoring.yml diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml new file mode 100644 index 0000000..dc47731 --- /dev/null +++ b/docker-compose.monitoring.yml @@ -0,0 +1,63 @@ +version: '3.4' +services: + # nginx + nginx-exporter: + image: nginx/nginx-prometheus-exporter + depends_on: + - nginx + command: + - -nginx.scrape-uri + - http://nginx:8081/stub_status + + # postgres + postgres-exporter: + image: wrouesnel/postgres_exporter:v0.8.0 + depends_on: + - postgres + env_file: env/postgres-exporter.env + + # redis + redis-worker-exporter: + image: oliver006/redis_exporter + depends_on: + - redis + env_file: env/redis-worker-exporter.env + redis-cache-exporter: + image: oliver006/redis_exporter + depends_on: + - redis-cache + env_file: env/redis-cache-exporter.env + + # prometheus + prometheus: + image: prom/prometheus:v2.22.0 + depends_on: + - postgres-exporter + - redis-cache-exporter + - redis-worker-exporter + - nginx-exporter + - netbox + ports: + - 9090 + volumes: + - ./monitoring/prometheus/:/etc/prometheus/ + - prometheus-data:/prometheus/data + + # grafana + grafana: + image: grafana/grafana:7.2.1 + depends_on: + - prometheus + ports: + - 3000 + volumes: + - ./monitoring/grafana/plugins/:/var/lib/grafana/plugins/:z,ro + - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:z,ro + - ./monitoring/grafana/dashboards/:/etc/grafana/dashboards/:z,ro + - grafana-data:/var/lib/grafana + +volumes: + prometheus-data: + driver: local + grafana-data: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 3c0af7d..8ad3efe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,14 +43,6 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro - nginx-exporter: - image: nginx/nginx-prometheus-exporter - depends_on: - - nginx - command: - - -nginx.scrape-uri - - http://nginx:8081/stub_status - # postgres postgres: image: postgres:12-alpine @@ -58,14 +50,8 @@ services: volumes: - netbox-postgres-data:/var/lib/postgresql/data - postgres-exporter: - image: wrouesnel/postgres_exporter:v0.8.0 - depends_on: - - postgres - env_file: env/postgres-exporter.env - # redis - redis: &redis + redis: image: redis:6-alpine command: - sh @@ -82,45 +68,6 @@ services: - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis-cache.env - redis-worker-exporter: - image: oliver006/redis_exporter - depends_on: - - redis - env_file: env/redis-worker-exporter.env - redis-cache-exporter: - image: oliver006/redis_exporter - depends_on: - - redis-cache - env_file: env/redis-cache-exporter.env - - # prometheus - prometheus: - image: prom/prometheus:v2.22.0 - depends_on: - - postgres-exporter - - redis-cache-exporter - - redis-worker-exporter - - nginx-exporter - - netbox - ports: - - 9090 - volumes: - - ./monitoring/prometheus/:/etc/prometheus/ - - prometheus-data:/prometheus/data - - # grafana - grafana: - image: grafana/grafana:7.2.1 - depends_on: - - prometheus - ports: - - 3000 - volumes: - - ./monitoring/grafana/plugins/:/var/lib/grafana/plugins/:z,ro - - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:z,ro - - ./monitoring/grafana/dashboards/:/etc/grafana/dashboards/:z,ro - - grafana-data:/var/lib/grafana - volumes: netbox-static-files: driver: local @@ -132,7 +79,3 @@ volumes: driver: local netbox-redis-data: driver: local - prometheus-data: - driver: local - grafana-data: - driver: local From e18d6c53b3148d627ce33cca703bf861cc93427d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 26 Oct 2020 15:16:49 +0100 Subject: [PATCH 31/36] Revert most changes --- README.md | 5 +- docker-compose.monitoring.yml | 63 - docker-compose.test.yml | 2 +- docker-compose.yml | 7 +- env/grafana.env | 3 - env/netbox.env | 2 +- env/postgres-exporter.env | 4 - env/redis-cache-exporter.env | 2 - env/redis-worker-exporter.env | 2 - env/{redis-worker.env => redis.env} | 0 monitoring/grafana/dashboards/netbox/.keep | 0 .../dashboards/nginx/nginx_1d070f2.json | 579 --- .../postgres/postgresql-database_rev4.json | 3395 ----------------- .../prometheus/prometheus-stats_rev2.json | 894 ----- ...is-exporter-helm-stable-redis-ha_rev1.json | 1411 ------- monitoring/grafana/plugins/.keep | 0 .../provisioning/dashboards/dashboards.yml | 9 - .../grafana/provisioning/datasources/prom.yml | 8 - .../grafana/provisioning/notifiers/.keep | 0 monitoring/grafana/provisioning/plugins/.keep | 0 monitoring/prometheus/prometheus.yml | 20 - 21 files changed, 5 insertions(+), 6401 deletions(-) delete mode 100644 docker-compose.monitoring.yml delete mode 100644 env/grafana.env delete mode 100644 env/postgres-exporter.env delete mode 100644 env/redis-cache-exporter.env delete mode 100644 env/redis-worker-exporter.env rename env/{redis-worker.env => redis.env} (100%) delete mode 100644 monitoring/grafana/dashboards/netbox/.keep delete mode 100644 monitoring/grafana/dashboards/nginx/nginx_1d070f2.json delete mode 100644 monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json delete mode 100644 monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json delete mode 100644 monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json delete mode 100644 monitoring/grafana/plugins/.keep delete mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml delete mode 100644 monitoring/grafana/provisioning/datasources/prom.yml delete mode 100644 monitoring/grafana/provisioning/notifiers/.keep delete mode 100644 monitoring/grafana/provisioning/plugins/.keep delete mode 100644 monitoring/prometheus/prometheus.yml diff --git a/README.md b/README.md index f41aafb..a425078 100644 --- a/README.md +++ b/README.md @@ -79,11 +79,8 @@ The default credentials are: ## Documentation Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly. -You'll learn how to read the integrated metrics using [Prometheus][prometheus] and [Grafana][grafana]. -It also covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration. +It covers advanced topics such as using files for secrets, deployment to Kubernetes and configuring NAPALM or LDAP. -[prometheus]: https://prometheus.io/ -[grafana]: https://grafana.com/grafana/ [netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/ ## Getting Help diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml deleted file mode 100644 index dc47731..0000000 --- a/docker-compose.monitoring.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '3.4' -services: - # nginx - nginx-exporter: - image: nginx/nginx-prometheus-exporter - depends_on: - - nginx - command: - - -nginx.scrape-uri - - http://nginx:8081/stub_status - - # postgres - postgres-exporter: - image: wrouesnel/postgres_exporter:v0.8.0 - depends_on: - - postgres - env_file: env/postgres-exporter.env - - # redis - redis-worker-exporter: - image: oliver006/redis_exporter - depends_on: - - redis - env_file: env/redis-worker-exporter.env - redis-cache-exporter: - image: oliver006/redis_exporter - depends_on: - - redis-cache - env_file: env/redis-cache-exporter.env - - # prometheus - prometheus: - image: prom/prometheus:v2.22.0 - depends_on: - - postgres-exporter - - redis-cache-exporter - - redis-worker-exporter - - nginx-exporter - - netbox - ports: - - 9090 - volumes: - - ./monitoring/prometheus/:/etc/prometheus/ - - prometheus-data:/prometheus/data - - # grafana - grafana: - image: grafana/grafana:7.2.1 - depends_on: - - prometheus - ports: - - 3000 - volumes: - - ./monitoring/grafana/plugins/:/var/lib/grafana/plugins/:z,ro - - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:z,ro - - ./monitoring/grafana/dashboards/:/etc/grafana/dashboards/:z,ro - - grafana-data:/var/lib/grafana - -volumes: - prometheus-data: - driver: local - grafana-data: - driver: local diff --git a/docker-compose.test.yml b/docker-compose.test.yml index b04d9c2..23f74ca 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -36,7 +36,7 @@ services: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis-worker.env + env_file: env/redis.env redis-cache: image: redis:6-alpine command: diff --git a/docker-compose.yml b/docker-compose.yml index 8ad3efe..8a2c575 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,9 +18,6 @@ services: - netbox-nginx-config:/etc/netbox-nginx:z - netbox-static-files:/opt/netbox/netbox/static:z - netbox-media-files:/opt/netbox/netbox/media:z - - type: tmpfs - target: /tmp/metrics - read_only: false netbox-worker: <<: *netbox depends_on: @@ -57,7 +54,7 @@ services: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis-worker.env + env_file: env/redis.env volumes: - netbox-redis-data:/data redis-cache: @@ -65,7 +62,7 @@ services: command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis-cache.env volumes: diff --git a/env/grafana.env b/env/grafana.env deleted file mode 100644 index c0c22bf..0000000 --- a/env/grafana.env +++ /dev/null @@ -1,3 +0,0 @@ -GF_SECURITY_ADMIN_USER=admin -GF_SECURITY_ADMIN_PASSWORD=admin -GF_SECURITY_SECRET_KEY=oew5RCBGGBba0MdsEKrj diff --git a/env/netbox.env b/env/netbox.env index eec999c..8da5259 100644 --- a/env/netbox.env +++ b/env/netbox.env @@ -16,7 +16,7 @@ EMAIL_SSL_CERTFILE= EMAIL_SSL_KEYFILE= MAX_PAGE_SIZE=1000 MEDIA_ROOT=/opt/netbox/netbox/media -METRICS_ENABLED=true +METRICS_ENABLED=false NAPALM_USERNAME= NAPALM_PASSWORD= NAPALM_TIMEOUT=10 diff --git a/env/postgres-exporter.env b/env/postgres-exporter.env deleted file mode 100644 index ab0e652..0000000 --- a/env/postgres-exporter.env +++ /dev/null @@ -1,4 +0,0 @@ -DATA_SOURCE_URI=postgres?sslmode=disable -DATA_SOURCE_USER=netbox -DATA_SOURCE_PASS=J5brHrAXFLQSif0K -PG_EXPORTER_AUTO_DISCOVER_DATABASES=true diff --git a/env/redis-cache-exporter.env b/env/redis-cache-exporter.env deleted file mode 100644 index 2894b8d..0000000 --- a/env/redis-cache-exporter.env +++ /dev/null @@ -1,2 +0,0 @@ -REDIS_ADDR=redis://redis-cache:6379 -REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 diff --git a/env/redis-worker-exporter.env b/env/redis-worker-exporter.env deleted file mode 100644 index 5d937c9..0000000 --- a/env/redis-worker-exporter.env +++ /dev/null @@ -1,2 +0,0 @@ -REDIS_ADDR=redis://redis:6379 -REDIS_PASSWORD=H733Kdjndks81 diff --git a/env/redis-worker.env b/env/redis.env similarity index 100% rename from env/redis-worker.env rename to env/redis.env diff --git a/monitoring/grafana/dashboards/netbox/.keep b/monitoring/grafana/dashboards/netbox/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json b/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json deleted file mode 100644 index 905ac25..0000000 --- a/monitoring/grafana/dashboards/nginx/nginx_1d070f2.json +++ /dev/null @@ -1,579 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Official dashboard for NGINX Prometheus exporter", - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 8, - "iteration": 1603022185715, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": "DS_PROMETHEUS", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 4, - "panels": [], - "title": "Status", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorPostfix": false, - "colorPrefix": false, - "colorValue": false, - "colors": [ - "#E02F44", - "#FF9830", - "#299c46" - ], - "datasource": "DS_PROMETHEUS", - "decimals": null, - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 8, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "repeat": "instance", - "repeatDirection": "h", - "scopedVars": { - "instance": { - "selected": false, - "text": "nginx-exporter:9113", - "value": "nginx-exporter:9113" - } - }, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "nginx_up{instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "1,1", - "timeFrom": null, - "timeShift": null, - "title": "NGINX Status for $instance", - "type": "singlestat", - "valueFontSize": "100%", - "valueMaps": [ - { - "op": "=", - "text": "Down", - "value": "0" - }, - { - "op": "=", - "text": "Up", - "value": "1" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "datasource": "DS_PROMETHEUS", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 6, - "panels": [], - "title": "Metrics", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "DS_PROMETHEUS", - "decimals": null, - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(nginx_connections_accepted{instance=~\"$instance\"}[5m])", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{instance}} accepted", - "refId": "A" - }, - { - "expr": "irate(nginx_connections_handled{instance=~\"$instance\"}[5m])", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{instance}} handled", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Processed connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "short", - "label": "Connections (rate)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "DS_PROMETHEUS", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "nginx_connections_active{instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} active", - "refId": "A" - }, - { - "expr": "nginx_connections_reading{instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} reading", - "refId": "B" - }, - { - "expr": "nginx_connections_waiting{instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} waiting", - "refId": "C" - }, - { - "expr": "nginx_connections_writing{instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} writing", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Active Connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "Connections", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "DS_PROMETHEUS", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(nginx_http_requests_total{instance=~\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}} total requests", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "nginx", - "prometheus", - "nginx prometheus exporter" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "DS_PROMETHEUS", - "value": "DS_PROMETHEUS" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "DS_PROMETHEUS", - "definition": "label_values(nginx_up, instance)", - "hide": 0, - "includeAll": true, - "label": "", - "multi": true, - "name": "instance", - "options": [], - "query": "label_values(nginx_up, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "NGINX Dashboard", - "uid": "MsjffzSZz", - "version": 1 -} diff --git a/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json b/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json deleted file mode 100644 index 9e3c0cd..0000000 --- a/monitoring/grafana/dashboards/postgres/postgresql-database_rev4.json +++ /dev/null @@ -1,3395 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "This dashboard works with postgres_exporter for prometheus", - "editable": true, - "gnetId": 9628, - "graphTooltip": 0, - "iteration": 1603018468350, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 34, - "panels": [], - "title": "General Counters, CPU, Memory and File Descriptor Stats", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "#7eb26d", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 36, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{short_version}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Version", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "name" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "description": "start time of the process", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "dateTimeFromNow", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "110%", - "prefix": "", - "prefixFontSize": "110%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "process_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Start Time", - "type": "singlestat", - "valueFontSize": "110%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "decbytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 1 - }, - "height": "200px", - "id": 10, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Current fetch data", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "decbytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 12, - "y": 1 - }, - "height": "200px", - "id": 11, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Current insert data", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "decbytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 16, - "y": 1 - }, - "height": "200px", - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Current update data", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 38, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Max Connections", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "Average user and system CPU time spent in seconds.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(rate(process_cpu_seconds_total{release=\"$release\", instance=\"$instance\"}[5m]) * 1000)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "CPU Time", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "Virtual and Resident memory size in bytes, averages over 5 min interval", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 3 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(rate(process_resident_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Mem", - "refId": "A" - }, - { - "expr": "avg(rate(process_virtual_memory_bytes{release=\"$release\", instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Virtual Mem", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "Number of open file descriptors", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 3 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{release=\"$release\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Open FD", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 32, - "panels": [], - "title": "Settings", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 11 - }, - "id": 40, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Shared Buffers", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 11 - }, - "id": 42, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Effective Cache", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 11 - }, - "id": 44, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Maintenance Work Mem", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 9, - "y": 11 - }, - "id": 46, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Work Mem", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 1, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 12, - "y": 11 - }, - "id": 48, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Max WAL Size", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 15, - "y": 11 - }, - "id": 50, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Random Page Cost", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 18, - "y": 11 - }, - "id": 52, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_seq_page_cost", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Seq Page Cost", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 20, - "y": 11 - }, - "id": 54, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Max Worker Processes", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 22, - "y": 11 - }, - "id": 56, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Max Parallel Workers", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 30, - "panels": [], - "title": "Database Stats", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 1, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 3, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=\"active\"} !=0", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{datname}}, s: {{state}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Active sessions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 15 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=~\"$datname\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{datname}} commits", - "refId": "A" - }, - { - "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=~\"$datname\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{datname}} rollbacks", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Transactions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 15 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Update data", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Fetch data (SELECT)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 22 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Insert data", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 22 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_locks_count{datname=~\"$datname\", instance=~\"$instance\", mode=~\"$mode\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}},{{mode}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Lock tables", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", instance=~\"$instance\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Return data", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 29 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}, s: {{state}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Idle sessions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 29 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", instance=~\"$instance\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{datname}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Delete data", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 2, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 62, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=~\"$datname\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ datname }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Cache Hit Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 4, - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 36 - }, - "hiddenSeries": false, - "id": 64, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "buffers_backend", - "refId": "A" - }, - { - "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "buffers_alloc", - "refId": "B" - }, - { - "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "backend_fsync", - "refId": "C" - }, - { - "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "buffers_checkpoint", - "refId": "D" - }, - { - "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "buffers_clean", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Buffers (bgwriter)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 36 - }, - "hiddenSeries": false, - "id": 66, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=~\"$datname\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{datname}} conflicts", - "refId": "B" - }, - { - "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=~\"$datname\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{datname}} deadlocks", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Conflicts/Deadlocks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 43 - }, - "id": 68, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=~\"$datname\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{datname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Temp File (Bytes)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "gridPos": { - "h": 7, - "w": 16, - "x": 8, - "y": 43 - }, - "id": 70, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", - "refId": "B" - }, - { - "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Checkpoint Stats", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "postgres", - "db", - "stats" - ], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 200, - "auto_min": "1s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1s", - "value": "1s" - }, - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "1s,5s,1m,5m,1h,6h,1d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "query_result(pg_exporter_last_scrape_duration_seconds)", - "refresh": 2, - "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Release", - "multi": false, - "name": "release", - "options": [], - "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", - "refresh": 2, - "regex": "/.*release=\"([^\"]+)/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "localhost:9090", - "value": "localhost:9090" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Instance", - "multi": false, - "name": "instance", - "options": [], - "query": "query_result(up{release=\"$release\"})", - "refresh": 1, - "regex": "/.*instance=\"([^\"]+).*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Database", - "multi": true, - "name": "datname", - "options": [], - "query": "label_values(datname)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Lock table", - "multi": true, - "name": "mode", - "options": [], - "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "DS_PROMETHEUS", - "value": "DS_PROMETHEUS" - }, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "PostgreSQL Database", - "uid": "000000039", - "version": 1 -} diff --git a/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json b/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json deleted file mode 100644 index c2f9b73..0000000 --- a/monitoring/grafana/dashboards/prometheus/prometheus-stats_rev2.json +++ /dev/null @@ -1,894 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "The official, pre-built Prometheus Stats Dashboard.", - "editable": true, - "gnetId": 2, - "graphTooltip": 0, - "links": [ - { - "icon": "info", - "tags": [], - "targetBlank": true, - "title": "Grafana Docs", - "tooltip": "", - "type": "link", - "url": "http://www.grafana.org/docs" - }, - { - "icon": "info", - "tags": [], - "targetBlank": true, - "title": "Prometheus Docs", - "type": "link", - "url": "http://prometheus.io/docs/introduction/overview/" - } - ], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 1, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 5, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Uptime", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 6, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "prometheus_local_storage_memory_series", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "1,5", - "title": "Local Storage Memory Series", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 7, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "prometheus_local_storage_indexing_queue_length", - "intervalFactor": 2, - "refId": "A", - "step": 4 - } - ], - "thresholds": "500,4000", - "title": "Interal Storage Queue Length", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "op": "=", - "text": "Empty", - "value": "0" - } - ], - "valueName": "current" - }, - { - "content": "\"Prometheus\nPrometheus\n\n

You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the Grafana and Prometheus projects.

", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 9, - "links": [], - "mode": "html", - "options": { - "content": "\"Prometheus\nPrometheus\n\n

You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the Grafana and Prometheus projects.

", - "mode": "html" - }, - "pluginVersion": "7.1.0", - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "prometheus": "#C15C17", - "{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 6, - "w": 18, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_local_storage_ingested_samples_total[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Samples ingested (rate-5m)", - "tooltip": { - "msResolution": false, - "ordering": "alphabetical", - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 18, - "y": 5 - }, - "id": 8, - "links": [], - "mode": "markdown", - "options": { - "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ", - "mode": "markdown" - }, - "pluginVersion": "7.1.0", - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": { - "prometheus": "#F9BA8F", - "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 10, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_target_interval_length_seconds_count[5m])", - "intervalFactor": 2, - "legendFormat": "{{job}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Target Scrapes (last 5m)", - "tooltip": { - "msResolution": false, - "ordering": "alphabetical", - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 10, - "y": 11 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{quantile}} ({{interval}})", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Scrape Duration", - "tooltip": { - "msResolution": false, - "ordering": "alphabetical", - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 11 - }, - "id": 11, - "links": [], - "mode": "markdown", - "options": { - "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ", - "mode": "markdown" - }, - "pluginVersion": "7.1.0", - "style": {}, - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 18, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_evaluator_duration_milliseconds{quantile!=\"0.01\", quantile!=\"0.05\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{quantile}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Rule Eval Duration", - "tooltip": { - "msResolution": false, - "ordering": "alphabetical", - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.", - "datasource": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 18 - }, - "id": 15, - "links": [], - "mode": "markdown", - "options": { - "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.", - "mode": "markdown" - }, - "pluginVersion": "7.1.0", - "style": {}, - "title": "", - "transparent": true, - "type": "text" - } - ], - "refresh": false, - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "DS_PROMETHEUS", - "value": "DS_PROMETHEUS" - }, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Prometheus Stats", - "uid": "HplbpgtGz", - "version": 1 -} diff --git a/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json b/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json deleted file mode 100644 index 79c822e..0000000 --- a/monitoring/grafana/dashboards/redis/redis-dashboard-for-prometheus-redis-exporter-helm-stable-redis-ha_rev1.json +++ /dev/null @@ -1,1411 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Redis Dashboard for Prometheus Redis Exporter 1.x, it works with helm stable/redis-ha exporter.\r\n\r\nIf you missing redis memory utilization, please modify \"maxmemory\" value in values.yaml", - "editable": true, - "gnetId": 11835, - "graphTooltip": 0, - "iteration": 1603019449855, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 0 - }, - "id": 9, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(max_over_time(redis_uptime_in_seconds{instance=~\"$instance\"}[$__interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Uptime", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 2, - "y": 0 - }, - "hideTimeOverride": true, - "id": 12, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "redis_connected_clients{instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": "", - "timeFrom": "1m", - "timeShift": null, - "title": "Clients", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_PROMETHEUS}", - "decimals": 0, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 0 - }, - "hideTimeOverride": true, - "id": 11, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "100 * (redis_memory_used_bytes{instance=~\"$instance\"} / redis_memory_max_bytes{instance=~\"$instance\"} )", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": "80,95", - "timeFrom": "1m", - "timeShift": null, - "title": "Memory Usage", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(redis_commands_processed_total{instance=~\"$instance\"}[1m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "A", - "refId": "A", - "step": 240, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commands Executed / sec", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "decimals": 2, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 0 - }, - "hiddenSeries": false, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": true, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(redis_keyspace_hits_total{instance=~\"$instance\"}[5m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "hits", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "irate(redis_keyspace_misses_total{instance=~\"$instance\"}[5m])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "misses", - "metric": "", - "refId": "B", - "step": 240, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Hits / Misses per Sec", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "max": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 7, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "redis_memory_used_bytes{instance=~\"$instance\"} ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "used", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "redis_memory_max_bytes{instance=~\"$instance\"} ", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "max", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(redis_net_input_bytes_total{instance=~\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ input }}", - "refId": "A", - "step": 240 - }, - { - "expr": "rate(redis_net_output_bytes_total{instance=~\"$instance\"}[5m])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ output }}", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network I/O", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 7, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 5, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ db }} ", - "refId": "A", - "step": 240, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total Items per DB", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 7, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "hiddenSeries": false, - "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "not expiring", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "expiring", - "metric": "", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Expiring vs Not-Expiring Keys", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "evicts": "#890F02", - "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", - "reclaims": "#3F6833" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "reclaims", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(redis_expired_keys_total{instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "expired", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum(rate(redis_evicted_keys_total{instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "evicted", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Expired / Evicted", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 8, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 14, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "topk(5, irate(redis_commands_total{instance=~\"$instance\"} [1m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ cmd }}", - "metric": "redis_command_calls_total", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Command Calls / sec", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "redis_connected_clients{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Redis connected clients", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "prometheus", - "redis" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "DS_PROMETHEUS", - "value": "DS_PROMETHEUS" - }, - "hide": 2, - "includeAll": false, - "label": "Prometheus", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "redis-worker-exporter:9121", - "value": "redis-worker-exporter:9121" - }, - "datasource": "$DS_PROMETHEUS", - "definition": "label_values(redis_up, instance)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(redis_up, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Redis Dashboard for Prometheus Redis Exporter (helm stable/redis-ha)", - "uid": "xDLNRKUWz", - "version": 1 -} diff --git a/monitoring/grafana/plugins/.keep b/monitoring/grafana/plugins/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml deleted file mode 100644 index 0198297..0000000 --- a/monitoring/grafana/provisioning/dashboards/dashboards.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: 1 - -providers: -- name: dashboards - type: file - updateIntervalSeconds: 30 - options: - path: /etc/grafana/dashboards - foldersFromFilesStructure: true diff --git a/monitoring/grafana/provisioning/datasources/prom.yml b/monitoring/grafana/provisioning/datasources/prom.yml deleted file mode 100644 index 9d7d545..0000000 --- a/monitoring/grafana/provisioning/datasources/prom.yml +++ /dev/null @@ -1,8 +0,0 @@ -# config file version -apiVersion: 1 - -datasources: - - name: DS_PROMETHEUS - type: prometheus - default: true - url: http://prometheus:9090 diff --git a/monitoring/grafana/provisioning/notifiers/.keep b/monitoring/grafana/provisioning/notifiers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/provisioning/plugins/.keep b/monitoring/grafana/provisioning/plugins/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml deleted file mode 100644 index 2a88357..0000000 --- a/monitoring/prometheus/prometheus.yml +++ /dev/null @@ -1,20 +0,0 @@ -scrape_configs: -- job_name: prometheus - static_configs: - - targets: ['localhost:9090'] - -- job_name: netbox - static_configs: - - targets: ['netbox:8001'] - -- job_name: nginx - static_configs: - - targets: ['nginx-exporter:9113'] - -- job_name: postgresql - static_configs: - - targets: ['postgres-exporter:9187'] - -- job_name: redis - static_configs: - - targets: ['redis-worker-exporter:9121', 'redis-cache-exporter:9121'] From 3d244a194606f263a5abed3245b8209175376e32 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Mon, 26 Oct 2020 15:23:17 +0100 Subject: [PATCH 32/36] Fix warning in Github Actions Our build script is using "set-env" which has been deprecated by Github. See this bolg entry: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index be65bd4..360f08c 100755 --- a/build.sh +++ b/build.sh @@ -216,7 +216,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi if [ -n "${GH_ACTION}" ]; then - echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + echo "FINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" >> $GITHUB_ENV echo "::set-output name=skipped::false" fi From 7d055fbcafde5268b09b9bae2ddd01866c3f7689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 26 Oct 2020 15:29:41 +0100 Subject: [PATCH 33/36] Mention monitoring in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a425078..7c4d20d 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The default credentials are: ## Documentation Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly. -It covers advanced topics such as using files for secrets, deployment to Kubernetes and configuring NAPALM or LDAP. +It covers advanced topics such as using files for secrets, deployment to Kubernetes, monitoring and configuring NAPALM or LDAP. [netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/ From 5512ea68b304183d8530bfbbfacf9ee98f25b931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 26 Oct 2020 15:40:22 +0100 Subject: [PATCH 34/36] Ignore prometheus.yml --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8e90907..6eada06 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ configuration/* !configuration/extra.py configuration/ldap/* !configuration/ldap/ldap_config.py +prometheus.yml From 911488242f0eb6ad0eddcb350e83dd2dd8c20e61 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Mon, 26 Oct 2020 15:42:31 +0100 Subject: [PATCH 35/36] Use release version of Python 3.9 in tests --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 243aa68..50d17e9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,7 +20,7 @@ jobs: docker_from: - '' # use the default of the build script - python:3.8-alpine - - python:3.9-rc-alpine + - python:3.9-alpine fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From a52a5547befddd4f93d5de521c32026d8631eb94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 26 Oct 2020 16:21:47 +0100 Subject: [PATCH 36/36] Preparation for 0.26.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d21d277..4e8f395 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.25.0 +0.26.0