* Add boto3 as a dependency * Add Amazon S3 backend for remote data sources * Update docs to include Amazon S3 support
This commit is contained in:
parent
5cd3ad0b12
commit
cacc418cd5
|
@ -2,6 +2,10 @@
|
|||
# https://github.com/mozilla/bleach
|
||||
bleach<6.0
|
||||
|
||||
# Python client for Amazon AWS API
|
||||
# https://github.com/boto/boto3
|
||||
boto3
|
||||
|
||||
# The Python web framework on which NetBox is built
|
||||
# https://github.com/django/django
|
||||
Django<4.2
|
||||
|
|
|
@ -14,15 +14,17 @@ The type of data source. Supported options include:
|
|||
|
||||
* Local directory
|
||||
* git repository
|
||||
* Amazon S3 bucket
|
||||
|
||||
### URL
|
||||
|
||||
The URL identifying the remote source. Some examples are included below.
|
||||
|
||||
| Type | Example URL |
|
||||
|------|-------------|
|
||||
| Local | file:///var/my/data/source/ |
|
||||
| git | https://https://github.com/my-organization/my-repo |
|
||||
| Type | Example URL |
|
||||
|-----------|----------------------------------------------------|
|
||||
| Local | file:///path/to/my/data/ |
|
||||
| git | https://github.com/my-organization/my-repo |
|
||||
| Amazon S3 | https://s3.us-east-2.amazonaws.com/my-bucket-name/ |
|
||||
|
||||
### Status
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ The static home view has been replaced with a fully customizable dashboard. User
|
|||
|
||||
#### Remote Data Sources ([#11558](https://github.com/netbox-community/netbox/issues/11558))
|
||||
|
||||
NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths and from git repositories is supported, and we expect to add support for additional backends in the near future.
|
||||
NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths, git repositories, and Amazon S3 buckets is supported, and we expect to introduce additional backends in the near future.
|
||||
|
||||
#### Configuration Template Rendering ([#11559](https://github.com/netbox-community/netbox/issues/11559))
|
||||
|
||||
|
|
|
@ -10,10 +10,12 @@ from utilities.choices import ChoiceSet
|
|||
class DataSourceTypeChoices(ChoiceSet):
|
||||
LOCAL = 'local'
|
||||
GIT = 'git'
|
||||
AMAZON_S3 = 'amazon-s3'
|
||||
|
||||
CHOICES = (
|
||||
(LOCAL, _('Local'), 'gray'),
|
||||
(GIT, _('Git'), 'blue'),
|
||||
(AMAZON_S3, _('Amazon S3'), 'blue'),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
@ -1,9 +1,14 @@
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlunparse, urlparse
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as Boto3Config
|
||||
from django import forms
|
||||
from django.conf import settings
|
||||
from django.utils.translation import gettext as _
|
||||
|
@ -115,3 +120,70 @@ class GitBackend(DataBackend):
|
|||
yield local_path.name
|
||||
|
||||
local_path.cleanup()
|
||||
|
||||
|
||||
@register_backend(DataSourceTypeChoices.AMAZON_S3)
|
||||
class S3Backend(DataBackend):
|
||||
parameters = {
|
||||
'aws_access_key_id': forms.CharField(
|
||||
label=_('AWS access key ID'),
|
||||
widget=forms.TextInput(attrs={'class': 'form-control'})
|
||||
),
|
||||
'aws_secret_access_key': forms.CharField(
|
||||
label=_('AWS secret access key'),
|
||||
widget=forms.TextInput(attrs={'class': 'form-control'})
|
||||
),
|
||||
}
|
||||
|
||||
REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com'
|
||||
|
||||
@contextmanager
|
||||
def fetch(self):
|
||||
local_path = tempfile.TemporaryDirectory()
|
||||
|
||||
# Build the S3 configuration
|
||||
s3_config = Boto3Config(
|
||||
proxies=settings.HTTP_PROXIES,
|
||||
)
|
||||
|
||||
# Initialize the S3 resource and bucket
|
||||
aws_access_key_id = self.params.get('aws_access_key_id')
|
||||
aws_secret_access_key = self.params.get('aws_secret_access_key')
|
||||
s3 = boto3.resource(
|
||||
's3',
|
||||
region_name=self._region_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
config=s3_config
|
||||
)
|
||||
bucket = s3.Bucket(self._bucket_name)
|
||||
|
||||
# Download all files within the specified path
|
||||
for obj in bucket.objects.filter(Prefix=self._remote_path):
|
||||
local_filename = os.path.join(local_path.name, obj.key)
|
||||
# Build local path
|
||||
Path(os.path.dirname(local_filename)).mkdir(parents=True, exist_ok=True)
|
||||
bucket.download_file(obj.key, local_filename)
|
||||
|
||||
yield local_path.name
|
||||
|
||||
local_path.cleanup()
|
||||
|
||||
@property
|
||||
def _region_name(self):
|
||||
domain = urlparse(self.url).netloc
|
||||
if m := re.match(self.REGION_REGEX, domain):
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
@property
|
||||
def _bucket_name(self):
|
||||
url_path = urlparse(self.url).path.lstrip('/')
|
||||
return url_path.split('/')[0]
|
||||
|
||||
@property
|
||||
def _remote_path(self):
|
||||
url_path = urlparse(self.url).path.lstrip('/')
|
||||
if '/' in url_path:
|
||||
return url_path.split('/', 1)[1]
|
||||
return ''
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
bleach==5.0.1
|
||||
boto3==1.26.91
|
||||
Django==4.1.7
|
||||
django-cors-headers==3.14.0
|
||||
django-debug-toolbar==3.8.1
|
||||
|
|
Loading…
Reference in New Issue