Saving running environment

scripts/bulk.py.new

@@ -0,0 +1,516 @@
+import netaddr
+import csv
+from dcim.choices import InterfaceTypeChoices, InterfaceModeChoices
+from dcim.models import Platform, DeviceRole, Site
+from ipam.models import IPAddress, VRF, Interface, Prefix, VLAN
+from tenancy.models import Tenant
+from virtualization.models import VirtualMachine, Cluster
+from virtualization.choices import VirtualMachineStatusChoices
+from extras.scripts import Script, TextVar, ChoiceVar, ObjectVar
+from extras.models import Tag
+from utilities.forms import APISelect
+
+
+class VM:
+
+    status: str
+    tenant: Tenant
+    cluster: Cluster
+    site: Site
+    csv_ip_address: str
+    ip_address: IPAddress
+
+    DEFAULT_TAGS = ['ansible', 'zero_day']
+    DEFAULT_DOMAIN_PRIVATE = 'privatedns.zone'
+    DEFAULT_DOMAIN_PUBLIC = 'publicdns.zone'
+
+    def __init__(self, status, tenant, cluster, datazone, env, platform, role, backup, vcpus, memory, disk, ip_address, hostname, extra_tags):
+        # IP address can first be created after vm
+        self.csv_ip_address = ip_address
+        self.set_cluster(cluster)
+        self.set_status(status)
+        self.set_tenant(tenant)
+        self.set_datazone(datazone)
+        self.set_env(env)
+        self.set_platform(platform)
+        self.set_role(role)
+        self.set_backup_tag(backup)
+        self.set_vcpus(vcpus)
+        self.set_memory(memory)
+        self.set_disk(disk)
+        self.set_hostname(hostname)
+        self.set_vlan(None)
+        self.set_extra_tags(extra_tags)
+
+    def set_extra_tags(self, extra_tags):
+        try:
+            if extra_tags is None:
+                self.extra_tags = extra_tags
+            else:
+                self.extra_tags = extra_tags.split(',')
+        except Exception as e:
+            raise Exception("Extra tags - {0}".format(e))
+
+    def set_vlan(self, vlan):
+        try:
+            self.vlan = VLAN.objects.get(
+                vid=vlan,
+                site=self.site
+            )
+        except Exception:
+            self.vlan = None
+
+    def generate_hostname(self):
+        # I now proclaim this VM, First of its Name, Queen of the Andals and the First Men, Protector of the Seven Kingdoms
+        vm_index = "001"
+        vm_search_for = "{0}-{1}-{2}-".format(self.site.slug, self.env.name.split('_')[1], self.role.name.split(':')[0])
+        vms = VirtualMachine.objects.filter(
+            name__startswith=vm_search_for
+        )
+        if len(vms) > 0:
+            # Get last of its kind
+            last_vm_index = int(vms[len(vms) - 1].name.split('-')[3]) + 1
+            if last_vm_index < 10:
+                vm_index = '00' + str(last_vm_index)
+            elif last_vm_index < 100:
+                vm_index = '0' + str(last_vm_index)
+            else:
+                vm_index = str(last_vm_index)
+
+        return "{0}{1}".format(vm_search_for, vm_index)
+
+    def get_vrf(self):
+        return VRF.objects.get(
+            name="global"
+        )
+
+    def get_fqdn(self):
+        return "{}.{}".format(self.hostname, self.DEFAULT_DOMAIN_PRIVATE if netaddr.IPNetwork(self.ip_address.address).is_private() is True else self.DEFAULT_DOMAIN_PUBLIC)
+
+    def set_ip_address(self, vm):
+        try:
+            if not isinstance(self.vlan, VLAN):
+                ip_check = IPAddress.objects.filter(address=self.csv_ip_address)
+                if len(ip_check) > 0:
+                    raise Exception(str(ip_check[0].address) + ' is already assigned to ' + str(ip_check[0].interface.name))
+                self.ip_address = IPAddress(
+                    address=self.csv_ip_address,
+                    vrf=self.get_vrf(),
+                    tenant=self.tenant,
+                    family=4,
+                )
+            else:
+                # Auto assign IPs from vlan
+                prefix = Prefix.objects.get(
+                    vlan=self.vlan,
+                    site=vm.site
+                )
+                prefix.is_pool = True
+                # Save as pool
+                prefix.save()
+
+                ip_address = prefix.get_first_available_ip()
+                self.ip_address = IPAddress(
+                    address=ip_address,
+                    vrf=self.get_vrf(),
+                    tenant=self.tenant,
+                    family=4,
+                )
+            self.ip_address.dns_name = self.get_fqdn()
+            self.ip_address.save()
+        except Exception as e:
+            self.ip_address = None
+            raise Exception("IP address - {0}".format(e))
+
+    def get_vlan(self):
+        return False
+
+    def set_hostname(self, hostname):
+        try:
+            self.hostname = hostname if hostname is not None else self.generate_hostname()
+        except Exception as e:
+            raise Exception("Hostname - {0}".format(e))
+
+    def set_disk(self, disk):
+        try:
+            self.disk = disk
+        except Exception as e:
+            raise Exception("Disk - {0}".format(e))
+
+    def set_memory(self, memory):
+        try:
+            self.memory = memory
+        except Exception as e:
+            raise Exception("Memory - {0}".format(e))
+
+    def set_vcpus(self, vcpus):
+        try:
+            self.vcpus = vcpus
+        except Exception as e:
+            raise Exception("vcpus - {0}".format(e))
+
+    def set_backup_tag(self, backup):
+        try:
+            if isinstance(backup, Tag):
+                self.backup = backup
+            else:
+                self.backup = Tag.objects.filter(name="{0}".format(backup))[0]
+        except Exception as e:
+            raise Exception("Tag backup {0} does not exist, {1}".format(backup, e))
+
+    def set_site(self, site):
+        try:
+            if isinstance(site, Site):
+                self.site = site
+            else:
+                raise Exception("site is not of instance 'Site'")
+        except Exception as e:
+            raise Exception('Site does not exist - ' + str(e))
+
+    def set_role(self, role):
+        try:
+            self.role = DeviceRole.objects.get(
+                name=role
+            )
+        except Exception as e:
+            raise Exception('Role does not exist - ' + str(e))
+
+    def set_platform(self, platform):
+        try:
+            if isinstance(platform, Platform):
+                self.platform = platform
+            else:
+                self.platform = Platform.objects.get(
+                    name=platform
+                )
+        except Exception as e:
+            raise Exception("Platform does not exist {0}".format(e))
+
+    def set_env(self, env):
+        try:
+            if isinstance(env, Tag):
+                self.env = env
+            else:
+                self.env = Tag.objects.get(name="env_{0}".format(env))
+        except Exception as e:
+            raise Exception("Tag env does not exist - {0}".format(e, env))
+
+    def set_datazone(self, datazone):
+        try:
+            self.datazone = Tag.objects.get(name="datazone_{0}".format(datazone))
+        except Exception as e:
+            raise Exception("Tag datazone does not exist - {0}".format(e))
+
+    def set_cluster(self, cluster):
+        try:
+            if isinstance(cluster, Cluster):
+                self.cluster = cluster
+            else:
+                self.cluster = Cluster.objects.get(
+                    name=cluster
+                )
+            self.set_site(self.cluster.site)
+        except Exception as e:
+            raise Exception("Cluster does not exist {0}".format(e))
+
+    def set_tenant(self, tenant):
+        try:
+            if isinstance(tenant, Tenant):
+                self.tenant = tenant
+            else:
+                self.tenant = Tenant.objects.get(
+                    slug=tenant
+                )
+        except Exception as e:
+            raise Exception("Tenant does not exist {0}".format(e))
+
+    def set_status(self, status):
+        try:
+            self.status = VirtualMachineStatusChoices.STATUS_STAGED if status == 'staged' else VirtualMachineStatusChoices.STATUS_PLANNED
+        except Exception as e:
+            raise Exception("Status does not exist {0}".format(e))
+
+    def get_ip_address(self):
+        return self.ip_address
+
+    def __create_ip_address(self, vm):
+        self.set_ip_address(vm)
+        vm.primary_ip4 = self.get_ip_address()
+        vm.save()
+
+    def __create_tags(self, vm: VirtualMachine):
+        vm.tags.add(self.datazone)
+        vm.tags.add(self.env)
+        vm.tags.add(self.backup)
+        for tag in self.DEFAULT_TAGS:
+            vm.tags.add(tag)
+        if self.extra_tags is not None:
+            for tag in self.extra_tags:
+                vm.tags.add(tag)
+        vm.save()
+        self.set_tags(vm.tags)
+
+    def set_tags(self, tags):
+        self.tags = tags
+
+    def get_tags(self):
+        list = []
+        for tag in self.tags.all():
+            list.append(tag.name)
+        return list
+
+    def __create_vm(self):
+        vm = VirtualMachine(
+            status=self.status,
+            cluster=self.cluster,
+            platform=self.platform,
+            role=self.role,
+            tenant=self.tenant,
+            name=self.hostname,
+            disk=self.disk,
+            memory=self.memory,
+            vcpus=self.vcpus,
+        )
+        vm.save()
+        return vm
+
+    def __create_interface(self, vm: VirtualMachine):
+        """
+        Setup interface and add IP address
+        """
+        try:
+
+            # Get net address tools
+            ip = netaddr.IPNetwork(vm.primary_ip4.address)
+            prefix_search = str(ip.network) + '/' + str(ip.prefixlen)
+
+            prefix = Prefix.objects.get(
+                prefix=prefix_search,
+                is_pool=True,
+                site=self.site,
+            )
+
+            interfaces = vm.get_config_context().get('interfaces')
+
+            interface = Interface(
+                name=interfaces['nic0']['name'],
+                mtu=interfaces['nic0']['mtu'],
+                virtual_machine=vm,
+                type=InterfaceTypeChoices.TYPE_VIRTUAL
+            )
+
+            # If we need anything other than Access, here is were to change it
+            if interfaces['nic0']['mode'] == "Access":
+                interface.mode = InterfaceModeChoices.MODE_ACCESS
+                interface.untagged_vlan = prefix.vlan
+
+            interface.save()
+
+            self.ip_address.interface = interface
+            self.ip_address.save()
+
+        except Exception as e:
+            raise Exception("Error while creating interface - {}".format(e))
+        return True
+
+    def create(self):
+        try:
+            vm = self.__create_vm()
+            self.__create_ip_address(vm)
+            self.__create_tags(vm)
+            self.__create_interface(vm)
+        except Exception as e:
+            raise e
+        return True
+
+
+class BulkDeployVM(Script):
+    """
+    Example CSV full:
+    status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags
+    staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,redirtp:v0.2.0,backup_nobackup,1,1024,10,odn1-vlb-redirtp-001,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001"
+    staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,consul:v1.0.1,backup_general_1,2,2048,20,odn1-vlb-consul-001,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002"
+    staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,rediast:v0.2.0,backup_general_4,4,4096,30,odn1-vlb-rediast-001,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003"
+
+    Example CSV minimal (If all defaults are set):
+    vcpus,memory,disk,ip_address,extra_tags
+    1,1024,10,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001"
+    2,2048,20,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002"
+    4,4096,30,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003"
+
+    ** Required Params **
+    Param: cluster      - vSphere cluster name
+    Param: env          - Adds 'env_xxx' tag
+    Param: platform     - VM Platform e.g base:v1.0.0-coreos
+    Param: backup       - Adds backup tag
+    Param: vcpus        - Virtual CPUs (hot add)
+    Param: memory       - Virtual memory (hot add)
+    Param: disk         - Disk2 size
+    Param: hostname     - VM hostname (Optional if 'role' is set)
+    Param: role         - VM Device role
+    Param: ip_address   - VM IP address
+
+    ** Optional Params **
+    Param: status       - VM status (default 'staged')
+    Param: tenant       - Netbox tenant (default slug:'patientsky-hosting')
+    Param: datazone     - Adds 'datazone_x' tag (default 'rr')
+    Param: extra_tags   - Adds extra tags to VM
+    """
+
+    DEFAULT_CSV_FIELDS = "env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags"
+    datazone_rr: bool = True
+
+    class Meta:
+        name = "Bulk deploy new VMs"
+        description = "Deploy new virtual machines from existing platforms"
+        fields = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_role']
+        field_order = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_role']
+        commit_default = False
+
+    vms = TextVar(
+        label="Import CSV",
+        description="CSV data",
+        required=True,
+        default=DEFAULT_CSV_FIELDS
+    )
+
+    default_status = ChoiceVar(
+        label="Default Status",
+        description="Default CSV field `status` if none given",
+        required=False,
+        choices=(
+            (VirtualMachineStatusChoices.STATUS_STAGED, 'Staged (Deploy now)'),
+            (VirtualMachineStatusChoices.STATUS_PLANNED, 'Planned (Save for later)')
+        )
+    )
+
+    default_tenant = ObjectVar(
+        label="Default Tenant",
+        default=1,
+        required=False,
+        description="Default CSV field `tenant` if none given",
+        queryset=Tenant.objects.all()
+    )
+
+    default_datazone = ChoiceVar(
+        label="Default Datazone",
+        description="Default CSV field `datazone` if none given",
+        default="rr",
+        required=False,
+        choices=(
+            ('rr', 'Round robin (1,2)'),
+            ('1', '1'),
+            ('2', '2')
+        )
+    )
+
+    default_cluster = ObjectVar(
+        label="Default Cluster",
+        description="Default CSV field `cluster` if none given",
+        required=False,
+        queryset=Cluster.objects.all()
+    )
+
+    default_env = ObjectVar(
+        label="Default Environment",
+        description="Default CSV field `env` if none given",
+        default="env_inf",
+        required=False,
+        queryset=Tag.objects.filter(
+            name__startswith='env_',
+        ).order_by('name'),
+    )
+
+    default_platform = ObjectVar(
+        label="Default Platform",
+        description="Default CSV field `platform` if none given",
+        required=False,
+        widget=APISelect(
+            api_url='/api/dcim/platforms/?name__ic=_',
+            display_field='display_name',
+        ),
+        queryset=Platform.objects.all().order_by('name')
+    )
+
+    default_role = ObjectVar(
+        label="Default Role",
+        description="Default CSV field `role` if none given",
+        default=None,
+        required=False,
+        widget=APISelect(
+            api_url='/api/dcim/device-roles/?vm_role=true',
+            display_field='display_name',
+        ),
+        queryset=DeviceRole.objects.filter(
+            vm_role=True
+        ).order_by('name')
+    )
+
+    default_backup = ObjectVar(
+        label="Default Backup",
+        description="Default CSV field `backup` if none given",
+        required=False,
+        queryset=Tag.objects.filter(
+            name__startswith='backup_',
+        ).order_by('name'),
+    )
+
+    def get_vm_data(self):
+        return self.vm_data
+
+    def set_csv_data(self, vms):
+        self.csv_raw_data = csv.DictReader(vms, delimiter=',')
+
+    def get_csv_raw_data(self):
+        return self.csv_raw_data
+
+    def set(self, data):
+        self.set_csv_data(data['vms'].splitlines())
+
+    def get_datazone(self, datazone):
+        if datazone == 'rr':
+            datazone = 1 if self.datazone_rr else 2
+            self.datazone_rr = not self.datazone_rr
+        return datazone
+
+    def run(self, data):
+        # Set data from raw csv
+        self.set(data)
+        line = 1
+        for raw_vm in self.get_csv_raw_data():
+            try:
+                vm = VM(
+                    status=raw_vm.get('status') if raw_vm.get('status') is not None else data['default_status'],
+                    tenant=raw_vm.get('tenant') if raw_vm.get('tenant') is not None else data['default_tenant'],
+                    datazone=raw_vm.get('datazone') if raw_vm.get('datazone') is not None else self.get_datazone(data['default_datazone']),
+                    cluster=raw_vm.get('cluster') if raw_vm.get('cluster') is not None else data['default_cluster'],
+                    env=raw_vm.get('env') if raw_vm.get('env') is not None else data['default_env'],
+                    platform=raw_vm.get('platform') if raw_vm.get('platform') is not None else data['default_platform'],
+                    role=raw_vm.get('role') if raw_vm.get('role') is not None else data['default_role'],
+                    backup=raw_vm.get('backup') if raw_vm.get('backup') is not None else data['default_backup'],
+                    vcpus=raw_vm.get('vcpus'),
+                    memory=raw_vm.get('memory'),
+                    disk=raw_vm.get('disk'),
+                    hostname=raw_vm.get('hostname'),
+                    ip_address=raw_vm.get('ip_address'),
+                    extra_tags=raw_vm.get('extra_tags'),
+                )
+                vm.create()
+                self.log_success(
+                    "{} `{}` for `{}`, `{}`, in cluster `{}`, env `{}`, datazone `{}`, backup `{}`".
+                    format(
+                        vm.status.capitalize(),
+                        vm.hostname,
+                        vm.tenant,
+                        vm.ip_address.address,
+                        vm.cluster,
+                        str(vm.env.name).split('_')[1],
+                        vm.datazone,
+                        vm.backup,
+                    )
+                )
+                line += 1
+            except Exception as e:
+                self.log_failure("CSV line {}, Error while creating VM \n`{}`".format(line, e))
+        return data['vms']
+