diff --git a/scripts/bulk.py b/scripts/bulk.py new file mode 100644 index 0000000..734cdd7 --- /dev/null +++ b/scripts/bulk.py @@ -0,0 +1,590 @@ +import netaddr +import csv +from dcim.choices import InterfaceTypeChoices, InterfaceModeChoices +from dcim.models import Platform, DeviceRole, Site +from ipam.models import IPAddress, VRF, Interface, Prefix, VLAN +from tenancy.models import Tenant +from virtualization.models import VirtualMachine, Cluster +from virtualization.choices import VirtualMachineStatusChoices +from extras.scripts import Script, TextVar, ChoiceVar, ObjectVar +from extras.models import Tag +from utilities.forms import APISelect + + +class VM: + + status: str + tenant: Tenant + cluster: Cluster + site: Site + csv_ip_address: str + ip_address: IPAddress + comment: str + + DEFAULT_TAGS = ['ansible', 'zero_day'] + DEFAULT_DOMAIN_PRIVATE = 'privatedns.zone' + DEFAULT_DOMAIN_PUBLIC = 'publicdns.zone' + + def __init__(self, status, tenant, cluster, datazone, env, platform, role, backup, backup_offsite, vcpus, memory, disk, ip_address, hostname, extra_tags): + # IP address can first be created after vm + self.csv_ip_address = ip_address + self.set_cluster(cluster) + self.set_status(status) + self.set_tenant(tenant) + self.set_datazone(datazone) + self.set_env(env) + self.set_platform(platform) + self.set_role(role) + self.set_backup_tag(backup) + self.set_backup_offsite_tag(backup_offsite) + self.set_vcpus(vcpus) + self.set_memory(memory) + self.set_disk(disk) + self.set_hostname(hostname) + self.set_vlan(None) + self.set_extra_tags(extra_tags) + self.set_comments() + + def set_comments(self): + try: + extra_tags = "" + for tag in self.extra_tags: + extra_tags += tag + "," + + if self.backup_offsite is not None: + self.comments = "status,tenant,cluster,datazone,env,platform,role,backup,backup_offsite,vcpus,memory,disk,hostname,ip_address,extra_tags\n" + self.comments += "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},\"{14}\"".format( + self.status, + self.tenant.slug, + self.cluster, + self.datazone.name.split('datazone_')[1], + self.env.name.split('env_')[1], + self.platform, + self.role, + self.backup.name, + self.backup_offsite.name if self.backup_offsite is not None else "", + self.vcpus, + self.memory, + self.disk, + self.hostname, + self.csv_ip_address, + extra_tags[:-1], + ) + else: + self.comments = "status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags\n" + self.comments += "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},\"{13}\"".format( + self.status, + self.tenant.slug, + self.cluster, + self.datazone.name.split('datazone_')[1], + self.env.name.split('env_')[1], + self.platform, + self.role, + self.backup.name, + self.vcpus, + self.memory, + self.disk, + self.hostname, + self.csv_ip_address, + extra_tags[:-1], + ) + + except Exception as e: + raise Exception("Comments - {0}".format(e)) + + def set_extra_tags(self, extra_tags): + try: + if extra_tags is None: + self.extra_tags = extra_tags + else: + self.extra_tags = extra_tags.split(',') + except Exception as e: + raise Exception("Extra tags - {0}".format(e)) + + def set_vlan(self, vlan): + try: + self.vlan = VLAN.objects.get( + vid=vlan, + site=self.site + ) + except Exception: + self.vlan = None + + def generate_hostname(self): + # I now proclaim this VM, First of its Name, Queen of the Andals and the First Men, Protector of the Seven Kingdoms + vm_index = "001" + vm_search_for = "{0}-{1}-{2}-".format(self.site.slug, self.env.name.split('_')[1], self.role.name.split(':')[0]) + vms = VirtualMachine.objects.filter( + name__startswith=vm_search_for + ) + if len(vms) > 0: + # Get last of its kind + last_vm_index = int(vms[len(vms) - 1].name.split('-')[3]) + 1 + if last_vm_index < 10: + vm_index = '00' + str(last_vm_index) + elif last_vm_index < 100: + vm_index = '0' + str(last_vm_index) + else: + vm_index = str(last_vm_index) + + return "{0}{1}".format(vm_search_for, vm_index) + + def get_vrf(self): + return VRF.objects.get( + name="global" + ) + + def get_fqdn(self): + return "{}.{}".format(self.hostname, self.DEFAULT_DOMAIN_PRIVATE if netaddr.IPNetwork(self.ip_address.address).is_private() is True else self.DEFAULT_DOMAIN_PUBLIC) + + def set_ip_address(self, vm): + try: + ip_check = IPAddress.objects.filter(address=self.csv_ip_address) + if len(ip_check) > 0: + raise Exception(str(ip_check[0].address) + ' is already assigned') + + if not isinstance(self.vlan, VLAN): + self.ip_address = IPAddress( + address=self.csv_ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + else: + # Auto assign IPs from vlan + prefix = Prefix.objects.get( + vlan=self.vlan, + site=vm.site + ) + prefix.is_pool = True + # Save as pool + prefix.save() + + ip_address = prefix.get_first_available_ip() + self.ip_address = IPAddress( + address=ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + self.ip_address.dns_name = self.get_fqdn() + self.ip_address.save() + + except Exception as e: + self.ip_address = None + raise Exception("IP address - {0}".format(e)) + + def get_vlan(self): + return False + + def set_hostname(self, hostname): + try: + self.hostname = hostname if hostname is not None else self.generate_hostname() + except Exception as e: + raise Exception("Hostname - {0}".format(e)) + + def set_disk(self, disk): + try: + self.disk = disk + except Exception as e: + raise Exception("Disk - {0}".format(e)) + + def set_memory(self, memory): + try: + self.memory = memory + except Exception as e: + raise Exception("Memory - {0}".format(e)) + + def set_vcpus(self, vcpus): + try: + self.vcpus = vcpus + except Exception as e: + raise Exception("vcpus - {0}".format(e)) + + def set_backup_tag(self, backup): + try: + if isinstance(backup, Tag): + self.backup = backup + else: + self.backup = Tag.objects.filter(name="{0}".format(backup))[0] + except Exception as e: + raise Exception("Tag backup {0} does not exist, {1}".format(backup, e)) + + def set_backup_offsite_tag(self, backup_offsite): + try: + if isinstance(backup_offsite, Tag): + self.backup_offsite = backup_offsite + else: + self.backup_offsite = Tag.objects.filter(name="{0}".format(backup_offsite))[0] + except Exception: + self.backup_offsite = None + + def set_site(self, site): + try: + if isinstance(site, Site): + self.site = site + else: + raise Exception("site is not of instance 'Site'") + except Exception as e: + raise Exception('Site does not exist - ' + str(e)) + + def set_role(self, role): + try: + self.role = DeviceRole.objects.get( + name=role + ) + except Exception as e: + raise Exception('Role does not exist - ' + str(e)) + + def set_platform(self, platform): + try: + if isinstance(platform, Platform): + self.platform = platform + else: + self.platform = Platform.objects.get( + name=platform + ) + except Exception as e: + raise Exception("Platform does not exist {0}".format(e)) + + def set_env(self, env): + try: + if isinstance(env, Tag): + self.env = env + else: + self.env = Tag.objects.get(name="env_{0}".format(env)) + except Exception as e: + raise Exception("Tag env does not exist - {0}".format(e, env)) + + def set_datazone(self, datazone): + try: + self.datazone = Tag.objects.get(name="datazone_{0}".format(datazone)) + except Exception as e: + raise Exception("Tag datazone does not exist - {0}".format(e)) + + def set_cluster(self, cluster): + try: + if isinstance(cluster, Cluster): + self.cluster = cluster + else: + self.cluster = Cluster.objects.get( + name=cluster + ) + self.set_site(self.cluster.site) + except Exception as e: + raise Exception("Cluster does not exist {0}".format(e)) + + def set_tenant(self, tenant): + try: + if isinstance(tenant, Tenant): + self.tenant = tenant + else: + self.tenant = Tenant.objects.get( + slug=tenant + ) + except Exception as e: + raise Exception("Tenant does not exist {0}".format(e)) + + def set_status(self, status): + try: + self.status = VirtualMachineStatusChoices.STATUS_STAGED if status == 'staged' else VirtualMachineStatusChoices.STATUS_PLANNED + except Exception as e: + raise Exception("Status does not exist {0}".format(e)) + + def get_ip_address(self): + return self.ip_address + + def __create_ip_address(self, vm): + self.set_ip_address(vm) + vm.primary_ip4 = self.get_ip_address() + vm.save() + + def __create_tags(self, vm: VirtualMachine): + vm.tags.add(self.datazone) + vm.tags.add(self.env) + vm.tags.add(self.backup) + if(self.backup_offsite is not None): + vm.tags.add(self.backup_offsite) + for tag in self.DEFAULT_TAGS: + vm.tags.add(tag) + if self.extra_tags is not None: + for tag in self.extra_tags: + vm.tags.add(tag) + vm.save() + self.set_tags(vm.tags) + + def set_tags(self, tags): + self.tags = tags + + def get_tags(self): + list = [] + for tag in self.tags.all(): + list.append(tag.name) + return list + + def __create_vm(self): + vm = VirtualMachine( + status=self.status, + cluster=self.cluster, + platform=self.platform, + role=self.role, + tenant=self.tenant, + name=self.hostname, + disk=self.disk, + memory=self.memory, + vcpus=self.vcpus, + comments=self.comments, + ) + vm.save() + return vm + + def __create_interface(self, vm: VirtualMachine): + """ + Setup interface and add IP address + """ + try: + + # Get net address tools + ip = netaddr.IPNetwork(vm.primary_ip4.address) + prefix_search = str(ip.network) + '/' + str(ip.prefixlen) + + prefix = Prefix.objects.get( + prefix=prefix_search, + is_pool=True, + site=self.site, + ) + + interfaces = vm.get_config_context().get('interfaces') + + interface = Interface( + name=interfaces['nic0']['name'], + mtu=interfaces['nic0']['mtu'], + virtual_machine=vm, + type=InterfaceTypeChoices.TYPE_VIRTUAL + ) + + # If we need anything other than Access, here is were to change it + if interfaces['nic0']['mode'] == "Access": + interface.mode = InterfaceModeChoices.MODE_ACCESS + interface.untagged_vlan = prefix.vlan + + interface.save() + + self.ip_address.interface = interface + self.ip_address.save() + + except Exception as e: + raise Exception("Error while creating interface - {}".format(e)) + return True + + def create(self): + try: + vm = self.__create_vm() + self.__create_ip_address(vm) + self.__create_tags(vm) + self.__create_interface(vm) + except Exception as e: + raise e + return True + + +class BulkDeployVM(Script): + """ + Example CSV full: + status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,redirtp:v0.2.0,backup_nobackup,1,1024,10,odn1-vlb-redirtp-001,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,consul:v1.0.1,backup_general_1,2,2048,20,odn1-vlb-consul-001,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,rediast:v0.2.0,backup_general_4,4,4096,30,odn1-vlb-rediast-001,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + Example CSV minimal (If all defaults are set): + vcpus,memory,disk,ip_address,extra_tags + 1,1024,10,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + 2,2048,20,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + 4,4096,30,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + ** Required Params ** + Param: cluster - vSphere cluster name + Param: env - Adds 'env_xxx' tag + Param: platform - VM Platform e.g base:v1.0.0-coreos + Param: backup - Adds backup tag + Param: backup_offsite - Adds offsite backup tag + Param: vcpus - Virtual CPUs (hot add) + Param: memory - Virtual memory (hot add) + Param: disk - Disk2 size + Param: hostname - VM hostname (Optional if 'role' is set) + Param: role - VM Device role + Param: ip_address - VM IP address + + ** Optional Params ** + Param: status - VM status (default 'staged') + Param: tenant - Netbox tenant (default slug:'patientsky-hosting') + Param: datazone - Adds 'datazone_x' tag (default 'rr') + Param: extra_tags - Adds extra tags to VM + """ + + DEFAULT_CSV_FIELDS = "vcpus,memory,disk,ip_address,extra_tags" + datazone_rr: bool = True + + class Meta: + name = "Bulk deploy new VMs" + description = "Deploy new virtual machines from existing platforms" + fields = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_backup_offsite', 'default_role'] + field_order = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_backup_offsite', 'default_role'] + commit_default = False + + vms = TextVar( + label="Import CSV", + description="CSV data", + required=True, + default=DEFAULT_CSV_FIELDS + ) + + default_status = ChoiceVar( + label="Default Status", + description="Default CSV field `status` if none given", + required=False, + choices=( + (VirtualMachineStatusChoices.STATUS_STAGED, 'Staged (Deploy now)'), + (VirtualMachineStatusChoices.STATUS_PLANNED, 'Planned (Save for later)') + ) + ) + + default_tenant = ObjectVar( + label="Default Tenant", + default=1, + required=False, + description="Default CSV field `tenant` if none given", + queryset=Tenant.objects.all() + ) + + default_datazone = ChoiceVar( + label="Default Datazone", + description="Default CSV field `datazone` if none given", + default="rr", + required=False, + choices=( + ('rr', 'Round robin (1,2)'), + ('1', '1'), + ('2', '2') + ) + ) + + default_cluster = ObjectVar( + label="Default Cluster", + description="Default CSV field `cluster` if none given", + required=False, + queryset=Cluster.objects.all() + ) + + default_env = ObjectVar( + label="Default Environment", + description="Default CSV field `env` if none given", + default="env_inf", + required=False, + queryset=Tag.objects.filter( + name__startswith='env_', + ).order_by('name'), + ) + + default_platform = ObjectVar( + label="Default Platform", + description="Default CSV field `platform` if none given", + required=False, + widget=APISelect( + api_url='/api/dcim/platforms/?name__ic=_', + display_field='display_name', + ), + queryset=Platform.objects.all().order_by('name') + ) + + default_role = ObjectVar( + label="Default Role", + description="Default CSV field `role` if none given", + default=None, + required=False, + widget=APISelect( + api_url='/api/dcim/device-roles/?vm_role=true', + display_field='display_name', + ), + queryset=DeviceRole.objects.filter( + vm_role=True + ).order_by('name') + ) + + default_backup = ObjectVar( + label="Default Backup", + description="Default CSV field `backup` if none given", + required=False, + queryset=Tag.objects.filter( + name__startswith='backup_', + ).order_by('name'), + ) + + default_backup_offsite = ObjectVar( + label="Default Offsite Backup", + description="Default CSV field `backup_offsite` if none given", + required=False, + queryset=Tag.objects.filter( + name__startswith='backup_off', + ).order_by('name'), + ) + + def get_vm_data(self): + return self.vm_data + + def set_csv_data(self, vms): + self.csv_raw_data = csv.DictReader(vms, delimiter=',') + + def get_csv_raw_data(self): + return self.csv_raw_data + + def set(self, data): + self.set_csv_data(data['vms'].splitlines()) + + def get_datazone(self, datazone): + if datazone == 'rr': + datazone = 1 if self.datazone_rr else 2 + self.datazone_rr = not self.datazone_rr + return datazone + + def run(self, data): + # Set data from raw csv + self.set(data) + line = 1 + for raw_vm in self.get_csv_raw_data(): + try: + vm = VM( + status=raw_vm.get('status') if raw_vm.get('status') is not None else data['default_status'], + tenant=raw_vm.get('tenant') if raw_vm.get('tenant') is not None else data['default_tenant'], + datazone=raw_vm.get('datazone') if raw_vm.get('datazone') is not None else self.get_datazone(data['default_datazone']), + cluster=raw_vm.get('cluster') if raw_vm.get('cluster') is not None else data['default_cluster'], + env=raw_vm.get('env') if raw_vm.get('env') is not None else data['default_env'], + platform=raw_vm.get('platform') if raw_vm.get('platform') is not None else data['default_platform'], + role=raw_vm.get('role') if raw_vm.get('role') is not None else data['default_role'], + backup=raw_vm.get('backup') if raw_vm.get('backup') is not None else data['default_backup'], + backup_offsite=raw_vm.get('backup_offsite') if raw_vm.get('backup_offsite') is not None else data['default_backup_offsite'], + vcpus=raw_vm.get('vcpus'), + memory=raw_vm.get('memory'), + disk=raw_vm.get('disk'), + hostname=raw_vm.get('hostname'), + ip_address=raw_vm.get('ip_address'), + extra_tags=raw_vm.get('extra_tags'), + ) + vm.create() + self.log_success( + "{} `{}` for `{}`, `{}`, in cluster `{}`, env `{}`, datazone `{}`, backup `{}`". + format( + vm.status.capitalize(), + vm.hostname, + vm.tenant, + vm.ip_address.address, + vm.cluster, + str(vm.env.name).split('_')[1], + vm.datazone, + vm.backup, + ) + ) + line += 1 + except Exception as e: + self.log_failure("CSV line {}, Error while creating VM \n`{}`".format(line, e)) + return data['vms'] diff --git a/scripts/bulk.py.backup b/scripts/bulk.py.backup new file mode 100644 index 0000000..c6e6908 --- /dev/null +++ b/scripts/bulk.py.backup @@ -0,0 +1,592 @@ +import netaddr +import csv +from dcim.choices import InterfaceTypeChoices, InterfaceModeChoices +from dcim.models import Platform, DeviceRole, Site +from ipam.models import IPAddress, VRF, Interface, Prefix, VLAN +from tenancy.models import Tenant +from virtualization.models import VirtualMachine, Cluster +from virtualization.choices import VirtualMachineStatusChoices +from extras.scripts import Script, TextVar, ChoiceVar, ObjectVar +from extras.models import Tag +from utilities.forms import APISelect + + +class VM: + + __staged_hostnames: [] + status: str + tenant: Tenant + cluster: Cluster + site: Site + csv_ip_address: str + ip_address: IPAddress + comment: str + + DEFAULT_TAGS = ['ansible', 'zero_day'] + DEFAULT_DOMAIN_PRIVATE = 'privatedns.zone' + DEFAULT_DOMAIN_PUBLIC = 'publicdns.zone' + + def __init__(self, status, tenant, cluster, datazone, env, platform, role, backup, backup_offsite, vcpus, memory, disk, ip_address, hostname, extra_tags): + # IP address can first be created after vm + self.csv_ip_address = ip_address + self.set_cluster(cluster) + self.set_status(status) + self.set_tenant(tenant) + self.set_datazone(datazone) + self.set_env(env) + self.set_platform(platform) + self.set_role(role) + self.set_backup_tag(backup) + self.set_backup_offsite_tag(backup_offsite) + self.set_vcpus(vcpus) + self.set_memory(memory) + self.set_disk(disk) + self.set_hostname(hostname) + self.set_vlan(None) + self.set_extra_tags(extra_tags) + self.set_comments() + + def set_comments(self): + try: + extra_tags = "" + for tag in self.extra_tags: + extra_tags += tag + "," + + if self.backup_offsite is not None: + self.comments = "status,tenant,cluster,datazone,env,platform,role,backup,backup_offsite,vcpus,memory,disk,hostname,ip_address,extra_tags\n" + self.comments += "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},\"{14}\"".format( + self.status, + self.tenant.slug, + self.cluster, + self.datazone.name.split('datazone_')[1], + self.env.name.split('env_')[1], + self.platform, + self.role, + self.backup.name.split('vsphere_tag_')[1], + self.backup_offsite.name.split('vsphere_tag_')[1] if self.backup_offsite is not None else "", + self.vcpus, + self.memory, + self.disk, + self.hostname, + self.csv_ip_address, + extra_tags[:-1], + ) + else: + self.comments = "status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags\n" + self.comments += "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},\"{13}\"".format( + self.status, + self.tenant.slug, + self.cluster, + self.datazone.name.split('datazone_')[1], + self.env.name.split('env_')[1], + self.platform, + self.role, + self.backup.name.split('vsphere_tag_')[1], + self.vcpus, + self.memory, + self.disk, + self.hostname, + self.csv_ip_address, + extra_tags[:-1], + ) + + except Exception as e: + raise Exception("Comments - {0}".format(e)) + + def set_extra_tags(self, extra_tags): + try: + if extra_tags is None: + self.extra_tags = extra_tags + else: + self.extra_tags = extra_tags.split(',') + except Exception as e: + raise Exception("Extra tags - {0}".format(e)) + + def set_vlan(self, vlan): + try: + self.vlan = VLAN.objects.get( + vid=vlan, + site=self.site + ) + except Exception: + self.vlan = None + + def generate_hostname(self): + # I now proclaim this VM, First of its Name, Queen of the Andals and the First Men, Protector of the Seven Kingdoms + vm_index = "001" + vm_search_for = "{0}-{1}-{2}-".format(self.site.slug, self.env.name.split('_')[1], self.role.name.split(':')[0]) + vms = VirtualMachine.objects.filter( + name__startswith=vm_search_for + ) + if len(vms) > 0: + # Get last of its kind + last_vm_index = int(vms[len(vms) - 1].name.split('-')[3]) + 1 + if last_vm_index < 10: + vm_index = '00' + str(last_vm_index) + elif last_vm_index < 100: + vm_index = '0' + str(last_vm_index) + else: + vm_index = str(last_vm_index) + # self.__staged_hostnames.append("{0}{1}".format(search_for, vm_index)) + return "{0}{1}".format(vm_search_for, vm_index) + + def get_vrf(self): + return VRF.objects.get( + name="global" + ) + + def get_fqdn(self): + return "{}.{}".format(self.hostname, self.DEFAULT_DOMAIN_PRIVATE if netaddr.IPNetwork(self.ip_address.address).is_private() is True else self.DEFAULT_DOMAIN_PUBLIC) + + def set_ip_address(self, vm): + try: + ip_check = IPAddress.objects.filter(address=self.csv_ip_address) + if len(ip_check) > 0: + raise Exception(str(ip_check[0].address) + ' is already assigned') + + if not isinstance(self.vlan, VLAN): + self.ip_address = IPAddress( + address=self.csv_ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + else: + # Auto assign IPs from vlan + prefix = Prefix.objects.get( + vlan=self.vlan, + site=vm.site + ) + prefix.is_pool = True + # Save as pool + prefix.save() + + ip_address = prefix.get_first_available_ip() + self.ip_address = IPAddress( + address=ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + self.ip_address.dns_name = self.get_fqdn() + self.ip_address.save() + + except Exception as e: + self.ip_address = None + raise Exception("IP address - {0}".format(e)) + + def get_vlan(self): + return False + + def set_hostname(self, hostname): + try: + self.hostname = hostname if hostname is not None else self.generate_hostname() + except Exception as e: + raise Exception("Hostname - {0}".format(e)) + + def set_disk(self, disk): + try: + self.disk = disk + except Exception as e: + raise Exception("Disk - {0}".format(e)) + + def set_memory(self, memory): + try: + self.memory = memory + except Exception as e: + raise Exception("Memory - {0}".format(e)) + + def set_vcpus(self, vcpus): + try: + self.vcpus = vcpus + except Exception as e: + raise Exception("vcpus - {0}".format(e)) + + def set_backup_tag(self, backup): + try: + if isinstance(backup, Tag): + self.backup = backup + else: + self.backup = Tag.objects.filter(name="vsphere_tag_{0}".format(backup))[0] + except Exception as e: + raise Exception("Tag backup {0} does not exist, {1}".format(backup, e)) + + def set_backup_offsite_tag(self, backup_offsite): + try: + if isinstance(backup_offsite, Tag): + self.backup_offsite = backup_offsite + else: + self.backup_offsite = Tag.objects.filter(name="vsphere_tag_{0}".format(backup_offsite))[0] + except Exception: + self.backup_offsite = None + + def set_site(self, site): + try: + if isinstance(site, Site): + self.site = site + else: + raise Exception("site is not of instance 'Site'") + except Exception as e: + raise Exception('Site does not exist - ' + str(e)) + + def set_role(self, role): + try: + self.role = DeviceRole.objects.get( + name=role + ) + except Exception as e: + raise Exception('Role does not exist - ' + str(e)) + + def set_platform(self, platform): + try: + if isinstance(platform, Platform): + self.platform = platform + else: + self.platform = Platform.objects.get( + name=platform + ) + except Exception as e: + raise Exception("Platform does not exist {0}".format(e)) + + def set_env(self, env): + try: + if isinstance(env, Tag): + self.env = env + else: + self.env = Tag.objects.get(name="env_{0}".format(env)) + except Exception as e: + raise Exception("Tag env does not exist - {0}".format(e, env)) + + def set_datazone(self, datazone): + try: + self.datazone = Tag.objects.get(name="datazone_{0}".format(datazone)) + except Exception as e: + raise Exception("Tag datazone does not exist - {0}".format(e)) + + def set_cluster(self, cluster): + try: + if isinstance(cluster, Cluster): + self.cluster = cluster + else: + self.cluster = Cluster.objects.get( + name=cluster + ) + self.set_site(self.cluster.site) + except Exception as e: + raise Exception("Cluster does not exist {0}".format(e)) + + def set_tenant(self, tenant): + try: + if isinstance(tenant, Tenant): + self.tenant = tenant + else: + self.tenant = Tenant.objects.get( + slug=tenant + ) + except Exception as e: + raise Exception("Tenant does not exist {0}".format(e)) + + def set_status(self, status): + try: + self.status = VirtualMachineStatusChoices.STATUS_STAGED if status == 'staged' else VirtualMachineStatusChoices.STATUS_PLANNED + except Exception as e: + raise Exception("Status does not exist {0}".format(e)) + + def get_ip_address(self): + return self.ip_address + + def __create_ip_address(self, vm): + self.set_ip_address(vm) + vm.primary_ip4 = self.get_ip_address() + vm.save() + + def __create_tags(self, vm: VirtualMachine): + vm.tags.add(self.datazone) + vm.tags.add(self.env) + vm.tags.add(self.backup) + if(self.backup_offsite is not None): + vm.tags.add(self.backup_offsite) + for tag in self.DEFAULT_TAGS: + vm.tags.add(tag) + if self.extra_tags is not None: + for tag in self.extra_tags: + vm.tags.add(tag) + vm.save() + self.set_tags(vm.tags) + + def set_tags(self, tags): + self.tags = tags + + def get_tags(self): + list = [] + for tag in self.tags.all(): + list.append(tag.name) + return list + + def __create_vm(self): + vm = VirtualMachine( + status=self.status, + cluster=self.cluster, + platform=self.platform, + role=self.role, + tenant=self.tenant, + name=self.hostname, + disk=self.disk, + memory=self.memory, + vcpus=self.vcpus, + comments=self.comments, + ) + vm.save() + return vm + + def __create_interface(self, vm: VirtualMachine): + """ + Setup interface and add IP address + """ + try: + + # Get net address tools + ip = netaddr.IPNetwork(vm.primary_ip4.address) + prefix_search = str(ip.network) + '/' + str(ip.prefixlen) + + prefix = Prefix.objects.get( + prefix=prefix_search, + is_pool=True, + site=self.site, + ) + + interfaces = vm.get_config_context().get('interfaces') + + interface = Interface( + name=interfaces['nic0']['name'], + mtu=interfaces['nic0']['mtu'], + virtual_machine=vm, + type=InterfaceTypeChoices.TYPE_VIRTUAL + ) + + # If we need anything other than Access, here is were to change it + if interfaces['nic0']['mode'] == "Access": + interface.mode = InterfaceModeChoices.MODE_ACCESS + interface.untagged_vlan = prefix.vlan + + interface.save() + + self.ip_address.interface = interface + self.ip_address.save() + + except Exception as e: + raise Exception("Error while creating interface - {}".format(e)) + return True + + def create(self): + try: + vm = self.__create_vm() + self.__create_ip_address(vm) + self.__create_tags(vm) + self.__create_interface(vm) + except Exception as e: + raise e + return True + + +class BulkDeployVM(Script): + """ + Example CSV full: + status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,redirtp:v0.2.0,nobackup,1,1024,10,odn1-vlb-redirtp-001,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,consul:v1.0.1,backup_general_1,2,2048,20,odn1-vlb-consul-001,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,rediast:v0.2.0,backup_general_4,4,4096,30,odn1-vlb-rediast-001,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + Example CSV minimal (If all defaults are set): + vcpus,memory,disk,ip_address,extra_tags + 1,1024,10,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + 2,2048,20,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + 4,4096,30,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + ** Required Params ** + Param: cluster - vSphere cluster name + Param: env - Adds 'env_xxx' tag + Param: platform - VM Platform e.g base:v1.0.0-coreos + Param: backup - Adds 'vsphere_tag_xxxx' tag + Param: backup_offsite - Adds 'vsphere_tag_xxxx' tag + Param: vcpus - Virtual CPUs (hot add) + Param: memory - Virtual memory (hot add) + Param: disk - Disk2 size + Param: hostname - VM hostname (Optional if 'role' is set) + Param: role - VM Device role + Param: ip_address - VM IP address + + ** Optional Params ** + Param: status - VM status (default 'staged') + Param: tenant - Netbox tenant (default slug:'patientsky-hosting') + Param: datazone - Adds 'datazone_x' tag (default 'rr') + Param: extra_tags - Adds extra tags to VM + """ + + DEFAULT_CSV_FIELDS = "vcpus,memory,disk,ip_address,extra_tags" + datazone_rr: bool = True + + class Meta: + name = "Bulk deploy new VMs" + description = "Deploy new virtual machines from existing platforms" + fields = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_backup_offsite', 'default_role'] + field_order = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_backup_offsite', 'default_role'] + commit_default = False + + vms = TextVar( + label="Import CSV", + description="CSV data", + required=True, + default=DEFAULT_CSV_FIELDS + ) + + default_status = ChoiceVar( + label="Default Status", + description="Default CSV field `status` if none given", + required=False, + choices=( + (VirtualMachineStatusChoices.STATUS_STAGED, 'Staged (Deploy now)'), + (VirtualMachineStatusChoices.STATUS_PLANNED, 'Planned (Save for later)') + ) + ) + + default_tenant = ObjectVar( + label="Default Tenant", + default=1, + required=False, + description="Default CSV field `tenant` if none given", + queryset=Tenant.objects.all() + ) + + default_datazone = ChoiceVar( + label="Default Datazone", + description="Default CSV field `datazone` if none given", + default="rr", + required=False, + choices=( + ('rr', 'Round robin (1,2)'), + ('1', '1'), + ('2', '2') + ) + ) + + default_cluster = ObjectVar( + label="Default Cluster", + description="Default CSV field `cluster` if none given", + required=False, + queryset=Cluster.objects.all() + ) + + default_env = ObjectVar( + label="Default Environment", + description="Default CSV field `env` if none given", + default="env_inf", + required=False, + queryset=Tag.objects.filter( + name__startswith='env_', + ).order_by('name'), + ) + + default_platform = ObjectVar( + label="Default Platform", + description="Default CSV field `platform` if none given", + required=False, + widget=APISelect( + api_url='/api/dcim/platforms/?name__ic=_', + display_field='display_name', + ), + queryset=Platform.objects.all().order_by('name') + ) + + default_role = ObjectVar( + label="Default Role", + description="Default CSV field `role` if none given", + default=None, + required=False, + widget=APISelect( + api_url='/api/dcim/device-roles/?vm_role=true', + display_field='display_name', + ), + queryset=DeviceRole.objects.filter( + vm_role=True + ).order_by('name') + ) + + default_backup = ObjectVar( + label="Default Backup", + description="Default CSV field `backup` if none given", + required=False, + queryset=Tag.objects.filter( + name__startswith='vsphere_tag_', + ).order_by('name'), + ) + + default_backup_offsite = ObjectVar( + label="Default Offsite Backup", + description="Default CSV field `backup_offsite` if none given", + required=False, + queryset=Tag.objects.filter( + name__startswith='vsphere_tag_backup_off', + ).order_by('name'), + ) + + def get_vm_data(self): + return self.vm_data + + def set_csv_data(self, vms): + self.csv_raw_data = csv.DictReader(vms, delimiter=',') + + def get_csv_raw_data(self): + return self.csv_raw_data + + def set(self, data): + self.set_csv_data(data['vms'].splitlines()) + + def get_datazone(self, datazone): + if datazone == 'rr': + datazone = 1 if self.datazone_rr else 2 + self.datazone_rr = not self.datazone_rr + return datazone + + def run(self, data): + # Set data from raw csv + self.set(data) + line = 1 + for raw_vm in self.get_csv_raw_data(): + try: + vm = VM( + status=raw_vm.get('status') if raw_vm.get('status') is not None else data['default_status'], + tenant=raw_vm.get('tenant') if raw_vm.get('tenant') is not None else data['default_tenant'], + datazone=raw_vm.get('datazone') if raw_vm.get('datazone') is not None else self.get_datazone(data['default_datazone']), + cluster=raw_vm.get('cluster') if raw_vm.get('cluster') is not None else data['default_cluster'], + env=raw_vm.get('env') if raw_vm.get('env') is not None else data['default_env'], + platform=raw_vm.get('platform') if raw_vm.get('platform') is not None else data['default_platform'], + role=raw_vm.get('role') if raw_vm.get('role') is not None else data['default_role'], + backup=raw_vm.get('backup') if raw_vm.get('backup') is not None else data['default_backup'], + backup_offsite=raw_vm.get('backup_offsite') if raw_vm.get('backup_offsite') is not None else data['default_backup_offsite'], + vcpus=raw_vm.get('vcpus'), + memory=raw_vm.get('memory'), + disk=raw_vm.get('disk'), + hostname=raw_vm.get('hostname'), + ip_address=raw_vm.get('ip_address'), + extra_tags=raw_vm.get('extra_tags'), + ) + vm.create() + self.log_success( + "{} `{}` for `{}`, `{}`, in cluster `{}`, env `{}`, datazone `{}`, backup `{}`". + format( + vm.status.capitalize(), + vm.hostname, + vm.tenant, + vm.ip_address.address, + vm.cluster, + str(vm.env.name).split('_')[1], + vm.datazone, + vm.backup, + ) + ) + line += 1 + except Exception as e: + self.log_failure("CSV line {}, Error while creating VM \n`{}`".format(line, e)) + return data['vms'] + diff --git a/scripts/bulk.py.new b/scripts/bulk.py.new new file mode 100644 index 0000000..d6dea86 --- /dev/null +++ b/scripts/bulk.py.new @@ -0,0 +1,516 @@ +import netaddr +import csv +from dcim.choices import InterfaceTypeChoices, InterfaceModeChoices +from dcim.models import Platform, DeviceRole, Site +from ipam.models import IPAddress, VRF, Interface, Prefix, VLAN +from tenancy.models import Tenant +from virtualization.models import VirtualMachine, Cluster +from virtualization.choices import VirtualMachineStatusChoices +from extras.scripts import Script, TextVar, ChoiceVar, ObjectVar +from extras.models import Tag +from utilities.forms import APISelect + + +class VM: + + status: str + tenant: Tenant + cluster: Cluster + site: Site + csv_ip_address: str + ip_address: IPAddress + + DEFAULT_TAGS = ['ansible', 'zero_day'] + DEFAULT_DOMAIN_PRIVATE = 'privatedns.zone' + DEFAULT_DOMAIN_PUBLIC = 'publicdns.zone' + + def __init__(self, status, tenant, cluster, datazone, env, platform, role, backup, vcpus, memory, disk, ip_address, hostname, extra_tags): + # IP address can first be created after vm + self.csv_ip_address = ip_address + self.set_cluster(cluster) + self.set_status(status) + self.set_tenant(tenant) + self.set_datazone(datazone) + self.set_env(env) + self.set_platform(platform) + self.set_role(role) + self.set_backup_tag(backup) + self.set_vcpus(vcpus) + self.set_memory(memory) + self.set_disk(disk) + self.set_hostname(hostname) + self.set_vlan(None) + self.set_extra_tags(extra_tags) + + def set_extra_tags(self, extra_tags): + try: + if extra_tags is None: + self.extra_tags = extra_tags + else: + self.extra_tags = extra_tags.split(',') + except Exception as e: + raise Exception("Extra tags - {0}".format(e)) + + def set_vlan(self, vlan): + try: + self.vlan = VLAN.objects.get( + vid=vlan, + site=self.site + ) + except Exception: + self.vlan = None + + def generate_hostname(self): + # I now proclaim this VM, First of its Name, Queen of the Andals and the First Men, Protector of the Seven Kingdoms + vm_index = "001" + vm_search_for = "{0}-{1}-{2}-".format(self.site.slug, self.env.name.split('_')[1], self.role.name.split(':')[0]) + vms = VirtualMachine.objects.filter( + name__startswith=vm_search_for + ) + if len(vms) > 0: + # Get last of its kind + last_vm_index = int(vms[len(vms) - 1].name.split('-')[3]) + 1 + if last_vm_index < 10: + vm_index = '00' + str(last_vm_index) + elif last_vm_index < 100: + vm_index = '0' + str(last_vm_index) + else: + vm_index = str(last_vm_index) + + return "{0}{1}".format(vm_search_for, vm_index) + + def get_vrf(self): + return VRF.objects.get( + name="global" + ) + + def get_fqdn(self): + return "{}.{}".format(self.hostname, self.DEFAULT_DOMAIN_PRIVATE if netaddr.IPNetwork(self.ip_address.address).is_private() is True else self.DEFAULT_DOMAIN_PUBLIC) + + def set_ip_address(self, vm): + try: + if not isinstance(self.vlan, VLAN): + ip_check = IPAddress.objects.filter(address=self.csv_ip_address) + if len(ip_check) > 0: + raise Exception(str(ip_check[0].address) + ' is already assigned to ' + str(ip_check[0].interface.name)) + self.ip_address = IPAddress( + address=self.csv_ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + else: + # Auto assign IPs from vlan + prefix = Prefix.objects.get( + vlan=self.vlan, + site=vm.site + ) + prefix.is_pool = True + # Save as pool + prefix.save() + + ip_address = prefix.get_first_available_ip() + self.ip_address = IPAddress( + address=ip_address, + vrf=self.get_vrf(), + tenant=self.tenant, + family=4, + ) + self.ip_address.dns_name = self.get_fqdn() + self.ip_address.save() + except Exception as e: + self.ip_address = None + raise Exception("IP address - {0}".format(e)) + + def get_vlan(self): + return False + + def set_hostname(self, hostname): + try: + self.hostname = hostname if hostname is not None else self.generate_hostname() + except Exception as e: + raise Exception("Hostname - {0}".format(e)) + + def set_disk(self, disk): + try: + self.disk = disk + except Exception as e: + raise Exception("Disk - {0}".format(e)) + + def set_memory(self, memory): + try: + self.memory = memory + except Exception as e: + raise Exception("Memory - {0}".format(e)) + + def set_vcpus(self, vcpus): + try: + self.vcpus = vcpus + except Exception as e: + raise Exception("vcpus - {0}".format(e)) + + def set_backup_tag(self, backup): + try: + if isinstance(backup, Tag): + self.backup = backup + else: + self.backup = Tag.objects.filter(name="{0}".format(backup))[0] + except Exception as e: + raise Exception("Tag backup {0} does not exist, {1}".format(backup, e)) + + def set_site(self, site): + try: + if isinstance(site, Site): + self.site = site + else: + raise Exception("site is not of instance 'Site'") + except Exception as e: + raise Exception('Site does not exist - ' + str(e)) + + def set_role(self, role): + try: + self.role = DeviceRole.objects.get( + name=role + ) + except Exception as e: + raise Exception('Role does not exist - ' + str(e)) + + def set_platform(self, platform): + try: + if isinstance(platform, Platform): + self.platform = platform + else: + self.platform = Platform.objects.get( + name=platform + ) + except Exception as e: + raise Exception("Platform does not exist {0}".format(e)) + + def set_env(self, env): + try: + if isinstance(env, Tag): + self.env = env + else: + self.env = Tag.objects.get(name="env_{0}".format(env)) + except Exception as e: + raise Exception("Tag env does not exist - {0}".format(e, env)) + + def set_datazone(self, datazone): + try: + self.datazone = Tag.objects.get(name="datazone_{0}".format(datazone)) + except Exception as e: + raise Exception("Tag datazone does not exist - {0}".format(e)) + + def set_cluster(self, cluster): + try: + if isinstance(cluster, Cluster): + self.cluster = cluster + else: + self.cluster = Cluster.objects.get( + name=cluster + ) + self.set_site(self.cluster.site) + except Exception as e: + raise Exception("Cluster does not exist {0}".format(e)) + + def set_tenant(self, tenant): + try: + if isinstance(tenant, Tenant): + self.tenant = tenant + else: + self.tenant = Tenant.objects.get( + slug=tenant + ) + except Exception as e: + raise Exception("Tenant does not exist {0}".format(e)) + + def set_status(self, status): + try: + self.status = VirtualMachineStatusChoices.STATUS_STAGED if status == 'staged' else VirtualMachineStatusChoices.STATUS_PLANNED + except Exception as e: + raise Exception("Status does not exist {0}".format(e)) + + def get_ip_address(self): + return self.ip_address + + def __create_ip_address(self, vm): + self.set_ip_address(vm) + vm.primary_ip4 = self.get_ip_address() + vm.save() + + def __create_tags(self, vm: VirtualMachine): + vm.tags.add(self.datazone) + vm.tags.add(self.env) + vm.tags.add(self.backup) + for tag in self.DEFAULT_TAGS: + vm.tags.add(tag) + if self.extra_tags is not None: + for tag in self.extra_tags: + vm.tags.add(tag) + vm.save() + self.set_tags(vm.tags) + + def set_tags(self, tags): + self.tags = tags + + def get_tags(self): + list = [] + for tag in self.tags.all(): + list.append(tag.name) + return list + + def __create_vm(self): + vm = VirtualMachine( + status=self.status, + cluster=self.cluster, + platform=self.platform, + role=self.role, + tenant=self.tenant, + name=self.hostname, + disk=self.disk, + memory=self.memory, + vcpus=self.vcpus, + ) + vm.save() + return vm + + def __create_interface(self, vm: VirtualMachine): + """ + Setup interface and add IP address + """ + try: + + # Get net address tools + ip = netaddr.IPNetwork(vm.primary_ip4.address) + prefix_search = str(ip.network) + '/' + str(ip.prefixlen) + + prefix = Prefix.objects.get( + prefix=prefix_search, + is_pool=True, + site=self.site, + ) + + interfaces = vm.get_config_context().get('interfaces') + + interface = Interface( + name=interfaces['nic0']['name'], + mtu=interfaces['nic0']['mtu'], + virtual_machine=vm, + type=InterfaceTypeChoices.TYPE_VIRTUAL + ) + + # If we need anything other than Access, here is were to change it + if interfaces['nic0']['mode'] == "Access": + interface.mode = InterfaceModeChoices.MODE_ACCESS + interface.untagged_vlan = prefix.vlan + + interface.save() + + self.ip_address.interface = interface + self.ip_address.save() + + except Exception as e: + raise Exception("Error while creating interface - {}".format(e)) + return True + + def create(self): + try: + vm = self.__create_vm() + self.__create_ip_address(vm) + self.__create_tags(vm) + self.__create_interface(vm) + except Exception as e: + raise e + return True + + +class BulkDeployVM(Script): + """ + Example CSV full: + status,tenant,cluster,datazone,env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,redirtp:v0.2.0,backup_nobackup,1,1024,10,odn1-vlb-redirtp-001,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,consul:v1.0.1,backup_general_1,2,2048,20,odn1-vlb-consul-001,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + staged,patientsky-hosting,odn1,1,vlb,base:v1.0.0-coreos,rediast:v0.2.0,backup_general_4,4,4096,30,odn1-vlb-rediast-001,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + Example CSV minimal (If all defaults are set): + vcpus,memory,disk,ip_address,extra_tags + 1,1024,10,10.50.61.10/24,"voip,test_tag,cluster_id_voip_galera_001" + 2,2048,20,10.50.61.11/24,"voip,test_tag,cluster_id_voip_galera_002" + 4,4096,30,10.50.61.12/24,"voip,test_tag,cluster_id_voip_galera_003" + + ** Required Params ** + Param: cluster - vSphere cluster name + Param: env - Adds 'env_xxx' tag + Param: platform - VM Platform e.g base:v1.0.0-coreos + Param: backup - Adds backup tag + Param: vcpus - Virtual CPUs (hot add) + Param: memory - Virtual memory (hot add) + Param: disk - Disk2 size + Param: hostname - VM hostname (Optional if 'role' is set) + Param: role - VM Device role + Param: ip_address - VM IP address + + ** Optional Params ** + Param: status - VM status (default 'staged') + Param: tenant - Netbox tenant (default slug:'patientsky-hosting') + Param: datazone - Adds 'datazone_x' tag (default 'rr') + Param: extra_tags - Adds extra tags to VM + """ + + DEFAULT_CSV_FIELDS = "env,platform,role,backup,vcpus,memory,disk,hostname,ip_address,extra_tags" + datazone_rr: bool = True + + class Meta: + name = "Bulk deploy new VMs" + description = "Deploy new virtual machines from existing platforms" + fields = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_role'] + field_order = ['vms', 'default_status', 'default_tenant', 'default_datazone', 'default_backup', 'default_role'] + commit_default = False + + vms = TextVar( + label="Import CSV", + description="CSV data", + required=True, + default=DEFAULT_CSV_FIELDS + ) + + default_status = ChoiceVar( + label="Default Status", + description="Default CSV field `status` if none given", + required=False, + choices=( + (VirtualMachineStatusChoices.STATUS_STAGED, 'Staged (Deploy now)'), + (VirtualMachineStatusChoices.STATUS_PLANNED, 'Planned (Save for later)') + ) + ) + + default_tenant = ObjectVar( + label="Default Tenant", + default=1, + required=False, + description="Default CSV field `tenant` if none given", + queryset=Tenant.objects.all() + ) + + default_datazone = ChoiceVar( + label="Default Datazone", + description="Default CSV field `datazone` if none given", + default="rr", + required=False, + choices=( + ('rr', 'Round robin (1,2)'), + ('1', '1'), + ('2', '2') + ) + ) + + default_cluster = ObjectVar( + label="Default Cluster", + description="Default CSV field `cluster` if none given", + required=False, + queryset=Cluster.objects.all() + ) + + default_env = ObjectVar( + label="Default Environment", + description="Default CSV field `env` if none given", + default="env_inf", + required=False, + queryset=Tag.objects.filter( + name__startswith='env_', + ).order_by('name'), + ) + + default_platform = ObjectVar( + label="Default Platform", + description="Default CSV field `platform` if none given", + required=False, + widget=APISelect( + api_url='/api/dcim/platforms/?name__ic=_', + display_field='display_name', + ), + queryset=Platform.objects.all().order_by('name') + ) + + default_role = ObjectVar( + label="Default Role", + description="Default CSV field `role` if none given", + default=None, + required=False, + widget=APISelect( + api_url='/api/dcim/device-roles/?vm_role=true', + display_field='display_name', + ), + queryset=DeviceRole.objects.filter( + vm_role=True + ).order_by('name') + ) + + default_backup = ObjectVar( + label="Default Backup", + description="Default CSV field `backup` if none given", + required=False, + queryset=Tag.objects.filter( + name__startswith='backup_', + ).order_by('name'), + ) + + def get_vm_data(self): + return self.vm_data + + def set_csv_data(self, vms): + self.csv_raw_data = csv.DictReader(vms, delimiter=',') + + def get_csv_raw_data(self): + return self.csv_raw_data + + def set(self, data): + self.set_csv_data(data['vms'].splitlines()) + + def get_datazone(self, datazone): + if datazone == 'rr': + datazone = 1 if self.datazone_rr else 2 + self.datazone_rr = not self.datazone_rr + return datazone + + def run(self, data): + # Set data from raw csv + self.set(data) + line = 1 + for raw_vm in self.get_csv_raw_data(): + try: + vm = VM( + status=raw_vm.get('status') if raw_vm.get('status') is not None else data['default_status'], + tenant=raw_vm.get('tenant') if raw_vm.get('tenant') is not None else data['default_tenant'], + datazone=raw_vm.get('datazone') if raw_vm.get('datazone') is not None else self.get_datazone(data['default_datazone']), + cluster=raw_vm.get('cluster') if raw_vm.get('cluster') is not None else data['default_cluster'], + env=raw_vm.get('env') if raw_vm.get('env') is not None else data['default_env'], + platform=raw_vm.get('platform') if raw_vm.get('platform') is not None else data['default_platform'], + role=raw_vm.get('role') if raw_vm.get('role') is not None else data['default_role'], + backup=raw_vm.get('backup') if raw_vm.get('backup') is not None else data['default_backup'], + vcpus=raw_vm.get('vcpus'), + memory=raw_vm.get('memory'), + disk=raw_vm.get('disk'), + hostname=raw_vm.get('hostname'), + ip_address=raw_vm.get('ip_address'), + extra_tags=raw_vm.get('extra_tags'), + ) + vm.create() + self.log_success( + "{} `{}` for `{}`, `{}`, in cluster `{}`, env `{}`, datazone `{}`, backup `{}`". + format( + vm.status.capitalize(), + vm.hostname, + vm.tenant, + vm.ip_address.address, + vm.cluster, + str(vm.env.name).split('_')[1], + vm.datazone, + vm.backup, + ) + ) + line += 1 + except Exception as e: + self.log_failure("CSV line {}, Error while creating VM \n`{}`".format(line, e)) + return data['vms'] + diff --git a/scripts/create_vms.py b/scripts/create_vms.py deleted file mode 100644 index 0a22557..0000000 --- a/scripts/create_vms.py +++ /dev/null @@ -1,469 +0,0 @@ -import netaddr - -from dcim.choices import InterfaceTypeChoices, InterfaceModeChoices -from dcim.models import Platform, DeviceRole, Site -from ipam.models import IPAddress, VRF, Interface, Prefix, VLAN -from tenancy.models import Tenant -from virtualization.models import VirtualMachine, Cluster -from virtualization.choices import VirtualMachineStatusChoices -from extras.scripts import Script, ObjectVar, ChoiceVar, TextVar, IntegerVar -from extras.models import Tag -from utilities.forms import APISelect - - -class DeployVM(Script): - - env = "" - interfaces = [] - time_zone = "" - dns_domain_private = "" - dns_domain_public = "" - dns_servers = [] - ntp_servers = [] - ssh_authorized_keys = [] - ssh_port = "" - _default_interface = {} - tags = [] - output = [] - success_log = "" - - class Meta: - name = "Deploy new VMs" - description = "Deploy new virtual machines from existing platforms and roles using AWX" - fields = ['persist_disk', 'status', 'health_check', 'serial', 'tenant', 'cluster', 'env', 'untagged_vlan', 'backup', 'ip_addresses', 'vm_count', 'vcpus', 'memory', 'platform', 'role', 'disk', 'ssh_authorized_keys', 'hostnames'] - field_order = ['status', 'tenant', 'cluster', 'env', 'platform', 'role', 'health_check', 'serial', 'persist_disk', 'backup', 'vm_count', 'vcpus', 'memory', 'disk', 'hostnames', 'untagged_vlan', 'ip_addresses', 'ssh_authorized_keys'] - commit_default = False - - health_check = ChoiceVar( - label="Health checks on deployment", - description="Deployment will fail if server does not pass Consul health checks", - required=True, - choices=( - ('True', 'Yes'), - ('False', 'No') - ) - ) - - serial = ChoiceVar( - label="Serial deployment", - description="VM will not be parallelized in deployment", - required=True, - choices=( - ('False', 'No'), - ('True', 'Yes'), - ) - ) - - persist_disk = ChoiceVar( - label="Persist volume on redeploy", - description="VM will persist disk2 in vSphere on redeploys", - required=True, - choices=( - ('False', 'No'), - ('True', 'Yes'), - ) - ) - - status = ChoiceVar( - label="VM Status", - description="Deploy VM now or later?", - required=True, - choices=( - (VirtualMachineStatusChoices.STATUS_STAGED, 'Staged (Deploy now)'), - (VirtualMachineStatusChoices.STATUS_PLANNED, 'Planned (Save for later)') - ) - ) - - tenant = ObjectVar( - default="patientsky-hosting", - description="Name of the tenant the VMs beloing to", - queryset=Tenant.objects.filter() - ) - - cluster = ObjectVar( - default="odn1", - description="Name of the vSphere cluster you are deploying to", - queryset=Cluster.objects.all() - ) - - env = ChoiceVar( - label="Environment", - description="Environment to deploy VM", - default="vlb", - choices=( - ('pno', 'pno'), - ('inf', 'inf'), - ('stg', 'stg'), - ('dev', 'dev'), - ('hem', 'hem'), - ('hov', 'hov'), - ('hpl', 'hpl'), - ('mgt', 'mgt'), - ('cse', 'cse'), - ('qua', 'qua'), - ('dmo', 'dmo'), - ('vlb', 'vlb'), - ('cmi', 'cmi') - ) - ) - - platform = ObjectVar( - description="Host OS to deploy", - queryset=Platform.objects.filter( - name__regex=r'^(base_.*)' - ).order_by('name') - ) - - role = ObjectVar( - label="VM Role", - description="VM Role", - queryset=DeviceRole.objects.filter( - vm_role=True - ).order_by('name') - ) - - backup = ChoiceVar( - label="Backup strategy", - description="The backup strategy deployed to this VM with Veeam", - required=True, - choices=( - ('nobackup', 'Never'), - ('backup_general_1', 'Daily'), - ('backup_general_4', 'Monthly') - ) - ) - - ip_addresses = TextVar( - required=False, - label="IP Addresses", - description="List of IP addresses to create w. prefix e.g 192.168.0.10/24. If none given, hosts will be assigned IPs 'automagically'" - ) - - untagged_vlan = ObjectVar( - required=False, - label="VLAN", - widget=APISelect(api_url='/api/ipam/vlans/', display_field='display_name'), - queryset=VLAN.objects.all(), - description="Choose VLAN for IP-addresses", - ) - - hostnames = TextVar( - required=True, - label="Hostnames", - description="List of hostnames to create." - ) - - ssh_authorized_keys = TextVar( - label="SSH Authorized Keys", - required=False, - description="List of accepted SSH keys - defaults to site config context 'ssh_authorized_keys'" - ) - - vm_count = ChoiceVar( - label="Number of VMs", - description="Number of VMs to deploy", - choices=( - ('1', '1'), - ('2', '2'), - ('3', '3'), - ('4', '4'), - ('5', '5'), - ('6', '6'), - ('7', '7'), - ('8', '8'), - ('9', '9'), - ('10', '10') - ) - ) - - vcpus = ChoiceVar( - label="Number of CPUs", - description="Number of virtual CPUs", - default="2", - choices=( - ('1', '1'), - ('2', '2'), - ('3', '3'), - ('4', '4'), - ('5', '5'), - ('6', '6'), - ('7', '7'), - ('8', '8') - ) - ) - - memory = ChoiceVar( - description="Amount of VM memory", - default="4096", - choices=( - ('1024', '1024'), - ('2048', '2048'), - ('4096', '4096'), - ('8192', '8192'), - ('16384', '16384'), - ('32768', '32768') - ) - ) - - disk = IntegerVar( - label="Disk size", - description="Disk size in GB", - default="20" - ) - - def appendLogSuccess(self, log: str, obj=None): - self.success_log += " {} `\n{}\n`".format(log, obj) - return self - - def flushLogSuccess(self): - self.log_success(self.success_log) - self.success_log = "" - return self - - def _generateHostname(self, cluster, env, descriptor): - - # I now proclaim this VM, First of its Name, Queen of the Andals and the First Men, Protector of the Seven Kingdoms - vm_index = "001" - - search_for = str(cluster) + '-' + env + '-' + descriptor + '-' - vms = VirtualMachine.objects.filter( - name__startswith=search_for - ) - - if len(vms) > 0: - # Get last of its kind - last_vm_index = int(vms[len(vms) - 1].name.split('-')[3]) + 1 - if last_vm_index < 10: - vm_index = '00' + str(last_vm_index) - elif last_vm_index < 100: - vm_index = '0' + str(last_vm_index) - else: - vm_index = str(last_vm_index) - - hostname = str(cluster) + '-' + env + '-' + descriptor + '-' + vm_index - - return hostname - - def __validateInput(self, data, base_context_data): - - try: - self.interfaces = base_context_data['interfaces'] - self.time_zone = base_context_data['time_zone'] - self.tags = base_context_data['tags'] - self.dns_domain_private = base_context_data['dns_domain_private'] - self.dns_domain_public = base_context_data['dns_domain_public'] - self.dns_servers = base_context_data['dns_servers'] - self.ntp_servers = base_context_data['ntp_servers'] - self.ssh_authorized_keys = base_context_data['ssh_authorized_keys'] - self.ssh_port = base_context_data['ssh_port'] - self.tags.update({'env_' + self.env: {'comments': 'Environment', 'color': '009688'}}) - self.tags.update({'vsphere_' + data['backup']: {'comments': 'Backup strategy', 'color': '009688'}}) - if (data['health_check'] == 'True'): - self.tags.update({'health_check': {'comments': 'Do health checks in deployment', 'color': '4caf50'}}) - if (data['serial'] == 'True'): - self.tags.update({'serial': {'comments': 'Do health checks in deployment', 'color': '4caf50'}}) - except Exception as error: - self.log_failure("Error when parsing context_data! Error: " + str(error)) - return False - - if self.interfaces is None: - self.log_failure("No interfaces object in context data!") - return False - - if data['ip_addresses'] != "" and len(data['ip_addresses'].splitlines()) != int(data['vm_count']): - self.log_failure("The number of IP addresses and VMs does not match!") - return False - - if data['hostnames'] != "" and len(data['hostnames'].splitlines()) != int(data['vm_count']): - self.log_failure("The number of hostnames and VMs does not match!") - return False - - if self.interfaces is None: - self.log_failure("No interfaces object in context data!") - return False - - return True - - def run(self, data): - - vrf = VRF.objects.get( - name="global" - ) - - self.env = data['env'] - - # Setup base virtual machine for copying config_context - base_vm = VirtualMachine( - cluster=data['cluster'], - platform=data['platform'], - role=data['role'], - tenant=data['tenant'], - name="script_temp" - ) - - base_vm.save() - - if self.__validateInput(data=data, base_context_data=base_vm.get_config_context()) is False: - return False - - # Delete base virtual machine for copying config_context - vm_delete = VirtualMachine.objects.get( - name="script_temp" - ) - vm_delete.delete() - - for i in range(0, int(data['vm_count'])): - - hostnames = data['hostnames'].splitlines() - ip_addresses = data['ip_addresses'].splitlines() - - if len(hostnames) > 0: - hostname = hostnames[i] - else: - hostname = self._generateHostname( - cluster=data['cluster'].name, - env=self.env, - descriptor="na" - ) - - # Check if VM exists - if len(VirtualMachine.objects.filter(name=hostname)) > 0: - self.log_failure("VM with hostname " + hostname + " already exists!") - return False - - if len(ip_addresses) > 0: - - # Check if IP address exists - ip_check = IPAddress.objects.filter( - address=ip_addresses[i] - ) - - if len(ip_check) > 0: - self.log_failure(str(ip_check[0].address) + ' is already assigned to ' + str(ip_check[0].interface.name)) - return False - - domain = self.dns_domain_private if netaddr.IPNetwork(ip_addresses[i]).is_private() is True else self.dns_domain_public - ip_address = IPAddress( - address=ip_addresses[i], - vrf=vrf, - tenant=data['tenant'], - family=4, - dns_name=hostname + '.' + domain, - ) - - ip_address.save() - self.appendLogSuccess(log="Created IP", obj=ip_addresses[i]).appendLogSuccess(log="DNS", obj=hostname + '.' + domain) - else: - if data['untagged_vlan'] is not None: - - try: - - # Auto assign IPs from vsphere_port_group - prefix = Prefix.objects.get( - vlan=data['untagged_vlan'], - site=Site.objects.get(name=data['cluster'].site.name), - is_pool=True - ) - - ip = prefix.get_first_available_ip() - - domain = self.dns_domain_private if netaddr.IPNetwork(ip).is_private() is True else self.dns_domain_public - ip_address = IPAddress( - address=ip, - vrf=vrf, - tenant=data['tenant'], - family=4, - dns_name=hostname + '.' + domain, - ) - - ip_address.save() - self.appendLogSuccess(log="Auto-assigned IP", obj=ip).appendLogSuccess(log="DNS", obj=hostname + '.' + domain) - except Exception: - self.log_failure("An error occurred while auto-assigning IP address. VLAN or Prefix not found!") - return False - else: - self.log_failure("No IP address choice was made") - return False - - vm = VirtualMachine( - status=data['status'], - cluster=data['cluster'], - platform=data['platform'], - role=data['role'], - tenant=data['tenant'], - name=hostname, - disk=data['disk'], - memory=data['memory'], - vcpus=data['vcpus'] - ) - - vm.primary_ip4 = ip_address - vm.save() - self.appendLogSuccess(log="for VM in ", obj=vm.cluster) - - self.__assignTags(vm=vm) - - # Assign IP to interface - if self.__setupInterface(ip_address=ip_address, data=data, vm=vm) is False: - return False - - self.flushLogSuccess() - - return self.success_log - - def __assignTags(self, vm: VirtualMachine): - """ - Assign tags from context data - """ - for tag in self.tags: - if len(Tag.objects.filter(name=tag)) == 0: - color = self.tags[tag]['color'] if 'color' in self.tags[tag] else '9e9e9e' - comments = self.tags[tag]['comments'] if 'comments' in self.tags[tag] else 'No comments' - newTag = Tag(comments=comments, name=tag, color=color) - newTag.save() - - vm.tags.add(tag) - - vm.save() - - def __setupInterface(self, ip_address: IPAddress, vm: VirtualMachine, data): - """ - Setup interface and add IP address - """ - - # Get net address tools - ip = netaddr.IPNetwork(ip_address.address) - prefix_search = str(ip.network) + '/' + str(ip.prefixlen) - - try: - prefix = Prefix.objects.get( - prefix=prefix_search, - is_pool=True - ) - except Exception: - self.log_failure("Prefix for IP " + ip_address.address + " was not found") - return False - - # Right now we dont support multiple nics. But the data model supports it - interface = Interface( - name=self.interfaces['nic0']['name'], - mtu=self.interfaces['nic0']['mtu'], - virtual_machine=vm, - type=InterfaceTypeChoices.TYPE_VIRTUAL - ) - - # If we need anything other than Access, here is were to change it - if self.interfaces['nic0']['mode'] == "Access": - interface.mode = InterfaceModeChoices.MODE_ACCESS - interface.untagged_vlan = prefix.vlan - self.interfaces['nic0']['vsphere_port_group'] = prefix.vlan.name - - interface.save() - - # Add interface to IP address - ip_address.interface = interface - ip_address.save() - - self.appendLogSuccess(log="with interface ", obj=interface).appendLogSuccess(log=", vlan ", obj=prefix.vlan.name) - - return True