ganeti.ovf

1 #!/usr/bin/python 2 # 3 4 # Copyright (C) 2011, 2012 Google Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions are 9 # met: 10 # 11 # 1. Redistributions of source code must retain the above copyright notice, 12 # this list of conditions and the following disclaimer. 13 # 14 # 2. Redistributions in binary form must reproduce the above copyright 15 # notice, this list of conditions and the following disclaimer in the 16 # documentation and/or other materials provided with the distribution. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 """Converter tools between ovf and ganeti config file 32 33 """ 34 35 # pylint: disable=F0401, E1101 36 37 # F0401 because ElementTree is not default for python 2.4 38 # E1101 makes no sense - pylint assumes that ElementTree object is a tuple 39 40 41 import ConfigParser 42 import errno 43 import logging 44 import os 45 import os.path 46 import re 47 import shutil 48 import tarfile 49 import tempfile 50 import xml.dom.minidom 51 import xml.parsers.expat 52 try: 53 import xml.etree.ElementTree as ET 54 except ImportError: 55 import elementtree.ElementTree as ET 56 57 try: 58 ParseError = ET.ParseError # pylint: disable=E1103 59 except AttributeError: 60 ParseError = None 61 62 from ganeti import constants 63 from ganeti import errors 64 from ganeti import utils 65 from ganeti import pathutils 66 67 68 # Schemas used in OVF format 69 GANETI_SCHEMA = "http://ganeti" 70 OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1" 71 RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" 72 "CIM_ResourceAllocationSettingData") 73 VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" 74 "CIM_VirtualSystemSettingData") 75 XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance" 76 77 # File extensions in OVF package 78 OVA_EXT = ".ova" 79 OVF_EXT = ".ovf" 80 MF_EXT = ".mf" 81 CERT_EXT = ".cert" 82 COMPRESSION_EXT = ".gz" 83 FILE_EXTENSIONS = [ 84 OVF_EXT, 85 MF_EXT, 86 CERT_EXT, 87 ] 88 89 COMPRESSION_TYPE = "gzip" 90 NO_COMPRESSION = [None, "identity"] 91 COMPRESS = "compression" 92 DECOMPRESS = "decompression" 93 ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS] 94 95 VMDK = "vmdk" 96 RAW = "raw" 97 COW = "cow" 98 ALLOWED_FORMATS = [RAW, COW, VMDK] 99 100 # ResourceType values 101 RASD_TYPE = { 102 "vcpus": "3", 103 "memory": "4", 104 "scsi-controller": "6", 105 "ethernet-adapter": "10", 106 "disk": "17", 107 } 108 109 SCSI_SUBTYPE = "lsilogic" 110 VS_TYPE = { 111 "ganeti": "ganeti-ovf", 112 "external": "vmx-04", 113 } 114 115 # AllocationUnits values and conversion 116 ALLOCATION_UNITS = { 117 "b": ["bytes", "b"], 118 "kb": ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"], 119 "mb": ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"], 120 "gb": ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"], 121 } 122 CONVERT_UNITS_TO_MB = { 123 "b": lambda x: x / (1024 * 1024), 124 "kb": lambda x: x / 1024, 125 "mb": lambda x: x, 126 "gb": lambda x: x * 1024, 127 } 128 129 # Names of the config fields 130 NAME = "name" 131 OS = "os" 132 HYPERV = "hypervisor" 133 VCPUS = "vcpus" 134 MEMORY = "memory" 135 AUTO_BALANCE = "auto_balance" 136 DISK_TEMPLATE = "disk_template" 137 TAGS = "tags" 138 VERSION = "version" 139 140 # Instance IDs of System and SCSI controller 141 INSTANCE_ID = { 142 "system": 0, 143 "vcpus": 1, 144 "memory": 2, 145 "scsi": 3, 146 } 147 148 # Disk format descriptions 149 DISK_FORMAT = { 150 RAW: "http://en.wikipedia.org/wiki/Byte", 151 VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html" 152 "#monolithicSparse", 153 COW: "http://www.gnome.org/~markmc/qcow-image-format.html", 154 }

155 156 157 -def CheckQemuImg():

158 """ Make sure that qemu-img is present before performing operations. 159 160 @raise errors.OpPrereqError: when qemu-img was not found in the system 161 162 """ 163 if not constants.QEMUIMG_PATH: 164 raise errors.OpPrereqError("qemu-img not found at build time, unable" 165 " to continue", errors.ECODE_STATE)

166

167 168 -def LinkFile(old_path, prefix=None, suffix=None, directory=None):

169 """Create link with a given prefix and suffix. 170 171 This is a wrapper over os.link. It tries to create a hard link for given file, 172 but instead of rising error when file exists, the function changes the name 173 a little bit. 174 175 @type old_path:string 176 @param old_path: path to the file that is to be linked 177 @type prefix: string 178 @param prefix: prefix of filename for the link 179 @type suffix: string 180 @param suffix: suffix of the filename for the link 181 @type directory: string 182 @param directory: directory of the link 183 184 @raise errors.OpPrereqError: when error on linking is different than 185 "File exists" 186 187 """ 188 assert(prefix is not None or suffix is not None) 189 if directory is None: 190 directory = os.getcwd() 191 new_path = utils.PathJoin(directory, "%s%s" % (prefix, suffix)) 192 counter = 1 193 while True: 194 try: 195 os.link(old_path, new_path) 196 break 197 except OSError, err: 198 if err.errno == errno.EEXIST: 199 new_path = utils.PathJoin(directory, 200 "%s_%s%s" % (prefix, counter, suffix)) 201 counter += 1 202 else: 203 raise errors.OpPrereqError("Error moving the file %s to %s location:" 204 " %s" % (old_path, new_path, err), 205 errors.ECODE_ENVIRON) 206 return new_path

207

208 209 -class OVFReader(object):

210 """Reader class for OVF files. 211 212 @type files_list: list 213 @ivar files_list: list of files in the OVF package 214 @type tree: ET.ElementTree 215 @ivar tree: XML tree of the .ovf file 216 @type schema_name: string 217 @ivar schema_name: name of the .ovf file 218 @type input_dir: string 219 @ivar input_dir: directory in which the .ovf file resides 220 221 """

222 - def __init__(self, input_path):

223 """Initialiaze the reader - load the .ovf file to XML parser. 224 225 It is assumed that names of manifesto (.mf), certificate (.cert) and ovf 226 files are the same. In order to account any other files as part of the ovf 227 package, they have to be explicitly mentioned in the Resources section 228 of the .ovf file. 229 230 @type input_path: string 231 @param input_path: absolute path to the .ovf file 232 233 @raise errors.OpPrereqError: when .ovf file is not a proper XML file or some 234 of the files mentioned in Resources section do not exist 235 236 """ 237 self.tree = ET.ElementTree() 238 try: 239 self.tree.parse(input_path) 240 except (ParseError, xml.parsers.expat.ExpatError), err: 241 raise errors.OpPrereqError("Error while reading %s file: %s" % 242 (OVF_EXT, err), errors.ECODE_ENVIRON) 243 244 # Create a list of all files in the OVF package 245 (input_dir, input_file) = os.path.split(input_path) 246 (input_name, _) = os.path.splitext(input_file) 247 files_directory = utils.ListVisibleFiles(input_dir) 248 files_list = [] 249 for file_name in files_directory: 250 (name, extension) = os.path.splitext(file_name) 251 if extension in FILE_EXTENSIONS and name == input_name: 252 files_list.append(file_name) 253 files_list += self._GetAttributes("{%s}References/{%s}File" % 254 (OVF_SCHEMA, OVF_SCHEMA), 255 "{%s}href" % OVF_SCHEMA) 256 for file_name in files_list: 257 file_path = utils.PathJoin(input_dir, file_name) 258 if not os.path.exists(file_path): 259 raise errors.OpPrereqError("File does not exist: %s" % file_path, 260 errors.ECODE_ENVIRON) 261 logging.info("Files in the OVF package: %s", " ".join(files_list)) 262 self.files_list = files_list 263 self.input_dir = input_dir 264 self.schema_name = input_name

265

266 - def _GetAttributes(self, path, attribute):

267 """Get specified attribute from all nodes accessible using given path. 268 269 Function follows the path from root node to the desired tags using path, 270 then reads the apropriate attribute values. 271 272 @type path: string 273 @param path: path of nodes to visit 274 @type attribute: string 275 @param attribute: attribute for which we gather the information 276 @rtype: list 277 @return: for each accessible tag with the attribute value set, value of the 278 attribute 279 280 """ 281 current_list = self.tree.findall(path) 282 results = [x.get(attribute) for x in current_list] 283 return filter(None, results)

284

285 - def _GetElementMatchingAttr(self, path, match_attr):

286 """Searches for element on a path that matches certain attribute value. 287 288 Function follows the path from root node to the desired tags using path, 289 then searches for the first one matching the attribute value. 290 291 @type path: string 292 @param path: path of nodes to visit 293 @type match_attr: tuple 294 @param match_attr: pair (attribute, value) for which we search 295 @rtype: ET.ElementTree or None 296 @return: first element matching match_attr or None if nothing matches 297 298 """ 299 potential_elements = self.tree.findall(path) 300 (attr, val) = match_attr 301 for elem in potential_elements: 302 if elem.get(attr) == val: 303 return elem 304 return None

305

306 - def _GetElementMatchingText(self, path, match_text):

307 """Searches for element on a path that matches certain text value. 308 309 Function follows the path from root node to the desired tags using path, 310 then searches for the first one matching the text value. 311 312 @type path: string 313 @param path: path of nodes to visit 314 @type match_text: tuple 315 @param match_text: pair (node, text) for which we search 316 @rtype: ET.ElementTree or None 317 @return: first element matching match_text or None if nothing matches 318 319 """ 320 potential_elements = self.tree.findall(path) 321 (node, text) = match_text 322 for elem in potential_elements: 323 if elem.findtext(node) == text: 324 return elem 325 return None

326 327 @staticmethod

328 - def _GetDictParameters(root, schema):

329 """Reads text in all children and creates the dictionary from the contents. 330 331 @type root: ET.ElementTree or None 332 @param root: father of the nodes we want to collect data about 333 @type schema: string 334 @param schema: schema name to be removed from the tag 335 @rtype: dict 336 @return: dictionary containing tags and their text contents, tags have their 337 schema fragment removed or empty dictionary, when root is None 338 339 """ 340 if root is None: 341 return {} 342 results = {} 343 for element in list(root): 344 pref_len = len("{%s}" % schema) 345 assert(schema in element.tag) 346 tag = element.tag[pref_len:] 347 results[tag] = element.text 348 return results

349

350 - def VerifyManifest(self):

351 """Verifies manifest for the OVF package, if one is given. 352 353 @raise errors.OpPrereqError: if SHA1 checksums do not match 354 355 """ 356 if "%s%s" % (self.schema_name, MF_EXT) in self.files_list: 357 logging.warning("Verifying SHA1 checksums, this may take a while") 358 manifest_filename = "%s%s" % (self.schema_name, MF_EXT) 359 manifest_path = utils.PathJoin(self.input_dir, manifest_filename) 360 manifest_content = utils.ReadFile(manifest_path).splitlines() 361 manifest_files = {} 362 regexp = r"SHA1\((\S+)\)= (\S+)" 363 for line in manifest_content: 364 match = re.match(regexp, line) 365 if match: 366 file_name = match.group(1) 367 sha1_sum = match.group(2) 368 manifest_files[file_name] = sha1_sum 369 files_with_paths = [utils.PathJoin(self.input_dir, file_name) 370 for file_name in self.files_list] 371 sha1_sums = utils.FingerprintFiles(files_with_paths) 372 for file_name, value in manifest_files.iteritems(): 373 if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value: 374 raise errors.OpPrereqError("SHA1 checksum of %s does not match the" 375 " value in manifest file" % file_name, 376 errors.ECODE_ENVIRON) 377 logging.info("SHA1 checksums verified")

378

379 - def GetInstanceName(self):

380 """Provides information about instance name. 381 382 @rtype: string 383 @return: instance name string 384 385 """ 386 find_name = "{%s}VirtualSystem/{%s}Name" % (OVF_SCHEMA, OVF_SCHEMA) 387 return self.tree.findtext(find_name)

388

389 - def GetDiskTemplate(self):

390 """Returns disk template from .ovf file 391 392 @rtype: string or None 393 @return: name of the template 394 """ 395 find_template = ("{%s}GanetiSection/{%s}DiskTemplate" % 396 (GANETI_SCHEMA, GANETI_SCHEMA)) 397 return self.tree.findtext(find_template)

398

399 - def GetHypervisorData(self):

400 """Provides hypervisor information - hypervisor name and options. 401 402 @rtype: dict 403 @return: dictionary containing name of the used hypervisor and all the 404 specified options 405 406 """ 407 hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" % 408 (GANETI_SCHEMA, GANETI_SCHEMA)) 409 hypervisor_data = self.tree.find(hypervisor_search) 410 if hypervisor_data is None: 411 return {"hypervisor_name": constants.VALUE_AUTO} 412 results = { 413 "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA, 414 default=constants.VALUE_AUTO), 415 } 416 parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA) 417 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) 418 return results

419

420 - def GetOSData(self):

421 """ Provides operating system information - os name and options. 422 423 @rtype: dict 424 @return: dictionary containing name and options for the chosen OS 425 426 """ 427 results = {} 428 os_search = ("{%s}GanetiSection/{%s}OperatingSystem" % 429 (GANETI_SCHEMA, GANETI_SCHEMA)) 430 os_data = self.tree.find(os_search) 431 if os_data is not None: 432 results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA) 433 parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA) 434 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) 435 return results

436

437 - def GetBackendData(self):

438 """ Provides backend information - vcpus, memory, auto balancing options. 439 440 @rtype: dict 441 @return: dictionary containing options for vcpus, memory and auto balance 442 settings 443 444 """ 445 results = {} 446 447 find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" % 448 (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) 449 match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"]) 450 vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus) 451 if vcpus is not None: 452 vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, 453 default=constants.VALUE_AUTO) 454 else: 455 vcpus_count = constants.VALUE_AUTO 456 results["vcpus"] = str(vcpus_count) 457 458 find_memory = find_vcpus 459 match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"]) 460 memory = self._GetElementMatchingText(find_memory, match_memory) 461 memory_raw = None 462 if memory is not None: 463 alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA) 464 matching_units = [units for units, variants in ALLOCATION_UNITS.items() 465 if alloc_units.lower() in variants] 466 if matching_units == []: 467 raise errors.OpPrereqError("Unit %s for RAM memory unknown" % 468 alloc_units, errors.ECODE_INVAL) 469 units = matching_units[0] 470 memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, 471 default=constants.VALUE_AUTO)) 472 memory_count = CONVERT_UNITS_TO_MB[units](memory_raw) 473 else: 474 memory_count = constants.VALUE_AUTO 475 results["memory"] = str(memory_count) 476 477 find_balance = ("{%s}GanetiSection/{%s}AutoBalance" % 478 (GANETI_SCHEMA, GANETI_SCHEMA)) 479 balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO) 480 results["auto_balance"] = balance 481 482 return results

483

484 - def GetTagsData(self):

485 """Provides tags information for instance. 486 487 @rtype: string or None 488 @return: string of comma-separated tags for the instance 489 490 """ 491 find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA) 492 results = self.tree.findtext(find_tags) 493 if results: 494 return results 495 else: 496 return None

497

498 - def GetVersionData(self):

499 """Provides version number read from .ovf file 500 501 @rtype: string 502 @return: string containing the version number 503 504 """ 505 find_version = ("{%s}GanetiSection/{%s}Version" % 506 (GANETI_SCHEMA, GANETI_SCHEMA)) 507 return self.tree.findtext(find_version)

508

509 - def GetNetworkData(self):

510 """Provides data about the network in the OVF instance. 511 512 The method gathers the data about networks used by OVF instance. It assumes 513 that 'name' tag means something - in essence, if it contains one of the 514 words 'bridged' or 'routed' then that will be the mode of this network in 515 Ganeti. The information about the network can be either in GanetiSection or 516 VirtualHardwareSection. 517 518 @rtype: dict 519 @return: dictionary containing all the network information 520 521 """ 522 results = {} 523 networks_search = ("{%s}NetworkSection/{%s}Network" % 524 (OVF_SCHEMA, OVF_SCHEMA)) 525 network_names = self._GetAttributes(networks_search, 526 "{%s}name" % OVF_SCHEMA) 527 required = ["ip", "mac", "link", "mode", "network"] 528 for (counter, network_name) in enumerate(network_names): 529 network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" 530 % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) 531 ganeti_search = ("{%s}GanetiSection/{%s}Network/{%s}Nic" % 532 (GANETI_SCHEMA, GANETI_SCHEMA, GANETI_SCHEMA)) 533 network_match = ("{%s}Connection" % RASD_SCHEMA, network_name) 534 ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name) 535 network_data = self._GetElementMatchingText(network_search, network_match) 536 network_ganeti_data = self._GetElementMatchingAttr(ganeti_search, 537 ganeti_match) 538 539 ganeti_data = {} 540 if network_ganeti_data is not None: 541 ganeti_data["mode"] = network_ganeti_data.findtext("{%s}Mode" % 542 GANETI_SCHEMA) 543 ganeti_data["mac"] = network_ganeti_data.findtext("{%s}MACAddress" % 544 GANETI_SCHEMA) 545 ganeti_data["ip"] = network_ganeti_data.findtext("{%s}IPAddress" % 546 GANETI_SCHEMA) 547 ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" % 548 GANETI_SCHEMA) 549 ganeti_data["network"] = network_ganeti_data.findtext("{%s}Net" % 550 GANETI_SCHEMA) 551 mac_data = None 552 if network_data is not None: 553 mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA) 554 555 network_name = network_name.lower() 556 557 # First, some not Ganeti-specific information is collected 558 if constants.NIC_MODE_BRIDGED in network_name: 559 results["nic%s_mode" % counter] = "bridged" 560 elif constants.NIC_MODE_ROUTED in network_name: 561 results["nic%s_mode" % counter] = "routed" 562 results["nic%s_mac" % counter] = mac_data 563 564 # GanetiSection data overrides 'manually' collected data 565 for name, value in ganeti_data.iteritems(): 566 results["nic%s_%s" % (counter, name)] = value 567 568 # Bridged network has no IP - unless specifically stated otherwise 569 if (results.get("nic%s_mode" % counter) == "bridged" and 570 not results.get("nic%s_ip" % counter)): 571 results["nic%s_ip" % counter] = constants.VALUE_NONE 572 573 for option in required: 574 if not results.get("nic%s_%s" % (counter, option)): 575 results["nic%s_%s" % (counter, option)] = constants.VALUE_AUTO 576 577 if network_names: 578 results["nic_count"] = str(len(network_names)) 579 return results

580

581 - def GetDisksNames(self):

582 """Provides list of file names for the disks used by the instance. 583 584 @rtype: list 585 @return: list of file names, as referenced in .ovf file 586 587 """ 588 results = [] 589 disks_search = "{%s}DiskSection/{%s}Disk" % (OVF_SCHEMA, OVF_SCHEMA) 590 disk_ids = self._GetAttributes(disks_search, "{%s}fileRef" % OVF_SCHEMA) 591 for disk in disk_ids: 592 disk_search = "{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA) 593 disk_match = ("{%s}id" % OVF_SCHEMA, disk) 594 disk_elem = self._GetElementMatchingAttr(disk_search, disk_match) 595 if disk_elem is None: 596 raise errors.OpPrereqError("%s file corrupted - disk %s not found in" 597 " references" % (OVF_EXT, disk), 598 errors.ECODE_ENVIRON) 599 disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA) 600 disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA) 601 results.append((disk_name, disk_compression)) 602 return results

603

604 605 -def SubElementText(parent, tag, text, attrib={}, **extra):

606 # pylint: disable=W0102 607 """This is just a wrapper on ET.SubElement that always has text content. 608 609 """ 610 if text is None: 611 return None 612 elem = ET.SubElement(parent, tag, attrib=attrib, **extra) 613 elem.text = str(text) 614 return elem

615

616 617 -class OVFWriter(object):

618 """Writer class for OVF files. 619 620 @type tree: ET.ElementTree 621 @ivar tree: XML tree that we are constructing 622 @type virtual_system_type: string 623 @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage 624 in VMWare this requires to be vmx 625 @type hardware_list: list 626 @ivar hardware_list: list of items prepared for VirtualHardwareSection 627 @type next_instance_id: int 628 @ivar next_instance_id: next instance id to be used when creating elements on 629 hardware_list 630 631 """

632 - def __init__(self, has_gnt_section):

633 """Initialize the writer - set the top element. 634 635 @type has_gnt_section: bool 636 @param has_gnt_section: if the Ganeti schema should be added - i.e. this 637 means that Ganeti section will be present 638 639 """ 640 env_attribs = { 641 "xmlns:xsi": XML_SCHEMA, 642 "xmlns:vssd": VSSD_SCHEMA, 643 "xmlns:rasd": RASD_SCHEMA, 644 "xmlns:ovf": OVF_SCHEMA, 645 "xmlns": OVF_SCHEMA, 646 "xml:lang": "en-US", 647 } 648 if has_gnt_section: 649 env_attribs["xmlns:gnt"] = GANETI_SCHEMA 650 self.virtual_system_type = VS_TYPE["ganeti"] 651 else: 652 self.virtual_system_type = VS_TYPE["external"] 653 self.tree = ET.Element("Envelope", attrib=env_attribs) 654 self.hardware_list = [] 655 # INSTANCE_ID contains statically assigned IDs, starting from 0 656 self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish

657

658 - def SaveDisksData(self, disks):

659 """Convert disk information to certain OVF sections. 660 661 @type disks: list 662 @param disks: list of dictionaries of disk options from config.ini 663 664 """ 665 references = ET.SubElement(self.tree, "References") 666 disk_section = ET.SubElement(self.tree, "DiskSection") 667 SubElementText(disk_section, "Info", "Virtual disk information") 668 for counter, disk in enumerate(disks): 669 file_id = "file%s" % counter 670 disk_id = "disk%s" % counter 671 file_attribs = { 672 "ovf:href": disk["path"], 673 "ovf:size": str(disk["real-size"]), 674 "ovf:id": file_id, 675 } 676 disk_attribs = { 677 "ovf:capacity": str(disk["virt-size"]), 678 "ovf:diskId": disk_id, 679 "ovf:fileRef": file_id, 680 "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]), 681 } 682 if "compression" in disk: 683 file_attribs["ovf:compression"] = disk["compression"] 684 ET.SubElement(references, "File", attrib=file_attribs) 685 ET.SubElement(disk_section, "Disk", attrib=disk_attribs) 686 687 # Item in VirtualHardwareSection creation 688 disk_item = ET.Element("Item") 689 SubElementText(disk_item, "rasd:ElementName", disk_id) 690 SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id) 691 SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id) 692 SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"]) 693 SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"]) 694 self.hardware_list.append(disk_item) 695 self.next_instance_id += 1

696

697 - def SaveNetworksData(self, networks):

698 """Convert network information to NetworkSection. 699 700 @type networks: list 701 @param networks: list of dictionaries of network options form config.ini 702 703 """ 704 network_section = ET.SubElement(self.tree, "NetworkSection") 705 SubElementText(network_section, "Info", "List of logical networks") 706 for counter, network in enumerate(networks): 707 network_name = "%s%s" % (network["mode"], counter) 708 network_attrib = {"ovf:name": network_name} 709 ET.SubElement(network_section, "Network", attrib=network_attrib) 710 711 # Item in VirtualHardwareSection creation 712 network_item = ET.Element("Item") 713 SubElementText(network_item, "rasd:Address", network["mac"]) 714 SubElementText(network_item, "rasd:Connection", network_name) 715 SubElementText(network_item, "rasd:ElementName", network_name) 716 SubElementText(network_item, "rasd:InstanceID", self.next_instance_id) 717 SubElementText(network_item, "rasd:ResourceType", 718 RASD_TYPE["ethernet-adapter"]) 719 self.hardware_list.append(network_item) 720 self.next_instance_id += 1

721 722 @staticmethod

723 - def _SaveNameAndParams(root, data):

724 """Save name and parameters information under root using data. 725 726 @type root: ET.Element 727 @param root: root element for the Name and Parameters 728 @type data: dict 729 @param data: data from which we gather the values 730 731 """ 732 assert(data.get("name")) 733 name = SubElementText(root, "gnt:Name", data["name"]) 734 params = ET.SubElement(root, "gnt:Parameters") 735 for name, value in data.iteritems(): 736 if name != "name": 737 SubElementText(params, "gnt:%s" % name, value)

738

739 - def SaveGanetiData(self, ganeti, networks):

740 """Convert Ganeti-specific information to GanetiSection. 741 742 @type ganeti: dict 743 @param ganeti: dictionary of Ganeti-specific options from config.ini 744 @type networks: list 745 @param networks: list of dictionaries of network options form config.ini 746 747 """ 748 ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection") 749 750 SubElementText(ganeti_section, "gnt:Version", ganeti.get("version")) 751 SubElementText(ganeti_section, "gnt:DiskTemplate", 752 ganeti.get("disk_template")) 753 SubElementText(ganeti_section, "gnt:AutoBalance", 754 ganeti.get("auto_balance")) 755 SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags")) 756 757 osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem") 758 self._SaveNameAndParams(osys, ganeti["os"]) 759 760 hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor") 761 self._SaveNameAndParams(hypervisor, ganeti["hypervisor"]) 762 763 network_section = ET.SubElement(ganeti_section, "gnt:Network") 764 for counter, network in enumerate(networks): 765 network_name = "%s%s" % (network["mode"], counter) 766 nic_attrib = {"ovf:name": network_name} 767 nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib) 768 SubElementText(nic, "gnt:Mode", network["mode"]) 769 SubElementText(nic, "gnt:MACAddress", network["mac"]) 770 SubElementText(nic, "gnt:IPAddress", network["ip"]) 771 SubElementText(nic, "gnt:Link", network["link"]) 772 SubElementText(nic, "gnt:Net", network["network"])

773

774 - def SaveVirtualSystemData(self, name, vcpus, memory):

775 """Convert virtual system information to OVF sections. 776 777 @type name: string 778 @param name: name of the instance 779 @type vcpus: int 780 @param vcpus: number of VCPUs 781 @type memory: int 782 @param memory: RAM memory in MB 783 784 """ 785 assert(vcpus > 0) 786 assert(memory > 0) 787 vs_attrib = {"ovf:id": name} 788 virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib) 789 SubElementText(virtual_system, "Info", "A virtual machine") 790 791 name_section = ET.SubElement(virtual_system, "Name") 792 name_section.text = name 793 os_attrib = {"ovf:id": "0"} 794 os_section = ET.SubElement(virtual_system, "OperatingSystemSection", 795 attrib=os_attrib) 796 SubElementText(os_section, "Info", "Installed guest operating system") 797 hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection") 798 SubElementText(hardware_section, "Info", "Virtual hardware requirements") 799 800 # System description 801 system = ET.SubElement(hardware_section, "System") 802 SubElementText(system, "vssd:ElementName", "Virtual Hardware Family") 803 SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"]) 804 SubElementText(system, "vssd:VirtualSystemIdentifier", name) 805 SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type) 806 807 # Item for vcpus 808 vcpus_item = ET.SubElement(hardware_section, "Item") 809 SubElementText(vcpus_item, "rasd:ElementName", 810 "%s virtual CPU(s)" % vcpus) 811 SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"]) 812 SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"]) 813 SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus) 814 815 # Item for memory 816 memory_item = ET.SubElement(hardware_section, "Item") 817 SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20") 818 SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory) 819 SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"]) 820 SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"]) 821 SubElementText(memory_item, "rasd:VirtualQuantity", memory) 822 823 # Item for scsi controller 824 scsi_item = ET.SubElement(hardware_section, "Item") 825 SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"]) 826 SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0") 827 SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"]) 828 SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE) 829 SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"]) 830 831 # Other items - from self.hardware_list 832 for item in self.hardware_list: 833 hardware_section.append(item)

834

835 - def PrettyXmlDump(self):

836 """Formatter of the XML file. 837 838 @rtype: string 839 @return: XML tree in the form of nicely-formatted string 840 841 """ 842 raw_string = ET.tostring(self.tree) 843 parsed_xml = xml.dom.minidom.parseString(raw_string) 844 xml_string = parsed_xml.toprettyxml(indent=" ") 845 text_re = re.compile(r">\n\s+([^<>\s].*?)\n\s+</", re.DOTALL) 846 return text_re.sub(r">\g<1></", xml_string)

847

848 849 -class Converter(object):

850 """Converter class for OVF packages. 851 852 Converter is a class above both ImporterOVF and ExporterOVF. It's purpose is 853 to provide a common interface for the two. 854 855 @type options: optparse.Values 856 @ivar options: options parsed from the command line 857 @type output_dir: string 858 @ivar output_dir: directory to which the results of conversion shall be 859 written 860 @type temp_file_manager: L{utils.TemporaryFileManager} 861 @ivar temp_file_manager: container for temporary files created during 862 conversion 863 @type temp_dir: string 864 @ivar temp_dir: temporary directory created then we deal with OVA 865 866 """

867 - def __init__(self, input_path, options):

868 """Initialize the converter. 869 870 @type input_path: string 871 @param input_path: path to the Converter input file 872 @type options: optparse.Values 873 @param options: command line options 874 875 @raise errors.OpPrereqError: if file does not exist 876 877 """ 878 input_path = os.path.abspath(input_path) 879 if not os.path.isfile(input_path): 880 raise errors.OpPrereqError("File does not exist: %s" % input_path, 881 errors.ECODE_ENVIRON) 882 self.options = options 883 self.temp_file_manager = utils.TemporaryFileManager() 884 self.temp_dir = None 885 self.output_dir = None 886 self._ReadInputData(input_path)

887

888 - def _ReadInputData(self, input_path):

889 """Reads the data on which the conversion will take place. 890 891 @type input_path: string 892 @param input_path: absolute path to the Converter input file 893 894 """ 895 raise NotImplementedError()

896

897 - def _CompressDisk(self, disk_path, compression, action):

898 """Performs (de)compression on the disk and returns the new path 899 900 @type disk_path: string 901 @param disk_path: path to the disk 902 @type compression: string 903 @param compression: compression type 904 @type action: string 905 @param action: whether the action is compression or decompression 906 @rtype: string 907 @return: new disk path after (de)compression 908 909 @raise errors.OpPrereqError: disk (de)compression failed or "compression" 910 is not supported 911 912 """ 913 assert(action in ALLOWED_ACTIONS) 914 # For now we only support gzip, as it is used in ovftool 915 if compression != COMPRESSION_TYPE: 916 raise errors.OpPrereqError("Unsupported compression type: %s" 917 % compression, errors.ECODE_INVAL) 918 disk_file = os.path.basename(disk_path) 919 if action == DECOMPRESS: 920 (disk_name, _) = os.path.splitext(disk_file) 921 prefix = disk_name 922 elif action == COMPRESS: 923 prefix = disk_file 924 new_path = utils.GetClosedTempfile(suffix=COMPRESSION_EXT, prefix=prefix, 925 dir=self.output_dir) 926 self.temp_file_manager.Add(new_path) 927 args = ["gzip", "-c", disk_path] 928 run_result = utils.RunCmd(args, output=new_path) 929 if run_result.failed: 930 raise errors.OpPrereqError("Disk %s failed with output: %s" 931 % (action, run_result.stderr), 932 errors.ECODE_ENVIRON) 933 logging.info("The %s of the disk is completed", action) 934 return (COMPRESSION_EXT, new_path)

935

936 - def _ConvertDisk(self, disk_format, disk_path):

937 """Performes conversion to specified format. 938 939 @type disk_format: string 940 @param disk_format: format to which the disk should be converted 941 @type disk_path: string 942 @param disk_path: path to the disk that should be converted 943 @rtype: string 944 @return path to the output disk 945 946 @raise errors.OpPrereqError: convertion of the disk failed 947 948 """ 949 CheckQemuImg() 950 disk_file = os.path.basename(disk_path) 951 (disk_name, disk_extension) = os.path.splitext(disk_file) 952 if disk_extension != disk_format: 953 logging.warning("Conversion of disk image to %s format, this may take" 954 " a while", disk_format) 955 956 new_disk_path = utils.GetClosedTempfile( 957 suffix=".%s" % disk_format, prefix=disk_name, dir=self.output_dir) 958 self.temp_file_manager.Add(new_disk_path) 959 args = [ 960 constants.QEMUIMG_PATH, 961 "convert", 962 "-O", 963 disk_format, 964 disk_path, 965 new_disk_path, 966 ] 967 run_result = utils.RunCmd(args, cwd=os.getcwd()) 968 if run_result.failed: 969 raise errors.OpPrereqError("Convertion to %s failed, qemu-img output was" 970 ": %s" % (disk_format, run_result.stderr), 971 errors.ECODE_ENVIRON) 972 return (".%s" % disk_format, new_disk_path)

973 974 @staticmethod

975 - def _GetDiskQemuInfo(disk_path, regexp):

976 """Figures out some information of the disk using qemu-img. 977 978 @type disk_path: string 979 @param disk_path: path to the disk we want to know the format of 980 @type regexp: string 981 @param regexp: string that has to be matched, it has to contain one group 982 @rtype: string 983 @return: disk format 984 985 @raise errors.OpPrereqError: format information cannot be retrieved 986 987 """ 988 CheckQemuImg() 989 args = [constants.QEMUIMG_PATH, "info", disk_path] 990 run_result = utils.RunCmd(args, cwd=os.getcwd()) 991 if run_result.failed: 992 raise errors.OpPrereqError("Gathering info about the disk using qemu-img" 993 " failed, output was: %s" % run_result.stderr, 994 errors.ECODE_ENVIRON) 995 result = run_result.output 996 regexp = r"%s" % regexp 997 match = re.search(regexp, result) 998 if match: 999 disk_format = match.group(1) 1000 else: 1001 raise errors.OpPrereqError("No file information matching %s found in:" 1002 " %s" % (regexp, result), 1003 errors.ECODE_ENVIRON) 1004 return disk_format

1005

1006 - def Parse(self):

1007 """Parses the data and creates a structure containing all required info. 1008 1009 """ 1010 raise NotImplementedError()

1011

1012 - def Save(self):

1013 """Saves the gathered configuration in an apropriate format. 1014 1015 """ 1016 raise NotImplementedError()

1017

1018 - def Cleanup(self):

1019 """Cleans the temporary directory, if one was created. 1020 1021 """ 1022 self.temp_file_manager.Cleanup() 1023 if self.temp_dir: 1024 shutil.rmtree(self.temp_dir) 1025 self.temp_dir = None

1026

1027 1028 -class OVFImporter(Converter):

1029 """Converter from OVF to Ganeti config file. 1030 1031 @type input_dir: string 1032 @ivar input_dir: directory in which the .ovf file resides 1033 @type output_dir: string 1034 @ivar output_dir: directory to which the results of conversion shall be 1035 written 1036 @type input_path: string 1037 @ivar input_path: complete path to the .ovf file 1038 @type ovf_reader: L{OVFReader} 1039 @ivar ovf_reader: OVF reader instance collects data from .ovf file 1040 @type results_name: string 1041 @ivar results_name: name of imported instance 1042 @type results_template: string 1043 @ivar results_template: disk template read from .ovf file or command line 1044 arguments 1045 @type results_hypervisor: dict 1046 @ivar results_hypervisor: hypervisor information gathered from .ovf file or 1047 command line arguments 1048 @type results_os: dict 1049 @ivar results_os: operating system information gathered from .ovf file or 1050 command line arguments 1051 @type results_backend: dict 1052 @ivar results_backend: backend information gathered from .ovf file or 1053 command line arguments 1054 @type results_tags: string 1055 @ivar results_tags: string containing instance-specific tags 1056 @type results_version: string 1057 @ivar results_version: version as required by Ganeti import 1058 @type results_network: dict 1059 @ivar results_network: network information gathered from .ovf file or command 1060 line arguments 1061 @type results_disk: dict 1062 @ivar results_disk: disk information gathered from .ovf file or command line 1063 arguments 1064 1065 """

1066 - def _ReadInputData(self, input_path):

1067 """Reads the data on which the conversion will take place. 1068 1069 @type input_path: string 1070 @param input_path: absolute path to the .ovf or .ova input file 1071 1072 @raise errors.OpPrereqError: if input file is neither .ovf nor .ova 1073 1074 """ 1075 (input_dir, input_file) = os.path.split(input_path) 1076 (_, input_extension) = os.path.splitext(input_file) 1077 1078 if input_extension == OVF_EXT: 1079 logging.info("%s file extension found, no unpacking necessary", OVF_EXT) 1080 self.input_dir = input_dir 1081 self.input_path = input_path 1082 self.temp_dir = None 1083 elif input_extension == OVA_EXT: 1084 logging.info("%s file extension found, proceeding to unpacking", OVA_EXT) 1085 self._UnpackOVA(input_path) 1086 else: 1087 raise errors.OpPrereqError("Unknown file extension; expected %s or %s" 1088 " file" % (OVA_EXT, OVF_EXT), 1089 errors.ECODE_INVAL) 1090 assert ((input_extension == OVA_EXT and self.temp_dir) or 1091 (input_extension == OVF_EXT and not self.temp_dir)) 1092 assert self.input_dir in self.input_path 1093 1094 if self.options.output_dir: 1095 self.output_dir = os.path.abspath(self.options.output_dir) 1096 if (os.path.commonprefix([pathutils.EXPORT_DIR, self.output_dir]) != 1097 pathutils.EXPORT_DIR): 1098 logging.warning("Export path is not under %s directory, import to" 1099 " Ganeti using gnt-backup may fail", 1100 pathutils.EXPORT_DIR) 1101 else: 1102 self.output_dir = pathutils.EXPORT_DIR 1103 1104 self.ovf_reader = OVFReader(self.input_path) 1105 self.ovf_reader.VerifyManifest()

1106

1107 - def _UnpackOVA(self, input_path):

1108 """Unpacks the .ova package into temporary directory. 1109 1110 @type input_path: string 1111 @param input_path: path to the .ova package file 1112 1113 @raise errors.OpPrereqError: if file is not a proper tarball, one of the 1114 files in the archive seem malicious (e.g. path starts with '../') or 1115 .ova package does not contain .ovf file 1116 1117 """ 1118 input_name = None 1119 if not tarfile.is_tarfile(input_path): 1120 raise errors.OpPrereqError("The provided %s file is not a proper tar" 1121 " archive" % OVA_EXT, errors.ECODE_ENVIRON) 1122 ova_content = tarfile.open(input_path) 1123 temp_dir = tempfile.mkdtemp() 1124 self.temp_dir = temp_dir 1125 for file_name in ova_content.getnames(): 1126 file_normname = os.path.normpath(file_name) 1127 try: 1128 utils.PathJoin(temp_dir, file_normname) 1129 except ValueError, err: 1130 raise errors.OpPrereqError("File %s inside %s package is not safe" % 1131 (file_name, OVA_EXT), errors.ECODE_ENVIRON) 1132 if file_name.endswith(OVF_EXT): 1133 input_name = file_name 1134 if not input_name: 1135 raise errors.OpPrereqError("No %s file in %s package found" % 1136 (OVF_EXT, OVA_EXT), errors.ECODE_ENVIRON) 1137 logging.warning("Unpacking the %s archive, this may take a while", 1138 input_path) 1139 self.input_dir = temp_dir 1140 self.input_path = utils.PathJoin(self.temp_dir, input_name) 1141 try: 1142 try: 1143 extract = ova_content.extractall 1144 except AttributeError: 1145 # This is a prehistorical case of using python < 2.5 1146 for member in ova_content.getmembers(): 1147 ova_content.extract(member, path=self.temp_dir) 1148 else: 1149 extract(self.temp_dir) 1150 except tarfile.TarError, err: 1151 raise errors.OpPrereqError("Error while extracting %s archive: %s" % 1152 (OVA_EXT, err), errors.ECODE_ENVIRON) 1153 logging.info("OVA package extracted to %s directory", self.temp_dir)

1154

1155 - def Parse(self):

1156 """Parses the data and creates a structure containing all required info. 1157 1158 The method reads the information given either as a command line option or as 1159 a part of the OVF description. 1160 1161 @raise errors.OpPrereqError: if some required part of the description of 1162 virtual instance is missing or unable to create output directory 1163 1164 """ 1165 self.results_name = self._GetInfo("instance name", self.options.name, 1166 self._ParseNameOptions, 1167 self.ovf_reader.GetInstanceName) 1168 if not self.results_name: 1169 raise errors.OpPrereqError("Name of instance not provided", 1170 errors.ECODE_INVAL) 1171 1172 self.output_dir = utils.PathJoin(self.output_dir, self.results_name) 1173 try: 1174 utils.Makedirs(self.output_dir) 1175 except OSError, err: 1176 raise errors.OpPrereqError("Failed to create directory %s: %s" % 1177 (self.output_dir, err), errors.ECODE_ENVIRON) 1178 1179 self.results_template = self._GetInfo( 1180 "disk template", self.options.disk_template, self._ParseTemplateOptions, 1181 self.ovf_reader.GetDiskTemplate) 1182 if not self.results_template: 1183 logging.info("Disk template not given") 1184 1185 self.results_hypervisor = self._GetInfo( 1186 "hypervisor", self.options.hypervisor, self._ParseHypervisorOptions, 1187 self.ovf_reader.GetHypervisorData) 1188 assert self.results_hypervisor["hypervisor_name"] 1189 if self.results_hypervisor["hypervisor_name"] == constants.VALUE_AUTO: 1190 logging.debug("Default hypervisor settings from the cluster will be used") 1191 1192 self.results_os = self._GetInfo( 1193 "OS", self.options.os, self._ParseOSOptions, self.ovf_reader.GetOSData) 1194 if not self.results_os.get("os_name"): 1195 raise errors.OpPrereqError("OS name must be provided", 1196 errors.ECODE_INVAL) 1197 1198 self.results_backend = self._GetInfo( 1199 "backend", self.options.beparams, 1200 self._ParseBackendOptions, self.ovf_reader.GetBackendData) 1201 assert self.results_backend.get("vcpus") 1202 assert self.results_backend.get("memory") 1203 assert self.results_backend.get("auto_balance") is not None 1204 1205 self.results_tags = self._GetInfo( 1206 "tags", self.options.tags, self._ParseTags, self.ovf_reader.GetTagsData) 1207 1208 ovf_version = self.ovf_reader.GetVersionData() 1209 if ovf_version: 1210 self.results_version = ovf_version 1211 else: 1212 self.results_version = constants.EXPORT_VERSION 1213 1214 self.results_network = self._GetInfo( 1215 "network", self.options.nics, self._ParseNicOptions, 1216 self.ovf_reader.GetNetworkData, ignore_test=self.options.no_nics) 1217 1218 self.results_disk = self._GetInfo( 1219 "disk", self.options.disks, self._ParseDiskOptions, self._GetDiskInfo, 1220 ignore_test=self.results_template == constants.DT_DISKLESS) 1221 1222 if not self.results_disk and not self.results_network: 1223 raise errors.OpPrereqError("Either disk specification or network" 1224 " description must be present", 1225 errors.ECODE_STATE)

1226 1227 @staticmethod

1228 - def _GetInfo(name, cmd_arg, cmd_function, nocmd_function, 1229 ignore_test=False):

1230 """Get information about some section - e.g. disk, network, hypervisor. 1231 1232 @type name: string 1233 @param name: name of the section 1234 @type cmd_arg: dict 1235 @param cmd_arg: command line argument specific for section 'name' 1236 @type cmd_function: callable 1237 @param cmd_function: function to call if 'cmd_args' exists 1238 @type nocmd_function: callable 1239 @param nocmd_function: function to call if 'cmd_args' is not there 1240 1241 """ 1242 if ignore_test: 1243 logging.info("Information for %s will be ignored", name) 1244 return {} 1245 if cmd_arg: 1246 logging.info("Information for %s will be parsed from command line", name) 1247 results = cmd_function() 1248 else: 1249 logging.info("Information for %s will be parsed from %s file", 1250 name, OVF_EXT) 1251 results = nocmd_function() 1252 logging.info("Options for %s were succesfully read", name) 1253 return results

1254

1255 - def _ParseNameOptions(self):

1256 """Returns name if one was given in command line. 1257 1258 @rtype: string 1259 @return: name of an instance 1260 1261 """ 1262 return self.options.name

1263

1264 - def _ParseTemplateOptions(self):

1265 """Returns disk template if one was given in command line. 1266 1267 @rtype: string 1268 @return: disk template name 1269 1270 """ 1271 return self.options.disk_template

1272

1273 - def _ParseHypervisorOptions(self):

1274 """Parses hypervisor options given in a command line. 1275 1276 @rtype: dict 1277 @return: dictionary containing name of the chosen hypervisor and all the 1278 options 1279 1280 """ 1281 assert type(self.options.hypervisor) is tuple 1282 assert len(self.options.hypervisor) == 2 1283 results = {} 1284 if self.options.hypervisor[0]: 1285 results["hypervisor_name"] = self.options.hypervisor[0] 1286 else: 1287 results["hypervisor_name"] = constants.VALUE_AUTO 1288 results.update(self.options.hypervisor[1]) 1289 return results

1290

1291 - def _ParseOSOptions(self):

1292 """Parses OS options given in command line. 1293 1294 @rtype: dict 1295 @return: dictionary containing name of chosen OS and all its options 1296 1297 """ 1298 assert self.options.os 1299 results = {} 1300 results["os_name"] = self.options.os 1301 results.update(self.options.osparams) 1302 return results

1303

1304 - def _ParseBackendOptions(self):

1305 """Parses backend options given in command line. 1306 1307 @rtype: dict 1308 @return: dictionary containing vcpus, memory and auto-balance options 1309 1310 """ 1311 assert self.options.beparams 1312 backend = {} 1313 backend.update(self.options.beparams) 1314 must_contain = ["vcpus", "memory", "auto_balance"] 1315 for element in must_contain: 1316 if backend.get(element) is None: 1317 backend[element] = constants.VALUE_AUTO 1318 return backend

1319

1320 - def _ParseTags(self):

1321 """Returns tags list given in command line. 1322 1323 @rtype: string 1324 @return: string containing comma-separated tags 1325 1326 """ 1327 return self.options.tags

1328

1329 - def _ParseNicOptions(self):

1330 """Parses network options given in a command line or as a dictionary. 1331 1332 @rtype: dict 1333 @return: dictionary of network-related options 1334 1335 """ 1336 assert self.options.nics 1337 results = {} 1338 for (nic_id, nic_desc) in self.options.nics: 1339 results["nic%s_mode" % nic_id] = \ 1340 nic_desc.get("mode", constants.VALUE_AUTO) 1341 results["nic%s_mac" % nic_id] = nic_desc.get("mac", constants.VALUE_AUTO) 1342 results["nic%s_link" % nic_id] = \ 1343 nic_desc.get("link", constants.VALUE_AUTO) 1344 results["nic%s_network" % nic_id] = \ 1345 nic_desc.get("network", constants.VALUE_AUTO) 1346 if nic_desc.get("mode") == "bridged": 1347 results["nic%s_ip" % nic_id] = constants.VALUE_NONE 1348 else: 1349 results["nic%s_ip" % nic_id] = constants.VALUE_AUTO 1350 results["nic_count"] = str(len(self.options.nics)) 1351 return results

1352

1353 - def _ParseDiskOptions(self):

1354 """Parses disk options given in a command line. 1355 1356 @rtype: dict 1357 @return: dictionary of disk-related options 1358 1359 @raise errors.OpPrereqError: disk description does not contain size 1360 information or size information is invalid or creation failed 1361 1362 """ 1363 CheckQemuImg() 1364 assert self.options.disks 1365 results = {} 1366 for (disk_id, disk_desc) in self.options.disks: 1367 results["disk%s_ivname" % disk_id] = "disk/%s" % disk_id 1368 if disk_desc.get("size"): 1369 try: 1370 disk_size = utils.ParseUnit(disk_desc["size"]) 1371 except ValueError: 1372 raise errors.OpPrereqError("Invalid disk size for disk %s: %s" % 1373 (disk_id, disk_desc["size"]), 1374 errors.ECODE_INVAL) 1375 new_path = utils.PathJoin(self.output_dir, str(disk_id)) 1376 args = [ 1377 constants.QEMUIMG_PATH, 1378 "create", 1379 "-f", 1380 "raw", 1381 new_path, 1382 disk_size, 1383 ] 1384 run_result = utils.RunCmd(args) 1385 if run_result.failed: 1386 raise errors.OpPrereqError("Creation of disk %s failed, output was:" 1387 " %s" % (new_path, run_result.stderr), 1388 errors.ECODE_ENVIRON) 1389 results["disk%s_size" % disk_id] = str(disk_size) 1390 results["disk%s_dump" % disk_id] = "disk%s.raw" % disk_id 1391 else: 1392 raise errors.OpPrereqError("Disks created for import must have their" 1393 " size specified", 1394 errors.ECODE_INVAL) 1395 results["disk_count"] = str(len(self.options.disks)) 1396 return results

1397

1398 - def _GetDiskInfo(self):

1399 """Gathers information about disks used by instance, perfomes conversion. 1400 1401 @rtype: dict 1402 @return: dictionary of disk-related options 1403 1404 @raise errors.OpPrereqError: disk is not in the same directory as .ovf file 1405 1406 """ 1407 results = {} 1408 disks_list = self.ovf_reader.GetDisksNames() 1409 for (counter, (disk_name, disk_compression)) in enumerate(disks_list): 1410 if os.path.dirname(disk_name): 1411 raise errors.OpPrereqError("Disks are not allowed to have absolute" 1412 " paths or paths outside main OVF" 1413 " directory", errors.ECODE_ENVIRON) 1414 disk, _ = os.path.splitext(disk_name) 1415 disk_path = utils.PathJoin(self.input_dir, disk_name) 1416 if disk_compression not in NO_COMPRESSION: 1417 _, disk_path = self._CompressDisk(disk_path, disk_compression, 1418 DECOMPRESS) 1419 disk, _ = os.path.splitext(disk) 1420 if self._GetDiskQemuInfo(disk_path, r"file format: (\S+)") != "raw": 1421 logging.info("Conversion to raw format is required") 1422 ext, new_disk_path = self._ConvertDisk("raw", disk_path) 1423 1424 final_disk_path = LinkFile(new_disk_path, prefix=disk, suffix=ext, 1425 directory=self.output_dir) 1426 final_name = os.path.basename(final_disk_path) 1427 disk_size = os.path.getsize(final_disk_path) / (1024 * 1024) 1428 results["disk%s_dump" % counter] = final_name 1429 results["disk%s_size" % counter] = str(disk_size) 1430 results["disk%s_ivname" % counter] = "disk/%s" % str(counter) 1431 if disks_list: 1432 results["disk_count"] = str(len(disks_list)) 1433 return results

1434

1435 - def Save(self):

1436 """Saves all the gathered information in a constant.EXPORT_CONF_FILE file. 1437 1438 @raise errors.OpPrereqError: when saving to config file failed 1439 1440 """ 1441 logging.info("Conversion was succesfull, saving %s in %s directory", 1442 constants.EXPORT_CONF_FILE, self.output_dir) 1443 results = { 1444 constants.INISECT_INS: {}, 1445 constants.INISECT_BEP: {}, 1446 constants.INISECT_EXP: {}, 1447 constants.INISECT_OSP: {}, 1448 constants.INISECT_HYP: {}, 1449 } 1450 1451 results[constants.INISECT_INS].update(self.results_disk) 1452 results[constants.INISECT_INS].update(self.results_network) 1453 results[constants.INISECT_INS]["hypervisor"] = \ 1454 self.results_hypervisor["hypervisor_name"] 1455 results[constants.INISECT_INS]["name"] = self.results_name 1456 if self.results_template: 1457 results[constants.INISECT_INS]["disk_template"] = self.results_template 1458 if self.results_tags: 1459 results[constants.INISECT_INS]["tags"] = self.results_tags 1460 1461 results[constants.INISECT_BEP].update(self.results_backend) 1462 1463 results[constants.INISECT_EXP]["os"] = self.results_os["os_name"] 1464 results[constants.INISECT_EXP]["version"] = self.results_version 1465 1466 del self.results_os["os_name"] 1467 results[constants.INISECT_OSP].update(self.results_os) 1468 1469 del self.results_hypervisor["hypervisor_name"] 1470 results[constants.INISECT_HYP].update(self.results_hypervisor) 1471 1472 output_file_name = utils.PathJoin(self.output_dir, 1473 constants.EXPORT_CONF_FILE) 1474 1475 output = [] 1476 for section, options in results.iteritems(): 1477 output.append("[%s]" % section) 1478 for name, value in options.iteritems(): 1479 if value is None: 1480 value = "" 1481 output.append("%s = %s" % (name, value)) 1482 output.append("") 1483 output_contents = "\n".join(output) 1484 1485 try: 1486 utils.WriteFile(output_file_name, data=output_contents) 1487 except errors.ProgrammerError, err: 1488 raise errors.OpPrereqError("Saving the config file failed: %s" % err, 1489 errors.ECODE_ENVIRON) 1490 1491 self.Cleanup()

1492

1493 1494 -class ConfigParserWithDefaults(ConfigParser.SafeConfigParser):

1495 """This is just a wrapper on SafeConfigParser, that uses default values 1496 1497 """

1498 - def get(self, section, options, raw=None, vars=None): # pylint: disable=W0622

1499 try: 1500 result = ConfigParser.SafeConfigParser.get(self, section, options, 1501 raw=raw, vars=vars) 1502 except ConfigParser.NoOptionError: 1503 result = None 1504 return result

1505

1506 - def getint(self, section, options):

1507 try: 1508 result = ConfigParser.SafeConfigParser.get(self, section, options) 1509 except ConfigParser.NoOptionError: 1510 result = 0 1511 return int(result)

1512

1513 1514 -class OVFExporter(Converter):

1515 """Converter from Ganeti config file to OVF 1516 1517 @type input_dir: string 1518 @ivar input_dir: directory in which the config.ini file resides 1519 @type output_dir: string 1520 @ivar output_dir: directory to which the results of conversion shall be 1521 written 1522 @type packed_dir: string 1523 @ivar packed_dir: if we want OVA package, this points to the real (i.e. not 1524 temp) output directory 1525 @type input_path: string 1526 @ivar input_path: complete path to the config.ini file 1527 @type output_path: string 1528 @ivar output_path: complete path to .ovf file 1529 @type config_parser: L{ConfigParserWithDefaults} 1530 @ivar config_parser: parser for the config.ini file 1531 @type reference_files: list 1532 @ivar reference_files: files referenced in the ovf file 1533 @type results_disk: list 1534 @ivar results_disk: list of dictionaries of disk options from config.ini 1535 @type results_network: list 1536 @ivar results_network: list of dictionaries of network options form config.ini 1537 @type results_name: string 1538 @ivar results_name: name of the instance 1539 @type results_vcpus: string 1540 @ivar results_vcpus: number of VCPUs 1541 @type results_memory: string 1542 @ivar results_memory: RAM memory in MB 1543 @type results_ganeti: dict 1544 @ivar results_ganeti: dictionary of Ganeti-specific options from config.ini 1545 1546 """

1547 - def _ReadInputData(self, input_path):

1548 """Reads the data on which the conversion will take place. 1549 1550 @type input_path: string 1551 @param input_path: absolute path to the config.ini input file 1552 1553 @raise errors.OpPrereqError: error when reading the config file 1554 1555 """ 1556 input_dir = os.path.dirname(input_path) 1557 self.input_path = input_path 1558 self.input_dir = input_dir 1559 if self.options.output_dir: 1560 self.output_dir = os.path.abspath(self.options.output_dir) 1561 else: 1562 self.output_dir = input_dir 1563 self.config_parser = ConfigParserWithDefaults() 1564 logging.info("Reading configuration from %s file", input_path) 1565 try: 1566 self.config_parser.read(input_path) 1567 except ConfigParser.MissingSectionHeaderError, err: 1568 raise errors.OpPrereqError("Error when trying to read %s: %s" % 1569 (input_path, err), errors.ECODE_ENVIRON) 1570 if self.options.ova_package: 1571 self.temp_dir = tempfile.mkdtemp() 1572 self.packed_dir = self.output_dir 1573 self.output_dir = self.temp_dir 1574 1575 self.ovf_writer = OVFWriter(not self.options.ext_usage)

1576

1577 - def _ParseName(self):

1578 """Parses name from command line options or config file. 1579 1580 @rtype: string 1581 @return: name of Ganeti instance 1582 1583 @raise errors.OpPrereqError: if name of the instance is not provided 1584 1585 """ 1586 if self.options.name: 1587 name = self.options.name 1588 else: 1589 name = self.config_parser.get(constants.INISECT_INS, NAME) 1590 if name is None: 1591 raise errors.OpPrereqError("No instance name found", 1592 errors.ECODE_ENVIRON) 1593 return name

1594

1595 - def _ParseVCPUs(self):

1596 """Parses vcpus number from config file. 1597 1598 @rtype: int 1599 @return: number of virtual CPUs 1600 1601 @raise errors.OpPrereqError: if number of VCPUs equals 0 1602 1603 """ 1604 vcpus = self.config_parser.getint(constants.INISECT_BEP, VCPUS) 1605 if vcpus == 0: 1606 raise errors.OpPrereqError("No CPU information found", 1607 errors.ECODE_ENVIRON) 1608 return vcpus

1609

1610 - def _ParseMemory(self):

1611 """Parses vcpus number from config file. 1612 1613 @rtype: int 1614 @return: amount of memory in MB 1615 1616 @raise errors.OpPrereqError: if amount of memory equals 0 1617 1618 """ 1619 memory = self.config_parser.getint(constants.INISECT_BEP, MEMORY) 1620 if memory == 0: 1621 raise errors.OpPrereqError("No memory information found", 1622 errors.ECODE_ENVIRON) 1623 return memory

1624

1625 - def _ParseGaneti(self):

1626 """Parses Ganeti data from config file. 1627 1628 @rtype: dictionary 1629 @return: dictionary of Ganeti-specific options 1630 1631 """ 1632 results = {} 1633 # hypervisor 1634 results["hypervisor"] = {} 1635 hyp_name = self.config_parser.get(constants.INISECT_INS, HYPERV) 1636 if hyp_name is None: 1637 raise errors.OpPrereqError("No hypervisor information found", 1638 errors.ECODE_ENVIRON) 1639 results["hypervisor"]["name"] = hyp_name 1640 pairs = self.config_parser.items(constants.INISECT_HYP) 1641 for (name, value) in pairs: 1642 results["hypervisor"][name] = value 1643 # os 1644 results["os"] = {} 1645 os_name = self.config_parser.get(constants.INISECT_EXP, OS) 1646 if os_name is None: 1647 raise errors.OpPrereqError("No operating system information found", 1648 errors.ECODE_ENVIRON) 1649 results["os"]["name"] = os_name 1650 pairs = self.config_parser.items(constants.INISECT_OSP) 1651 for (name, value) in pairs: 1652 results["os"][name] = value 1653 # other 1654 others = [ 1655 (constants.INISECT_INS, DISK_TEMPLATE, "disk_template"), 1656 (constants.INISECT_BEP, AUTO_BALANCE, "auto_balance"), 1657 (constants.INISECT_INS, TAGS, "tags"), 1658 (constants.INISECT_EXP, VERSION, "version"), 1659 ] 1660 for (section, element, name) in others: 1661 results[name] = self.config_parser.get(section, element) 1662 return results

1663

1664 - def _ParseNetworks(self):

1665 """Parses network data from config file. 1666 1667 @rtype: list 1668 @return: list of dictionaries of network options 1669 1670 @raise errors.OpPrereqError: then network mode is not recognized 1671 1672 """ 1673 results = [] 1674 counter = 0 1675 while True: 1676 data_link = \ 1677 self.config_parser.get(constants.INISECT_INS, 1678 "nic%s_link" % counter) 1679 if data_link is None: 1680 break 1681 results.append({ 1682 "mode": self.config_parser.get(constants.INISECT_INS, 1683 "nic%s_mode" % counter), 1684 "mac": self.config_parser.get(constants.INISECT_INS, 1685 "nic%s_mac" % counter), 1686 "ip": self.config_parser.get(constants.INISECT_INS, 1687 "nic%s_ip" % counter), 1688 "network": self.config_parser.get(constants.INISECT_INS, 1689 "nic%s_network" % counter), 1690 "link": data_link, 1691 }) 1692 if results[counter]["mode"] not in constants.NIC_VALID_MODES: 1693 raise errors.OpPrereqError("Network mode %s not recognized" 1694 % results[counter]["mode"], 1695 errors.ECODE_INVAL) 1696 counter += 1 1697 return results

1698

1699 - def _GetDiskOptions(self, disk_file, compression):

1700 """Convert the disk and gather disk info for .ovf file. 1701 1702 @type disk_file: string 1703 @param disk_file: name of the disk (without the full path) 1704 @type compression: bool 1705 @param compression: whether the disk should be compressed or not 1706 1707 @raise errors.OpPrereqError: when disk image does not exist 1708 1709 """ 1710 disk_path = utils.PathJoin(self.input_dir, disk_file) 1711 results = {} 1712 if not os.path.isfile(disk_path): 1713 raise errors.OpPrereqError("Disk image does not exist: %s" % disk_path, 1714 errors.ECODE_ENVIRON) 1715 if os.path.dirname(disk_file): 1716 raise errors.OpPrereqError("Path for the disk: %s contains a directory" 1717 " name" % disk_path, errors.ECODE_ENVIRON) 1718 disk_name, _ = os.path.splitext(disk_file) 1719 ext, new_disk_path = self._ConvertDisk(self.options.disk_format, disk_path) 1720 results["format"] = self.options.disk_format 1721 results["virt-size"] = self._GetDiskQemuInfo( 1722 new_disk_path, r"virtual size: \S+ \((\d+) bytes\)") 1723 if compression: 1724 ext2, new_disk_path = self._CompressDisk(new_disk_path, "gzip", 1725 COMPRESS) 1726 disk_name, _ = os.path.splitext(disk_name) 1727 results["compression"] = "gzip" 1728 ext += ext2 1729 final_disk_path = LinkFile(new_disk_path, prefix=disk_name, suffix=ext, 1730 directory=self.output_dir) 1731 final_disk_name = os.path.basename(final_disk_path) 1732 results["real-size"] = os.path.getsize(final_disk_path) 1733 results["path"] = final_disk_name 1734 self.references_files.append(final_disk_path) 1735 return results

1736

1737 - def _ParseDisks(self):

1738 """Parses disk data from config file. 1739 1740 @rtype: list 1741 @return: list of dictionaries of disk options 1742 1743 """ 1744 results = [] 1745 counter = 0 1746 while True: 1747 disk_file = \ 1748 self.config_parser.get(constants.INISECT_INS, "disk%s_dump" % counter) 1749 if disk_file is None: 1750 break 1751 results.append(self._GetDiskOptions(disk_file, self.options.compression)) 1752 counter += 1 1753 return results

1754

1755 - def Parse(self):

1756 """Parses the data and creates a structure containing all required info. 1757 1758 """ 1759 try: 1760 utils.Makedirs(self.output_dir) 1761 except OSError, err: 1762 raise errors.OpPrereqError("Failed to create directory %s: %s" % 1763 (self.output_dir, err), errors.ECODE_ENVIRON) 1764 1765 self.references_files = [] 1766 self.results_name = self._ParseName() 1767 self.results_vcpus = self._ParseVCPUs() 1768 self.results_memory = self._ParseMemory() 1769 if not self.options.ext_usage: 1770 self.results_ganeti = self._ParseGaneti() 1771 self.results_network = self._ParseNetworks() 1772 self.results_disk = self._ParseDisks()

1773

1774 - def _PrepareManifest(self, path):

1775 """Creates manifest for all the files in OVF package. 1776 1777 @type path: string 1778 @param path: path to manifesto file 1779 1780 @raise errors.OpPrereqError: if error occurs when writing file 1781 1782 """ 1783 logging.info("Preparing manifest for the OVF package") 1784 lines = [] 1785 files_list = [self.output_path] 1786 files_list.extend(self.references_files) 1787 logging.warning("Calculating SHA1 checksums, this may take a while") 1788 sha1_sums = utils.FingerprintFiles(files_list) 1789 for file_path, value in sha1_sums.iteritems(): 1790 file_name = os.path.basename(file_path) 1791 lines.append("SHA1(%s)= %s" % (file_name, value)) 1792 lines.append("") 1793 data = "\n".join(lines) 1794 try: 1795 utils.WriteFile(path, data=data) 1796 except errors.ProgrammerError, err: 1797 raise errors.OpPrereqError("Saving the manifest file failed: %s" % err, 1798 errors.ECODE_ENVIRON)

1799 1800 @staticmethod

1801 - def _PrepareTarFile(tar_path, files_list):

1802 """Creates tarfile from the files in OVF package. 1803 1804 @type tar_path: string 1805 @param tar_path: path to the resulting file 1806 @type files_list: list 1807 @param files_list: list of files in the OVF package 1808 1809 """ 1810 logging.info("Preparing tarball for the OVF package") 1811 open(tar_path, mode="w").close() 1812 ova_package = tarfile.open(name=tar_path, mode="w") 1813 for file_path in files_list: 1814 file_name = os.path.basename(file_path) 1815 ova_package.add(file_path, arcname=file_name) 1816 ova_package.close()

1817

1818 - def Save(self):

1819 """Saves the gathered configuration in an apropriate format. 1820 1821 @raise errors.OpPrereqError: if unable to create output directory 1822 1823 """ 1824 output_file = "%s%s" % (self.results_name, OVF_EXT) 1825 output_path = utils.PathJoin(self.output_dir, output_file) 1826 self.ovf_writer = OVFWriter(not self.options.ext_usage) 1827 logging.info("Saving read data to %s", output_path) 1828 1829 self.output_path = utils.PathJoin(self.output_dir, output_file) 1830 files_list = [self.output_path] 1831 1832 self.ovf_writer.SaveDisksData(self.results_disk) 1833 self.ovf_writer.SaveNetworksData(self.results_network) 1834 if not self.options.ext_usage: 1835 self.ovf_writer.SaveGanetiData(self.results_ganeti, self.results_network) 1836 1837 self.ovf_writer.SaveVirtualSystemData(self.results_name, self.results_vcpus, 1838 self.results_memory) 1839 1840 data = self.ovf_writer.PrettyXmlDump() 1841 utils.WriteFile(self.output_path, data=data) 1842 1843 manifest_file = "%s%s" % (self.results_name, MF_EXT) 1844 manifest_path = utils.PathJoin(self.output_dir, manifest_file) 1845 self._PrepareManifest(manifest_path) 1846 files_list.append(manifest_path) 1847 1848 files_list.extend(self.references_files) 1849 1850 if self.options.ova_package: 1851 ova_file = "%s%s" % (self.results_name, OVA_EXT) 1852 packed_path = utils.PathJoin(self.packed_dir, ova_file) 1853 try: 1854 utils.Makedirs(self.packed_dir) 1855 except OSError, err: 1856 raise errors.OpPrereqError("Failed to create directory %s: %s" % 1857 (self.packed_dir, err), 1858 errors.ECODE_ENVIRON) 1859 self._PrepareTarFile(packed_path, files_list) 1860 logging.info("Creation of the OVF package was successfull") 1861 self.Cleanup()

1862

Source Code for Module ganeti.ovf