ganeti.storage.container

1 # 2 # 3 4 # Copyright (C) 2009, 2011, 2012 Google Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions are 9 # met: 10 # 11 # 1. Redistributions of source code must retain the above copyright notice, 12 # this list of conditions and the following disclaimer. 13 # 14 # 2. Redistributions in binary form must reproduce the above copyright 15 # notice, this list of conditions and the following disclaimer in the 16 # documentation and/or other materials provided with the distribution. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 """Storage container abstraction. 32 33 """ 34 35 # pylint: disable=W0232,R0201 36 37 # W0232, since we use these as singletons rather than object holding 38 # data 39 40 # R0201, for the same reason 41 42 # TODO: FileStorage initialised with paths whereas the others not 43 44 import logging 45 46 from ganeti import errors 47 from ganeti import constants 48 from ganeti import utils

49 50 51 -def _ParseSize(value):

52 return int(round(float(value), 0))

53

54 55 -class _Base(object):

56 """Base class for storage abstraction. 57 58 """

59 - def List(self, name, fields):

60 """Returns a list of all entities within the storage unit. 61 62 @type name: string or None 63 @param name: Entity name or None for all 64 @type fields: list 65 @param fields: List with all requested result fields (order is preserved) 66 67 """ 68 raise NotImplementedError()

69

70 - def Modify(self, name, changes): # pylint: disable=W0613

71 """Modifies an entity within the storage unit. 72 73 @type name: string 74 @param name: Entity name 75 @type changes: dict 76 @param changes: New field values 77 78 """ 79 # Don't raise an error if no changes are requested 80 if changes: 81 raise errors.ProgrammerError("Unable to modify the following" 82 "fields: %r" % (changes.keys(), ))

83

84 - def Execute(self, name, op):

85 """Executes an operation on an entity within the storage unit. 86 87 @type name: string 88 @param name: Entity name 89 @type op: string 90 @param op: Operation name 91 92 """ 93 raise NotImplementedError()

94

95 96 -class FileStorage(_Base): # pylint: disable=W0223

97 """File storage unit. 98 99 """

100 - def __init__(self, paths):

101 """Initializes this class. 102 103 @type paths: list 104 @param paths: List of file storage paths 105 106 """ 107 super(FileStorage, self).__init__() 108 109 self._paths = paths

110

111 - def List(self, name, fields):

112 """Returns a list of all entities within the storage unit. 113 114 See L{_Base.List}. 115 116 """ 117 rows = [] 118 119 if name is None: 120 paths = self._paths 121 else: 122 paths = [name] 123 124 for path in paths: 125 rows.append(self._ListInner(path, fields)) 126 127 return rows

128 129 @staticmethod

130 - def _ListInner(path, fields):

131 """Gathers requested information from directory. 132 133 @type path: string 134 @param path: Path to directory 135 @type fields: list 136 @param fields: Requested fields 137 138 """ 139 values = [] 140 141 # Pre-calculate information in case it's requested more than once 142 if constants.SF_USED in fields: 143 dirsize = utils.CalculateDirectorySize(path) 144 else: 145 dirsize = None 146 147 if constants.SF_FREE in fields or constants.SF_SIZE in fields: 148 fsstats = utils.GetFilesystemStats(path) 149 else: 150 fsstats = None 151 152 # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields. 153 for field_name in fields: 154 if field_name == constants.SF_NAME: 155 values.append(path) 156 157 elif field_name == constants.SF_USED: 158 values.append(dirsize) 159 160 elif field_name == constants.SF_FREE: 161 values.append(fsstats[1]) 162 163 elif field_name == constants.SF_SIZE: 164 values.append(fsstats[0]) 165 166 elif field_name == constants.SF_ALLOCATABLE: 167 values.append(True) 168 169 else: 170 raise errors.StorageError("Unknown field: %r" % field_name) 171 172 return values

173

174 175 -class _LvmBase(_Base): # pylint: disable=W0223

176 """Base class for LVM storage containers. 177 178 @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_* 179 constants, lvm command output fields (list), and conversion 180 function or static value (for static value, the lvm output field 181 can be an empty list) 182 183 """ 184 LIST_SEP = "|" 185 LIST_COMMAND = None 186 LIST_FIELDS = None 187

188 - def List(self, name, wanted_field_names):

189 """Returns a list of all entities within the storage unit. 190 191 See L{_Base.List}. 192 193 """ 194 # Get needed LVM fields 195 lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names) 196 197 # Build LVM command 198 cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP, 199 lvm_fields, name) 200 201 # Run LVM command 202 cmd_result = self._RunListCommand(cmd_args) 203 204 # Split and rearrange LVM command output 205 return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP, 206 len(lvm_fields)), 207 self.LIST_FIELDS, 208 wanted_field_names, 209 lvm_fields)

210 211 @staticmethod

212 - def _GetLvmFields(fields_def, wanted_field_names):

213 """Returns unique list of fields wanted from LVM command. 214 215 @type fields_def: list 216 @param fields_def: Field definitions 217 @type wanted_field_names: list 218 @param wanted_field_names: List of requested fields 219 220 """ 221 field_to_idx = dict([(field_name, idx) 222 for (idx, (field_name, _, _)) in 223 enumerate(fields_def)]) 224 225 lvm_fields = [] 226 227 for field_name in wanted_field_names: 228 try: 229 idx = field_to_idx[field_name] 230 except IndexError: 231 raise errors.StorageError("Unknown field: %r" % field_name) 232 233 (_, lvm_names, _) = fields_def[idx] 234 235 lvm_fields.extend(lvm_names) 236 237 return utils.UniqueSequence(lvm_fields)

238 239 @classmethod

240 - def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):

241 """Builds the final result list. 242 243 @type cmd_result: iterable 244 @param cmd_result: Iterable of LVM command output (iterable of lists) 245 @type fields_def: list 246 @param fields_def: Field definitions 247 @type wanted_field_names: list 248 @param wanted_field_names: List of requested fields 249 @type lvm_fields: list 250 @param lvm_fields: LVM fields 251 252 """ 253 lvm_name_to_idx = dict([(lvm_name, idx) 254 for (idx, lvm_name) in enumerate(lvm_fields)]) 255 field_to_idx = dict([(field_name, idx) 256 for (idx, (field_name, _, _)) in 257 enumerate(fields_def)]) 258 259 data = [] 260 for raw_data in cmd_result: 261 row = [] 262 263 for field_name in wanted_field_names: 264 (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]] 265 266 values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names] 267 268 if callable(mapper): 269 # we got a function, call it with all the declared fields 270 val = mapper(*values) # pylint: disable=W0142 271 elif len(values) == 1: 272 assert mapper is None, ("Invalid mapper value (neither callable" 273 " nor None) for one-element fields") 274 # we don't have a function, but we had a single field 275 # declared, pass it unchanged 276 val = values[0] 277 else: 278 # let's make sure there are no fields declared (cannot map > 279 # 1 field without a function) 280 assert not values, "LVM storage has multi-fields without a function" 281 val = mapper 282 283 row.append(val) 284 285 data.append(row) 286 287 return data

288 289 @staticmethod

290 - def _BuildListCommand(cmd, sep, options, name):

291 """Builds LVM command line. 292 293 @type cmd: string 294 @param cmd: Command name 295 @type sep: string 296 @param sep: Field separator character 297 @type options: list of strings 298 @param options: Wanted LVM fields 299 @type name: name or None 300 @param name: Name of requested entity 301 302 """ 303 args = [cmd, 304 "--noheadings", "--units=m", "--nosuffix", 305 "--separator", sep, 306 "--options", ",".join(options)] 307 308 if name is not None: 309 args.append(name) 310 311 return args

312 313 @staticmethod

314 - def _RunListCommand(args):

315 """Run LVM command. 316 317 """ 318 result = utils.RunCmd(args) 319 320 if result.failed: 321 raise errors.StorageError("Failed to run %r, command output: %s" % 322 (args[0], result.output)) 323 324 return result.stdout

325 326 @staticmethod

327 - def _SplitList(data, sep, fieldcount):

328 """Splits LVM command output into rows and fields. 329 330 @type data: string 331 @param data: LVM command output 332 @type sep: string 333 @param sep: Field separator character 334 @type fieldcount: int 335 @param fieldcount: Expected number of fields 336 337 """ 338 for line in data.splitlines(): 339 fields = line.strip().split(sep) 340 341 if len(fields) != fieldcount: 342 logging.warning("Invalid line returned from lvm command: %s", line) 343 continue 344 345 yield fields

346

347 348 -def _LvmPvGetAllocatable(attr):

349 """Determines whether LVM PV is allocatable. 350 351 @rtype: bool 352 353 """ 354 if attr: 355 return (attr[0] == "a") 356 else: 357 logging.warning("Invalid PV attribute: %r", attr) 358 return False

359

360 361 -class LvmPvStorage(_LvmBase): # pylint: disable=W0223

362 """LVM Physical Volume storage unit. 363 364 """ 365 LIST_COMMAND = "pvs" 366 367 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 368 # definitions. 369 LIST_FIELDS = [ 370 (constants.SF_NAME, ["pv_name"], None), 371 (constants.SF_SIZE, ["pv_size"], _ParseSize), 372 (constants.SF_USED, ["pv_used"], _ParseSize), 373 (constants.SF_FREE, ["pv_free"], _ParseSize), 374 (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable), 375 ] 376

377 - def _SetAllocatable(self, name, allocatable):

378 """Sets the "allocatable" flag on a physical volume. 379 380 @type name: string 381 @param name: Physical volume name 382 @type allocatable: bool 383 @param allocatable: Whether to set the "allocatable" flag 384 385 """ 386 args = ["pvchange", "--allocatable"] 387 388 if allocatable: 389 args.append("y") 390 else: 391 args.append("n") 392 393 args.append(name) 394 395 result = utils.RunCmd(args) 396 if result.failed: 397 raise errors.StorageError("Failed to modify physical volume," 398 " pvchange output: %s" % 399 result.output)

400

401 - def Modify(self, name, changes):

402 """Modifies flags on a physical volume. 403 404 See L{_Base.Modify}. 405 406 """ 407 if constants.SF_ALLOCATABLE in changes: 408 self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE]) 409 del changes[constants.SF_ALLOCATABLE] 410 411 # Other changes will be handled (and maybe refused) by the base class. 412 return _LvmBase.Modify(self, name, changes)

413

414 415 -class LvmVgStorage(_LvmBase):

416 """LVM Volume Group storage unit. 417 418 """ 419 LIST_COMMAND = "vgs" 420 VGREDUCE_COMMAND = "vgreduce" 421 422 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 423 # definitions. 424 LIST_FIELDS = [ 425 (constants.SF_NAME, ["vg_name"], None), 426 (constants.SF_SIZE, ["vg_size"], _ParseSize), 427 (constants.SF_FREE, ["vg_free"], _ParseSize), 428 (constants.SF_USED, ["vg_size", "vg_free"], 429 lambda x, y: _ParseSize(x) - _ParseSize(y)), 430 (constants.SF_ALLOCATABLE, [], True), 431 ] 432

433 - def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd):

434 """Runs "vgreduce --removemissing" on a volume group. 435 436 @type name: string 437 @param name: Volume group name 438 439 """ 440 # Ignoring vgreduce exit code. Older versions exit with an error even tough 441 # the VG is already consistent. This was fixed in later versions, but we 442 # cannot depend on it. 443 result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name]) 444 445 # Keep output in case something went wrong 446 vgreduce_output = result.output 447 448 # work around newer LVM version 449 if ("Wrote out consistent volume group" not in vgreduce_output or 450 "vgreduce --removemissing --force" in vgreduce_output): 451 # we need to re-run with --force 452 result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", 453 "--force", name]) 454 vgreduce_output += "\n" + result.output 455 456 result = _runcmd_fn([self.LIST_COMMAND, "--noheadings", 457 "--nosuffix", name]) 458 # we also need to check the output 459 if result.failed or "Couldn't find device with uuid" in result.output: 460 raise errors.StorageError(("Volume group '%s' still not consistent," 461 " 'vgreduce' output: %r," 462 " 'vgs' output: %r") % 463 (name, vgreduce_output, result.output))

464

465 - def Execute(self, name, op):

466 """Executes an operation on a virtual volume. 467 468 See L{_Base.Execute}. 469 470 """ 471 if op == constants.SO_FIX_CONSISTENCY: 472 return self._RemoveMissing(name) 473 474 return _LvmBase.Execute(self, name, op)

475 476 477 # Lookup table for storage types 478 _STORAGE_TYPES = { 479 constants.ST_FILE: FileStorage, 480 constants.ST_LVM_PV: LvmPvStorage, 481 constants.ST_LVM_VG: LvmVgStorage, 482 constants.ST_SHARED_FILE: FileStorage, 483 constants.ST_GLUSTER: FileStorage, 484 }

485 486 487 -def GetStorageClass(name):

488 """Returns the class for a storage type. 489 490 @type name: string 491 @param name: Storage type 492 493 """ 494 try: 495 return _STORAGE_TYPES[name] 496 except KeyError: 497 raise errors.StorageError("Unknown storage type: %r" % name)

498

499 500 -def GetStorage(name, *args):

501 """Factory function for storage methods. 502 503 @type name: string 504 @param name: Storage type 505 506 """ 507 return GetStorageClass(name)(*args)

508

Source Code for Module ganeti.storage.container