ganeti.storage

1 # 2 # 3 4 # Copyright (C) 2009, 2011, 2012 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 22 """Storage container abstraction. 23 24 """ 25 26 # pylint: disable=W0232,R0201 27 28 # W0232, since we use these as singletons rather than object holding 29 # data 30 31 # R0201, for the same reason 32 33 # TODO: FileStorage initialised with paths whereas the others not 34 35 import logging 36 37 from ganeti import errors 38 from ganeti import constants 39 from ganeti import utils

40 41 42 -def _ParseSize(value):

43 return int(round(float(value), 0))

44

45 46 -class _Base:

47 """Base class for storage abstraction. 48 49 """

50 - def List(self, name, fields):

51 """Returns a list of all entities within the storage unit. 52 53 @type name: string or None 54 @param name: Entity name or None for all 55 @type fields: list 56 @param fields: List with all requested result fields (order is preserved) 57 58 """ 59 raise NotImplementedError()

60

61 - def Modify(self, name, changes): # pylint: disable=W0613

62 """Modifies an entity within the storage unit. 63 64 @type name: string 65 @param name: Entity name 66 @type changes: dict 67 @param changes: New field values 68 69 """ 70 # Don't raise an error if no changes are requested 71 if changes: 72 raise errors.ProgrammerError("Unable to modify the following" 73 "fields: %r" % (changes.keys(), ))

74

75 - def Execute(self, name, op):

76 """Executes an operation on an entity within the storage unit. 77 78 @type name: string 79 @param name: Entity name 80 @type op: string 81 @param op: Operation name 82 83 """ 84 raise NotImplementedError()

85

86 87 -class FileStorage(_Base): # pylint: disable=W0223

88 """File storage unit. 89 90 """

91 - def __init__(self, paths):

92 """Initializes this class. 93 94 @type paths: list 95 @param paths: List of file storage paths 96 97 """ 98 self._paths = paths

99

100 - def List(self, name, fields):

101 """Returns a list of all entities within the storage unit. 102 103 See L{_Base.List}. 104 105 """ 106 rows = [] 107 108 if name is None: 109 paths = self._paths 110 else: 111 paths = [name] 112 113 for path in paths: 114 rows.append(self._ListInner(path, fields)) 115 116 return rows

117 118 @staticmethod

119 - def _ListInner(path, fields):

120 """Gathers requested information from directory. 121 122 @type path: string 123 @param path: Path to directory 124 @type fields: list 125 @param fields: Requested fields 126 127 """ 128 values = [] 129 130 # Pre-calculate information in case it's requested more than once 131 if constants.SF_USED in fields: 132 dirsize = utils.CalculateDirectorySize(path) 133 else: 134 dirsize = None 135 136 if constants.SF_FREE in fields or constants.SF_SIZE in fields: 137 fsstats = utils.GetFilesystemStats(path) 138 else: 139 fsstats = None 140 141 # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields. 142 for field_name in fields: 143 if field_name == constants.SF_NAME: 144 values.append(path) 145 146 elif field_name == constants.SF_USED: 147 values.append(dirsize) 148 149 elif field_name == constants.SF_FREE: 150 values.append(fsstats[1]) 151 152 elif field_name == constants.SF_SIZE: 153 values.append(fsstats[0]) 154 155 elif field_name == constants.SF_ALLOCATABLE: 156 values.append(True) 157 158 else: 159 raise errors.StorageError("Unknown field: %r" % field_name) 160 161 return values

162

163 164 -class _LvmBase(_Base): # pylint: disable=W0223

165 """Base class for LVM storage containers. 166 167 @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_* 168 constants, lvm command output fields (list), and conversion 169 function or static value (for static value, the lvm output field 170 can be an empty list) 171 172 """ 173 LIST_SEP = "|" 174 LIST_COMMAND = None 175 LIST_FIELDS = None 176

177 - def List(self, name, wanted_field_names):

178 """Returns a list of all entities within the storage unit. 179 180 See L{_Base.List}. 181 182 """ 183 # Get needed LVM fields 184 lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names) 185 186 # Build LVM command 187 cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP, 188 lvm_fields, name) 189 190 # Run LVM command 191 cmd_result = self._RunListCommand(cmd_args) 192 193 # Split and rearrange LVM command output 194 return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP, 195 len(lvm_fields)), 196 self.LIST_FIELDS, 197 wanted_field_names, 198 lvm_fields)

199 200 @staticmethod

201 - def _GetLvmFields(fields_def, wanted_field_names):

202 """Returns unique list of fields wanted from LVM command. 203 204 @type fields_def: list 205 @param fields_def: Field definitions 206 @type wanted_field_names: list 207 @param wanted_field_names: List of requested fields 208 209 """ 210 field_to_idx = dict([(field_name, idx) 211 for (idx, (field_name, _, _)) in 212 enumerate(fields_def)]) 213 214 lvm_fields = [] 215 216 for field_name in wanted_field_names: 217 try: 218 idx = field_to_idx[field_name] 219 except IndexError: 220 raise errors.StorageError("Unknown field: %r" % field_name) 221 222 (_, lvm_names, _) = fields_def[idx] 223 224 lvm_fields.extend(lvm_names) 225 226 return utils.UniqueSequence(lvm_fields)

227 228 @classmethod

229 - def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):

230 """Builds the final result list. 231 232 @type cmd_result: iterable 233 @param cmd_result: Iterable of LVM command output (iterable of lists) 234 @type fields_def: list 235 @param fields_def: Field definitions 236 @type wanted_field_names: list 237 @param wanted_field_names: List of requested fields 238 @type lvm_fields: list 239 @param lvm_fields: LVM fields 240 241 """ 242 lvm_name_to_idx = dict([(lvm_name, idx) 243 for (idx, lvm_name) in enumerate(lvm_fields)]) 244 field_to_idx = dict([(field_name, idx) 245 for (idx, (field_name, _, _)) in 246 enumerate(fields_def)]) 247 248 data = [] 249 for raw_data in cmd_result: 250 row = [] 251 252 for field_name in wanted_field_names: 253 (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]] 254 255 values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names] 256 257 if callable(mapper): 258 # we got a function, call it with all the declared fields 259 val = mapper(*values) # pylint: disable=W0142 260 elif len(values) == 1: 261 assert mapper is None, ("Invalid mapper value (neither callable" 262 " nor None) for one-element fields") 263 # we don't have a function, but we had a single field 264 # declared, pass it unchanged 265 val = values[0] 266 else: 267 # let's make sure there are no fields declared (cannot map > 268 # 1 field without a function) 269 assert not values, "LVM storage has multi-fields without a function" 270 val = mapper 271 272 row.append(val) 273 274 data.append(row) 275 276 return data

277 278 @staticmethod

279 - def _BuildListCommand(cmd, sep, options, name):

280 """Builds LVM command line. 281 282 @type cmd: string 283 @param cmd: Command name 284 @type sep: string 285 @param sep: Field separator character 286 @type options: list of strings 287 @param options: Wanted LVM fields 288 @type name: name or None 289 @param name: Name of requested entity 290 291 """ 292 args = [cmd, 293 "--noheadings", "--units=m", "--nosuffix", 294 "--separator", sep, 295 "--options", ",".join(options)] 296 297 if name is not None: 298 args.append(name) 299 300 return args

301 302 @staticmethod

303 - def _RunListCommand(args):

304 """Run LVM command. 305 306 """ 307 result = utils.RunCmd(args) 308 309 if result.failed: 310 raise errors.StorageError("Failed to run %r, command output: %s" % 311 (args[0], result.output)) 312 313 return result.stdout

314 315 @staticmethod

316 - def _SplitList(data, sep, fieldcount):

317 """Splits LVM command output into rows and fields. 318 319 @type data: string 320 @param data: LVM command output 321 @type sep: string 322 @param sep: Field separator character 323 @type fieldcount: int 324 @param fieldcount: Expected number of fields 325 326 """ 327 for line in data.splitlines(): 328 fields = line.strip().split(sep) 329 330 if len(fields) != fieldcount: 331 logging.warning("Invalid line returned from lvm command: %s", line) 332 continue 333 334 yield fields

335

336 337 -def _LvmPvGetAllocatable(attr):

338 """Determines whether LVM PV is allocatable. 339 340 @rtype: bool 341 342 """ 343 if attr: 344 return (attr[0] == "a") 345 else: 346 logging.warning("Invalid PV attribute: %r", attr) 347 return False

348

349 350 -class LvmPvStorage(_LvmBase): # pylint: disable=W0223

351 """LVM Physical Volume storage unit. 352 353 """ 354 LIST_COMMAND = "pvs" 355 356 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 357 # definitions. 358 LIST_FIELDS = [ 359 (constants.SF_NAME, ["pv_name"], None), 360 (constants.SF_SIZE, ["pv_size"], _ParseSize), 361 (constants.SF_USED, ["pv_used"], _ParseSize), 362 (constants.SF_FREE, ["pv_free"], _ParseSize), 363 (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable), 364 ] 365

366 - def _SetAllocatable(self, name, allocatable):

367 """Sets the "allocatable" flag on a physical volume. 368 369 @type name: string 370 @param name: Physical volume name 371 @type allocatable: bool 372 @param allocatable: Whether to set the "allocatable" flag 373 374 """ 375 args = ["pvchange", "--allocatable"] 376 377 if allocatable: 378 args.append("y") 379 else: 380 args.append("n") 381 382 args.append(name) 383 384 result = utils.RunCmd(args) 385 if result.failed: 386 raise errors.StorageError("Failed to modify physical volume," 387 " pvchange output: %s" % 388 result.output)

389

390 - def Modify(self, name, changes):

391 """Modifies flags on a physical volume. 392 393 See L{_Base.Modify}. 394 395 """ 396 if constants.SF_ALLOCATABLE in changes: 397 self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE]) 398 del changes[constants.SF_ALLOCATABLE] 399 400 # Other changes will be handled (and maybe refused) by the base class. 401 return _LvmBase.Modify(self, name, changes)

402

403 404 -class LvmVgStorage(_LvmBase):

405 """LVM Volume Group storage unit. 406 407 """ 408 LIST_COMMAND = "vgs" 409 VGREDUCE_COMMAND = "vgreduce" 410 411 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 412 # definitions. 413 LIST_FIELDS = [ 414 (constants.SF_NAME, ["vg_name"], None), 415 (constants.SF_SIZE, ["vg_size"], _ParseSize), 416 (constants.SF_FREE, ["vg_free"], _ParseSize), 417 (constants.SF_USED, ["vg_size", "vg_free"], 418 lambda x, y: _ParseSize(x) - _ParseSize(y)), 419 (constants.SF_ALLOCATABLE, [], True), 420 ] 421

422 - def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd):

423 """Runs "vgreduce --removemissing" on a volume group. 424 425 @type name: string 426 @param name: Volume group name 427 428 """ 429 # Ignoring vgreduce exit code. Older versions exit with an error even tough 430 # the VG is already consistent. This was fixed in later versions, but we 431 # cannot depend on it. 432 result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name]) 433 434 # Keep output in case something went wrong 435 vgreduce_output = result.output 436 437 # work around newer LVM version 438 if ("Wrote out consistent volume group" not in vgreduce_output or 439 "vgreduce --removemissing --force" in vgreduce_output): 440 # we need to re-run with --force 441 result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", 442 "--force", name]) 443 vgreduce_output += "\n" + result.output 444 445 result = _runcmd_fn([self.LIST_COMMAND, "--noheadings", 446 "--nosuffix", name]) 447 # we also need to check the output 448 if result.failed or "Couldn't find device with uuid" in result.output: 449 raise errors.StorageError(("Volume group '%s' still not consistent," 450 " 'vgreduce' output: %r," 451 " 'vgs' output: %r") % 452 (name, vgreduce_output, result.output))

453

454 - def Execute(self, name, op):

455 """Executes an operation on a virtual volume. 456 457 See L{_Base.Execute}. 458 459 """ 460 if op == constants.SO_FIX_CONSISTENCY: 461 return self._RemoveMissing(name) 462 463 return _LvmBase.Execute(self, name, op)

464 465 466 # Lookup table for storage types 467 _STORAGE_TYPES = { 468 constants.ST_FILE: FileStorage, 469 constants.ST_LVM_PV: LvmPvStorage, 470 constants.ST_LVM_VG: LvmVgStorage, 471 }

472 473 474 -def GetStorageClass(name):

475 """Returns the class for a storage type. 476 477 @type name: string 478 @param name: Storage type 479 480 """ 481 try: 482 return _STORAGE_TYPES[name] 483 except KeyError: 484 raise errors.StorageError("Unknown storage type: %r" % name)

485

486 487 -def GetStorage(name, *args):

488 """Factory function for storage methods. 489 490 @type name: string 491 @param name: Storage type 492 493 """ 494 return GetStorageClass(name)(*args)

495

Source Code for Module ganeti.storage