ganeti.storage

1 # 2 # 3 4 # Copyright (C) 2009, 2011 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 22 """Storage container abstraction. 23 24 """ 25 26 # pylint: disable-msg=W0232,R0201 27 28 # W0232, since we use these as singletons rather than object holding 29 # data 30 31 # R0201, for the same reason 32 33 # TODO: FileStorage initialised with paths whereas the others not 34 35 import logging 36 37 from ganeti import errors 38 from ganeti import constants 39 from ganeti import utils

40 41 42 -def _ParseSize(value):

43 return int(round(float(value), 0))

44

45 46 -class _Base:

47 """Base class for storage abstraction. 48 49 """

50 - def List(self, name, fields):

51 """Returns a list of all entities within the storage unit. 52 53 @type name: string or None 54 @param name: Entity name or None for all 55 @type fields: list 56 @param fields: List with all requested result fields (order is preserved) 57 58 """ 59 raise NotImplementedError()

60

61 - def Modify(self, name, changes): # pylint: disable-msg=W0613

62 """Modifies an entity within the storage unit. 63 64 @type name: string 65 @param name: Entity name 66 @type changes: dict 67 @param changes: New field values 68 69 """ 70 # Don't raise an error if no changes are requested 71 if changes: 72 raise errors.ProgrammerError("Unable to modify the following" 73 "fields: %r" % (changes.keys(), ))

74

75 - def Execute(self, name, op):

76 """Executes an operation on an entity within the storage unit. 77 78 @type name: string 79 @param name: Entity name 80 @type op: string 81 @param op: Operation name 82 83 """ 84 raise NotImplementedError()

85

86 87 -class FileStorage(_Base): # pylint: disable-msg=W0223

88 """File storage unit. 89 90 """

91 - def __init__(self, paths):

92 """Initializes this class. 93 94 @type paths: list 95 @param paths: List of file storage paths 96 97 """ 98 self._paths = paths

99

100 - def List(self, name, fields):

101 """Returns a list of all entities within the storage unit. 102 103 See L{_Base.List}. 104 105 """ 106 rows = [] 107 108 if name is None: 109 paths = self._paths 110 else: 111 paths = [name] 112 113 for path in paths: 114 rows.append(self._ListInner(path, fields)) 115 116 return rows

117 118 @staticmethod

119 - def _ListInner(path, fields):

120 """Gathers requested information from directory. 121 122 @type path: string 123 @param path: Path to directory 124 @type fields: list 125 @param fields: Requested fields 126 127 """ 128 values = [] 129 130 # Pre-calculate information in case it's requested more than once 131 if constants.SF_USED in fields: 132 dirsize = utils.CalculateDirectorySize(path) 133 else: 134 dirsize = None 135 136 if constants.SF_FREE in fields or constants.SF_SIZE in fields: 137 fsstats = utils.GetFilesystemStats(path) 138 else: 139 fsstats = None 140 141 # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields. 142 for field_name in fields: 143 if field_name == constants.SF_NAME: 144 values.append(path) 145 146 elif field_name == constants.SF_USED: 147 values.append(dirsize) 148 149 elif field_name == constants.SF_FREE: 150 values.append(fsstats[1]) 151 152 elif field_name == constants.SF_SIZE: 153 values.append(fsstats[0]) 154 155 elif field_name == constants.SF_ALLOCATABLE: 156 values.append(True) 157 158 else: 159 raise errors.StorageError("Unknown field: %r" % field_name) 160 161 return values

162

163 164 -class _LvmBase(_Base): # pylint: disable-msg=W0223

165 """Base class for LVM storage containers. 166 167 @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_* 168 constants, lvm command output fields (list), and conversion 169 function or static value (for static value, the lvm output field 170 can be an empty list) 171 172 """ 173 LIST_SEP = "|" 174 LIST_COMMAND = None 175 LIST_FIELDS = None 176

177 - def List(self, name, wanted_field_names):

178 """Returns a list of all entities within the storage unit. 179 180 See L{_Base.List}. 181 182 """ 183 # Get needed LVM fields 184 lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names) 185 186 # Build LVM command 187 cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP, 188 lvm_fields, name) 189 190 # Run LVM command 191 cmd_result = self._RunListCommand(cmd_args) 192 193 # Split and rearrange LVM command output 194 return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP, 195 len(lvm_fields)), 196 self.LIST_FIELDS, 197 wanted_field_names, 198 lvm_fields)

199 200 @staticmethod

201 - def _GetLvmFields(fields_def, wanted_field_names):

202 """Returns unique list of fields wanted from LVM command. 203 204 @type fields_def: list 205 @param fields_def: Field definitions 206 @type wanted_field_names: list 207 @param wanted_field_names: List of requested fields 208 209 """ 210 field_to_idx = dict([(field_name, idx) 211 for (idx, (field_name, _, _)) in 212 enumerate(fields_def)]) 213 214 lvm_fields = [] 215 216 for field_name in wanted_field_names: 217 try: 218 idx = field_to_idx[field_name] 219 except IndexError: 220 raise errors.StorageError("Unknown field: %r" % field_name) 221 222 (_, lvm_names, _) = fields_def[idx] 223 224 lvm_fields.extend(lvm_names) 225 226 return utils.UniqueSequence(lvm_fields)

227 228 @classmethod

229 - def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):

230 """Builds the final result list. 231 232 @type cmd_result: iterable 233 @param cmd_result: Iterable of LVM command output (iterable of lists) 234 @type fields_def: list 235 @param fields_def: Field definitions 236 @type wanted_field_names: list 237 @param wanted_field_names: List of requested fields 238 @type lvm_fields: list 239 @param lvm_fields: LVM fields 240 241 """ 242 lvm_name_to_idx = dict([(lvm_name, idx) 243 for (idx, lvm_name) in enumerate(lvm_fields)]) 244 field_to_idx = dict([(field_name, idx) 245 for (idx, (field_name, _, _)) in 246 enumerate(fields_def)]) 247 248 data = [] 249 for raw_data in cmd_result: 250 row = [] 251 252 for field_name in wanted_field_names: 253 (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]] 254 255 values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names] 256 257 if callable(mapper): 258 # we got a function, call it with all the declared fields 259 val = mapper(*values) # pylint: disable-msg=W0142 260 elif len(values) == 1: 261 assert mapper is None, ("Invalid mapper value (neither callable" 262 " nor None) for one-element fields") 263 # we don't have a function, but we had a single field 264 # declared, pass it unchanged 265 val = values[0] 266 else: 267 # let's make sure there are no fields declared (cannot map > 268 # 1 field without a function) 269 assert not values, "LVM storage has multi-fields without a function" 270 val = mapper 271 272 row.append(val) 273 274 data.append(row) 275 276 return data

277 278 @staticmethod

279 - def _BuildListCommand(cmd, sep, options, name):

280 """Builds LVM command line. 281 282 @type cmd: string 283 @param cmd: Command name 284 @type sep: string 285 @param sep: Field separator character 286 @type options: list of strings 287 @param options: Wanted LVM fields 288 @type name: name or None 289 @param name: Name of requested entity 290 291 """ 292 args = [cmd, 293 "--noheadings", "--units=m", "--nosuffix", 294 "--separator", sep, 295 "--options", ",".join(options)] 296 297 if name is not None: 298 args.append(name) 299 300 return args

301 302 @staticmethod

303 - def _RunListCommand(args):

304 """Run LVM command. 305 306 """ 307 result = utils.RunCmd(args) 308 309 if result.failed: 310 raise errors.StorageError("Failed to run %r, command output: %s" % 311 (args[0], result.output)) 312 313 return result.stdout

314 315 @staticmethod

316 - def _SplitList(data, sep, fieldcount):

317 """Splits LVM command output into rows and fields. 318 319 @type data: string 320 @param data: LVM command output 321 @type sep: string 322 @param sep: Field separator character 323 @type fieldcount: int 324 @param fieldcount: Expected number of fields 325 326 """ 327 for line in data.splitlines(): 328 fields = line.strip().split(sep) 329 330 if len(fields) != fieldcount: 331 logging.warning("Invalid line returned from lvm command: %s", line) 332 continue 333 334 yield fields

335

336 337 -def _LvmPvGetAllocatable(attr):

338 """Determines whether LVM PV is allocatable. 339 340 @rtype: bool 341 342 """ 343 if attr: 344 return (attr[0] == "a") 345 else: 346 logging.warning("Invalid PV attribute: %r", attr) 347 return False

348

349 350 -class LvmPvStorage(_LvmBase): # pylint: disable-msg=W0223

351 """LVM Physical Volume storage unit. 352 353 """ 354 LIST_COMMAND = "pvs" 355 356 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 357 # definitions. 358 LIST_FIELDS = [ 359 (constants.SF_NAME, ["pv_name"], None), 360 (constants.SF_SIZE, ["pv_size"], _ParseSize), 361 (constants.SF_USED, ["pv_used"], _ParseSize), 362 (constants.SF_FREE, ["pv_free"], _ParseSize), 363 (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable), 364 ] 365

366 - def _SetAllocatable(self, name, allocatable):

367 """Sets the "allocatable" flag on a physical volume. 368 369 @type name: string 370 @param name: Physical volume name 371 @type allocatable: bool 372 @param allocatable: Whether to set the "allocatable" flag 373 374 """ 375 args = ["pvchange", "--allocatable"] 376 377 if allocatable: 378 args.append("y") 379 else: 380 args.append("n") 381 382 args.append(name) 383 384 result = utils.RunCmd(args) 385 if result.failed: 386 raise errors.StorageError("Failed to modify physical volume," 387 " pvchange output: %s" % 388 result.output)

389

390 - def Modify(self, name, changes):

391 """Modifies flags on a physical volume. 392 393 See L{_Base.Modify}. 394 395 """ 396 if constants.SF_ALLOCATABLE in changes: 397 self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE]) 398 del changes[constants.SF_ALLOCATABLE] 399 400 # Other changes will be handled (and maybe refused) by the base class. 401 return _LvmBase.Modify(self, name, changes)

402

403 404 -class LvmVgStorage(_LvmBase):

405 """LVM Volume Group storage unit. 406 407 """ 408 LIST_COMMAND = "vgs" 409 410 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 411 # definitions. 412 LIST_FIELDS = [ 413 (constants.SF_NAME, ["vg_name"], None), 414 (constants.SF_SIZE, ["vg_size"], _ParseSize), 415 (constants.SF_FREE, ["vg_free"], _ParseSize), 416 (constants.SF_USED, ["vg_size", "vg_free"], 417 lambda x, y: _ParseSize(x) - _ParseSize(y)), 418 (constants.SF_ALLOCATABLE, [], True), 419 ] 420

421 - def _RemoveMissing(self, name):

422 """Runs "vgreduce --removemissing" on a volume group. 423 424 @type name: string 425 @param name: Volume group name 426 427 """ 428 # Ignoring vgreduce exit code. Older versions exit with an error even tough 429 # the VG is already consistent. This was fixed in later versions, but we 430 # cannot depend on it. 431 result = utils.RunCmd(["vgreduce", "--removemissing", name]) 432 433 # Keep output in case something went wrong 434 vgreduce_output = result.output 435 436 result = utils.RunCmd(["vgs", "--noheadings", "--nosuffix", name]) 437 if result.failed: 438 raise errors.StorageError(("Volume group '%s' still not consistent," 439 " 'vgreduce' output: %r," 440 " 'vgs' output: %r") % 441 (name, vgreduce_output, result.output))

442

443 - def Execute(self, name, op):

444 """Executes an operation on a virtual volume. 445 446 See L{_Base.Execute}. 447 448 """ 449 if op == constants.SO_FIX_CONSISTENCY: 450 return self._RemoveMissing(name) 451 452 return _LvmBase.Execute(self, name, op)

453 454 455 # Lookup table for storage types 456 _STORAGE_TYPES = { 457 constants.ST_FILE: FileStorage, 458 constants.ST_LVM_PV: LvmPvStorage, 459 constants.ST_LVM_VG: LvmVgStorage, 460 }

461 462 463 -def GetStorageClass(name):

464 """Returns the class for a storage type. 465 466 @type name: string 467 @param name: Storage type 468 469 """ 470 try: 471 return _STORAGE_TYPES[name] 472 except KeyError: 473 raise errors.StorageError("Unknown storage type: %r" % name)

474

475 476 -def GetStorage(name, *args):

477 """Factory function for storage methods. 478 479 @type name: string 480 @param name: Storage type 481 482 """ 483 return GetStorageClass(name)(*args)

484

Source Code for Module ganeti.storage