ganeti.storage

1 # 2 # 3 4 # Copyright (C) 2009 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 22 """Storage container abstraction. 23 24 """ 25 26 # pylint: disable-msg=W0232,R0201 27 28 # W0232, since we use these as singletons rather than object holding 29 # data 30 31 # R0201, for the same reason 32 33 # TODO: FileStorage initialised with paths whereas the others not 34 35 import logging 36 37 from ganeti import errors 38 from ganeti import constants 39 from ganeti import utils

40 41 42 -def _ParseSize(value):

43 return int(round(float(value), 0))

44

45 46 -class _Base:

47 """Base class for storage abstraction. 48 49 """

50 - def List(self, name, fields):

51 """Returns a list of all entities within the storage unit. 52 53 @type name: string or None 54 @param name: Entity name or None for all 55 @type fields: list 56 @param fields: List with all requested result fields (order is preserved) 57 58 """ 59 raise NotImplementedError()

60

61 - def Modify(self, name, changes): # pylint: disable-msg=W0613

62 """Modifies an entity within the storage unit. 63 64 @type name: string 65 @param name: Entity name 66 @type changes: dict 67 @param changes: New field values 68 69 """ 70 # Don't raise an error if no changes are requested 71 if changes: 72 raise errors.ProgrammerError("Unable to modify the following" 73 "fields: %r" % (changes.keys(), ))

74

75 - def Execute(self, name, op):

76 """Executes an operation on an entity within the storage unit. 77 78 @type name: string 79 @param name: Entity name 80 @type op: string 81 @param op: Operation name 82 83 """ 84 raise NotImplementedError()

85

86 87 -class FileStorage(_Base): # pylint: disable-msg=W0223

88 """File storage unit. 89 90 """

91 - def __init__(self, paths):

92 """Initializes this class. 93 94 @type paths: list 95 @param paths: List of file storage paths 96 97 """ 98 self._paths = paths

99

100 - def List(self, name, fields):

101 """Returns a list of all entities within the storage unit. 102 103 See L{_Base.List}. 104 105 """ 106 rows = [] 107 108 if name is None: 109 paths = self._paths 110 else: 111 paths = [name] 112 113 for path in paths: 114 rows.append(self._ListInner(path, fields)) 115 116 return rows

117 118 @staticmethod

119 - def _ListInner(path, fields):

120 """Gathers requested information from directory. 121 122 @type path: string 123 @param path: Path to directory 124 @type fields: list 125 @param fields: Requested fields 126 127 """ 128 values = [] 129 130 # Pre-calculate information in case it's requested more than once 131 if constants.SF_USED in fields: 132 dirsize = utils.CalculateDirectorySize(path) 133 else: 134 dirsize = None 135 136 if constants.SF_FREE in fields or constants.SF_SIZE in fields: 137 fsstats = utils.GetFilesystemStats(path) 138 else: 139 fsstats = None 140 141 # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields. 142 for field_name in fields: 143 if field_name == constants.SF_NAME: 144 values.append(path) 145 146 elif field_name == constants.SF_USED: 147 values.append(dirsize) 148 149 elif field_name == constants.SF_FREE: 150 values.append(fsstats[1]) 151 152 elif field_name == constants.SF_SIZE: 153 values.append(fsstats[0]) 154 155 elif field_name == constants.SF_ALLOCATABLE: 156 values.append(True) 157 158 else: 159 raise errors.StorageError("Unknown field: %r" % field_name) 160 161 return values

162

163 164 -class _LvmBase(_Base): # pylint: disable-msg=W0223

165 """Base class for LVM storage containers. 166 167 @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_* 168 constants, lvm command output fields (list), and conversion 169 function or static value (for static value, the lvm output field 170 can be an empty list) 171 172 """ 173 LIST_SEP = "|" 174 LIST_COMMAND = None 175 LIST_FIELDS = None 176

177 - def List(self, name, wanted_field_names):

178 """Returns a list of all entities within the storage unit. 179 180 See L{_Base.List}. 181 182 """ 183 # Get needed LVM fields 184 lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names) 185 186 # Build LVM command 187 cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP, 188 lvm_fields, name) 189 190 # Run LVM command 191 cmd_result = self._RunListCommand(cmd_args) 192 193 # Split and rearrange LVM command output 194 return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP, 195 len(lvm_fields)), 196 self.LIST_FIELDS, 197 wanted_field_names, 198 lvm_fields)

199 200 @staticmethod

201 - def _GetLvmFields(fields_def, wanted_field_names):

202 """Returns unique list of fields wanted from LVM command. 203 204 @type fields_def: list 205 @param fields_def: Field definitions 206 @type wanted_field_names: list 207 @param wanted_field_names: List of requested fields 208 209 """ 210 field_to_idx = dict([(field_name, idx) 211 for (idx, (field_name, _, _)) in 212 enumerate(fields_def)]) 213 214 lvm_fields = [] 215 216 for field_name in wanted_field_names: 217 try: 218 idx = field_to_idx[field_name] 219 except IndexError: 220 raise errors.StorageError("Unknown field: %r" % field_name) 221 222 (_, lvm_names, _) = fields_def[idx] 223 224 lvm_fields.extend(lvm_names) 225 226 return utils.UniqueSequence(lvm_fields)

227 228 @classmethod

229 - def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):

230 """Builds the final result list. 231 232 @type cmd_result: iterable 233 @param cmd_result: Iterable of LVM command output (iterable of lists) 234 @type fields_def: list 235 @param fields_def: Field definitions 236 @type wanted_field_names: list 237 @param wanted_field_names: List of requested fields 238 @type lvm_fields: list 239 @param lvm_fields: LVM fields 240 241 """ 242 lvm_name_to_idx = dict([(lvm_name, idx) 243 for (idx, lvm_name) in enumerate(lvm_fields)]) 244 field_to_idx = dict([(field_name, idx) 245 for (idx, (field_name, _, _)) in 246 enumerate(fields_def)]) 247 248 data = [] 249 for raw_data in cmd_result: 250 row = [] 251 252 for field_name in wanted_field_names: 253 (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]] 254 255 values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names] 256 257 if callable(mapper): 258 # we got a function, call it with all the declared fields 259 val = mapper(*values) # pylint: disable-msg=W0142 260 elif len(values) == 1: 261 # we don't have a function, but we had a single field 262 # declared, pass it unchanged 263 val = values[0] 264 else: 265 # let's make sure there are no fields declared (cannot map > 266 # 1 field without a function) 267 assert not values, "LVM storage has multi-fields without a function" 268 val = mapper 269 270 row.append(val) 271 272 data.append(row) 273 274 return data

275 276 @staticmethod

277 - def _BuildListCommand(cmd, sep, options, name):

278 """Builds LVM command line. 279 280 @type cmd: string 281 @param cmd: Command name 282 @type sep: string 283 @param sep: Field separator character 284 @type options: list of strings 285 @param options: Wanted LVM fields 286 @type name: name or None 287 @param name: Name of requested entity 288 289 """ 290 args = [cmd, 291 "--noheadings", "--units=m", "--nosuffix", 292 "--separator", sep, 293 "--options", ",".join(options)] 294 295 if name is not None: 296 args.append(name) 297 298 return args

299 300 @staticmethod

301 - def _RunListCommand(args):

302 """Run LVM command. 303 304 """ 305 result = utils.RunCmd(args) 306 307 if result.failed: 308 raise errors.StorageError("Failed to run %r, command output: %s" % 309 (args[0], result.output)) 310 311 return result.stdout

312 313 @staticmethod

314 - def _SplitList(data, sep, fieldcount):

315 """Splits LVM command output into rows and fields. 316 317 @type data: string 318 @param data: LVM command output 319 @type sep: string 320 @param sep: Field separator character 321 @type fieldcount: int 322 @param fieldcount: Expected number of fields 323 324 """ 325 for line in data.splitlines(): 326 fields = line.strip().split(sep) 327 328 if len(fields) != fieldcount: 329 logging.warning("Invalid line returned from lvm command: %s", line) 330 continue 331 332 yield fields

333

334 335 -class LvmPvStorage(_LvmBase): # pylint: disable-msg=W0223

336 """LVM Physical Volume storage unit. 337 338 """ 339 @staticmethod

340 - def _GetAllocatable(attr):

341 if attr: 342 return (attr[0] == "a") 343 else: 344 logging.warning("Invalid PV attribute: %r", attr) 345 return False

346 347 LIST_COMMAND = "pvs" 348 349 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 350 # definitions. 351 LIST_FIELDS = [ 352 (constants.SF_NAME, ["pv_name"], None), 353 (constants.SF_SIZE, ["pv_size"], _ParseSize), 354 (constants.SF_USED, ["pv_used"], _ParseSize), 355 (constants.SF_FREE, ["pv_free"], _ParseSize), 356 (constants.SF_ALLOCATABLE, ["pv_attr"], _GetAllocatable), 357 ] 358

359 - def _SetAllocatable(self, name, allocatable):

360 """Sets the "allocatable" flag on a physical volume. 361 362 @type name: string 363 @param name: Physical volume name 364 @type allocatable: bool 365 @param allocatable: Whether to set the "allocatable" flag 366 367 """ 368 args = ["pvchange", "--allocatable"] 369 370 if allocatable: 371 args.append("y") 372 else: 373 args.append("n") 374 375 args.append(name) 376 377 result = utils.RunCmd(args) 378 if result.failed: 379 raise errors.StorageError("Failed to modify physical volume," 380 " pvchange output: %s" % 381 result.output)

382

383 - def Modify(self, name, changes):

384 """Modifies flags on a physical volume. 385 386 See L{_Base.Modify}. 387 388 """ 389 if constants.SF_ALLOCATABLE in changes: 390 self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE]) 391 del changes[constants.SF_ALLOCATABLE] 392 393 # Other changes will be handled (and maybe refused) by the base class. 394 return _LvmBase.Modify(self, name, changes)

395

396 397 -class LvmVgStorage(_LvmBase):

398 """LVM Volume Group storage unit. 399 400 """ 401 LIST_COMMAND = "vgs" 402 403 # Make sure to update constants.VALID_STORAGE_FIELDS when changing field 404 # definitions. 405 LIST_FIELDS = [ 406 (constants.SF_NAME, ["vg_name"], None), 407 (constants.SF_SIZE, ["vg_size"], _ParseSize), 408 (constants.SF_FREE, ["vg_free"], _ParseSize), 409 (constants.SF_USED, ["vg_size", "vg_free"], 410 lambda x, y: _ParseSize(x) - _ParseSize(y)), 411 (constants.SF_ALLOCATABLE, [], True), 412 ] 413

414 - def _RemoveMissing(self, name):

415 """Runs "vgreduce --removemissing" on a volume group. 416 417 @type name: string 418 @param name: Volume group name 419 420 """ 421 # Ignoring vgreduce exit code. Older versions exit with an error even tough 422 # the VG is already consistent. This was fixed in later versions, but we 423 # cannot depend on it. 424 result = utils.RunCmd(["vgreduce", "--removemissing", name]) 425 426 # Keep output in case something went wrong 427 vgreduce_output = result.output 428 429 result = utils.RunCmd(["vgs", "--noheadings", "--nosuffix", name]) 430 if result.failed: 431 raise errors.StorageError(("Volume group '%s' still not consistent," 432 " 'vgreduce' output: %r," 433 " 'vgs' output: %r") % 434 (name, vgreduce_output, result.output))

435

436 - def Execute(self, name, op):

437 """Executes an operation on a virtual volume. 438 439 See L{_Base.Execute}. 440 441 """ 442 if op == constants.SO_FIX_CONSISTENCY: 443 return self._RemoveMissing(name) 444 445 return _LvmBase.Execute(self, name, op)

446 447 448 # Lookup table for storage types 449 _STORAGE_TYPES = { 450 constants.ST_FILE: FileStorage, 451 constants.ST_LVM_PV: LvmPvStorage, 452 constants.ST_LVM_VG: LvmVgStorage, 453 }

454 455 456 -def GetStorageClass(name):

457 """Returns the class for a storage type. 458 459 @type name: string 460 @param name: Storage type 461 462 """ 463 try: 464 return _STORAGE_TYPES[name] 465 except KeyError: 466 raise errors.StorageError("Unknown storage type: %r" % name)

467

468 469 -def GetStorage(name, *args):

470 """Factory function for storage methods. 471 472 @type name: string 473 @param name: Storage type 474 475 """ 476 return GetStorageClass(name)(*args)

477

Source Code for Module ganeti.storage