Package ganeti :: Package utils :: Module io
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.io

   1  # 
   2  # 
   3   
   4  # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. 
   5  # All rights reserved. 
   6  # 
   7  # Redistribution and use in source and binary forms, with or without 
   8  # modification, are permitted provided that the following conditions are 
   9  # met: 
  10  # 
  11  # 1. Redistributions of source code must retain the above copyright notice, 
  12  # this list of conditions and the following disclaimer. 
  13  # 
  14  # 2. Redistributions in binary form must reproduce the above copyright 
  15  # notice, this list of conditions and the following disclaimer in the 
  16  # documentation and/or other materials provided with the distribution. 
  17  # 
  18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 
  19  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
  20  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
  21  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
  22  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
  23  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
  24  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
  25  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
  26  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
  27  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
  28  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
  29   
  30  """Utility functions for I/O. 
  31   
  32  """ 
  33   
  34  import os 
  35  import logging 
  36  import shutil 
  37  import tempfile 
  38  import errno 
  39  import time 
  40  import stat 
  41  import grp 
  42  import pwd 
  43   
  44  from ganeti import errors 
  45  from ganeti import constants 
  46  from ganeti import pathutils 
  47  from ganeti.utils import filelock 
  48   
  49  #: Directory used by fsck(8) to store recovered data, usually at a file 
  50  #: system's root directory 
  51  _LOST_AND_FOUND = "lost+found" 
  52   
  53  # Possible values for keep_perms in WriteFile() 
  54  KP_NEVER = 0 
  55  KP_ALWAYS = 1 
  56  KP_IF_EXISTS = 2 
  57   
  58  KEEP_PERMS_VALUES = [ 
  59    KP_NEVER, 
  60    KP_ALWAYS, 
  61    KP_IF_EXISTS, 
  62    ] 
  63   
  64   
65 -def ErrnoOrStr(err):
66 """Format an EnvironmentError exception. 67 68 If the L{err} argument has an errno attribute, it will be looked up 69 and converted into a textual C{E...} description. Otherwise the 70 string representation of the error will be returned. 71 72 @type err: L{EnvironmentError} 73 @param err: the exception to format 74 75 """ 76 if hasattr(err, "errno"): 77 detail = errno.errorcode[err.errno] 78 else: 79 detail = str(err) 80 return detail
81 82
83 -class FileStatHelper(object):
84 """Helper to store file handle's C{fstat}. 85 86 Useful in combination with L{ReadFile}'s C{preread} parameter. 87 88 """
89 - def __init__(self):
90 """Initializes this class. 91 92 """ 93 self.st = None
94
95 - def __call__(self, fh):
96 """Calls C{fstat} on file handle. 97 98 """ 99 self.st = os.fstat(fh.fileno())
100 101
102 -def ReadFile(file_name, size=-1, preread=None):
103 """Reads a file. 104 105 @type size: int 106 @param size: Read at most size bytes (if negative, entire file) 107 @type preread: callable receiving file handle as single parameter 108 @param preread: Function called before file is read 109 @rtype: str 110 @return: the (possibly partial) content of the file 111 112 """ 113 f = open(file_name, "r") 114 try: 115 if preread: 116 preread(f) 117 118 return f.read(size) 119 finally: 120 f.close()
121 122
123 -def WriteFile(file_name, fn=None, data=None, 124 mode=None, uid=-1, gid=-1, 125 atime=None, mtime=None, close=True, 126 dry_run=False, backup=False, 127 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
128 """(Over)write a file atomically. 129 130 The file_name and either fn (a function taking one argument, the 131 file descriptor, and which should write the data to it) or data (the 132 contents of the file) must be passed. The other arguments are 133 optional and allow setting the file mode, owner and group, and the 134 mtime/atime of the file. 135 136 If the function doesn't raise an exception, it has succeeded and the 137 target file has the new contents. If the function has raised an 138 exception, an existing target file should be unmodified and the 139 temporary file should be removed. 140 141 @type file_name: str 142 @param file_name: the target filename 143 @type fn: callable 144 @param fn: content writing function, called with 145 file descriptor as parameter 146 @type data: str 147 @param data: contents of the file 148 @type mode: int 149 @param mode: file mode 150 @type uid: int 151 @param uid: the owner of the file 152 @type gid: int 153 @param gid: the group of the file 154 @type atime: int 155 @param atime: a custom access time to be set on the file 156 @type mtime: int 157 @param mtime: a custom modification time to be set on the file 158 @type close: boolean 159 @param close: whether to close file after writing it 160 @type prewrite: callable 161 @param prewrite: function to be called before writing content 162 @type postwrite: callable 163 @param postwrite: function to be called after writing content 164 @type keep_perms: members of L{KEEP_PERMS_VALUES} 165 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are 166 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and 167 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner, 168 group, and mode are taken from the file, and if the file doesn't 169 exist, they are taken from the other parameters. It is an error to 170 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid}, 171 or C{mode} are set to non-default values. 172 173 @rtype: None or int 174 @return: None if the 'close' parameter evaluates to True, 175 otherwise the file descriptor 176 177 @raise errors.ProgrammerError: if any of the arguments are not valid 178 179 """ 180 if not os.path.isabs(file_name): 181 raise errors.ProgrammerError("Path passed to WriteFile is not" 182 " absolute: '%s'" % file_name) 183 184 if [fn, data].count(None) != 1: 185 raise errors.ProgrammerError("fn or data required") 186 187 if [atime, mtime].count(None) == 1: 188 raise errors.ProgrammerError("Both atime and mtime must be either" 189 " set or None") 190 191 if not keep_perms in KEEP_PERMS_VALUES: 192 raise errors.ProgrammerError("Invalid value for keep_perms: %s" % 193 keep_perms) 194 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None): 195 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid'," 196 " and 'mode' cannot be set") 197 198 if backup and not dry_run and os.path.isfile(file_name): 199 CreateBackup(file_name) 200 201 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS: 202 # os.stat() raises an exception if the file doesn't exist 203 try: 204 file_stat = os.stat(file_name) 205 mode = stat.S_IMODE(file_stat.st_mode) 206 uid = file_stat.st_uid 207 gid = file_stat.st_gid 208 except OSError: 209 if keep_perms == KP_ALWAYS: 210 raise 211 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist 212 213 # Whether temporary file needs to be removed (e.g. if any error occurs) 214 do_remove = True 215 216 # Function result 217 result = None 218 219 (dir_name, base_name) = os.path.split(file_name) 220 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name, 221 dir=dir_name) 222 try: 223 try: 224 if uid != -1 or gid != -1: 225 os.chown(new_name, uid, gid) 226 if mode: 227 os.chmod(new_name, mode) 228 if callable(prewrite): 229 prewrite(fd) 230 if data is not None: 231 if isinstance(data, unicode): 232 data = data.encode() 233 assert isinstance(data, str) 234 to_write = len(data) 235 offset = 0 236 while offset < to_write: 237 written = os.write(fd, buffer(data, offset)) 238 assert written >= 0 239 assert written <= to_write - offset 240 offset += written 241 assert offset == to_write 242 else: 243 fn(fd) 244 if callable(postwrite): 245 postwrite(fd) 246 os.fsync(fd) 247 if atime is not None and mtime is not None: 248 os.utime(new_name, (atime, mtime)) 249 finally: 250 # Close file unless the file descriptor should be returned 251 if close: 252 os.close(fd) 253 else: 254 result = fd 255 256 # Rename file to destination name 257 if not dry_run: 258 os.rename(new_name, file_name) 259 # Successful, no need to remove anymore 260 do_remove = False 261 finally: 262 if do_remove: 263 RemoveFile(new_name) 264 265 return result
266 267
268 -def GetFileID(path=None, fd=None):
269 """Returns the file 'id', i.e. the dev/inode and mtime information. 270 271 Either the path to the file or the fd must be given. 272 273 @param path: the file path 274 @param fd: a file descriptor 275 @return: a tuple of (device number, inode number, mtime) 276 277 """ 278 if [path, fd].count(None) != 1: 279 raise errors.ProgrammerError("One and only one of fd/path must be given") 280 281 if fd is None: 282 st = os.stat(path) 283 else: 284 st = os.fstat(fd) 285 286 return (st.st_dev, st.st_ino, st.st_mtime)
287 288
289 -def VerifyFileID(fi_disk, fi_ours):
290 """Verifies that two file IDs are matching. 291 292 Differences in the inode/device are not accepted, but and older 293 timestamp for fi_disk is accepted. 294 295 @param fi_disk: tuple (dev, inode, mtime) representing the actual 296 file data 297 @param fi_ours: tuple (dev, inode, mtime) representing the last 298 written file data 299 @rtype: boolean 300 301 """ 302 (d1, i1, m1) = fi_disk 303 (d2, i2, m2) = fi_ours 304 305 return (d1, i1) == (d2, i2) and m1 <= m2
306 307
308 -def SafeWriteFile(file_name, file_id, **kwargs):
309 """Wraper over L{WriteFile} that locks the target file. 310 311 By keeping the target file locked during WriteFile, we ensure that 312 cooperating writers will safely serialise access to the file. 313 314 @type file_name: str 315 @param file_name: the target filename 316 @type file_id: tuple 317 @param file_id: a result from L{GetFileID} 318 319 """ 320 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT) 321 try: 322 filelock.LockFile(fd) 323 if file_id is not None: 324 disk_id = GetFileID(fd=fd) 325 if not VerifyFileID(disk_id, file_id): 326 raise errors.LockError("Cannot overwrite file %s, it has been modified" 327 " since last written" % file_name) 328 return WriteFile(file_name, **kwargs) 329 finally: 330 os.close(fd)
331 332
333 -def ReadOneLineFile(file_name, strict=False):
334 """Return the first non-empty line from a file. 335 336 @type strict: boolean 337 @param strict: if True, abort if the file has more than one 338 non-empty line 339 340 """ 341 file_lines = ReadFile(file_name).splitlines() 342 full_lines = filter(bool, file_lines) 343 if not file_lines or not full_lines: 344 raise errors.GenericError("No data in one-liner file %s" % file_name) 345 elif strict and len(full_lines) > 1: 346 raise errors.GenericError("Too many lines in one-liner file %s" % 347 file_name) 348 return full_lines[0]
349 350
351 -def RemoveFile(filename):
352 """Remove a file ignoring some errors. 353 354 Remove a file, ignoring non-existing ones or directories. Other 355 errors are passed. 356 357 @type filename: str 358 @param filename: the file to be removed 359 360 """ 361 try: 362 os.unlink(filename) 363 except OSError, err: 364 if err.errno not in (errno.ENOENT, errno.EISDIR): 365 raise
366 367
368 -def RemoveDir(dirname):
369 """Remove an empty directory. 370 371 Remove a directory, ignoring non-existing ones. 372 Other errors are passed. This includes the case, 373 where the directory is not empty, so it can't be removed. 374 375 @type dirname: str 376 @param dirname: the empty directory to be removed 377 378 """ 379 try: 380 os.rmdir(dirname) 381 except OSError, err: 382 if err.errno != errno.ENOENT: 383 raise
384 385
386 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None, 387 dir_gid=None):
388 """Renames a file. 389 390 This just creates the very least directory if it does not exist and C{mkdir} 391 is set to true. 392 393 @type old: string 394 @param old: Original path 395 @type new: string 396 @param new: New path 397 @type mkdir: bool 398 @param mkdir: Whether to create target directory if it doesn't exist 399 @type mkdir_mode: int 400 @param mkdir_mode: Mode for newly created directories 401 @type dir_uid: int 402 @param dir_uid: The uid for the (if fresh created) dir 403 @type dir_gid: int 404 @param dir_gid: The gid for the (if fresh created) dir 405 406 """ 407 try: 408 return os.rename(old, new) 409 except OSError, err: 410 # In at least one use case of this function, the job queue, directory 411 # creation is very rare. Checking for the directory before renaming is not 412 # as efficient. 413 if mkdir and err.errno == errno.ENOENT: 414 # Create directory and try again 415 dir_path = os.path.dirname(new) 416 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid) 417 418 return os.rename(old, new) 419 420 raise
421 422
423 -def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True, 424 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
425 """Enforces that given path has given permissions. 426 427 @param path: The path to the file 428 @param mode: The mode of the file 429 @param uid: The uid of the owner of this file 430 @param gid: The gid of the owner of this file 431 @param must_exist: Specifies if non-existance of path will be an error 432 @param _chmod_fn: chmod function to use (unittest only) 433 @param _chown_fn: chown function to use (unittest only) 434 435 """ 436 logging.debug("Checking %s", path) 437 438 # chown takes -1 if you want to keep one part of the ownership, however 439 # None is Python standard for that. So we remap them here. 440 if uid is None: 441 uid = -1 442 if gid is None: 443 gid = -1 444 445 try: 446 st = _stat_fn(path) 447 448 fmode = stat.S_IMODE(st[stat.ST_MODE]) 449 if fmode != mode: 450 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode) 451 _chmod_fn(path, mode) 452 453 if max(uid, gid) > -1: 454 fuid = st[stat.ST_UID] 455 fgid = st[stat.ST_GID] 456 if fuid != uid or fgid != gid: 457 logging.debug("Changing owner of %s from UID %s/GID %s to" 458 " UID %s/GID %s", path, fuid, fgid, uid, gid) 459 _chown_fn(path, uid, gid) 460 except EnvironmentError, err: 461 if err.errno == errno.ENOENT: 462 if must_exist: 463 raise errors.GenericError("Path %s should exist, but does not" % path) 464 else: 465 raise errors.GenericError("Error while changing permissions on %s: %s" % 466 (path, err))
467 468
469 -def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat, 470 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
471 """Enforces that given path is a dir and has given mode, uid and gid set. 472 473 @param path: The path to the file 474 @param mode: The mode of the file 475 @param uid: The uid of the owner of this file 476 @param gid: The gid of the owner of this file 477 @param _lstat_fn: Stat function to use (unittest only) 478 @param _mkdir_fn: mkdir function to use (unittest only) 479 @param _perm_fn: permission setter function to use (unittest only) 480 481 """ 482 logging.debug("Checking directory %s", path) 483 try: 484 # We don't want to follow symlinks 485 st = _lstat_fn(path) 486 except EnvironmentError, err: 487 if err.errno != errno.ENOENT: 488 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err)) 489 _mkdir_fn(path) 490 else: 491 if not stat.S_ISDIR(st[stat.ST_MODE]): 492 raise errors.GenericError(("Path %s is expected to be a directory, but " 493 "isn't") % path) 494 495 _perm_fn(path, mode, uid=uid, gid=gid)
496 497
498 -def Makedirs(path, mode=0750):
499 """Super-mkdir; create a leaf directory and all intermediate ones. 500 501 This is a wrapper around C{os.makedirs} adding error handling not implemented 502 before Python 2.5. 503 504 """ 505 try: 506 os.makedirs(path, mode) 507 except OSError, err: 508 # Ignore EEXIST. This is only handled in os.makedirs as included in 509 # Python 2.5 and above. 510 if err.errno != errno.EEXIST or not os.path.exists(path): 511 raise
512 513
514 -def TimestampForFilename():
515 """Returns the current time formatted for filenames. 516 517 The format doesn't contain colons as some shells and applications treat them 518 as separators. Uses the local timezone. 519 520 """ 521 return time.strftime("%Y-%m-%d_%H_%M_%S")
522 523
524 -def CreateBackup(file_name):
525 """Creates a backup of a file. 526 527 @type file_name: str 528 @param file_name: file to be backed up 529 @rtype: str 530 @return: the path to the newly created backup 531 @raise errors.ProgrammerError: for invalid file names 532 533 """ 534 if not os.path.isfile(file_name): 535 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % 536 file_name) 537 538 prefix = ("%s.backup-%s." % 539 (os.path.basename(file_name), TimestampForFilename())) 540 dir_name = os.path.dirname(file_name) 541 542 fsrc = open(file_name, "rb") 543 try: 544 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) 545 fdst = os.fdopen(fd, "wb") 546 try: 547 logging.debug("Backing up %s at %s", file_name, backup_name) 548 shutil.copyfileobj(fsrc, fdst) 549 finally: 550 fdst.close() 551 finally: 552 fsrc.close() 553 554 return backup_name
555 556
557 -def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
558 """Returns a list of visible files in a directory. 559 560 @type path: str 561 @param path: the directory to enumerate 562 @rtype: list 563 @return: the list of all files not starting with a dot 564 @raise ProgrammerError: if L{path} is not an absolue and normalized path 565 566 """ 567 if not IsNormAbsPath(path): 568 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" 569 " absolute/normalized: '%s'" % path) 570 571 mountpoint = _is_mountpoint(path) 572 573 def fn(name): 574 """File name filter. 575 576 Ignores files starting with a dot (".") as by Unix convention they're 577 considered hidden. The "lost+found" directory found at the root of some 578 filesystems is also hidden. 579 580 """ 581 return not (name.startswith(".") or 582 (mountpoint and name == _LOST_AND_FOUND and 583 os.path.isdir(os.path.join(path, name))))
584 585 return filter(fn, os.listdir(path)) 586 587
588 -def EnsureDirs(dirs):
589 """Make required directories, if they don't exist. 590 591 @param dirs: list of tuples (dir_name, dir_mode) 592 @type dirs: list of (string, integer) 593 594 """ 595 for dir_name, dir_mode in dirs: 596 try: 597 os.mkdir(dir_name, dir_mode) 598 except EnvironmentError, err: 599 if err.errno != errno.EEXIST: 600 raise errors.GenericError("Cannot create needed directory" 601 " '%s': %s" % (dir_name, err)) 602 try: 603 os.chmod(dir_name, dir_mode) 604 except EnvironmentError, err: 605 raise errors.GenericError("Cannot change directory permissions on" 606 " '%s': %s" % (dir_name, err)) 607 if not os.path.isdir(dir_name): 608 raise errors.GenericError("%s is not a directory" % dir_name)
609 610
611 -def FindFile(name, search_path, test=os.path.exists):
612 """Look for a filesystem object in a given path. 613 614 This is an abstract method to search for filesystem object (files, 615 dirs) under a given search path. 616 617 @type name: str 618 @param name: the name to look for 619 @type search_path: str 620 @param search_path: location to start at 621 @type test: callable 622 @param test: a function taking one argument that should return True 623 if the a given object is valid; the default value is 624 os.path.exists, causing only existing files to be returned 625 @rtype: str or None 626 @return: full path to the object if found, None otherwise 627 628 """ 629 # validate the filename mask 630 if constants.EXT_PLUGIN_MASK.match(name) is None: 631 logging.critical("Invalid value passed for external script name: '%s'", 632 name) 633 return None 634 635 for dir_name in search_path: 636 # FIXME: investigate switch to PathJoin 637 item_name = os.path.sep.join([dir_name, name]) 638 # check the user test and that we're indeed resolving to the given 639 # basename 640 if test(item_name) and os.path.basename(item_name) == name: 641 return item_name 642 return None
643 644
645 -def IsNormAbsPath(path):
646 """Check whether a path is absolute and also normalized 647 648 This avoids things like /dir/../../other/path to be valid. 649 650 """ 651 return os.path.normpath(path) == path and os.path.isabs(path)
652 653
654 -def IsBelowDir(root, other_path):
655 """Check whether a path is below a root dir. 656 657 This works around the nasty byte-byte comparison of commonprefix. 658 659 """ 660 if not (os.path.isabs(root) and os.path.isabs(other_path)): 661 raise ValueError("Provided paths '%s' and '%s' are not absolute" % 662 (root, other_path)) 663 664 norm_other = os.path.normpath(other_path) 665 666 if norm_other == os.sep: 667 # The root directory can never be below another path 668 return False 669 670 norm_root = os.path.normpath(root) 671 672 if norm_root == os.sep: 673 # This is the root directory, no need to add another slash 674 prepared_root = norm_root 675 else: 676 prepared_root = "%s%s" % (norm_root, os.sep) 677 678 return os.path.commonprefix([prepared_root, norm_other]) == prepared_root
679 680
681 -def PathJoin(*args):
682 """Safe-join a list of path components. 683 684 Requirements: 685 - the first argument must be an absolute path 686 - no component in the path must have backtracking (e.g. /../), 687 since we check for normalization at the end 688 689 @param args: the path components to be joined 690 @raise ValueError: for invalid paths 691 692 """ 693 # ensure we're having at least two paths passed in 694 if len(args) <= 1: 695 raise errors.ProgrammerError("PathJoin requires two arguments") 696 # ensure the first component is an absolute and normalized path name 697 root = args[0] 698 if not IsNormAbsPath(root): 699 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) 700 result = os.path.join(*args) 701 # ensure that the whole path is normalized 702 if not IsNormAbsPath(result): 703 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) 704 # check that we're still under the original prefix 705 if not IsBelowDir(root, result): 706 raise ValueError("Error: path joining resulted in different prefix" 707 " (%s != %s)" % (result, root)) 708 return result
709 710
711 -def TailFile(fname, lines=20):
712 """Return the last lines from a file. 713 714 @note: this function will only read and parse the last 4KB of 715 the file; if the lines are very long, it could be that less 716 than the requested number of lines are returned 717 718 @param fname: the file name 719 @type lines: int 720 @param lines: the (maximum) number of lines to return 721 722 """ 723 fd = open(fname, "r") 724 try: 725 fd.seek(0, 2) 726 pos = fd.tell() 727 pos = max(0, pos - 4096) 728 fd.seek(pos, 0) 729 raw_data = fd.read() 730 finally: 731 fd.close() 732 733 rows = raw_data.splitlines() 734 return rows[-lines:]
735 736
737 -def BytesToMebibyte(value):
738 """Converts bytes to mebibytes. 739 740 @type value: int 741 @param value: Value in bytes 742 @rtype: int 743 @return: Value in mebibytes 744 745 """ 746 return int(round(value / (1024.0 * 1024.0), 0))
747 748
749 -def CalculateDirectorySize(path):
750 """Calculates the size of a directory recursively. 751 752 @type path: string 753 @param path: Path to directory 754 @rtype: int 755 @return: Size in mebibytes 756 757 """ 758 size = 0 759 760 for (curpath, _, files) in os.walk(path): 761 for filename in files: 762 st = os.lstat(PathJoin(curpath, filename)) 763 size += st.st_size 764 765 return BytesToMebibyte(size)
766 767
768 -def GetFilesystemStats(path):
769 """Returns the total and free space on a filesystem. 770 771 @type path: string 772 @param path: Path on filesystem to be examined 773 @rtype: int 774 @return: tuple of (Total space, Free space) in mebibytes 775 776 """ 777 st = os.statvfs(path) 778 779 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize) 780 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize) 781 return (tsize, fsize)
782 783
784 -def ReadPidFile(pidfile):
785 """Read a pid from a file. 786 787 @type pidfile: string 788 @param pidfile: path to the file containing the pid 789 @rtype: int 790 @return: The process id, if the file exists and contains a valid PID, 791 otherwise 0 792 793 """ 794 try: 795 raw_data = ReadOneLineFile(pidfile) 796 except EnvironmentError, err: 797 if err.errno != errno.ENOENT: 798 logging.exception("Can't read pid file") 799 return 0 800 801 return _ParsePidFileContents(raw_data)
802 803
804 -def _ParsePidFileContents(data):
805 """Tries to extract a process ID from a PID file's content. 806 807 @type data: string 808 @rtype: int 809 @return: Zero if nothing could be read, PID otherwise 810 811 """ 812 try: 813 pid = int(data) 814 except (TypeError, ValueError): 815 logging.info("Can't parse pid file contents", exc_info=True) 816 return 0 817 else: 818 return pid
819 820
821 -def ReadLockedPidFile(path):
822 """Reads a locked PID file. 823 824 This can be used together with L{utils.process.StartDaemon}. 825 826 @type path: string 827 @param path: Path to PID file 828 @return: PID as integer or, if file was unlocked or couldn't be opened, None 829 830 """ 831 try: 832 fd = os.open(path, os.O_RDONLY) 833 except EnvironmentError, err: 834 if err.errno == errno.ENOENT: 835 # PID file doesn't exist 836 return None 837 raise 838 839 try: 840 try: 841 # Try to acquire lock 842 filelock.LockFile(fd) 843 except errors.LockError: 844 # Couldn't lock, daemon is running 845 return int(os.read(fd, 100)) 846 finally: 847 os.close(fd) 848 849 return None
850 851
852 -def _SplitSshKey(key):
853 """Splits a line for SSH's C{authorized_keys} file. 854 855 If the line has no options (e.g. no C{command="..."}), only the significant 856 parts, the key type and its hash, are used. Otherwise the whole line is used 857 (split at whitespace). 858 859 @type key: string 860 @param key: Key line 861 @rtype: tuple 862 863 """ 864 parts = key.split() 865 866 if parts and parts[0] in constants.SSHAK_ALL: 867 # If the key has no options in front of it, we only want the significant 868 # fields 869 return (False, parts[:2]) 870 else: 871 # Can't properly split the line, so use everything 872 return (True, parts)
873 874
875 -def AddAuthorizedKey(file_obj, key):
876 """Adds an SSH public key to an authorized_keys file. 877 878 @type file_obj: str or file handle 879 @param file_obj: path to authorized_keys file 880 @type key: str 881 @param key: string containing key 882 883 """ 884 key_fields = _SplitSshKey(key) 885 886 if isinstance(file_obj, basestring): 887 f = open(file_obj, "a+") 888 else: 889 f = file_obj 890 891 try: 892 nl = True 893 for line in f: 894 # Ignore whitespace changes 895 if _SplitSshKey(line) == key_fields: 896 break 897 nl = line.endswith("\n") 898 else: 899 if not nl: 900 f.write("\n") 901 f.write(key.rstrip("\r\n")) 902 f.write("\n") 903 f.flush() 904 finally: 905 f.close()
906 907
908 -def RemoveAuthorizedKey(file_name, key):
909 """Removes an SSH public key from an authorized_keys file. 910 911 @type file_name: str 912 @param file_name: path to authorized_keys file 913 @type key: str 914 @param key: string containing key 915 916 """ 917 key_fields = _SplitSshKey(key) 918 919 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name)) 920 try: 921 out = os.fdopen(fd, "w") 922 try: 923 f = open(file_name, "r") 924 try: 925 for line in f: 926 # Ignore whitespace changes while comparing lines 927 if _SplitSshKey(line) != key_fields: 928 out.write(line) 929 930 out.flush() 931 os.rename(tmpname, file_name) 932 finally: 933 f.close() 934 finally: 935 out.close() 936 except: 937 RemoveFile(tmpname) 938 raise
939 940
941 -def DaemonPidFileName(name):
942 """Compute a ganeti pid file absolute path 943 944 @type name: str 945 @param name: the daemon name 946 @rtype: str 947 @return: the full path to the pidfile corresponding to the given 948 daemon name 949 950 """ 951 return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
952 953
954 -def WritePidFile(pidfile):
955 """Write the current process pidfile. 956 957 @type pidfile: string 958 @param pidfile: the path to the file to be written 959 @raise errors.LockError: if the pid file already exists and 960 points to a live process 961 @rtype: int 962 @return: the file descriptor of the lock file; do not close this unless 963 you want to unlock the pid file 964 965 """ 966 # We don't rename nor truncate the file to not drop locks under 967 # existing processes 968 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600) 969 970 # Lock the PID file (and fail if not possible to do so). Any code 971 # wanting to send a signal to the daemon should try to lock the PID 972 # file before reading it. If acquiring the lock succeeds, the daemon is 973 # no longer running and the signal should not be sent. 974 try: 975 filelock.LockFile(fd_pidfile) 976 except errors.LockError: 977 msg = ["PID file '%s' is already locked by another process" % pidfile] 978 # Try to read PID file 979 pid = _ParsePidFileContents(os.read(fd_pidfile, 100)) 980 if pid > 0: 981 msg.append(", PID read from file is %s" % pid) 982 raise errors.PidFileLockError("".join(msg)) 983 984 os.write(fd_pidfile, "%d\n" % os.getpid()) 985 986 return fd_pidfile
987 988
989 -def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
990 """Reads the watcher pause file. 991 992 @type filename: string 993 @param filename: Path to watcher pause file 994 @type now: None, float or int 995 @param now: Current time as Unix timestamp 996 @type remove_after: int 997 @param remove_after: Remove watcher pause file after specified amount of 998 seconds past the pause end time 999 1000 """ 1001 if now is None: 1002 now = time.time() 1003 1004 try: 1005 value = ReadFile(filename) 1006 except IOError, err: 1007 if err.errno != errno.ENOENT: 1008 raise 1009 value = None 1010 1011 if value is not None: 1012 try: 1013 value = int(value) 1014 except ValueError: 1015 logging.warning(("Watcher pause file (%s) contains invalid value," 1016 " removing it"), filename) 1017 RemoveFile(filename) 1018 value = None 1019 1020 if value is not None: 1021 # Remove file if it's outdated 1022 if now > (value + remove_after): 1023 RemoveFile(filename) 1024 value = None 1025 1026 elif now > value: 1027 value = None 1028 1029 return value
1030 1031
1032 -def NewUUID():
1033 """Returns a random UUID. 1034 1035 @note: This is a Linux-specific method as it uses the /proc 1036 filesystem. 1037 @rtype: str 1038 1039 """ 1040 return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n")
1041 1042
1043 -class TemporaryFileManager(object):
1044 """Stores the list of files to be deleted and removes them on demand. 1045 1046 """ 1047
1048 - def __init__(self):
1049 self._files = []
1050
1051 - def __del__(self):
1052 self.Cleanup()
1053
1054 - def Add(self, filename):
1055 """Add file to list of files to be deleted. 1056 1057 @type filename: string 1058 @param filename: path to filename to be added 1059 1060 """ 1061 self._files.append(filename)
1062
1063 - def Remove(self, filename):
1064 """Remove file from list of files to be deleted. 1065 1066 @type filename: string 1067 @param filename: path to filename to be deleted 1068 1069 """ 1070 self._files.remove(filename)
1071
1072 - def Cleanup(self):
1073 """Delete all files marked for deletion 1074 1075 """ 1076 while self._files: 1077 RemoveFile(self._files.pop())
1078 1079
1080 -def IsUserInGroup(uid, gid):
1081 """Returns True if the user belongs to the group. 1082 1083 @type uid: int 1084 @param uid: the user id 1085 @type gid: int 1086 @param gid: the group id 1087 @rtype: bool 1088 1089 """ 1090 user = pwd.getpwuid(uid) 1091 group = grp.getgrgid(gid) 1092 return user.pw_gid == gid or user.pw_name in group.gr_mem
1093 1094
1095 -def CanRead(username, filename):
1096 """Returns True if the user can access (read) the file. 1097 1098 @type username: string 1099 @param username: the name of the user 1100 @type filename: string 1101 @param filename: the name of the file 1102 @rtype: bool 1103 1104 """ 1105 filestats = os.stat(filename) 1106 user = pwd.getpwnam(username) 1107 uid = user.pw_uid 1108 user_readable = filestats.st_mode & stat.S_IRUSR != 0 1109 group_readable = filestats.st_mode & stat.S_IRGRP != 0 1110 return ((filestats.st_uid == uid and user_readable) 1111 or (filestats.st_uid != uid and 1112 IsUserInGroup(uid, filestats.st_gid) and group_readable))
1113