Package ganeti :: Package utils :: Module io
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.io

   1  # 
   2  # 
   3   
   4  # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. 
   5  # All rights reserved. 
   6  # 
   7  # Redistribution and use in source and binary forms, with or without 
   8  # modification, are permitted provided that the following conditions are 
   9  # met: 
  10  # 
  11  # 1. Redistributions of source code must retain the above copyright notice, 
  12  # this list of conditions and the following disclaimer. 
  13  # 
  14  # 2. Redistributions in binary form must reproduce the above copyright 
  15  # notice, this list of conditions and the following disclaimer in the 
  16  # documentation and/or other materials provided with the distribution. 
  17  # 
  18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 
  19  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
  20  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
  21  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
  22  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
  23  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
  24  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
  25  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
  26  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
  27  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
  28  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
  29   
  30  """Utility functions for I/O. 
  31   
  32  """ 
  33   
  34  import os 
  35  import logging 
  36  import shutil 
  37  import tempfile 
  38  import errno 
  39  import time 
  40  import stat 
  41  import grp 
  42  import pwd 
  43   
  44  from ganeti import errors 
  45  from ganeti import constants 
  46  from ganeti import pathutils 
  47  from ganeti.utils import filelock 
  48   
  49  #: Directory used by fsck(8) to store recovered data, usually at a file 
  50  #: system's root directory 
  51  _LOST_AND_FOUND = "lost+found" 
  52   
  53  # Possible values for keep_perms in WriteFile() 
  54  KP_NEVER = 0 
  55  KP_ALWAYS = 1 
  56  KP_IF_EXISTS = 2 
  57   
  58  KEEP_PERMS_VALUES = [ 
  59    KP_NEVER, 
  60    KP_ALWAYS, 
  61    KP_IF_EXISTS, 
  62    ] 
  63   
  64   
65 -def ErrnoOrStr(err):
66 """Format an EnvironmentError exception. 67 68 If the L{err} argument has an errno attribute, it will be looked up 69 and converted into a textual C{E...} description. Otherwise the 70 string representation of the error will be returned. 71 72 @type err: L{EnvironmentError} 73 @param err: the exception to format 74 75 """ 76 if hasattr(err, "errno"): 77 detail = errno.errorcode[err.errno] 78 else: 79 detail = str(err) 80 return detail
81 82
83 -class FileStatHelper(object):
84 """Helper to store file handle's C{fstat}. 85 86 Useful in combination with L{ReadFile}'s C{preread} parameter. 87 88 """
89 - def __init__(self):
90 """Initializes this class. 91 92 """ 93 self.st = None
94
95 - def __call__(self, fh):
96 """Calls C{fstat} on file handle. 97 98 """ 99 self.st = os.fstat(fh.fileno())
100 101
102 -def ReadFile(file_name, size=-1, preread=None):
103 """Reads a file. 104 105 @type size: int 106 @param size: Read at most size bytes (if negative, entire file) 107 @type preread: callable receiving file handle as single parameter 108 @param preread: Function called before file is read 109 @rtype: str 110 @return: the (possibly partial) content of the file 111 112 """ 113 f = open(file_name, "r") 114 try: 115 if preread: 116 preread(f) 117 118 return f.read(size) 119 finally: 120 f.close()
121 122
123 -def WriteFile(file_name, fn=None, data=None, 124 mode=None, uid=-1, gid=-1, 125 atime=None, mtime=None, close=True, 126 dry_run=False, backup=False, 127 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
128 """(Over)write a file atomically. 129 130 The file_name and either fn (a function taking one argument, the 131 file descriptor, and which should write the data to it) or data (the 132 contents of the file) must be passed. The other arguments are 133 optional and allow setting the file mode, owner and group, and the 134 mtime/atime of the file. 135 136 If the function doesn't raise an exception, it has succeeded and the 137 target file has the new contents. If the function has raised an 138 exception, an existing target file should be unmodified and the 139 temporary file should be removed. 140 141 @type file_name: str 142 @param file_name: the target filename 143 @type fn: callable 144 @param fn: content writing function, called with 145 file descriptor as parameter 146 @type data: str 147 @param data: contents of the file 148 @type mode: int 149 @param mode: file mode 150 @type uid: int 151 @param uid: the owner of the file 152 @type gid: int 153 @param gid: the group of the file 154 @type atime: int 155 @param atime: a custom access time to be set on the file 156 @type mtime: int 157 @param mtime: a custom modification time to be set on the file 158 @type close: boolean 159 @param close: whether to close file after writing it 160 @type prewrite: callable 161 @param prewrite: function to be called before writing content 162 @type postwrite: callable 163 @param postwrite: function to be called after writing content 164 @type keep_perms: members of L{KEEP_PERMS_VALUES} 165 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are 166 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and 167 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner, 168 group, and mode are taken from the file, and if the file doesn't 169 exist, they are taken from the other parameters. It is an error to 170 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid}, 171 or C{mode} are set to non-default values. 172 173 @rtype: None or int 174 @return: None if the 'close' parameter evaluates to True, 175 otherwise the file descriptor 176 177 @raise errors.ProgrammerError: if any of the arguments are not valid 178 179 """ 180 if not os.path.isabs(file_name): 181 raise errors.ProgrammerError("Path passed to WriteFile is not" 182 " absolute: '%s'" % file_name) 183 184 if [fn, data].count(None) != 1: 185 raise errors.ProgrammerError("fn or data required") 186 187 if [atime, mtime].count(None) == 1: 188 raise errors.ProgrammerError("Both atime and mtime must be either" 189 " set or None") 190 191 if not keep_perms in KEEP_PERMS_VALUES: 192 raise errors.ProgrammerError("Invalid value for keep_perms: %s" % 193 keep_perms) 194 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None): 195 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid'," 196 " and 'mode' cannot be set") 197 198 if backup and not dry_run and os.path.isfile(file_name): 199 CreateBackup(file_name) 200 201 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS: 202 # os.stat() raises an exception if the file doesn't exist 203 try: 204 file_stat = os.stat(file_name) 205 mode = stat.S_IMODE(file_stat.st_mode) 206 uid = file_stat.st_uid 207 gid = file_stat.st_gid 208 except OSError: 209 if keep_perms == KP_ALWAYS: 210 raise 211 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist 212 213 # Whether temporary file needs to be removed (e.g. if any error occurs) 214 do_remove = True 215 216 # Function result 217 result = None 218 219 (dir_name, base_name) = os.path.split(file_name) 220 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name, 221 dir=dir_name) 222 try: 223 try: 224 if uid != -1 or gid != -1: 225 os.chown(new_name, uid, gid) 226 if mode: 227 os.chmod(new_name, mode) 228 if callable(prewrite): 229 prewrite(fd) 230 if data is not None: 231 if isinstance(data, unicode): 232 data = data.encode() 233 assert isinstance(data, str) 234 to_write = len(data) 235 offset = 0 236 while offset < to_write: 237 written = os.write(fd, buffer(data, offset)) 238 assert written >= 0 239 assert written <= to_write - offset 240 offset += written 241 assert offset == to_write 242 else: 243 fn(fd) 244 if callable(postwrite): 245 postwrite(fd) 246 os.fsync(fd) 247 if atime is not None and mtime is not None: 248 os.utime(new_name, (atime, mtime)) 249 finally: 250 # Close file unless the file descriptor should be returned 251 if close: 252 os.close(fd) 253 else: 254 result = fd 255 256 # Rename file to destination name 257 if not dry_run: 258 os.rename(new_name, file_name) 259 # Successful, no need to remove anymore 260 do_remove = False 261 finally: 262 if do_remove: 263 RemoveFile(new_name) 264 265 return result
266 267
268 -def GetFileID(path=None, fd=None):
269 """Returns the file 'id', i.e. the dev/inode and mtime information. 270 271 Either the path to the file or the fd must be given. 272 273 @param path: the file path 274 @param fd: a file descriptor 275 @return: a tuple of (device number, inode number, mtime) 276 277 """ 278 if [path, fd].count(None) != 1: 279 raise errors.ProgrammerError("One and only one of fd/path must be given") 280 281 if fd is None: 282 st = os.stat(path) 283 else: 284 st = os.fstat(fd) 285 286 return (st.st_dev, st.st_ino, st.st_mtime)
287 288
289 -def VerifyFileID(fi_disk, fi_ours):
290 """Verifies that two file IDs are matching. 291 292 Differences in the inode/device are not accepted, but and older 293 timestamp for fi_disk is accepted. 294 295 @param fi_disk: tuple (dev, inode, mtime) representing the actual 296 file data 297 @param fi_ours: tuple (dev, inode, mtime) representing the last 298 written file data 299 @rtype: boolean 300 301 """ 302 (d1, i1, m1) = fi_disk 303 (d2, i2, m2) = fi_ours 304 305 return (d1, i1) == (d2, i2) and m1 <= m2
306 307
308 -def SafeWriteFile(file_name, file_id, **kwargs):
309 """Wraper over L{WriteFile} that locks the target file. 310 311 By keeping the target file locked during WriteFile, we ensure that 312 cooperating writers will safely serialise access to the file. 313 314 @type file_name: str 315 @param file_name: the target filename 316 @type file_id: tuple 317 @param file_id: a result from L{GetFileID} 318 319 """ 320 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT) 321 try: 322 filelock.LockFile(fd) 323 if file_id is not None: 324 disk_id = GetFileID(fd=fd) 325 if not VerifyFileID(disk_id, file_id): 326 raise errors.LockError("Cannot overwrite file %s, it has been modified" 327 " since last written" % file_name) 328 return WriteFile(file_name, **kwargs) 329 finally: 330 os.close(fd)
331 332
333 -def ReadOneLineFile(file_name, strict=False):
334 """Return the first non-empty line from a file. 335 336 @type strict: boolean 337 @param strict: if True, abort if the file has more than one 338 non-empty line 339 340 """ 341 file_lines = ReadFile(file_name).splitlines() 342 full_lines = filter(bool, file_lines) 343 if not file_lines or not full_lines: 344 raise errors.GenericError("No data in one-liner file %s" % file_name) 345 elif strict and len(full_lines) > 1: 346 raise errors.GenericError("Too many lines in one-liner file %s" % 347 file_name) 348 return full_lines[0]
349 350
351 -def RemoveFile(filename):
352 """Remove a file ignoring some errors. 353 354 Remove a file, ignoring non-existing ones or directories. Other 355 errors are passed. 356 357 @type filename: str 358 @param filename: the file to be removed 359 360 """ 361 try: 362 os.unlink(filename) 363 except OSError, err: 364 if err.errno not in (errno.ENOENT, errno.EISDIR): 365 raise
366 367
368 -def RemoveDir(dirname):
369 """Remove an empty directory. 370 371 Remove a directory, ignoring non-existing ones. 372 Other errors are passed. This includes the case, 373 where the directory is not empty, so it can't be removed. 374 375 @type dirname: str 376 @param dirname: the empty directory to be removed 377 378 """ 379 try: 380 os.rmdir(dirname) 381 except OSError, err: 382 if err.errno != errno.ENOENT: 383 raise
384 385
386 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None, 387 dir_gid=None):
388 """Renames a file. 389 390 This just creates the very least directory if it does not exist and C{mkdir} 391 is set to true. 392 393 @type old: string 394 @param old: Original path 395 @type new: string 396 @param new: New path 397 @type mkdir: bool 398 @param mkdir: Whether to create target directory if it doesn't exist 399 @type mkdir_mode: int 400 @param mkdir_mode: Mode for newly created directories 401 @type dir_uid: int 402 @param dir_uid: The uid for the (if fresh created) dir 403 @type dir_gid: int 404 @param dir_gid: The gid for the (if fresh created) dir 405 406 """ 407 try: 408 return os.rename(old, new) 409 except OSError, err: 410 # In at least one use case of this function, the job queue, directory 411 # creation is very rare. Checking for the directory before renaming is not 412 # as efficient. 413 if mkdir and err.errno == errno.ENOENT: 414 # Create directory and try again 415 dir_path = os.path.dirname(new) 416 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid) 417 418 return os.rename(old, new) 419 420 raise
421 422
423 -def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True, 424 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
425 """Enforces that given path has given permissions. 426 427 @param path: The path to the file 428 @param mode: The mode of the file 429 @param uid: The uid of the owner of this file 430 @param gid: The gid of the owner of this file 431 @param must_exist: Specifies if non-existance of path will be an error 432 @param _chmod_fn: chmod function to use (unittest only) 433 @param _chown_fn: chown function to use (unittest only) 434 435 """ 436 logging.debug("Checking %s", path) 437 438 # chown takes -1 if you want to keep one part of the ownership, however 439 # None is Python standard for that. So we remap them here. 440 if uid is None: 441 uid = -1 442 if gid is None: 443 gid = -1 444 445 try: 446 st = _stat_fn(path) 447 448 fmode = stat.S_IMODE(st[stat.ST_MODE]) 449 if fmode != mode: 450 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode) 451 _chmod_fn(path, mode) 452 453 if max(uid, gid) > -1: 454 fuid = st[stat.ST_UID] 455 fgid = st[stat.ST_GID] 456 if fuid != uid or fgid != gid: 457 logging.debug("Changing owner of %s from UID %s/GID %s to" 458 " UID %s/GID %s", path, fuid, fgid, uid, gid) 459 _chown_fn(path, uid, gid) 460 except EnvironmentError, err: 461 if err.errno == errno.ENOENT: 462 if must_exist: 463 raise errors.GenericError("Path %s should exist, but does not" % path) 464 else: 465 raise errors.GenericError("Error while changing permissions on %s: %s" % 466 (path, err))
467 468
469 -def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat, 470 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
471 """Enforces that given path is a dir and has given mode, uid and gid set. 472 473 @param path: The path to the file 474 @param mode: The mode of the file 475 @param uid: The uid of the owner of this file 476 @param gid: The gid of the owner of this file 477 @param _lstat_fn: Stat function to use (unittest only) 478 @param _mkdir_fn: mkdir function to use (unittest only) 479 @param _perm_fn: permission setter function to use (unittest only) 480 481 """ 482 logging.debug("Checking directory %s", path) 483 try: 484 # We don't want to follow symlinks 485 st = _lstat_fn(path) 486 except EnvironmentError, err: 487 if err.errno != errno.ENOENT: 488 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err)) 489 _mkdir_fn(path) 490 else: 491 if not stat.S_ISDIR(st[stat.ST_MODE]): 492 raise errors.GenericError(("Path %s is expected to be a directory, but " 493 "isn't") % path) 494 495 _perm_fn(path, mode, uid=uid, gid=gid)
496 497
498 -def Makedirs(path, mode=0750):
499 """Super-mkdir; create a leaf directory and all intermediate ones. 500 501 This is a wrapper around C{os.makedirs} adding error handling not implemented 502 before Python 2.5. 503 504 """ 505 try: 506 os.makedirs(path, mode) 507 except OSError, err: 508 # Ignore EEXIST. This is only handled in os.makedirs as included in 509 # Python 2.5 and above. 510 if err.errno != errno.EEXIST or not os.path.exists(path): 511 raise
512 513
514 -def TimestampForFilename():
515 """Returns the current time formatted for filenames. 516 517 The format doesn't contain colons as some shells and applications treat them 518 as separators. Uses the local timezone. 519 520 """ 521 return time.strftime("%Y-%m-%d_%H_%M_%S")
522 523
524 -def CreateBackup(file_name):
525 """Creates a backup of a file. 526 527 @type file_name: str 528 @param file_name: file to be backed up 529 @rtype: str 530 @return: the path to the newly created backup 531 @raise errors.ProgrammerError: for invalid file names 532 533 """ 534 if not os.path.isfile(file_name): 535 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % 536 file_name) 537 538 prefix = ("%s.backup-%s." % 539 (os.path.basename(file_name), TimestampForFilename())) 540 dir_name = os.path.dirname(file_name) 541 542 fsrc = open(file_name, "rb") 543 try: 544 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) 545 fdst = os.fdopen(fd, "wb") 546 try: 547 logging.debug("Backing up %s at %s", file_name, backup_name) 548 shutil.copyfileobj(fsrc, fdst) 549 finally: 550 fdst.close() 551 finally: 552 fsrc.close() 553 554 return backup_name
555 556
557 -def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
558 """Returns a list of visible files in a directory. 559 560 @type path: str 561 @param path: the directory to enumerate 562 @rtype: list 563 @return: the list of all files not starting with a dot 564 @raise ProgrammerError: if L{path} is not an absolue and normalized path 565 566 """ 567 if not IsNormAbsPath(path): 568 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" 569 " absolute/normalized: '%s'" % path) 570 571 mountpoint = _is_mountpoint(path) 572 573 def fn(name): 574 """File name filter. 575 576 Ignores files starting with a dot (".") as by Unix convention they're 577 considered hidden. The "lost+found" directory found at the root of some 578 filesystems is also hidden. 579 580 """ 581 return not (name.startswith(".") or 582 (mountpoint and name == _LOST_AND_FOUND and 583 os.path.isdir(os.path.join(path, name))))
584 585 return filter(fn, os.listdir(path)) 586 587
588 -def EnsureDirs(dirs):
589 """Make required directories, if they don't exist. 590 591 @param dirs: list of tuples (dir_name, dir_mode) 592 @type dirs: list of (string, integer) 593 594 """ 595 for dir_name, dir_mode in dirs: 596 try: 597 os.mkdir(dir_name, dir_mode) 598 except EnvironmentError, err: 599 if err.errno != errno.EEXIST: 600 raise errors.GenericError("Cannot create needed directory" 601 " '%s': %s" % (dir_name, err)) 602 try: 603 os.chmod(dir_name, dir_mode) 604 except EnvironmentError, err: 605 raise errors.GenericError("Cannot change directory permissions on" 606 " '%s': %s" % (dir_name, err)) 607 if not os.path.isdir(dir_name): 608 raise errors.GenericError("%s is not a directory" % dir_name)
609 610
611 -def FindFile(name, search_path, test=os.path.exists):
612 """Look for a filesystem object in a given path. 613 614 This is an abstract method to search for filesystem object (files, 615 dirs) under a given search path. 616 617 @type name: str 618 @param name: the name to look for 619 @type search_path: str 620 @param search_path: location to start at 621 @type test: callable 622 @param test: a function taking one argument that should return True 623 if the a given object is valid; the default value is 624 os.path.exists, causing only existing files to be returned 625 @rtype: str or None 626 @return: full path to the object if found, None otherwise 627 628 """ 629 # validate the filename mask 630 if constants.EXT_PLUGIN_MASK.match(name) is None: 631 logging.critical("Invalid value passed for external script name: '%s'", 632 name) 633 return None 634 635 for dir_name in search_path: 636 # FIXME: investigate switch to PathJoin 637 item_name = os.path.sep.join([dir_name, name]) 638 # check the user test and that we're indeed resolving to the given 639 # basename 640 if test(item_name) and os.path.basename(item_name) == name: 641 return item_name 642 return None
643 644
645 -def IsNormAbsPath(path):
646 """Check whether a path is absolute and also normalized 647 648 This avoids things like /dir/../../other/path to be valid. 649 650 """ 651 return os.path.normpath(path) == path and os.path.isabs(path)
652 653
654 -def IsBelowDir(root, other_path):
655 """Check whether a path is below a root dir. 656 657 This works around the nasty byte-byte comparison of commonprefix. 658 659 """ 660 if not (os.path.isabs(root) and os.path.isabs(other_path)): 661 raise ValueError("Provided paths '%s' and '%s' are not absolute" % 662 (root, other_path)) 663 664 norm_other = os.path.normpath(other_path) 665 666 if norm_other == os.sep: 667 # The root directory can never be below another path 668 return False 669 670 norm_root = os.path.normpath(root) 671 672 if norm_root == os.sep: 673 # This is the root directory, no need to add another slash 674 prepared_root = norm_root 675 else: 676 prepared_root = "%s%s" % (norm_root, os.sep) 677 678 return os.path.commonprefix([prepared_root, norm_other]) == prepared_root
679 680
681 -def PathJoin(*args):
682 """Safe-join a list of path components. 683 684 Requirements: 685 - the first argument must be an absolute path 686 - no component in the path must have backtracking (e.g. /../), 687 since we check for normalization at the end 688 689 @param args: the path components to be joined 690 @raise ValueError: for invalid paths 691 692 """ 693 # ensure we're having at least one path passed in 694 assert args 695 # ensure the first component is an absolute and normalized path name 696 root = args[0] 697 if not IsNormAbsPath(root): 698 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) 699 result = os.path.join(*args) 700 # ensure that the whole path is normalized 701 if not IsNormAbsPath(result): 702 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) 703 # check that we're still under the original prefix 704 if not IsBelowDir(root, result): 705 raise ValueError("Error: path joining resulted in different prefix" 706 " (%s != %s)" % (result, root)) 707 return result
708 709
710 -def TailFile(fname, lines=20):
711 """Return the last lines from a file. 712 713 @note: this function will only read and parse the last 4KB of 714 the file; if the lines are very long, it could be that less 715 than the requested number of lines are returned 716 717 @param fname: the file name 718 @type lines: int 719 @param lines: the (maximum) number of lines to return 720 721 """ 722 fd = open(fname, "r") 723 try: 724 fd.seek(0, 2) 725 pos = fd.tell() 726 pos = max(0, pos - 4096) 727 fd.seek(pos, 0) 728 raw_data = fd.read() 729 finally: 730 fd.close() 731 732 rows = raw_data.splitlines() 733 return rows[-lines:]
734 735
736 -def BytesToMebibyte(value):
737 """Converts bytes to mebibytes. 738 739 @type value: int 740 @param value: Value in bytes 741 @rtype: int 742 @return: Value in mebibytes 743 744 """ 745 return int(round(value / (1024.0 * 1024.0), 0))
746 747
748 -def CalculateDirectorySize(path):
749 """Calculates the size of a directory recursively. 750 751 @type path: string 752 @param path: Path to directory 753 @rtype: int 754 @return: Size in mebibytes 755 756 """ 757 size = 0 758 759 for (curpath, _, files) in os.walk(path): 760 for filename in files: 761 st = os.lstat(PathJoin(curpath, filename)) 762 size += st.st_size 763 764 return BytesToMebibyte(size)
765 766
767 -def GetFilesystemStats(path):
768 """Returns the total and free space on a filesystem. 769 770 @type path: string 771 @param path: Path on filesystem to be examined 772 @rtype: int 773 @return: tuple of (Total space, Free space) in mebibytes 774 775 """ 776 st = os.statvfs(path) 777 778 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize) 779 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize) 780 return (tsize, fsize)
781 782
783 -def ReadPidFile(pidfile):
784 """Read a pid from a file. 785 786 @type pidfile: string 787 @param pidfile: path to the file containing the pid 788 @rtype: int 789 @return: The process id, if the file exists and contains a valid PID, 790 otherwise 0 791 792 """ 793 try: 794 raw_data = ReadOneLineFile(pidfile) 795 except EnvironmentError, err: 796 if err.errno != errno.ENOENT: 797 logging.exception("Can't read pid file") 798 return 0 799 800 return _ParsePidFileContents(raw_data)
801 802
803 -def _ParsePidFileContents(data):
804 """Tries to extract a process ID from a PID file's content. 805 806 @type data: string 807 @rtype: int 808 @return: Zero if nothing could be read, PID otherwise 809 810 """ 811 try: 812 pid = int(data) 813 except (TypeError, ValueError): 814 logging.info("Can't parse pid file contents", exc_info=True) 815 return 0 816 else: 817 return pid
818 819
820 -def ReadLockedPidFile(path):
821 """Reads a locked PID file. 822 823 This can be used together with L{utils.process.StartDaemon}. 824 825 @type path: string 826 @param path: Path to PID file 827 @return: PID as integer or, if file was unlocked or couldn't be opened, None 828 829 """ 830 try: 831 fd = os.open(path, os.O_RDONLY) 832 except EnvironmentError, err: 833 if err.errno == errno.ENOENT: 834 # PID file doesn't exist 835 return None 836 raise 837 838 try: 839 try: 840 # Try to acquire lock 841 filelock.LockFile(fd) 842 except errors.LockError: 843 # Couldn't lock, daemon is running 844 return int(os.read(fd, 100)) 845 finally: 846 os.close(fd) 847 848 return None
849 850
851 -def _SplitSshKey(key):
852 """Splits a line for SSH's C{authorized_keys} file. 853 854 If the line has no options (e.g. no C{command="..."}), only the significant 855 parts, the key type and its hash, are used. Otherwise the whole line is used 856 (split at whitespace). 857 858 @type key: string 859 @param key: Key line 860 @rtype: tuple 861 862 """ 863 parts = key.split() 864 865 if parts and parts[0] in constants.SSHAK_ALL: 866 # If the key has no options in front of it, we only want the significant 867 # fields 868 return (False, parts[:2]) 869 else: 870 # Can't properly split the line, so use everything 871 return (True, parts)
872 873
874 -def AddAuthorizedKey(file_obj, key):
875 """Adds an SSH public key to an authorized_keys file. 876 877 @type file_obj: str or file handle 878 @param file_obj: path to authorized_keys file 879 @type key: str 880 @param key: string containing key 881 882 """ 883 key_fields = _SplitSshKey(key) 884 885 if isinstance(file_obj, basestring): 886 f = open(file_obj, "a+") 887 else: 888 f = file_obj 889 890 try: 891 nl = True 892 for line in f: 893 # Ignore whitespace changes 894 if _SplitSshKey(line) == key_fields: 895 break 896 nl = line.endswith("\n") 897 else: 898 if not nl: 899 f.write("\n") 900 f.write(key.rstrip("\r\n")) 901 f.write("\n") 902 f.flush() 903 finally: 904 f.close()
905 906
907 -def RemoveAuthorizedKey(file_name, key):
908 """Removes an SSH public key from an authorized_keys file. 909 910 @type file_name: str 911 @param file_name: path to authorized_keys file 912 @type key: str 913 @param key: string containing key 914 915 """ 916 key_fields = _SplitSshKey(key) 917 918 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name)) 919 try: 920 out = os.fdopen(fd, "w") 921 try: 922 f = open(file_name, "r") 923 try: 924 for line in f: 925 # Ignore whitespace changes while comparing lines 926 if _SplitSshKey(line) != key_fields: 927 out.write(line) 928 929 out.flush() 930 os.rename(tmpname, file_name) 931 finally: 932 f.close() 933 finally: 934 out.close() 935 except: 936 RemoveFile(tmpname) 937 raise
938 939
940 -def DaemonPidFileName(name):
941 """Compute a ganeti pid file absolute path 942 943 @type name: str 944 @param name: the daemon name 945 @rtype: str 946 @return: the full path to the pidfile corresponding to the given 947 daemon name 948 949 """ 950 return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
951 952
953 -def WritePidFile(pidfile):
954 """Write the current process pidfile. 955 956 @type pidfile: string 957 @param pidfile: the path to the file to be written 958 @raise errors.LockError: if the pid file already exists and 959 points to a live process 960 @rtype: int 961 @return: the file descriptor of the lock file; do not close this unless 962 you want to unlock the pid file 963 964 """ 965 # We don't rename nor truncate the file to not drop locks under 966 # existing processes 967 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600) 968 969 # Lock the PID file (and fail if not possible to do so). Any code 970 # wanting to send a signal to the daemon should try to lock the PID 971 # file before reading it. If acquiring the lock succeeds, the daemon is 972 # no longer running and the signal should not be sent. 973 try: 974 filelock.LockFile(fd_pidfile) 975 except errors.LockError: 976 msg = ["PID file '%s' is already locked by another process" % pidfile] 977 # Try to read PID file 978 pid = _ParsePidFileContents(os.read(fd_pidfile, 100)) 979 if pid > 0: 980 msg.append(", PID read from file is %s" % pid) 981 raise errors.PidFileLockError("".join(msg)) 982 983 os.write(fd_pidfile, "%d\n" % os.getpid()) 984 985 return fd_pidfile
986 987
988 -def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
989 """Reads the watcher pause file. 990 991 @type filename: string 992 @param filename: Path to watcher pause file 993 @type now: None, float or int 994 @param now: Current time as Unix timestamp 995 @type remove_after: int 996 @param remove_after: Remove watcher pause file after specified amount of 997 seconds past the pause end time 998 999 """ 1000 if now is None: 1001 now = time.time() 1002 1003 try: 1004 value = ReadFile(filename) 1005 except IOError, err: 1006 if err.errno != errno.ENOENT: 1007 raise 1008 value = None 1009 1010 if value is not None: 1011 try: 1012 value = int(value) 1013 except ValueError: 1014 logging.warning(("Watcher pause file (%s) contains invalid value," 1015 " removing it"), filename) 1016 RemoveFile(filename) 1017 value = None 1018 1019 if value is not None: 1020 # Remove file if it's outdated 1021 if now > (value + remove_after): 1022 RemoveFile(filename) 1023 value = None 1024 1025 elif now > value: 1026 value = None 1027 1028 return value
1029 1030
1031 -def NewUUID():
1032 """Returns a random UUID. 1033 1034 @note: This is a Linux-specific method as it uses the /proc 1035 filesystem. 1036 @rtype: str 1037 1038 """ 1039 return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n")
1040 1041
1042 -class TemporaryFileManager(object):
1043 """Stores the list of files to be deleted and removes them on demand. 1044 1045 """ 1046
1047 - def __init__(self):
1048 self._files = []
1049
1050 - def __del__(self):
1051 self.Cleanup()
1052
1053 - def Add(self, filename):
1054 """Add file to list of files to be deleted. 1055 1056 @type filename: string 1057 @param filename: path to filename to be added 1058 1059 """ 1060 self._files.append(filename)
1061
1062 - def Remove(self, filename):
1063 """Remove file from list of files to be deleted. 1064 1065 @type filename: string 1066 @param filename: path to filename to be deleted 1067 1068 """ 1069 self._files.remove(filename)
1070
1071 - def Cleanup(self):
1072 """Delete all files marked for deletion 1073 1074 """ 1075 while self._files: 1076 RemoveFile(self._files.pop())
1077 1078
1079 -def IsUserInGroup(uid, gid):
1080 """Returns True if the user belongs to the group. 1081 1082 @type uid: int 1083 @param uid: the user id 1084 @type gid: int 1085 @param gid: the group id 1086 @rtype: bool 1087 1088 """ 1089 user = pwd.getpwuid(uid) 1090 group = grp.getgrgid(gid) 1091 return user.pw_gid == gid or user.pw_name in group.gr_mem
1092 1093
1094 -def CanRead(username, filename):
1095 """Returns True if the user can access (read) the file. 1096 1097 @type username: string 1098 @param username: the name of the user 1099 @type filename: string 1100 @param filename: the name of the file 1101 @rtype: bool 1102 1103 """ 1104 filestats = os.stat(filename) 1105 user = pwd.getpwnam(username) 1106 uid = user.pw_uid 1107 user_readable = filestats.st_mode & stat.S_IRUSR != 0 1108 group_readable = filestats.st_mode & stat.S_IRGRP != 0 1109 return ((filestats.st_uid == uid and user_readable) 1110 or (filestats.st_uid != uid and 1111 IsUserInGroup(uid, filestats.st_gid) and group_readable))
1112