Package ganeti :: Package utils :: Module io
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.io

   1  # 
   2  # 
   3   
   4  # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2014 Google Inc. 
   5  # All rights reserved. 
   6  # 
   7  # Redistribution and use in source and binary forms, with or without 
   8  # modification, are permitted provided that the following conditions are 
   9  # met: 
  10  # 
  11  # 1. Redistributions of source code must retain the above copyright notice, 
  12  # this list of conditions and the following disclaimer. 
  13  # 
  14  # 2. Redistributions in binary form must reproduce the above copyright 
  15  # notice, this list of conditions and the following disclaimer in the 
  16  # documentation and/or other materials provided with the distribution. 
  17  # 
  18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 
  19  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
  20  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
  21  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
  22  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
  23  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
  24  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
  25  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
  26  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
  27  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
  28  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
  29   
  30  """Utility functions for I/O. 
  31   
  32  """ 
  33   
  34  import os 
  35  import re 
  36  import logging 
  37  import shutil 
  38  import tempfile 
  39  import errno 
  40  import time 
  41  import stat 
  42  import grp 
  43  import pwd 
  44   
  45  from ganeti import errors 
  46  from ganeti import constants 
  47  from ganeti import pathutils 
  48  from ganeti.utils import filelock 
  49   
  50  #: Directory used by fsck(8) to store recovered data, usually at a file 
  51  #: system's root directory 
  52  _LOST_AND_FOUND = "lost+found" 
  53   
  54  # Possible values for keep_perms in WriteFile() 
  55  KP_NEVER = 0 
  56  KP_ALWAYS = 1 
  57  KP_IF_EXISTS = 2 
  58   
  59  KEEP_PERMS_VALUES = [ 
  60    KP_NEVER, 
  61    KP_ALWAYS, 
  62    KP_IF_EXISTS, 
  63    ] 
  64   
  65   
66 -def ErrnoOrStr(err):
67 """Format an EnvironmentError exception. 68 69 If the L{err} argument has an errno attribute, it will be looked up 70 and converted into a textual C{E...} description. Otherwise the 71 string representation of the error will be returned. 72 73 @type err: L{EnvironmentError} 74 @param err: the exception to format 75 76 """ 77 if hasattr(err, "errno"): 78 detail = errno.errorcode[err.errno] 79 else: 80 detail = str(err) 81 return detail
82 83
84 -class FileStatHelper(object):
85 """Helper to store file handle's C{fstat}. 86 87 Useful in combination with L{ReadFile}'s C{preread} parameter. 88 89 """
90 - def __init__(self):
91 """Initializes this class. 92 93 """ 94 self.st = None
95
96 - def __call__(self, fh):
97 """Calls C{fstat} on file handle. 98 99 """ 100 self.st = os.fstat(fh.fileno())
101 102
103 -def ReadFile(file_name, size=-1, preread=None):
104 """Reads a file. 105 106 @type size: int 107 @param size: Read at most size bytes (if negative, entire file) 108 @type preread: callable receiving file handle as single parameter 109 @param preread: Function called before file is read 110 @rtype: str 111 @return: the (possibly partial) content of the file 112 @raise IOError: if the file cannot be opened 113 114 """ 115 f = open(file_name, "r") 116 try: 117 if preread: 118 preread(f) 119 120 return f.read(size) 121 finally: 122 f.close()
123 124
125 -def WriteFile(file_name, fn=None, data=None, 126 mode=None, uid=-1, gid=-1, 127 atime=None, mtime=None, close=True, 128 dry_run=False, backup=False, 129 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
130 """(Over)write a file atomically. 131 132 The file_name and either fn (a function taking one argument, the 133 file descriptor, and which should write the data to it) or data (the 134 contents of the file) must be passed. The other arguments are 135 optional and allow setting the file mode, owner and group, and the 136 mtime/atime of the file. 137 138 If the function doesn't raise an exception, it has succeeded and the 139 target file has the new contents. If the function has raised an 140 exception, an existing target file should be unmodified and the 141 temporary file should be removed. 142 143 @type file_name: str 144 @param file_name: the target filename 145 @type fn: callable 146 @param fn: content writing function, called with 147 file descriptor as parameter 148 @type data: str 149 @param data: contents of the file 150 @type mode: int 151 @param mode: file mode 152 @type uid: int 153 @param uid: the owner of the file 154 @type gid: int 155 @param gid: the group of the file 156 @type atime: int 157 @param atime: a custom access time to be set on the file 158 @type mtime: int 159 @param mtime: a custom modification time to be set on the file 160 @type close: boolean 161 @param close: whether to close file after writing it 162 @type prewrite: callable 163 @param prewrite: function to be called before writing content 164 @type postwrite: callable 165 @param postwrite: function to be called after writing content 166 @type keep_perms: members of L{KEEP_PERMS_VALUES} 167 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are 168 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and 169 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner, 170 group, and mode are taken from the file, and if the file doesn't 171 exist, they are taken from the other parameters. It is an error to 172 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid}, 173 or C{mode} are set to non-default values. 174 175 @rtype: None or int 176 @return: None if the 'close' parameter evaluates to True, 177 otherwise the file descriptor 178 179 @raise errors.ProgrammerError: if any of the arguments are not valid 180 181 """ 182 if not os.path.isabs(file_name): 183 raise errors.ProgrammerError("Path passed to WriteFile is not" 184 " absolute: '%s'" % file_name) 185 186 if [fn, data].count(None) != 1: 187 raise errors.ProgrammerError("fn or data required") 188 189 if [atime, mtime].count(None) == 1: 190 raise errors.ProgrammerError("Both atime and mtime must be either" 191 " set or None") 192 193 if not keep_perms in KEEP_PERMS_VALUES: 194 raise errors.ProgrammerError("Invalid value for keep_perms: %s" % 195 keep_perms) 196 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None): 197 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid'," 198 " and 'mode' cannot be set") 199 200 if backup and not dry_run and os.path.isfile(file_name): 201 CreateBackup(file_name) 202 203 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS: 204 # os.stat() raises an exception if the file doesn't exist 205 try: 206 file_stat = os.stat(file_name) 207 mode = stat.S_IMODE(file_stat.st_mode) 208 uid = file_stat.st_uid 209 gid = file_stat.st_gid 210 except OSError: 211 if keep_perms == KP_ALWAYS: 212 raise 213 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist 214 215 # Whether temporary file needs to be removed (e.g. if any error occurs) 216 do_remove = True 217 218 # Function result 219 result = None 220 221 (dir_name, base_name) = os.path.split(file_name) 222 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name, 223 dir=dir_name) 224 try: 225 try: 226 if uid != -1 or gid != -1: 227 os.chown(new_name, uid, gid) 228 if mode: 229 os.chmod(new_name, mode) 230 if callable(prewrite): 231 prewrite(fd) 232 if data is not None: 233 if isinstance(data, unicode): 234 data = data.encode() 235 assert isinstance(data, str) 236 to_write = len(data) 237 offset = 0 238 while offset < to_write: 239 written = os.write(fd, buffer(data, offset)) 240 assert written >= 0 241 assert written <= to_write - offset 242 offset += written 243 assert offset == to_write 244 else: 245 fn(fd) 246 if callable(postwrite): 247 postwrite(fd) 248 os.fsync(fd) 249 if atime is not None and mtime is not None: 250 os.utime(new_name, (atime, mtime)) 251 finally: 252 # Close file unless the file descriptor should be returned 253 if close: 254 os.close(fd) 255 else: 256 result = fd 257 258 # Rename file to destination name 259 if not dry_run: 260 os.rename(new_name, file_name) 261 # Successful, no need to remove anymore 262 do_remove = False 263 finally: 264 if do_remove: 265 RemoveFile(new_name) 266 267 return result
268 269
270 -def GetFileID(path=None, fd=None):
271 """Returns the file 'id', i.e. the dev/inode and mtime information. 272 273 Either the path to the file or the fd must be given. 274 275 @param path: the file path 276 @param fd: a file descriptor 277 @return: a tuple of (device number, inode number, mtime) 278 279 """ 280 if [path, fd].count(None) != 1: 281 raise errors.ProgrammerError("One and only one of fd/path must be given") 282 283 if fd is None: 284 st = os.stat(path) 285 else: 286 st = os.fstat(fd) 287 288 return (st.st_dev, st.st_ino, st.st_mtime)
289 290
291 -def VerifyFileID(fi_disk, fi_ours):
292 """Verifies that two file IDs are matching. 293 294 Differences in the inode/device are not accepted, but and older 295 timestamp for fi_disk is accepted. 296 297 @param fi_disk: tuple (dev, inode, mtime) representing the actual 298 file data 299 @param fi_ours: tuple (dev, inode, mtime) representing the last 300 written file data 301 @rtype: boolean 302 303 """ 304 (d1, i1, m1) = fi_disk 305 (d2, i2, m2) = fi_ours 306 307 return (d1, i1) == (d2, i2) and m1 <= m2
308 309
310 -def SafeWriteFile(file_name, file_id, **kwargs):
311 """Wraper over L{WriteFile} that locks the target file. 312 313 By keeping the target file locked during WriteFile, we ensure that 314 cooperating writers will safely serialise access to the file. 315 316 @type file_name: str 317 @param file_name: the target filename 318 @type file_id: tuple 319 @param file_id: a result from L{GetFileID} 320 321 """ 322 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT) 323 try: 324 filelock.LockFile(fd) 325 if file_id is not None: 326 disk_id = GetFileID(fd=fd) 327 if not VerifyFileID(disk_id, file_id): 328 raise errors.LockError("Cannot overwrite file %s, it has been modified" 329 " since last written" % file_name) 330 return WriteFile(file_name, **kwargs) 331 finally: 332 os.close(fd)
333 334
335 -def ReadOneLineFile(file_name, strict=False):
336 """Return the first non-empty line from a file. 337 338 @type strict: boolean 339 @param strict: if True, abort if the file has more than one 340 non-empty line 341 342 """ 343 file_lines = ReadFile(file_name).splitlines() 344 full_lines = filter(bool, file_lines) 345 if not file_lines or not full_lines: 346 raise errors.GenericError("No data in one-liner file %s" % file_name) 347 elif strict and len(full_lines) > 1: 348 raise errors.GenericError("Too many lines in one-liner file %s" % 349 file_name) 350 return full_lines[0]
351 352
353 -def RemoveFile(filename):
354 """Remove a file ignoring some errors. 355 356 Remove a file, ignoring non-existing ones or directories. Other 357 errors are passed. 358 359 @type filename: str 360 @param filename: the file to be removed 361 362 """ 363 try: 364 os.unlink(filename) 365 except OSError, err: 366 if err.errno not in (errno.ENOENT, errno.EISDIR): 367 raise
368 369
370 -def RemoveDir(dirname):
371 """Remove an empty directory. 372 373 Remove a directory, ignoring non-existing ones. 374 Other errors are passed. This includes the case, 375 where the directory is not empty, so it can't be removed. 376 377 @type dirname: str 378 @param dirname: the empty directory to be removed 379 380 """ 381 try: 382 os.rmdir(dirname) 383 except OSError, err: 384 if err.errno != errno.ENOENT: 385 raise
386 387
388 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None, 389 dir_gid=None):
390 """Renames a file. 391 392 This just creates the very least directory if it does not exist and C{mkdir} 393 is set to true. 394 395 @type old: string 396 @param old: Original path 397 @type new: string 398 @param new: New path 399 @type mkdir: bool 400 @param mkdir: Whether to create target directory if it doesn't exist 401 @type mkdir_mode: int 402 @param mkdir_mode: Mode for newly created directories 403 @type dir_uid: int 404 @param dir_uid: The uid for the (if fresh created) dir 405 @type dir_gid: int 406 @param dir_gid: The gid for the (if fresh created) dir 407 408 """ 409 try: 410 return os.rename(old, new) 411 except OSError, err: 412 # In at least one use case of this function, the job queue, directory 413 # creation is very rare. Checking for the directory before renaming is not 414 # as efficient. 415 if mkdir and err.errno == errno.ENOENT: 416 # Create directory and try again 417 dir_path = os.path.dirname(new) 418 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid) 419 420 return os.rename(old, new) 421 422 raise
423 424
425 -def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True, 426 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
427 """Enforces that given path has given permissions. 428 429 @param path: The path to the file 430 @param mode: The mode of the file 431 @param uid: The uid of the owner of this file 432 @param gid: The gid of the owner of this file 433 @param must_exist: Specifies if non-existance of path will be an error 434 @param _chmod_fn: chmod function to use (unittest only) 435 @param _chown_fn: chown function to use (unittest only) 436 437 """ 438 logging.debug("Checking %s", path) 439 440 # chown takes -1 if you want to keep one part of the ownership, however 441 # None is Python standard for that. So we remap them here. 442 if uid is None: 443 uid = -1 444 if gid is None: 445 gid = -1 446 447 try: 448 st = _stat_fn(path) 449 450 fmode = stat.S_IMODE(st[stat.ST_MODE]) 451 if fmode != mode: 452 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode) 453 _chmod_fn(path, mode) 454 455 if max(uid, gid) > -1: 456 fuid = st[stat.ST_UID] 457 fgid = st[stat.ST_GID] 458 if fuid != uid or fgid != gid: 459 logging.debug("Changing owner of %s from UID %s/GID %s to" 460 " UID %s/GID %s", path, fuid, fgid, uid, gid) 461 _chown_fn(path, uid, gid) 462 except EnvironmentError, err: 463 if err.errno == errno.ENOENT: 464 if must_exist: 465 raise errors.GenericError("Path %s should exist, but does not" % path) 466 else: 467 raise errors.GenericError("Error while changing permissions on %s: %s" % 468 (path, err))
469 470
471 -def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat, 472 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
473 """Enforces that given path is a dir and has given mode, uid and gid set. 474 475 @param path: The path to the file 476 @param mode: The mode of the file 477 @param uid: The uid of the owner of this file 478 @param gid: The gid of the owner of this file 479 @param _lstat_fn: Stat function to use (unittest only) 480 @param _mkdir_fn: mkdir function to use (unittest only) 481 @param _perm_fn: permission setter function to use (unittest only) 482 483 """ 484 logging.debug("Checking directory %s", path) 485 try: 486 # We don't want to follow symlinks 487 st = _lstat_fn(path) 488 except EnvironmentError, err: 489 if err.errno != errno.ENOENT: 490 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err)) 491 _mkdir_fn(path) 492 else: 493 if not stat.S_ISDIR(st[stat.ST_MODE]): 494 raise errors.GenericError(("Path %s is expected to be a directory, but " 495 "isn't") % path) 496 497 _perm_fn(path, mode, uid=uid, gid=gid)
498 499
500 -def Makedirs(path, mode=0750):
501 """Super-mkdir; create a leaf directory and all intermediate ones. 502 503 This is a wrapper around C{os.makedirs} adding error handling not implemented 504 before Python 2.5. 505 506 """ 507 try: 508 os.makedirs(path, mode) 509 except OSError, err: 510 # Ignore EEXIST. This is only handled in os.makedirs as included in 511 # Python 2.5 and above. 512 if err.errno != errno.EEXIST or not os.path.exists(path): 513 raise
514 515
516 -def TimestampForFilename():
517 """Returns the current time formatted for filenames. 518 519 The format doesn't contain colons as some shells and applications treat them 520 as separators. Uses the local timezone. 521 522 """ 523 return time.strftime("%Y-%m-%d_%H_%M_%S")
524 525
526 -def CreateBackup(file_name):
527 """Creates a backup of a file. 528 529 @type file_name: str 530 @param file_name: file to be backed up 531 @rtype: str 532 @return: the path to the newly created backup 533 @raise errors.ProgrammerError: for invalid file names 534 535 """ 536 if not os.path.isfile(file_name): 537 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % 538 file_name) 539 540 prefix = ("%s.backup-%s." % 541 (os.path.basename(file_name), TimestampForFilename())) 542 dir_name = os.path.dirname(file_name) 543 544 fsrc = open(file_name, "rb") 545 try: 546 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) 547 fdst = os.fdopen(fd, "wb") 548 try: 549 logging.debug("Backing up %s at %s", file_name, backup_name) 550 shutil.copyfileobj(fsrc, fdst) 551 finally: 552 fdst.close() 553 finally: 554 fsrc.close() 555 556 return backup_name
557 558
559 -def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
560 """Returns a list of visible files in a directory. 561 562 @type path: str 563 @param path: the directory to enumerate 564 @rtype: list 565 @return: the list of all files not starting with a dot 566 @raise ProgrammerError: if L{path} is not an absolue and normalized path 567 568 """ 569 if not IsNormAbsPath(path): 570 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" 571 " absolute/normalized: '%s'" % path) 572 573 mountpoint = _is_mountpoint(path) 574 575 def fn(name): 576 """File name filter. 577 578 Ignores files starting with a dot (".") as by Unix convention they're 579 considered hidden. The "lost+found" directory found at the root of some 580 filesystems is also hidden. 581 582 """ 583 return not (name.startswith(".") or 584 (mountpoint and name == _LOST_AND_FOUND and 585 os.path.isdir(os.path.join(path, name))))
586 587 return filter(fn, os.listdir(path)) 588 589
590 -def EnsureDirs(dirs):
591 """Make required directories, if they don't exist. 592 593 @param dirs: list of tuples (dir_name, dir_mode) 594 @type dirs: list of (string, integer) 595 596 """ 597 for dir_name, dir_mode in dirs: 598 try: 599 os.mkdir(dir_name, dir_mode) 600 except EnvironmentError, err: 601 if err.errno != errno.EEXIST: 602 raise errors.GenericError("Cannot create needed directory" 603 " '%s': %s" % (dir_name, err)) 604 try: 605 os.chmod(dir_name, dir_mode) 606 except EnvironmentError, err: 607 raise errors.GenericError("Cannot change directory permissions on" 608 " '%s' to 0%o: %s" % (dir_name, dir_mode, err)) 609 if not os.path.isdir(dir_name): 610 raise errors.GenericError("%s is not a directory" % dir_name)
611 612
613 -def FindFile(name, search_path, test=os.path.exists):
614 """Look for a filesystem object in a given path. 615 616 This is an abstract method to search for filesystem object (files, 617 dirs) under a given search path. 618 619 @type name: str 620 @param name: the name to look for 621 @type search_path: iterable of string 622 @param search_path: locations to start at 623 @type test: callable 624 @param test: a function taking one argument that should return True 625 if the a given object is valid; the default value is 626 os.path.exists, causing only existing files to be returned 627 @rtype: str or None 628 @return: full path to the object if found, None otherwise 629 630 """ 631 # validate the filename mask 632 if constants.EXT_PLUGIN_MASK.match(name) is None: 633 logging.critical("Invalid value passed for external script name: '%s'", 634 name) 635 return None 636 637 for dir_name in search_path: 638 # FIXME: investigate switch to PathJoin 639 item_name = os.path.sep.join([dir_name, name]) 640 # check the user test and that we're indeed resolving to the given 641 # basename 642 if test(item_name) and os.path.basename(item_name) == name: 643 return item_name 644 return None
645 646
647 -def IsNormAbsPath(path):
648 """Check whether a path is absolute and also normalized 649 650 This avoids things like /dir/../../other/path to be valid. 651 652 """ 653 return os.path.normpath(path) == path and os.path.isabs(path)
654 655
656 -def IsBelowDir(root, other_path):
657 """Check whether a path is below a root dir. 658 659 This works around the nasty byte-byte comparison of commonprefix. 660 661 """ 662 if not (os.path.isabs(root) and os.path.isabs(other_path)): 663 raise ValueError("Provided paths '%s' and '%s' are not absolute" % 664 (root, other_path)) 665 666 norm_other = os.path.normpath(other_path) 667 668 if norm_other == os.sep: 669 # The root directory can never be below another path 670 return False 671 672 norm_root = os.path.normpath(root) 673 674 if norm_root == os.sep: 675 # This is the root directory, no need to add another slash 676 prepared_root = norm_root 677 else: 678 prepared_root = "%s%s" % (norm_root, os.sep) 679 680 return os.path.commonprefix([prepared_root, norm_other]) == prepared_root
681 682 683 URL_RE = re.compile(r'(https?|ftps?)://') 684 685
686 -def IsUrl(path):
687 """Check whether a path is a HTTP URL. 688 689 """ 690 return URL_RE.match(path)
691 692
693 -def PathJoin(*args):
694 """Safe-join a list of path components. 695 696 Requirements: 697 - the first argument must be an absolute path 698 - no component in the path must have backtracking (e.g. /../), 699 since we check for normalization at the end 700 701 @param args: the path components to be joined 702 @raise ValueError: for invalid paths 703 704 """ 705 # ensure we're having at least two paths passed in 706 if len(args) <= 1: 707 raise errors.ProgrammerError("PathJoin requires two arguments") 708 # ensure the first component is an absolute and normalized path name 709 root = args[0] 710 if not IsNormAbsPath(root): 711 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) 712 result = os.path.join(*args) 713 # ensure that the whole path is normalized 714 if not IsNormAbsPath(result): 715 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) 716 # check that we're still under the original prefix 717 if not IsBelowDir(root, result): 718 raise ValueError("Error: path joining resulted in different prefix" 719 " (%s != %s)" % (result, root)) 720 return result
721 722
723 -def TailFile(fname, lines=20):
724 """Return the last lines from a file. 725 726 @note: this function will only read and parse the last 4KB of 727 the file; if the lines are very long, it could be that less 728 than the requested number of lines are returned 729 730 @param fname: the file name 731 @type lines: int 732 @param lines: the (maximum) number of lines to return 733 734 """ 735 fd = open(fname, "r") 736 try: 737 fd.seek(0, 2) 738 pos = fd.tell() 739 pos = max(0, pos - 4096) 740 fd.seek(pos, 0) 741 raw_data = fd.read() 742 finally: 743 fd.close() 744 745 rows = raw_data.splitlines() 746 return rows[-lines:]
747 748
749 -def BytesToMebibyte(value):
750 """Converts bytes to mebibytes. 751 752 @type value: int 753 @param value: Value in bytes 754 @rtype: int 755 @return: Value in mebibytes 756 757 """ 758 return int(round(value / (1024.0 * 1024.0), 0))
759 760
761 -def CalculateDirectorySize(path):
762 """Calculates the size of a directory recursively. 763 764 @type path: string 765 @param path: Path to directory 766 @rtype: int 767 @return: Size in mebibytes 768 769 """ 770 size = 0 771 772 for (curpath, _, files) in os.walk(path): 773 for filename in files: 774 st = os.lstat(PathJoin(curpath, filename)) 775 size += st.st_size 776 777 return BytesToMebibyte(size)
778 779
780 -def GetFilesystemStats(path):
781 """Returns the total and free space on a filesystem. 782 783 @type path: string 784 @param path: Path on filesystem to be examined 785 @rtype: int 786 @return: tuple of (Total space, Free space) in mebibytes 787 788 """ 789 st = os.statvfs(path) 790 791 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize) 792 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize) 793 return (tsize, fsize)
794 795
796 -def ReadPidFile(pidfile):
797 """Read a pid from a file. 798 799 @type pidfile: string 800 @param pidfile: path to the file containing the pid 801 @rtype: int 802 @return: The process id, if the file exists and contains a valid PID, 803 otherwise 0 804 805 """ 806 try: 807 raw_data = ReadOneLineFile(pidfile) 808 except EnvironmentError, err: 809 if err.errno != errno.ENOENT: 810 logging.exception("Can't read pid file") 811 return 0 812 813 return _ParsePidFileContents(raw_data)
814 815
816 -def _ParsePidFileContents(data):
817 """Tries to extract a process ID from a PID file's content. 818 819 @type data: string 820 @rtype: int 821 @return: Zero if nothing could be read, PID otherwise 822 823 """ 824 try: 825 pid = int(data) 826 except (TypeError, ValueError): 827 logging.info("Can't parse pid file contents", exc_info=True) 828 return 0 829 else: 830 return pid
831 832
833 -def ReadLockedPidFile(path):
834 """Reads a locked PID file. 835 836 This can be used together with L{utils.process.StartDaemon}. 837 838 @type path: string 839 @param path: Path to PID file 840 @return: PID as integer or, if file was unlocked or couldn't be opened, None 841 842 """ 843 try: 844 fd = os.open(path, os.O_RDONLY) 845 except EnvironmentError, err: 846 if err.errno == errno.ENOENT: 847 # PID file doesn't exist 848 return None 849 raise 850 851 try: 852 try: 853 # Try to acquire lock 854 filelock.LockFile(fd) 855 except errors.LockError: 856 # Couldn't lock, daemon is running 857 return int(os.read(fd, 100)) 858 finally: 859 os.close(fd) 860 861 return None
862 863
864 -def DaemonPidFileName(name):
865 """Compute a ganeti pid file absolute path 866 867 @type name: str 868 @param name: the daemon name 869 @rtype: str 870 @return: the full path to the pidfile corresponding to the given 871 daemon name 872 873 """ 874 return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
875 876
877 -def WritePidFile(pidfile):
878 """Write the current process pidfile. 879 880 @type pidfile: string 881 @param pidfile: the path to the file to be written 882 @raise errors.LockError: if the pid file already exists and 883 points to a live process 884 @rtype: int 885 @return: the file descriptor of the lock file; do not close this unless 886 you want to unlock the pid file 887 888 """ 889 # We don't rename nor truncate the file to not drop locks under 890 # existing processes 891 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600) 892 893 # Lock the PID file (and fail if not possible to do so). Any code 894 # wanting to send a signal to the daemon should try to lock the PID 895 # file before reading it. If acquiring the lock succeeds, the daemon is 896 # no longer running and the signal should not be sent. 897 try: 898 filelock.LockFile(fd_pidfile) 899 except errors.LockError: 900 msg = ["PID file '%s' is already locked by another process" % pidfile] 901 # Try to read PID file 902 pid = _ParsePidFileContents(os.read(fd_pidfile, 100)) 903 if pid > 0: 904 msg.append(", PID read from file is %s" % pid) 905 raise errors.PidFileLockError("".join(msg)) 906 907 os.write(fd_pidfile, "%d\n" % os.getpid()) 908 909 return fd_pidfile
910 911
912 -def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
913 """Reads the watcher pause file. 914 915 @type filename: string 916 @param filename: Path to watcher pause file 917 @type now: None, float or int 918 @param now: Current time as Unix timestamp 919 @type remove_after: int 920 @param remove_after: Remove watcher pause file after specified amount of 921 seconds past the pause end time 922 923 """ 924 if now is None: 925 now = time.time() 926 927 try: 928 value = ReadFile(filename) 929 except IOError, err: 930 if err.errno != errno.ENOENT: 931 raise 932 value = None 933 934 if value is not None: 935 try: 936 value = int(value) 937 except ValueError: 938 logging.warning(("Watcher pause file (%s) contains invalid value," 939 " removing it"), filename) 940 RemoveFile(filename) 941 value = None 942 943 if value is not None: 944 # Remove file if it's outdated 945 if now > (value + remove_after): 946 RemoveFile(filename) 947 value = None 948 949 elif now > value: 950 value = None 951 952 return value
953 954
955 -def NewUUID():
956 """Returns a random UUID. 957 958 @note: This is a Linux-specific method as it uses the /proc 959 filesystem. 960 @rtype: str 961 962 """ 963 return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n")
964 965
966 -class TemporaryFileManager(object):
967 """Stores the list of files to be deleted and removes them on demand. 968 969 """ 970
971 - def __init__(self):
972 self._files = []
973
974 - def __del__(self):
975 self.Cleanup()
976
977 - def Add(self, filename):
978 """Add file to list of files to be deleted. 979 980 @type filename: string 981 @param filename: path to filename to be added 982 983 """ 984 self._files.append(filename)
985
986 - def Remove(self, filename):
987 """Remove file from list of files to be deleted. 988 989 @type filename: string 990 @param filename: path to filename to be deleted 991 992 """ 993 self._files.remove(filename)
994
995 - def Cleanup(self):
996 """Delete all files marked for deletion 997 998 """ 999 while self._files: 1000 RemoveFile(self._files.pop())
1001 1002
1003 -def IsUserInGroup(uid, gid):
1004 """Returns True if the user belongs to the group. 1005 1006 @type uid: int 1007 @param uid: the user id 1008 @type gid: int 1009 @param gid: the group id 1010 @rtype: bool 1011 1012 """ 1013 user = pwd.getpwuid(uid) 1014 group = grp.getgrgid(gid) 1015 return user.pw_gid == gid or user.pw_name in group.gr_mem
1016 1017
1018 -def CanRead(username, filename):
1019 """Returns True if the user can access (read) the file. 1020 1021 @type username: string 1022 @param username: the name of the user 1023 @type filename: string 1024 @param filename: the name of the file 1025 @rtype: bool 1026 1027 """ 1028 filestats = os.stat(filename) 1029 user = pwd.getpwnam(username) 1030 uid = user.pw_uid 1031 user_readable = filestats.st_mode & stat.S_IRUSR != 0 1032 group_readable = filestats.st_mode & stat.S_IRGRP != 0 1033 return ((filestats.st_uid == uid and user_readable) 1034 or (filestats.st_uid != uid and 1035 IsUserInGroup(uid, filestats.st_gid) and group_readable))
1036