Package ganeti :: Package utils :: Module io
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.io

  1  # 
  2  # 
  3   
  4  # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, but 
 12  # WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
 14  # General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the Free Software 
 18  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
 19  # 02110-1301, USA. 
 20   
 21  """Utility functions for I/O. 
 22   
 23  """ 
 24   
 25  import os 
 26  import logging 
 27  import shutil 
 28  import tempfile 
 29  import errno 
 30  import time 
 31   
 32  from ganeti import errors 
 33  from ganeti import constants 
 34  from ganeti.utils import filelock 
 35   
 36   
 37  #: Path generating random UUID 
 38  _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid" 
 39   
 40   
41 -def ReadFile(file_name, size=-1, preread=None):
42 """Reads a file. 43 44 @type size: int 45 @param size: Read at most size bytes (if negative, entire file) 46 @type preread: callable receiving file handle as single parameter 47 @param preread: Function called before file is read 48 @rtype: str 49 @return: the (possibly partial) content of the file 50 51 """ 52 f = open(file_name, "r") 53 try: 54 if preread: 55 preread(f) 56 57 return f.read(size) 58 finally: 59 f.close()
60 61
62 -def WriteFile(file_name, fn=None, data=None, 63 mode=None, uid=-1, gid=-1, 64 atime=None, mtime=None, close=True, 65 dry_run=False, backup=False, 66 prewrite=None, postwrite=None):
67 """(Over)write a file atomically. 68 69 The file_name and either fn (a function taking one argument, the 70 file descriptor, and which should write the data to it) or data (the 71 contents of the file) must be passed. The other arguments are 72 optional and allow setting the file mode, owner and group, and the 73 mtime/atime of the file. 74 75 If the function doesn't raise an exception, it has succeeded and the 76 target file has the new contents. If the function has raised an 77 exception, an existing target file should be unmodified and the 78 temporary file should be removed. 79 80 @type file_name: str 81 @param file_name: the target filename 82 @type fn: callable 83 @param fn: content writing function, called with 84 file descriptor as parameter 85 @type data: str 86 @param data: contents of the file 87 @type mode: int 88 @param mode: file mode 89 @type uid: int 90 @param uid: the owner of the file 91 @type gid: int 92 @param gid: the group of the file 93 @type atime: int 94 @param atime: a custom access time to be set on the file 95 @type mtime: int 96 @param mtime: a custom modification time to be set on the file 97 @type close: boolean 98 @param close: whether to close file after writing it 99 @type prewrite: callable 100 @param prewrite: function to be called before writing content 101 @type postwrite: callable 102 @param postwrite: function to be called after writing content 103 104 @rtype: None or int 105 @return: None if the 'close' parameter evaluates to True, 106 otherwise the file descriptor 107 108 @raise errors.ProgrammerError: if any of the arguments are not valid 109 110 """ 111 if not os.path.isabs(file_name): 112 raise errors.ProgrammerError("Path passed to WriteFile is not" 113 " absolute: '%s'" % file_name) 114 115 if [fn, data].count(None) != 1: 116 raise errors.ProgrammerError("fn or data required") 117 118 if [atime, mtime].count(None) == 1: 119 raise errors.ProgrammerError("Both atime and mtime must be either" 120 " set or None") 121 122 if backup and not dry_run and os.path.isfile(file_name): 123 CreateBackup(file_name) 124 125 # Whether temporary file needs to be removed (e.g. if any error occurs) 126 do_remove = True 127 128 # Function result 129 result = None 130 131 (dir_name, base_name) = os.path.split(file_name) 132 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name, 133 dir=dir_name) 134 try: 135 try: 136 if uid != -1 or gid != -1: 137 os.chown(new_name, uid, gid) 138 if mode: 139 os.chmod(new_name, mode) 140 if callable(prewrite): 141 prewrite(fd) 142 if data is not None: 143 if isinstance(data, unicode): 144 data = data.encode() 145 assert isinstance(data, str) 146 to_write = len(data) 147 offset = 0 148 while offset < to_write: 149 written = os.write(fd, buffer(data, offset)) 150 assert written >= 0 151 assert written <= to_write - offset 152 offset += written 153 assert offset == to_write 154 else: 155 fn(fd) 156 if callable(postwrite): 157 postwrite(fd) 158 os.fsync(fd) 159 if atime is not None and mtime is not None: 160 os.utime(new_name, (atime, mtime)) 161 finally: 162 # Close file unless the file descriptor should be returned 163 if close: 164 os.close(fd) 165 else: 166 result = fd 167 168 # Rename file to destination name 169 if not dry_run: 170 os.rename(new_name, file_name) 171 # Successful, no need to remove anymore 172 do_remove = False 173 finally: 174 if do_remove: 175 RemoveFile(new_name) 176 177 return result
178 179
180 -def GetFileID(path=None, fd=None):
181 """Returns the file 'id', i.e. the dev/inode and mtime information. 182 183 Either the path to the file or the fd must be given. 184 185 @param path: the file path 186 @param fd: a file descriptor 187 @return: a tuple of (device number, inode number, mtime) 188 189 """ 190 if [path, fd].count(None) != 1: 191 raise errors.ProgrammerError("One and only one of fd/path must be given") 192 193 if fd is None: 194 st = os.stat(path) 195 else: 196 st = os.fstat(fd) 197 198 return (st.st_dev, st.st_ino, st.st_mtime)
199 200
201 -def VerifyFileID(fi_disk, fi_ours):
202 """Verifies that two file IDs are matching. 203 204 Differences in the inode/device are not accepted, but and older 205 timestamp for fi_disk is accepted. 206 207 @param fi_disk: tuple (dev, inode, mtime) representing the actual 208 file data 209 @param fi_ours: tuple (dev, inode, mtime) representing the last 210 written file data 211 @rtype: boolean 212 213 """ 214 (d1, i1, m1) = fi_disk 215 (d2, i2, m2) = fi_ours 216 217 return (d1, i1) == (d2, i2) and m1 <= m2
218 219
220 -def SafeWriteFile(file_name, file_id, **kwargs):
221 """Wraper over L{WriteFile} that locks the target file. 222 223 By keeping the target file locked during WriteFile, we ensure that 224 cooperating writers will safely serialise access to the file. 225 226 @type file_name: str 227 @param file_name: the target filename 228 @type file_id: tuple 229 @param file_id: a result from L{GetFileID} 230 231 """ 232 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT) 233 try: 234 filelock.LockFile(fd) 235 if file_id is not None: 236 disk_id = GetFileID(fd=fd) 237 if not VerifyFileID(disk_id, file_id): 238 raise errors.LockError("Cannot overwrite file %s, it has been modified" 239 " since last written" % file_name) 240 return WriteFile(file_name, **kwargs) 241 finally: 242 os.close(fd)
243 244
245 -def ReadOneLineFile(file_name, strict=False):
246 """Return the first non-empty line from a file. 247 248 @type strict: boolean 249 @param strict: if True, abort if the file has more than one 250 non-empty line 251 252 """ 253 file_lines = ReadFile(file_name).splitlines() 254 full_lines = filter(bool, file_lines) 255 if not file_lines or not full_lines: 256 raise errors.GenericError("No data in one-liner file %s" % file_name) 257 elif strict and len(full_lines) > 1: 258 raise errors.GenericError("Too many lines in one-liner file %s" % 259 file_name) 260 return full_lines[0]
261 262
263 -def RemoveFile(filename):
264 """Remove a file ignoring some errors. 265 266 Remove a file, ignoring non-existing ones or directories. Other 267 errors are passed. 268 269 @type filename: str 270 @param filename: the file to be removed 271 272 """ 273 try: 274 os.unlink(filename) 275 except OSError, err: 276 if err.errno not in (errno.ENOENT, errno.EISDIR): 277 raise
278 279
280 -def RemoveDir(dirname):
281 """Remove an empty directory. 282 283 Remove a directory, ignoring non-existing ones. 284 Other errors are passed. This includes the case, 285 where the directory is not empty, so it can't be removed. 286 287 @type dirname: str 288 @param dirname: the empty directory to be removed 289 290 """ 291 try: 292 os.rmdir(dirname) 293 except OSError, err: 294 if err.errno != errno.ENOENT: 295 raise
296 297
298 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None, 299 dir_gid=None):
300 """Renames a file. 301 302 @type old: string 303 @param old: Original path 304 @type new: string 305 @param new: New path 306 @type mkdir: bool 307 @param mkdir: Whether to create target directory if it doesn't exist 308 @type mkdir_mode: int 309 @param mkdir_mode: Mode for newly created directories 310 @type dir_uid: int 311 @param dir_uid: The uid for the (if fresh created) dir 312 @type dir_gid: int 313 @param dir_gid: The gid for the (if fresh created) dir 314 315 """ 316 try: 317 return os.rename(old, new) 318 except OSError, err: 319 # In at least one use case of this function, the job queue, directory 320 # creation is very rare. Checking for the directory before renaming is not 321 # as efficient. 322 if mkdir and err.errno == errno.ENOENT: 323 # Create directory and try again 324 dir_path = os.path.dirname(new) 325 Makedirs(dir_path, mode=mkdir_mode) 326 if not (dir_uid is None or dir_gid is None): 327 os.chown(dir_path, dir_uid, dir_gid) 328 329 return os.rename(old, new) 330 331 raise
332 333
334 -def Makedirs(path, mode=0750):
335 """Super-mkdir; create a leaf directory and all intermediate ones. 336 337 This is a wrapper around C{os.makedirs} adding error handling not implemented 338 before Python 2.5. 339 340 """ 341 try: 342 os.makedirs(path, mode) 343 except OSError, err: 344 # Ignore EEXIST. This is only handled in os.makedirs as included in 345 # Python 2.5 and above. 346 if err.errno != errno.EEXIST or not os.path.exists(path): 347 raise
348 349
350 -def TimestampForFilename():
351 """Returns the current time formatted for filenames. 352 353 The format doesn't contain colons as some shells and applications treat them 354 as separators. Uses the local timezone. 355 356 """ 357 return time.strftime("%Y-%m-%d_%H_%M_%S")
358 359
360 -def CreateBackup(file_name):
361 """Creates a backup of a file. 362 363 @type file_name: str 364 @param file_name: file to be backed up 365 @rtype: str 366 @return: the path to the newly created backup 367 @raise errors.ProgrammerError: for invalid file names 368 369 """ 370 if not os.path.isfile(file_name): 371 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % 372 file_name) 373 374 prefix = ("%s.backup-%s." % 375 (os.path.basename(file_name), TimestampForFilename())) 376 dir_name = os.path.dirname(file_name) 377 378 fsrc = open(file_name, "rb") 379 try: 380 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) 381 fdst = os.fdopen(fd, "wb") 382 try: 383 logging.debug("Backing up %s at %s", file_name, backup_name) 384 shutil.copyfileobj(fsrc, fdst) 385 finally: 386 fdst.close() 387 finally: 388 fsrc.close() 389 390 return backup_name
391 392
393 -def ListVisibleFiles(path):
394 """Returns a list of visible files in a directory. 395 396 @type path: str 397 @param path: the directory to enumerate 398 @rtype: list 399 @return: the list of all files not starting with a dot 400 @raise ProgrammerError: if L{path} is not an absolue and normalized path 401 402 """ 403 if not IsNormAbsPath(path): 404 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" 405 " absolute/normalized: '%s'" % path) 406 files = [i for i in os.listdir(path) if not i.startswith(".")] 407 return files
408 409
410 -def EnsureDirs(dirs):
411 """Make required directories, if they don't exist. 412 413 @param dirs: list of tuples (dir_name, dir_mode) 414 @type dirs: list of (string, integer) 415 416 """ 417 for dir_name, dir_mode in dirs: 418 try: 419 os.mkdir(dir_name, dir_mode) 420 except EnvironmentError, err: 421 if err.errno != errno.EEXIST: 422 raise errors.GenericError("Cannot create needed directory" 423 " '%s': %s" % (dir_name, err)) 424 try: 425 os.chmod(dir_name, dir_mode) 426 except EnvironmentError, err: 427 raise errors.GenericError("Cannot change directory permissions on" 428 " '%s': %s" % (dir_name, err)) 429 if not os.path.isdir(dir_name): 430 raise errors.GenericError("%s is not a directory" % dir_name)
431 432
433 -def FindFile(name, search_path, test=os.path.exists):
434 """Look for a filesystem object in a given path. 435 436 This is an abstract method to search for filesystem object (files, 437 dirs) under a given search path. 438 439 @type name: str 440 @param name: the name to look for 441 @type search_path: str 442 @param search_path: location to start at 443 @type test: callable 444 @param test: a function taking one argument that should return True 445 if the a given object is valid; the default value is 446 os.path.exists, causing only existing files to be returned 447 @rtype: str or None 448 @return: full path to the object if found, None otherwise 449 450 """ 451 # validate the filename mask 452 if constants.EXT_PLUGIN_MASK.match(name) is None: 453 logging.critical("Invalid value passed for external script name: '%s'", 454 name) 455 return None 456 457 for dir_name in search_path: 458 # FIXME: investigate switch to PathJoin 459 item_name = os.path.sep.join([dir_name, name]) 460 # check the user test and that we're indeed resolving to the given 461 # basename 462 if test(item_name) and os.path.basename(item_name) == name: 463 return item_name 464 return None
465 466
467 -def IsNormAbsPath(path):
468 """Check whether a path is absolute and also normalized 469 470 This avoids things like /dir/../../other/path to be valid. 471 472 """ 473 return os.path.normpath(path) == path and os.path.isabs(path)
474 475
476 -def PathJoin(*args):
477 """Safe-join a list of path components. 478 479 Requirements: 480 - the first argument must be an absolute path 481 - no component in the path must have backtracking (e.g. /../), 482 since we check for normalization at the end 483 484 @param args: the path components to be joined 485 @raise ValueError: for invalid paths 486 487 """ 488 # ensure we're having at least one path passed in 489 assert args 490 # ensure the first component is an absolute and normalized path name 491 root = args[0] 492 if not IsNormAbsPath(root): 493 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) 494 result = os.path.join(*args) 495 # ensure that the whole path is normalized 496 if not IsNormAbsPath(result): 497 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) 498 # check that we're still under the original prefix 499 prefix = os.path.commonprefix([root, result]) 500 if prefix != root: 501 raise ValueError("Error: path joining resulted in different prefix" 502 " (%s != %s)" % (prefix, root)) 503 return result
504 505
506 -def TailFile(fname, lines=20):
507 """Return the last lines from a file. 508 509 @note: this function will only read and parse the last 4KB of 510 the file; if the lines are very long, it could be that less 511 than the requested number of lines are returned 512 513 @param fname: the file name 514 @type lines: int 515 @param lines: the (maximum) number of lines to return 516 517 """ 518 fd = open(fname, "r") 519 try: 520 fd.seek(0, 2) 521 pos = fd.tell() 522 pos = max(0, pos - 4096) 523 fd.seek(pos, 0) 524 raw_data = fd.read() 525 finally: 526 fd.close() 527 528 rows = raw_data.splitlines() 529 return rows[-lines:]
530 531
532 -def BytesToMebibyte(value):
533 """Converts bytes to mebibytes. 534 535 @type value: int 536 @param value: Value in bytes 537 @rtype: int 538 @return: Value in mebibytes 539 540 """ 541 return int(round(value / (1024.0 * 1024.0), 0))
542 543
544 -def CalculateDirectorySize(path):
545 """Calculates the size of a directory recursively. 546 547 @type path: string 548 @param path: Path to directory 549 @rtype: int 550 @return: Size in mebibytes 551 552 """ 553 size = 0 554 555 for (curpath, _, files) in os.walk(path): 556 for filename in files: 557 st = os.lstat(PathJoin(curpath, filename)) 558 size += st.st_size 559 560 return BytesToMebibyte(size)
561 562
563 -def GetFilesystemStats(path):
564 """Returns the total and free space on a filesystem. 565 566 @type path: string 567 @param path: Path on filesystem to be examined 568 @rtype: int 569 @return: tuple of (Total space, Free space) in mebibytes 570 571 """ 572 st = os.statvfs(path) 573 574 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize) 575 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize) 576 return (tsize, fsize)
577 578
579 -def ReadPidFile(pidfile):
580 """Read a pid from a file. 581 582 @type pidfile: string 583 @param pidfile: path to the file containing the pid 584 @rtype: int 585 @return: The process id, if the file exists and contains a valid PID, 586 otherwise 0 587 588 """ 589 try: 590 raw_data = ReadOneLineFile(pidfile) 591 except EnvironmentError, err: 592 if err.errno != errno.ENOENT: 593 logging.exception("Can't read pid file") 594 return 0 595 596 try: 597 pid = int(raw_data) 598 except (TypeError, ValueError), err: 599 logging.info("Can't parse pid file contents", exc_info=True) 600 return 0 601 602 return pid
603 604
605 -def ReadLockedPidFile(path):
606 """Reads a locked PID file. 607 608 This can be used together with L{utils.process.StartDaemon}. 609 610 @type path: string 611 @param path: Path to PID file 612 @return: PID as integer or, if file was unlocked or couldn't be opened, None 613 614 """ 615 try: 616 fd = os.open(path, os.O_RDONLY) 617 except EnvironmentError, err: 618 if err.errno == errno.ENOENT: 619 # PID file doesn't exist 620 return None 621 raise 622 623 try: 624 try: 625 # Try to acquire lock 626 filelock.LockFile(fd) 627 except errors.LockError: 628 # Couldn't lock, daemon is running 629 return int(os.read(fd, 100)) 630 finally: 631 os.close(fd) 632 633 return None
634 635
636 -def AddAuthorizedKey(file_obj, key):
637 """Adds an SSH public key to an authorized_keys file. 638 639 @type file_obj: str or file handle 640 @param file_obj: path to authorized_keys file 641 @type key: str 642 @param key: string containing key 643 644 """ 645 key_fields = key.split() 646 647 if isinstance(file_obj, basestring): 648 f = open(file_obj, "a+") 649 else: 650 f = file_obj 651 652 try: 653 nl = True 654 for line in f: 655 # Ignore whitespace changes 656 if line.split() == key_fields: 657 break 658 nl = line.endswith("\n") 659 else: 660 if not nl: 661 f.write("\n") 662 f.write(key.rstrip("\r\n")) 663 f.write("\n") 664 f.flush() 665 finally: 666 f.close()
667 668
669 -def RemoveAuthorizedKey(file_name, key):
670 """Removes an SSH public key from an authorized_keys file. 671 672 @type file_name: str 673 @param file_name: path to authorized_keys file 674 @type key: str 675 @param key: string containing key 676 677 """ 678 key_fields = key.split() 679 680 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name)) 681 try: 682 out = os.fdopen(fd, "w") 683 try: 684 f = open(file_name, "r") 685 try: 686 for line in f: 687 # Ignore whitespace changes while comparing lines 688 if line.split() != key_fields: 689 out.write(line) 690 691 out.flush() 692 os.rename(tmpname, file_name) 693 finally: 694 f.close() 695 finally: 696 out.close() 697 except: 698 RemoveFile(tmpname) 699 raise
700 701
702 -def DaemonPidFileName(name):
703 """Compute a ganeti pid file absolute path 704 705 @type name: str 706 @param name: the daemon name 707 @rtype: str 708 @return: the full path to the pidfile corresponding to the given 709 daemon name 710 711 """ 712 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
713 714
715 -def WritePidFile(pidfile):
716 """Write the current process pidfile. 717 718 @type pidfile: string 719 @param pidfile: the path to the file to be written 720 @raise errors.LockError: if the pid file already exists and 721 points to a live process 722 @rtype: int 723 @return: the file descriptor of the lock file; do not close this unless 724 you want to unlock the pid file 725 726 """ 727 # We don't rename nor truncate the file to not drop locks under 728 # existing processes 729 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600) 730 731 # Lock the PID file (and fail if not possible to do so). Any code 732 # wanting to send a signal to the daemon should try to lock the PID 733 # file before reading it. If acquiring the lock succeeds, the daemon is 734 # no longer running and the signal should not be sent. 735 filelock.LockFile(fd_pidfile) 736 737 os.write(fd_pidfile, "%d\n" % os.getpid()) 738 739 return fd_pidfile
740 741
742 -def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
743 """Reads the watcher pause file. 744 745 @type filename: string 746 @param filename: Path to watcher pause file 747 @type now: None, float or int 748 @param now: Current time as Unix timestamp 749 @type remove_after: int 750 @param remove_after: Remove watcher pause file after specified amount of 751 seconds past the pause end time 752 753 """ 754 if now is None: 755 now = time.time() 756 757 try: 758 value = ReadFile(filename) 759 except IOError, err: 760 if err.errno != errno.ENOENT: 761 raise 762 value = None 763 764 if value is not None: 765 try: 766 value = int(value) 767 except ValueError: 768 logging.warning(("Watcher pause file (%s) contains invalid value," 769 " removing it"), filename) 770 RemoveFile(filename) 771 value = None 772 773 if value is not None: 774 # Remove file if it's outdated 775 if now > (value + remove_after): 776 RemoveFile(filename) 777 value = None 778 779 elif now > value: 780 value = None 781 782 return value
783 784
785 -def NewUUID():
786 """Returns a random UUID. 787 788 @note: This is a Linux-specific method as it uses the /proc 789 filesystem. 790 @rtype: str 791 792 """ 793 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
794