1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 """Utility functions for I/O.
22
23 """
24
25 import os
26 import logging
27 import shutil
28 import tempfile
29 import errno
30 import time
31
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
35
36
37
38 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
39
40
41 -def ReadFile(file_name, size=-1, preread=None):
42 """Reads a file.
43
44 @type size: int
45 @param size: Read at most size bytes (if negative, entire file)
46 @type preread: callable receiving file handle as single parameter
47 @param preread: Function called before file is read
48 @rtype: str
49 @return: the (possibly partial) content of the file
50
51 """
52 f = open(file_name, "r")
53 try:
54 if preread:
55 preread(f)
56
57 return f.read(size)
58 finally:
59 f.close()
60
61
62 -def WriteFile(file_name, fn=None, data=None,
63 mode=None, uid=-1, gid=-1,
64 atime=None, mtime=None, close=True,
65 dry_run=False, backup=False,
66 prewrite=None, postwrite=None):
67 """(Over)write a file atomically.
68
69 The file_name and either fn (a function taking one argument, the
70 file descriptor, and which should write the data to it) or data (the
71 contents of the file) must be passed. The other arguments are
72 optional and allow setting the file mode, owner and group, and the
73 mtime/atime of the file.
74
75 If the function doesn't raise an exception, it has succeeded and the
76 target file has the new contents. If the function has raised an
77 exception, an existing target file should be unmodified and the
78 temporary file should be removed.
79
80 @type file_name: str
81 @param file_name: the target filename
82 @type fn: callable
83 @param fn: content writing function, called with
84 file descriptor as parameter
85 @type data: str
86 @param data: contents of the file
87 @type mode: int
88 @param mode: file mode
89 @type uid: int
90 @param uid: the owner of the file
91 @type gid: int
92 @param gid: the group of the file
93 @type atime: int
94 @param atime: a custom access time to be set on the file
95 @type mtime: int
96 @param mtime: a custom modification time to be set on the file
97 @type close: boolean
98 @param close: whether to close file after writing it
99 @type prewrite: callable
100 @param prewrite: function to be called before writing content
101 @type postwrite: callable
102 @param postwrite: function to be called after writing content
103
104 @rtype: None or int
105 @return: None if the 'close' parameter evaluates to True,
106 otherwise the file descriptor
107
108 @raise errors.ProgrammerError: if any of the arguments are not valid
109
110 """
111 if not os.path.isabs(file_name):
112 raise errors.ProgrammerError("Path passed to WriteFile is not"
113 " absolute: '%s'" % file_name)
114
115 if [fn, data].count(None) != 1:
116 raise errors.ProgrammerError("fn or data required")
117
118 if [atime, mtime].count(None) == 1:
119 raise errors.ProgrammerError("Both atime and mtime must be either"
120 " set or None")
121
122 if backup and not dry_run and os.path.isfile(file_name):
123 CreateBackup(file_name)
124
125
126 do_remove = True
127
128
129 result = None
130
131 (dir_name, base_name) = os.path.split(file_name)
132 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
133 dir=dir_name)
134 try:
135 try:
136 if uid != -1 or gid != -1:
137 os.chown(new_name, uid, gid)
138 if mode:
139 os.chmod(new_name, mode)
140 if callable(prewrite):
141 prewrite(fd)
142 if data is not None:
143 if isinstance(data, unicode):
144 data = data.encode()
145 assert isinstance(data, str)
146 to_write = len(data)
147 offset = 0
148 while offset < to_write:
149 written = os.write(fd, buffer(data, offset))
150 assert written >= 0
151 assert written <= to_write - offset
152 offset += written
153 assert offset == to_write
154 else:
155 fn(fd)
156 if callable(postwrite):
157 postwrite(fd)
158 os.fsync(fd)
159 if atime is not None and mtime is not None:
160 os.utime(new_name, (atime, mtime))
161 finally:
162
163 if close:
164 os.close(fd)
165 else:
166 result = fd
167
168
169 if not dry_run:
170 os.rename(new_name, file_name)
171
172 do_remove = False
173 finally:
174 if do_remove:
175 RemoveFile(new_name)
176
177 return result
178
179
181 """Returns the file 'id', i.e. the dev/inode and mtime information.
182
183 Either the path to the file or the fd must be given.
184
185 @param path: the file path
186 @param fd: a file descriptor
187 @return: a tuple of (device number, inode number, mtime)
188
189 """
190 if [path, fd].count(None) != 1:
191 raise errors.ProgrammerError("One and only one of fd/path must be given")
192
193 if fd is None:
194 st = os.stat(path)
195 else:
196 st = os.fstat(fd)
197
198 return (st.st_dev, st.st_ino, st.st_mtime)
199
200
202 """Verifies that two file IDs are matching.
203
204 Differences in the inode/device are not accepted, but and older
205 timestamp for fi_disk is accepted.
206
207 @param fi_disk: tuple (dev, inode, mtime) representing the actual
208 file data
209 @param fi_ours: tuple (dev, inode, mtime) representing the last
210 written file data
211 @rtype: boolean
212
213 """
214 (d1, i1, m1) = fi_disk
215 (d2, i2, m2) = fi_ours
216
217 return (d1, i1) == (d2, i2) and m1 <= m2
218
219
221 """Wraper over L{WriteFile} that locks the target file.
222
223 By keeping the target file locked during WriteFile, we ensure that
224 cooperating writers will safely serialise access to the file.
225
226 @type file_name: str
227 @param file_name: the target filename
228 @type file_id: tuple
229 @param file_id: a result from L{GetFileID}
230
231 """
232 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
233 try:
234 filelock.LockFile(fd)
235 if file_id is not None:
236 disk_id = GetFileID(fd=fd)
237 if not VerifyFileID(disk_id, file_id):
238 raise errors.LockError("Cannot overwrite file %s, it has been modified"
239 " since last written" % file_name)
240 return WriteFile(file_name, **kwargs)
241 finally:
242 os.close(fd)
243
244
246 """Return the first non-empty line from a file.
247
248 @type strict: boolean
249 @param strict: if True, abort if the file has more than one
250 non-empty line
251
252 """
253 file_lines = ReadFile(file_name).splitlines()
254 full_lines = filter(bool, file_lines)
255 if not file_lines or not full_lines:
256 raise errors.GenericError("No data in one-liner file %s" % file_name)
257 elif strict and len(full_lines) > 1:
258 raise errors.GenericError("Too many lines in one-liner file %s" %
259 file_name)
260 return full_lines[0]
261
262
264 """Remove a file ignoring some errors.
265
266 Remove a file, ignoring non-existing ones or directories. Other
267 errors are passed.
268
269 @type filename: str
270 @param filename: the file to be removed
271
272 """
273 try:
274 os.unlink(filename)
275 except OSError, err:
276 if err.errno not in (errno.ENOENT, errno.EISDIR):
277 raise
278
279
281 """Remove an empty directory.
282
283 Remove a directory, ignoring non-existing ones.
284 Other errors are passed. This includes the case,
285 where the directory is not empty, so it can't be removed.
286
287 @type dirname: str
288 @param dirname: the empty directory to be removed
289
290 """
291 try:
292 os.rmdir(dirname)
293 except OSError, err:
294 if err.errno != errno.ENOENT:
295 raise
296
297
298 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
299 dir_gid=None):
300 """Renames a file.
301
302 @type old: string
303 @param old: Original path
304 @type new: string
305 @param new: New path
306 @type mkdir: bool
307 @param mkdir: Whether to create target directory if it doesn't exist
308 @type mkdir_mode: int
309 @param mkdir_mode: Mode for newly created directories
310 @type dir_uid: int
311 @param dir_uid: The uid for the (if fresh created) dir
312 @type dir_gid: int
313 @param dir_gid: The gid for the (if fresh created) dir
314
315 """
316 try:
317 return os.rename(old, new)
318 except OSError, err:
319
320
321
322 if mkdir and err.errno == errno.ENOENT:
323
324 dir_path = os.path.dirname(new)
325 Makedirs(dir_path, mode=mkdir_mode)
326 if not (dir_uid is None or dir_gid is None):
327 os.chown(dir_path, dir_uid, dir_gid)
328
329 return os.rename(old, new)
330
331 raise
332
333
335 """Super-mkdir; create a leaf directory and all intermediate ones.
336
337 This is a wrapper around C{os.makedirs} adding error handling not implemented
338 before Python 2.5.
339
340 """
341 try:
342 os.makedirs(path, mode)
343 except OSError, err:
344
345
346 if err.errno != errno.EEXIST or not os.path.exists(path):
347 raise
348
349
351 """Returns the current time formatted for filenames.
352
353 The format doesn't contain colons as some shells and applications treat them
354 as separators. Uses the local timezone.
355
356 """
357 return time.strftime("%Y-%m-%d_%H_%M_%S")
358
359
361 """Creates a backup of a file.
362
363 @type file_name: str
364 @param file_name: file to be backed up
365 @rtype: str
366 @return: the path to the newly created backup
367 @raise errors.ProgrammerError: for invalid file names
368
369 """
370 if not os.path.isfile(file_name):
371 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
372 file_name)
373
374 prefix = ("%s.backup-%s." %
375 (os.path.basename(file_name), TimestampForFilename()))
376 dir_name = os.path.dirname(file_name)
377
378 fsrc = open(file_name, "rb")
379 try:
380 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
381 fdst = os.fdopen(fd, "wb")
382 try:
383 logging.debug("Backing up %s at %s", file_name, backup_name)
384 shutil.copyfileobj(fsrc, fdst)
385 finally:
386 fdst.close()
387 finally:
388 fsrc.close()
389
390 return backup_name
391
392
394 """Returns a list of visible files in a directory.
395
396 @type path: str
397 @param path: the directory to enumerate
398 @rtype: list
399 @return: the list of all files not starting with a dot
400 @raise ProgrammerError: if L{path} is not an absolue and normalized path
401
402 """
403 if not IsNormAbsPath(path):
404 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
405 " absolute/normalized: '%s'" % path)
406 files = [i for i in os.listdir(path) if not i.startswith(".")]
407 return files
408
409
411 """Make required directories, if they don't exist.
412
413 @param dirs: list of tuples (dir_name, dir_mode)
414 @type dirs: list of (string, integer)
415
416 """
417 for dir_name, dir_mode in dirs:
418 try:
419 os.mkdir(dir_name, dir_mode)
420 except EnvironmentError, err:
421 if err.errno != errno.EEXIST:
422 raise errors.GenericError("Cannot create needed directory"
423 " '%s': %s" % (dir_name, err))
424 try:
425 os.chmod(dir_name, dir_mode)
426 except EnvironmentError, err:
427 raise errors.GenericError("Cannot change directory permissions on"
428 " '%s': %s" % (dir_name, err))
429 if not os.path.isdir(dir_name):
430 raise errors.GenericError("%s is not a directory" % dir_name)
431
432
433 -def FindFile(name, search_path, test=os.path.exists):
434 """Look for a filesystem object in a given path.
435
436 This is an abstract method to search for filesystem object (files,
437 dirs) under a given search path.
438
439 @type name: str
440 @param name: the name to look for
441 @type search_path: str
442 @param search_path: location to start at
443 @type test: callable
444 @param test: a function taking one argument that should return True
445 if the a given object is valid; the default value is
446 os.path.exists, causing only existing files to be returned
447 @rtype: str or None
448 @return: full path to the object if found, None otherwise
449
450 """
451
452 if constants.EXT_PLUGIN_MASK.match(name) is None:
453 logging.critical("Invalid value passed for external script name: '%s'",
454 name)
455 return None
456
457 for dir_name in search_path:
458
459 item_name = os.path.sep.join([dir_name, name])
460
461
462 if test(item_name) and os.path.basename(item_name) == name:
463 return item_name
464 return None
465
466
468 """Check whether a path is absolute and also normalized
469
470 This avoids things like /dir/../../other/path to be valid.
471
472 """
473 return os.path.normpath(path) == path and os.path.isabs(path)
474
475
477 """Safe-join a list of path components.
478
479 Requirements:
480 - the first argument must be an absolute path
481 - no component in the path must have backtracking (e.g. /../),
482 since we check for normalization at the end
483
484 @param args: the path components to be joined
485 @raise ValueError: for invalid paths
486
487 """
488
489 assert args
490
491 root = args[0]
492 if not IsNormAbsPath(root):
493 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
494 result = os.path.join(*args)
495
496 if not IsNormAbsPath(result):
497 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
498
499 prefix = os.path.commonprefix([root, result])
500 if prefix != root:
501 raise ValueError("Error: path joining resulted in different prefix"
502 " (%s != %s)" % (prefix, root))
503 return result
504
505
507 """Return the last lines from a file.
508
509 @note: this function will only read and parse the last 4KB of
510 the file; if the lines are very long, it could be that less
511 than the requested number of lines are returned
512
513 @param fname: the file name
514 @type lines: int
515 @param lines: the (maximum) number of lines to return
516
517 """
518 fd = open(fname, "r")
519 try:
520 fd.seek(0, 2)
521 pos = fd.tell()
522 pos = max(0, pos - 4096)
523 fd.seek(pos, 0)
524 raw_data = fd.read()
525 finally:
526 fd.close()
527
528 rows = raw_data.splitlines()
529 return rows[-lines:]
530
531
533 """Converts bytes to mebibytes.
534
535 @type value: int
536 @param value: Value in bytes
537 @rtype: int
538 @return: Value in mebibytes
539
540 """
541 return int(round(value / (1024.0 * 1024.0), 0))
542
543
545 """Calculates the size of a directory recursively.
546
547 @type path: string
548 @param path: Path to directory
549 @rtype: int
550 @return: Size in mebibytes
551
552 """
553 size = 0
554
555 for (curpath, _, files) in os.walk(path):
556 for filename in files:
557 st = os.lstat(PathJoin(curpath, filename))
558 size += st.st_size
559
560 return BytesToMebibyte(size)
561
562
564 """Returns the total and free space on a filesystem.
565
566 @type path: string
567 @param path: Path on filesystem to be examined
568 @rtype: int
569 @return: tuple of (Total space, Free space) in mebibytes
570
571 """
572 st = os.statvfs(path)
573
574 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
575 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
576 return (tsize, fsize)
577
578
580 """Read a pid from a file.
581
582 @type pidfile: string
583 @param pidfile: path to the file containing the pid
584 @rtype: int
585 @return: The process id, if the file exists and contains a valid PID,
586 otherwise 0
587
588 """
589 try:
590 raw_data = ReadOneLineFile(pidfile)
591 except EnvironmentError, err:
592 if err.errno != errno.ENOENT:
593 logging.exception("Can't read pid file")
594 return 0
595
596 try:
597 pid = int(raw_data)
598 except (TypeError, ValueError), err:
599 logging.info("Can't parse pid file contents", exc_info=True)
600 return 0
601
602 return pid
603
604
606 """Reads a locked PID file.
607
608 This can be used together with L{utils.process.StartDaemon}.
609
610 @type path: string
611 @param path: Path to PID file
612 @return: PID as integer or, if file was unlocked or couldn't be opened, None
613
614 """
615 try:
616 fd = os.open(path, os.O_RDONLY)
617 except EnvironmentError, err:
618 if err.errno == errno.ENOENT:
619
620 return None
621 raise
622
623 try:
624 try:
625
626 filelock.LockFile(fd)
627 except errors.LockError:
628
629 return int(os.read(fd, 100))
630 finally:
631 os.close(fd)
632
633 return None
634
635
637 """Adds an SSH public key to an authorized_keys file.
638
639 @type file_obj: str or file handle
640 @param file_obj: path to authorized_keys file
641 @type key: str
642 @param key: string containing key
643
644 """
645 key_fields = key.split()
646
647 if isinstance(file_obj, basestring):
648 f = open(file_obj, "a+")
649 else:
650 f = file_obj
651
652 try:
653 nl = True
654 for line in f:
655
656 if line.split() == key_fields:
657 break
658 nl = line.endswith("\n")
659 else:
660 if not nl:
661 f.write("\n")
662 f.write(key.rstrip("\r\n"))
663 f.write("\n")
664 f.flush()
665 finally:
666 f.close()
667
668
670 """Removes an SSH public key from an authorized_keys file.
671
672 @type file_name: str
673 @param file_name: path to authorized_keys file
674 @type key: str
675 @param key: string containing key
676
677 """
678 key_fields = key.split()
679
680 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
681 try:
682 out = os.fdopen(fd, "w")
683 try:
684 f = open(file_name, "r")
685 try:
686 for line in f:
687
688 if line.split() != key_fields:
689 out.write(line)
690
691 out.flush()
692 os.rename(tmpname, file_name)
693 finally:
694 f.close()
695 finally:
696 out.close()
697 except:
698 RemoveFile(tmpname)
699 raise
700
701
703 """Compute a ganeti pid file absolute path
704
705 @type name: str
706 @param name: the daemon name
707 @rtype: str
708 @return: the full path to the pidfile corresponding to the given
709 daemon name
710
711 """
712 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
713
714
716 """Write the current process pidfile.
717
718 @type pidfile: string
719 @param pidfile: the path to the file to be written
720 @raise errors.LockError: if the pid file already exists and
721 points to a live process
722 @rtype: int
723 @return: the file descriptor of the lock file; do not close this unless
724 you want to unlock the pid file
725
726 """
727
728
729 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
730
731
732
733
734
735 filelock.LockFile(fd_pidfile)
736
737 os.write(fd_pidfile, "%d\n" % os.getpid())
738
739 return fd_pidfile
740
741
743 """Reads the watcher pause file.
744
745 @type filename: string
746 @param filename: Path to watcher pause file
747 @type now: None, float or int
748 @param now: Current time as Unix timestamp
749 @type remove_after: int
750 @param remove_after: Remove watcher pause file after specified amount of
751 seconds past the pause end time
752
753 """
754 if now is None:
755 now = time.time()
756
757 try:
758 value = ReadFile(filename)
759 except IOError, err:
760 if err.errno != errno.ENOENT:
761 raise
762 value = None
763
764 if value is not None:
765 try:
766 value = int(value)
767 except ValueError:
768 logging.warning(("Watcher pause file (%s) contains invalid value,"
769 " removing it"), filename)
770 RemoveFile(filename)
771 value = None
772
773 if value is not None:
774
775 if now > (value + remove_after):
776 RemoveFile(filename)
777 value = None
778
779 elif now > value:
780 value = None
781
782 return value
783
784
786 """Returns a random UUID.
787
788 @note: This is a Linux-specific method as it uses the /proc
789 filesystem.
790 @rtype: str
791
792 """
793 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
794