1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 """Utility functions for I/O.
22
23 """
24
25 import os
26 import logging
27 import shutil
28 import tempfile
29 import errno
30 import time
31
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
35
36
37
38 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
39
40
42 """Reads a file.
43
44 @type size: int
45 @param size: Read at most size bytes (if negative, entire file)
46 @rtype: str
47 @return: the (possibly partial) content of the file
48
49 """
50 f = open(file_name, "r")
51 try:
52 return f.read(size)
53 finally:
54 f.close()
55
56
57 -def WriteFile(file_name, fn=None, data=None,
58 mode=None, uid=-1, gid=-1,
59 atime=None, mtime=None, close=True,
60 dry_run=False, backup=False,
61 prewrite=None, postwrite=None):
62 """(Over)write a file atomically.
63
64 The file_name and either fn (a function taking one argument, the
65 file descriptor, and which should write the data to it) or data (the
66 contents of the file) must be passed. The other arguments are
67 optional and allow setting the file mode, owner and group, and the
68 mtime/atime of the file.
69
70 If the function doesn't raise an exception, it has succeeded and the
71 target file has the new contents. If the function has raised an
72 exception, an existing target file should be unmodified and the
73 temporary file should be removed.
74
75 @type file_name: str
76 @param file_name: the target filename
77 @type fn: callable
78 @param fn: content writing function, called with
79 file descriptor as parameter
80 @type data: str
81 @param data: contents of the file
82 @type mode: int
83 @param mode: file mode
84 @type uid: int
85 @param uid: the owner of the file
86 @type gid: int
87 @param gid: the group of the file
88 @type atime: int
89 @param atime: a custom access time to be set on the file
90 @type mtime: int
91 @param mtime: a custom modification time to be set on the file
92 @type close: boolean
93 @param close: whether to close file after writing it
94 @type prewrite: callable
95 @param prewrite: function to be called before writing content
96 @type postwrite: callable
97 @param postwrite: function to be called after writing content
98
99 @rtype: None or int
100 @return: None if the 'close' parameter evaluates to True,
101 otherwise the file descriptor
102
103 @raise errors.ProgrammerError: if any of the arguments are not valid
104
105 """
106 if not os.path.isabs(file_name):
107 raise errors.ProgrammerError("Path passed to WriteFile is not"
108 " absolute: '%s'" % file_name)
109
110 if [fn, data].count(None) != 1:
111 raise errors.ProgrammerError("fn or data required")
112
113 if [atime, mtime].count(None) == 1:
114 raise errors.ProgrammerError("Both atime and mtime must be either"
115 " set or None")
116
117 if backup and not dry_run and os.path.isfile(file_name):
118 CreateBackup(file_name)
119
120
121 do_remove = True
122
123
124 result = None
125
126 (dir_name, base_name) = os.path.split(file_name)
127 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
128 dir=dir_name)
129 try:
130 try:
131 if uid != -1 or gid != -1:
132 os.chown(new_name, uid, gid)
133 if mode:
134 os.chmod(new_name, mode)
135 if callable(prewrite):
136 prewrite(fd)
137 if data is not None:
138 if isinstance(data, unicode):
139 data = data.encode()
140 assert isinstance(data, str)
141 to_write = len(data)
142 offset = 0
143 while offset < to_write:
144 written = os.write(fd, buffer(data, offset))
145 assert written >= 0
146 assert written <= to_write - offset
147 offset += written
148 assert offset == to_write
149 else:
150 fn(fd)
151 if callable(postwrite):
152 postwrite(fd)
153 os.fsync(fd)
154 if atime is not None and mtime is not None:
155 os.utime(new_name, (atime, mtime))
156 finally:
157
158 if close:
159 os.close(fd)
160 else:
161 result = fd
162
163
164 if not dry_run:
165 os.rename(new_name, file_name)
166
167 do_remove = False
168 finally:
169 if do_remove:
170 RemoveFile(new_name)
171
172 return result
173
174
176 """Returns the file 'id', i.e. the dev/inode and mtime information.
177
178 Either the path to the file or the fd must be given.
179
180 @param path: the file path
181 @param fd: a file descriptor
182 @return: a tuple of (device number, inode number, mtime)
183
184 """
185 if [path, fd].count(None) != 1:
186 raise errors.ProgrammerError("One and only one of fd/path must be given")
187
188 if fd is None:
189 st = os.stat(path)
190 else:
191 st = os.fstat(fd)
192
193 return (st.st_dev, st.st_ino, st.st_mtime)
194
195
197 """Verifies that two file IDs are matching.
198
199 Differences in the inode/device are not accepted, but and older
200 timestamp for fi_disk is accepted.
201
202 @param fi_disk: tuple (dev, inode, mtime) representing the actual
203 file data
204 @param fi_ours: tuple (dev, inode, mtime) representing the last
205 written file data
206 @rtype: boolean
207
208 """
209 (d1, i1, m1) = fi_disk
210 (d2, i2, m2) = fi_ours
211
212 return (d1, i1) == (d2, i2) and m1 <= m2
213
214
216 """Wraper over L{WriteFile} that locks the target file.
217
218 By keeping the target file locked during WriteFile, we ensure that
219 cooperating writers will safely serialise access to the file.
220
221 @type file_name: str
222 @param file_name: the target filename
223 @type file_id: tuple
224 @param file_id: a result from L{GetFileID}
225
226 """
227 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
228 try:
229 filelock.LockFile(fd)
230 if file_id is not None:
231 disk_id = GetFileID(fd=fd)
232 if not VerifyFileID(disk_id, file_id):
233 raise errors.LockError("Cannot overwrite file %s, it has been modified"
234 " since last written" % file_name)
235 return WriteFile(file_name, **kwargs)
236 finally:
237 os.close(fd)
238
239
241 """Return the first non-empty line from a file.
242
243 @type strict: boolean
244 @param strict: if True, abort if the file has more than one
245 non-empty line
246
247 """
248 file_lines = ReadFile(file_name).splitlines()
249 full_lines = filter(bool, file_lines)
250 if not file_lines or not full_lines:
251 raise errors.GenericError("No data in one-liner file %s" % file_name)
252 elif strict and len(full_lines) > 1:
253 raise errors.GenericError("Too many lines in one-liner file %s" %
254 file_name)
255 return full_lines[0]
256
257
259 """Remove a file ignoring some errors.
260
261 Remove a file, ignoring non-existing ones or directories. Other
262 errors are passed.
263
264 @type filename: str
265 @param filename: the file to be removed
266
267 """
268 try:
269 os.unlink(filename)
270 except OSError, err:
271 if err.errno not in (errno.ENOENT, errno.EISDIR):
272 raise
273
274
276 """Remove an empty directory.
277
278 Remove a directory, ignoring non-existing ones.
279 Other errors are passed. This includes the case,
280 where the directory is not empty, so it can't be removed.
281
282 @type dirname: str
283 @param dirname: the empty directory to be removed
284
285 """
286 try:
287 os.rmdir(dirname)
288 except OSError, err:
289 if err.errno != errno.ENOENT:
290 raise
291
292
293 -def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
294 dir_gid=None):
295 """Renames a file.
296
297 @type old: string
298 @param old: Original path
299 @type new: string
300 @param new: New path
301 @type mkdir: bool
302 @param mkdir: Whether to create target directory if it doesn't exist
303 @type mkdir_mode: int
304 @param mkdir_mode: Mode for newly created directories
305 @type dir_uid: int
306 @param dir_uid: The uid for the (if fresh created) dir
307 @type dir_gid: int
308 @param dir_gid: The gid for the (if fresh created) dir
309
310 """
311 try:
312 return os.rename(old, new)
313 except OSError, err:
314
315
316
317 if mkdir and err.errno == errno.ENOENT:
318
319 dir_path = os.path.dirname(new)
320 Makedirs(dir_path, mode=mkdir_mode)
321 if not (dir_uid is None or dir_gid is None):
322 os.chown(dir_path, dir_uid, dir_gid)
323
324 return os.rename(old, new)
325
326 raise
327
328
330 """Super-mkdir; create a leaf directory and all intermediate ones.
331
332 This is a wrapper around C{os.makedirs} adding error handling not implemented
333 before Python 2.5.
334
335 """
336 try:
337 os.makedirs(path, mode)
338 except OSError, err:
339
340
341 if err.errno != errno.EEXIST or not os.path.exists(path):
342 raise
343
344
346 """Returns the current time formatted for filenames.
347
348 The format doesn't contain colons as some shells and applications treat them
349 as separators. Uses the local timezone.
350
351 """
352 return time.strftime("%Y-%m-%d_%H_%M_%S")
353
354
356 """Creates a backup of a file.
357
358 @type file_name: str
359 @param file_name: file to be backed up
360 @rtype: str
361 @return: the path to the newly created backup
362 @raise errors.ProgrammerError: for invalid file names
363
364 """
365 if not os.path.isfile(file_name):
366 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
367 file_name)
368
369 prefix = ("%s.backup-%s." %
370 (os.path.basename(file_name), TimestampForFilename()))
371 dir_name = os.path.dirname(file_name)
372
373 fsrc = open(file_name, 'rb')
374 try:
375 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
376 fdst = os.fdopen(fd, 'wb')
377 try:
378 logging.debug("Backing up %s at %s", file_name, backup_name)
379 shutil.copyfileobj(fsrc, fdst)
380 finally:
381 fdst.close()
382 finally:
383 fsrc.close()
384
385 return backup_name
386
387
389 """Returns a list of visible files in a directory.
390
391 @type path: str
392 @param path: the directory to enumerate
393 @rtype: list
394 @return: the list of all files not starting with a dot
395 @raise ProgrammerError: if L{path} is not an absolue and normalized path
396
397 """
398 if not IsNormAbsPath(path):
399 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
400 " absolute/normalized: '%s'" % path)
401 files = [i for i in os.listdir(path) if not i.startswith(".")]
402 return files
403
404
406 """Make required directories, if they don't exist.
407
408 @param dirs: list of tuples (dir_name, dir_mode)
409 @type dirs: list of (string, integer)
410
411 """
412 for dir_name, dir_mode in dirs:
413 try:
414 os.mkdir(dir_name, dir_mode)
415 except EnvironmentError, err:
416 if err.errno != errno.EEXIST:
417 raise errors.GenericError("Cannot create needed directory"
418 " '%s': %s" % (dir_name, err))
419 try:
420 os.chmod(dir_name, dir_mode)
421 except EnvironmentError, err:
422 raise errors.GenericError("Cannot change directory permissions on"
423 " '%s': %s" % (dir_name, err))
424 if not os.path.isdir(dir_name):
425 raise errors.GenericError("%s is not a directory" % dir_name)
426
427
428 -def FindFile(name, search_path, test=os.path.exists):
429 """Look for a filesystem object in a given path.
430
431 This is an abstract method to search for filesystem object (files,
432 dirs) under a given search path.
433
434 @type name: str
435 @param name: the name to look for
436 @type search_path: str
437 @param search_path: location to start at
438 @type test: callable
439 @param test: a function taking one argument that should return True
440 if the a given object is valid; the default value is
441 os.path.exists, causing only existing files to be returned
442 @rtype: str or None
443 @return: full path to the object if found, None otherwise
444
445 """
446
447 if constants.EXT_PLUGIN_MASK.match(name) is None:
448 logging.critical("Invalid value passed for external script name: '%s'",
449 name)
450 return None
451
452 for dir_name in search_path:
453
454 item_name = os.path.sep.join([dir_name, name])
455
456
457 if test(item_name) and os.path.basename(item_name) == name:
458 return item_name
459 return None
460
461
463 """Check whether a path is absolute and also normalized
464
465 This avoids things like /dir/../../other/path to be valid.
466
467 """
468 return os.path.normpath(path) == path and os.path.isabs(path)
469
470
472 """Safe-join a list of path components.
473
474 Requirements:
475 - the first argument must be an absolute path
476 - no component in the path must have backtracking (e.g. /../),
477 since we check for normalization at the end
478
479 @param args: the path components to be joined
480 @raise ValueError: for invalid paths
481
482 """
483
484 assert args
485
486 root = args[0]
487 if not IsNormAbsPath(root):
488 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
489 result = os.path.join(*args)
490
491 if not IsNormAbsPath(result):
492 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
493
494 prefix = os.path.commonprefix([root, result])
495 if prefix != root:
496 raise ValueError("Error: path joining resulted in different prefix"
497 " (%s != %s)" % (prefix, root))
498 return result
499
500
502 """Return the last lines from a file.
503
504 @note: this function will only read and parse the last 4KB of
505 the file; if the lines are very long, it could be that less
506 than the requested number of lines are returned
507
508 @param fname: the file name
509 @type lines: int
510 @param lines: the (maximum) number of lines to return
511
512 """
513 fd = open(fname, "r")
514 try:
515 fd.seek(0, 2)
516 pos = fd.tell()
517 pos = max(0, pos-4096)
518 fd.seek(pos, 0)
519 raw_data = fd.read()
520 finally:
521 fd.close()
522
523 rows = raw_data.splitlines()
524 return rows[-lines:]
525
526
528 """Converts bytes to mebibytes.
529
530 @type value: int
531 @param value: Value in bytes
532 @rtype: int
533 @return: Value in mebibytes
534
535 """
536 return int(round(value / (1024.0 * 1024.0), 0))
537
538
540 """Calculates the size of a directory recursively.
541
542 @type path: string
543 @param path: Path to directory
544 @rtype: int
545 @return: Size in mebibytes
546
547 """
548 size = 0
549
550 for (curpath, _, files) in os.walk(path):
551 for filename in files:
552 st = os.lstat(PathJoin(curpath, filename))
553 size += st.st_size
554
555 return BytesToMebibyte(size)
556
557
559 """Returns the total and free space on a filesystem.
560
561 @type path: string
562 @param path: Path on filesystem to be examined
563 @rtype: int
564 @return: tuple of (Total space, Free space) in mebibytes
565
566 """
567 st = os.statvfs(path)
568
569 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
570 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
571 return (tsize, fsize)
572
573
575 """Read a pid from a file.
576
577 @type pidfile: string
578 @param pidfile: path to the file containing the pid
579 @rtype: int
580 @return: The process id, if the file exists and contains a valid PID,
581 otherwise 0
582
583 """
584 try:
585 raw_data = ReadOneLineFile(pidfile)
586 except EnvironmentError, err:
587 if err.errno != errno.ENOENT:
588 logging.exception("Can't read pid file")
589 return 0
590
591 try:
592 pid = int(raw_data)
593 except (TypeError, ValueError), err:
594 logging.info("Can't parse pid file contents", exc_info=True)
595 return 0
596
597 return pid
598
599
601 """Reads a locked PID file.
602
603 This can be used together with L{utils.process.StartDaemon}.
604
605 @type path: string
606 @param path: Path to PID file
607 @return: PID as integer or, if file was unlocked or couldn't be opened, None
608
609 """
610 try:
611 fd = os.open(path, os.O_RDONLY)
612 except EnvironmentError, err:
613 if err.errno == errno.ENOENT:
614
615 return None
616 raise
617
618 try:
619 try:
620
621 filelock.LockFile(fd)
622 except errors.LockError:
623
624 return int(os.read(fd, 100))
625 finally:
626 os.close(fd)
627
628 return None
629
630
632 """Adds an SSH public key to an authorized_keys file.
633
634 @type file_obj: str or file handle
635 @param file_obj: path to authorized_keys file
636 @type key: str
637 @param key: string containing key
638
639 """
640 key_fields = key.split()
641
642 if isinstance(file_obj, basestring):
643 f = open(file_obj, 'a+')
644 else:
645 f = file_obj
646
647 try:
648 nl = True
649 for line in f:
650
651 if line.split() == key_fields:
652 break
653 nl = line.endswith('\n')
654 else:
655 if not nl:
656 f.write("\n")
657 f.write(key.rstrip('\r\n'))
658 f.write("\n")
659 f.flush()
660 finally:
661 f.close()
662
663
665 """Removes an SSH public key from an authorized_keys file.
666
667 @type file_name: str
668 @param file_name: path to authorized_keys file
669 @type key: str
670 @param key: string containing key
671
672 """
673 key_fields = key.split()
674
675 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
676 try:
677 out = os.fdopen(fd, 'w')
678 try:
679 f = open(file_name, 'r')
680 try:
681 for line in f:
682
683 if line.split() != key_fields:
684 out.write(line)
685
686 out.flush()
687 os.rename(tmpname, file_name)
688 finally:
689 f.close()
690 finally:
691 out.close()
692 except:
693 RemoveFile(tmpname)
694 raise
695
696
698 """Compute a ganeti pid file absolute path
699
700 @type name: str
701 @param name: the daemon name
702 @rtype: str
703 @return: the full path to the pidfile corresponding to the given
704 daemon name
705
706 """
707 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
708
709
711 """Write the current process pidfile.
712
713 @type pidfile: string
714 @param pidfile: the path to the file to be written
715 @raise errors.LockError: if the pid file already exists and
716 points to a live process
717 @rtype: int
718 @return: the file descriptor of the lock file; do not close this unless
719 you want to unlock the pid file
720
721 """
722
723
724 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
725
726
727
728
729
730 filelock.LockFile(fd_pidfile)
731
732 os.write(fd_pidfile, "%d\n" % os.getpid())
733
734 return fd_pidfile
735
736
738 """Reads the watcher pause file.
739
740 @type filename: string
741 @param filename: Path to watcher pause file
742 @type now: None, float or int
743 @param now: Current time as Unix timestamp
744 @type remove_after: int
745 @param remove_after: Remove watcher pause file after specified amount of
746 seconds past the pause end time
747
748 """
749 if now is None:
750 now = time.time()
751
752 try:
753 value = ReadFile(filename)
754 except IOError, err:
755 if err.errno != errno.ENOENT:
756 raise
757 value = None
758
759 if value is not None:
760 try:
761 value = int(value)
762 except ValueError:
763 logging.warning(("Watcher pause file (%s) contains invalid value,"
764 " removing it"), filename)
765 RemoveFile(filename)
766 value = None
767
768 if value is not None:
769
770 if now > (value + remove_after):
771 RemoveFile(filename)
772 value = None
773
774 elif now > value:
775 value = None
776
777 return value
778
779
781 """Returns a random UUID.
782
783 @note: This is a Linux-specific method as it uses the /proc
784 filesystem.
785 @rtype: str
786
787 """
788 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
789