1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Ganeti utility module.
32
33 This module holds functions that can be used in both daemons (all) and
34 the command line scripts.
35
36 """
37
38
39
40 import os
41 import re
42 import errno
43 import pwd
44 import time
45 import itertools
46 import select
47 import logging
48 import signal
49
50 from ganeti import errors
51 from ganeti import constants
52 from ganeti import compat
53 from ganeti import pathutils
54
55 from ganeti.utils.algo import *
56 from ganeti.utils.filelock import *
57 from ganeti.utils.hash import *
58 from ganeti.utils.io import *
59 from ganeti.utils.livelock import *
60 from ganeti.utils.log import *
61 from ganeti.utils.lvm import *
62 from ganeti.utils.mlock import *
63 from ganeti.utils.nodesetup import *
64 from ganeti.utils.process import *
65 from ganeti.utils.retry import *
66 from ganeti.utils.security import *
67 from ganeti.utils.storage import *
68 from ganeti.utils.text import *
69 from ganeti.utils.wrapper import *
70 from ganeti.utils.version import *
71 from ganeti.utils.x509 import *
72 from ganeti.utils.bitarrays import *
73
74
75 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
76
77 UUID_RE = re.compile(constants.UUID_REGEX)
78
79
81 """Force the values of a dict to have certain types.
82
83 @type target: dict
84 @param target: the dict to update
85 @type key_types: dict
86 @param key_types: dict mapping target dict keys to types
87 in constants.ENFORCEABLE_TYPES
88 @type allowed_values: list
89 @keyword allowed_values: list of specially allowed values
90
91 """
92 if allowed_values is None:
93 allowed_values = []
94
95 if not isinstance(target, dict):
96 msg = "Expected dictionary, got '%s'" % target
97 raise errors.TypeEnforcementError(msg)
98
99 for key in target:
100 if key not in key_types:
101 msg = "Unknown parameter '%s'" % key
102 raise errors.TypeEnforcementError(msg)
103
104 if target[key] in allowed_values:
105 continue
106
107 ktype = key_types[key]
108 if ktype not in constants.ENFORCEABLE_TYPES:
109 msg = "'%s' has non-enforceable type %s" % (key, ktype)
110 raise errors.ProgrammerError(msg)
111
112 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
113 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
114 pass
115 elif not isinstance(target[key], basestring):
116 if isinstance(target[key], bool) and not target[key]:
117 target[key] = ""
118 else:
119 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
120 raise errors.TypeEnforcementError(msg)
121 elif ktype == constants.VTYPE_BOOL:
122 if isinstance(target[key], basestring) and target[key]:
123 if target[key].lower() == constants.VALUE_FALSE:
124 target[key] = False
125 elif target[key].lower() == constants.VALUE_TRUE:
126 target[key] = True
127 else:
128 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
129 raise errors.TypeEnforcementError(msg)
130 elif target[key]:
131 target[key] = True
132 else:
133 target[key] = False
134 elif ktype == constants.VTYPE_SIZE:
135 try:
136 target[key] = ParseUnit(target[key])
137 except errors.UnitParseError, err:
138 msg = "'%s' (value %s) is not a valid size. error: %s" % \
139 (key, target[key], err)
140 raise errors.TypeEnforcementError(msg)
141 elif ktype == constants.VTYPE_INT:
142 try:
143 target[key] = int(target[key])
144 except (ValueError, TypeError):
145 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
146 raise errors.TypeEnforcementError(msg)
147 elif ktype == constants.VTYPE_FLOAT:
148 try:
149 target[key] = float(target[key])
150 except (ValueError, TypeError):
151 msg = "'%s' (value %s) is not a valid float" % (key, target[key])
152 raise errors.TypeEnforcementError(msg)
153
154
156 """Validate the given service name.
157
158 @type name: number or string
159 @param name: Service name or port specification
160
161 """
162 try:
163 numport = int(name)
164 except (ValueError, TypeError):
165
166 valid = _VALID_SERVICE_NAME_RE.match(name)
167 else:
168
169
170 valid = (numport >= 0 and numport < (1 << 16))
171
172 if not valid:
173 raise errors.OpPrereqError("Invalid service name '%s'" % name,
174 errors.ECODE_INVAL)
175
176 return name
177
178
180 """Helper functions to compute which keys a invalid.
181
182 @param key_path: The current key path (if any)
183 @param options: The user provided options
184 @param defaults: The default dictionary
185 @return: A list of invalid keys
186
187 """
188 defaults_keys = frozenset(defaults.keys())
189 invalid = []
190 for key, value in options.items():
191 if key_path:
192 new_path = "%s/%s" % (key_path, key)
193 else:
194 new_path = key
195
196 if key not in defaults_keys:
197 invalid.append(new_path)
198 elif isinstance(value, dict):
199 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
200
201 return invalid
202
203
205 """Verify a dict has only keys set which also are in the defaults dict.
206
207 @param options: The user provided options
208 @param defaults: The default dictionary
209 @raise error.OpPrereqError: If one of the keys is not supported
210
211 """
212 invalid = _ComputeMissingKeys("", options, defaults)
213
214 if invalid:
215 raise errors.OpPrereqError("Provided option keys not supported: %s" %
216 CommaJoin(invalid), errors.ECODE_INVAL)
217
218
220 """List volume groups and their size
221
222 @rtype: dict
223 @return:
224 Dictionary with keys volume name and values
225 the size of the volume
226
227 """
228 command = "vgs --noheadings --units m --nosuffix -o name,size"
229 result = RunCmd(command)
230 retval = {}
231 if result.failed:
232 return retval
233
234 for line in result.stdout.splitlines():
235 try:
236 name, size = line.split()
237 size = int(float(size))
238 except (IndexError, ValueError), err:
239 logging.error("Invalid output from vgs (%s): %s", err, line)
240 continue
241
242 retval[name] = size
243
244 return retval
245
246
248 """Check whether the given bridge exists in the system
249
250 @type bridge: str
251 @param bridge: the bridge name to check
252 @rtype: boolean
253 @return: True if it does
254
255 """
256 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
257
258
260 """Try to convert a value ignoring errors.
261
262 This function tries to apply function I{fn} to I{val}. If no
263 C{ValueError} or C{TypeError} exceptions are raised, it will return
264 the result, else it will return the original value. Any other
265 exceptions are propagated to the caller.
266
267 @type fn: callable
268 @param fn: function to apply to the value
269 @param val: the value to be converted
270 @return: The converted value if the conversion was successful,
271 otherwise the original value.
272
273 """
274 try:
275 nv = fn(val)
276 except (ValueError, TypeError):
277 nv = val
278 return nv
279
280
282 """Parse a CPU mask definition and return the list of CPU IDs.
283
284 CPU mask format: comma-separated list of CPU IDs
285 or dash-separated ID ranges
286 Example: "0-2,5" -> "0,1,2,5"
287
288 @type cpu_mask: str
289 @param cpu_mask: CPU mask definition
290 @rtype: list of int
291 @return: list of CPU IDs
292
293 """
294 if not cpu_mask:
295 return []
296 cpu_list = []
297 for range_def in cpu_mask.split(","):
298 boundaries = range_def.split("-")
299 n_elements = len(boundaries)
300 if n_elements > 2:
301 raise errors.ParseError("Invalid CPU ID range definition"
302 " (only one hyphen allowed): %s" % range_def)
303 try:
304 lower = int(boundaries[0])
305 except (ValueError, TypeError), err:
306 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
307 " CPU ID range: %s" % str(err))
308 try:
309 higher = int(boundaries[-1])
310 except (ValueError, TypeError), err:
311 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
312 " CPU ID range: %s" % str(err))
313 if lower > higher:
314 raise errors.ParseError("Invalid CPU ID range definition"
315 " (%d > %d): %s" % (lower, higher, range_def))
316 cpu_list.extend(range(lower, higher + 1))
317 return cpu_list
318
319
321 """Parse a multiple CPU mask definition and return the list of CPU IDs.
322
323 CPU mask format: colon-separated list of comma-separated list of CPU IDs
324 or dash-separated ID ranges, with optional "all" as CPU value
325 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
326
327 @type cpu_mask: str
328 @param cpu_mask: multiple CPU mask definition
329 @rtype: list of lists of int
330 @return: list of lists of CPU IDs
331
332 """
333 if not cpu_mask:
334 return []
335 cpu_list = []
336 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
337 if range_def == constants.CPU_PINNING_ALL:
338 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
339 else:
340
341 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
342
343 return cpu_list
344
345
347 """Try to get the homedir of the given user.
348
349 The user can be passed either as a string (denoting the name) or as
350 an integer (denoting the user id). If the user is not found, the
351 C{default} argument is returned, which defaults to C{None}.
352
353 """
354 try:
355 if isinstance(user, basestring):
356 result = pwd.getpwnam(user)
357 elif isinstance(user, (int, long)):
358 result = pwd.getpwuid(user)
359 else:
360 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
361 type(user))
362 except KeyError:
363 return default
364 return result.pw_dir
365
366
368 """Returns the first non-existing integer from seq.
369
370 The seq argument should be a sorted list of positive integers. The
371 first time the index of an element is smaller than the element
372 value, the index will be returned.
373
374 The base argument is used to start at a different offset,
375 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
376
377 Example: C{[0, 1, 3]} will return I{2}.
378
379 @type seq: sequence
380 @param seq: the sequence to be analyzed.
381 @type base: int
382 @param base: use this value as the base index of the sequence
383 @rtype: int
384 @return: the first non-used index in the sequence
385
386 """
387 for idx, elem in enumerate(seq):
388 assert elem >= base, "Passed element is higher than base offset"
389 if elem > idx + base:
390
391 return idx + base
392 return None
393
394
396 """Waits for a condition to occur on the socket.
397
398 Immediately returns at the first interruption.
399
400 @type fdobj: integer or object supporting a fileno() method
401 @param fdobj: entity to wait for events on
402 @type event: integer
403 @param event: ORed condition (see select module)
404 @type timeout: float or None
405 @param timeout: Timeout in seconds
406 @rtype: int or None
407 @return: None for timeout, otherwise occured conditions
408
409 """
410 check = (event | select.POLLPRI |
411 select.POLLNVAL | select.POLLHUP | select.POLLERR)
412
413 if timeout is not None:
414
415 timeout *= 1000
416
417 poller = select.poll()
418 poller.register(fdobj, event)
419 try:
420
421
422
423 io_events = poller.poll(timeout)
424 except select.error, err:
425 if err[0] != errno.EINTR:
426 raise
427 io_events = []
428 if io_events and io_events[0][1] & check:
429 return io_events[0][1]
430 else:
431 return None
432
433
435 """Retry helper for WaitForFdCondition.
436
437 This class contains the retried and wait functions that make sure
438 WaitForFdCondition can continue waiting until the timeout is actually
439 expired.
440
441 """
442
444 self.timeout = timeout
445
446 - def Poll(self, fdobj, event):
452
454 self.timeout = timeout
455
456
458 """Waits for a condition to occur on the socket.
459
460 Retries until the timeout is expired, even if interrupted.
461
462 @type fdobj: integer or object supporting a fileno() method
463 @param fdobj: entity to wait for events on
464 @type event: integer
465 @param event: ORed condition (see select module)
466 @type timeout: float or None
467 @param timeout: Timeout in seconds
468 @rtype: int or None
469 @return: None for timeout, otherwise occured conditions
470
471 """
472 if timeout is not None:
473 retrywaiter = FdConditionWaiterHelper(timeout)
474 try:
475 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
476 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
477 except RetryTimeout:
478 result = None
479 else:
480 result = None
481 while result is None:
482 result = SingleWaitForFdCondition(fdobj, event, timeout)
483 return result
484
485
487 """Check for and start daemon if not alive.
488
489 @type name: string
490 @param name: daemon name
491
492 @rtype: bool
493 @return: 'True' if daemon successfully started,
494 'False' otherwise
495
496 """
497 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
498 if result.failed:
499 logging.error("Can't start daemon '%s', failure %s, output: %s",
500 name, result.fail_reason, result.output)
501 return False
502
503 return True
504
505
507 """Stop daemon
508
509 @type name: string
510 @param name: daemon name
511
512 @rtype: bool
513 @return: 'True' if daemon successfully stopped,
514 'False' otherwise
515
516 """
517 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
518 if result.failed:
519 logging.error("Can't stop daemon '%s', failure %s, output: %s",
520 name, result.fail_reason, result.output)
521 return False
522
523 return True
524
525
527 """Splits time as floating point number into a tuple.
528
529 @param value: Time in seconds
530 @type value: int or float
531 @return: Tuple containing (seconds, microseconds)
532
533 """
534 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
535
536 assert 0 <= seconds, \
537 "Seconds must be larger than or equal to 0, but are %s" % seconds
538 assert 0 <= microseconds <= 999999, \
539 "Microseconds must be 0-999999, but are %s" % microseconds
540
541 return (int(seconds), int(microseconds))
542
543
545 """Merges a tuple into time as a floating point number.
546
547 @param timetuple: Time as tuple, (seconds, microseconds)
548 @type timetuple: tuple
549 @return: Time as a floating point number expressed in seconds
550
551 """
552 (seconds, microseconds) = timetuple
553
554 assert 0 <= seconds, \
555 "Seconds must be larger than or equal to 0, but are %s" % seconds
556 assert 0 <= microseconds <= 999999, \
557 "Microseconds must be 0-999999, but are %s" % microseconds
558
559 return float(seconds) + (float(microseconds) * 0.000001)
560
561
563 """Return the current timestamp expressed as number of nanoseconds since the
564 unix epoch
565
566 @return: nanoseconds since the Unix epoch
567
568 """
569 return int(time.time() * 1000000000)
570
571
573 """Tries to find an item in a dictionary matching a name.
574
575 Callers have to ensure the data names aren't contradictory (e.g. a regexp
576 that matches a string). If the name isn't a direct key, all regular
577 expression objects in the dictionary are matched against it.
578
579 @type data: dict
580 @param data: Dictionary containing data
581 @type name: string
582 @param name: Name to look for
583 @rtype: tuple; (value in dictionary, matched groups as list)
584
585 """
586 if name in data:
587 return (data[name], [])
588
589 for key, value in data.items():
590
591 if hasattr(key, "match"):
592 m = key.match(name)
593 if m:
594 return (value, list(m.groups()))
595
596 return None
597
598
600 """Returns the list of mounted filesystems.
601
602 This function is Linux-specific.
603
604 @param filename: path of mounts file (/proc/mounts by default)
605 @rtype: list of tuples
606 @return: list of mount entries (device, mountpoint, fstype, options)
607
608 """
609
610 data = []
611 mountlines = ReadFile(filename).splitlines()
612 for line in mountlines:
613 device, mountpoint, fstype, options, _ = line.split(None, 4)
614 data.append((device, mountpoint, fstype, options))
615
616 return data
617
618
620 """Signal Handled decoration.
621
622 This special decorator installs a signal handler and then calls the target
623 function. The function must accept a 'signal_handlers' keyword argument,
624 which will contain a dict indexed by signal number, with SignalHandler
625 objects as values.
626
627 The decorator can be safely stacked with iself, to handle multiple signals
628 with different handlers.
629
630 @type signums: list
631 @param signums: signals to intercept
632
633 """
634 def wrap(fn):
635 def sig_function(*args, **kwargs):
636 assert "signal_handlers" not in kwargs or \
637 kwargs["signal_handlers"] is None or \
638 isinstance(kwargs["signal_handlers"], dict), \
639 "Wrong signal_handlers parameter in original function call"
640 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
641 signal_handlers = kwargs["signal_handlers"]
642 else:
643 signal_handlers = {}
644 kwargs["signal_handlers"] = signal_handlers
645 sighandler = SignalHandler(signums)
646 try:
647 for sig in signums:
648 signal_handlers[sig] = sighandler
649 return fn(*args, **kwargs)
650 finally:
651 sighandler.Reset()
652 return sig_function
653 return wrap
654
655
657 """Checks whether a timeout has expired.
658
659 """
660 return _time_fn() > (epoch + timeout)
661
662
673 else:
674
677
679 """Initializes this class.
680
681 """
682 (read_fd, write_fd) = os.pipe()
683
684
685
686
687 self._read_fh = os.fdopen(read_fd, "r", 0)
688 self._write_fh = os.fdopen(write_fd, "w", 0)
689
690 self._previous = self._SetWakeupFd(self._write_fh.fileno())
691
692
693 self.fileno = self._read_fh.fileno
694 self.read = self._read_fh.read
695
697 """Restores the previous wakeup file descriptor.
698
699 """
700 if hasattr(self, "_previous") and self._previous is not None:
701 self._SetWakeupFd(self._previous)
702 self._previous = None
703
705 """Notifies the wakeup file descriptor.
706
707 """
708 self._write_fh.write(chr(0))
709
711 """Called before object deletion.
712
713 """
714 self.Reset()
715
716
718 """Generic signal handler class.
719
720 It automatically restores the original handler when deconstructed or
721 when L{Reset} is called. You can either pass your own handler
722 function in or query the L{called} attribute to detect whether the
723 signal was sent.
724
725 @type signum: list
726 @ivar signum: the signals we handle
727 @type called: boolean
728 @ivar called: tracks whether any of the signals have been raised
729
730 """
731 - def __init__(self, signum, handler_fn=None, wakeup=None):
732 """Constructs a new SignalHandler instance.
733
734 @type signum: int or list of ints
735 @param signum: Single signal number or set of signal numbers
736 @type handler_fn: callable
737 @param handler_fn: Signal handling function
738
739 """
740 assert handler_fn is None or callable(handler_fn)
741
742 self.signum = set(signum)
743 self.called = False
744
745 self._handler_fn = handler_fn
746 self._wakeup = wakeup
747
748 self._previous = {}
749 try:
750 for signum in self.signum:
751
752 prev_handler = signal.signal(signum, self._HandleSignal)
753 try:
754 self._previous[signum] = prev_handler
755 except:
756
757 signal.signal(signum, prev_handler)
758 raise
759 except:
760
761 self.Reset()
762
763
764 raise
765
768
770 """Restore previous handler.
771
772 This will reset all the signals to their previous handlers.
773
774 """
775 for signum, prev_handler in self._previous.items():
776 signal.signal(signum, prev_handler)
777
778 del self._previous[signum]
779
781 """Unsets the L{called} flag.
782
783 This function can be used in case a signal may arrive several times.
784
785 """
786 self.called = False
787
789 """Actual signal handling function.
790
791 """
792
793
794 self.called = True
795
796 if self._wakeup:
797
798 self._wakeup.Notify()
799
800 if self._handler_fn:
801 self._handler_fn(signum, frame)
802
803
805 """A simple field set.
806
807 Among the features are:
808 - checking if a string is among a list of static string or regex objects
809 - checking if a whole list of string matches
810 - returning the matching groups from a regex match
811
812 Internally, all fields are held as regular expression objects.
813
814 """
816 self.items = [re.compile("^%s$" % value) for value in items]
817
819 """Extend the field set with the items from another one"""
820 self.items.extend(other_set.items)
821
823 """Checks if a field matches the current set
824
825 @type field: str
826 @param field: the string to match
827 @return: either None or a regular expression match object
828
829 """
830 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
831 return m
832 return None
833
835 """Returns the list of fields not matching the current set
836
837 @type items: list
838 @param items: the list of fields to check
839 @rtype: list
840 @return: list of non-matching fields
841
842 """
843 return [val for val in items if not self.Matches(val)]
844
845
888
889
891 """Checks if the instance has only disks of any of the dev_types.
892
893 @type disks_info: list of L{Disk}
894 @param disks_info: all the disks of the instance.
895 @type dev_types: list of disk templates
896 @param dev_types: the disk type required.
897
898 @rtype: bool
899 @return: True iff the instance only has disks of type dev_type.
900 """
901
902 assert not isinstance(dev_types, str)
903
904 if not disks_info and constants.DT_DISKLESS not in dev_types:
905 return False
906
907 for disk in disks_info:
908 if disk.dev_type not in dev_types:
909 return False
910
911 return True
912
913
915 """Checks if the instance has some disks of any types in dev_types.
916
917 @type disks_info: list of L{Disk}
918 @param disks_info: all the disks of the instance.
919 @type dev_types: list of disk template
920 @param dev_types: the disk type required.
921
922 @rtype: bool
923 @return: True if the instance has disks of type dev_types or the instance has
924 no disks and the dev_types allow DT_DISKLESS.
925 """
926
927 assert not isinstance(dev_types, str)
928
929 if not disks_info and constants.DT_DISKLESS in dev_types:
930 return True
931
932 for disk in disks_info:
933 if disk.dev_type in dev_types:
934 return True
935
936 return False
937
938
940 """Gives a summary disk template from disk devtypes.
941
942 @type disk_types: list of string
943 @param disk_types: all the dev_types of the instance.
944 @rtype disk template
945 @returns the summarized disk template of the disk types.
946
947 """
948 disk_types = set(dev_type for dev_type in disk_types)
949 if not disk_types:
950 return constants.DT_DISKLESS
951 elif len(disk_types) > 1:
952 return constants.DT_MIXED
953 else:
954 return disk_types.pop()
955
956
958 """Gives a summary disk template from disks.
959
960 @type disks_info: list of L{Disk}
961 @param disks_info: all the disks of the instance.
962 @rtype disk template
963 @returns the summarized disk template of the disk types.
964
965 """
966 return GetDiskTemplateString(d.dev_type for d in disks_info)
967