1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Ganeti utility module.
32
33 This module holds functions that can be used in both daemons (all) and
34 the command line scripts.
35
36 """
37
38
39
40 import os
41 import re
42 import errno
43 import pwd
44 import time
45 import itertools
46 import select
47 import logging
48 import signal
49
50 from ganeti import errors
51 from ganeti import constants
52 from ganeti import compat
53 from ganeti import pathutils
54
55 from ganeti.utils.algo import *
56 from ganeti.utils.filelock import *
57 from ganeti.utils.hash import *
58 from ganeti.utils.io import *
59 from ganeti.utils.livelock import *
60 from ganeti.utils.log import *
61 from ganeti.utils.lvm import *
62 from ganeti.utils.mlock import *
63 from ganeti.utils.nodesetup import *
64 from ganeti.utils.process import *
65 from ganeti.utils.retry import *
66 from ganeti.utils.security import *
67 from ganeti.utils.storage import *
68 from ganeti.utils.text import *
69 from ganeti.utils.wrapper import *
70 from ganeti.utils.version import *
71 from ganeti.utils.x509 import *
72
73
74 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
75
76 UUID_RE = re.compile(constants.UUID_REGEX)
77
78
80 """Force the values of a dict to have certain types.
81
82 @type target: dict
83 @param target: the dict to update
84 @type key_types: dict
85 @param key_types: dict mapping target dict keys to types
86 in constants.ENFORCEABLE_TYPES
87 @type allowed_values: list
88 @keyword allowed_values: list of specially allowed values
89
90 """
91 if allowed_values is None:
92 allowed_values = []
93
94 if not isinstance(target, dict):
95 msg = "Expected dictionary, got '%s'" % target
96 raise errors.TypeEnforcementError(msg)
97
98 for key in target:
99 if key not in key_types:
100 msg = "Unknown parameter '%s'" % key
101 raise errors.TypeEnforcementError(msg)
102
103 if target[key] in allowed_values:
104 continue
105
106 ktype = key_types[key]
107 if ktype not in constants.ENFORCEABLE_TYPES:
108 msg = "'%s' has non-enforceable type %s" % (key, ktype)
109 raise errors.ProgrammerError(msg)
110
111 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
112 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
113 pass
114 elif not isinstance(target[key], basestring):
115 if isinstance(target[key], bool) and not target[key]:
116 target[key] = ""
117 else:
118 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
119 raise errors.TypeEnforcementError(msg)
120 elif ktype == constants.VTYPE_BOOL:
121 if isinstance(target[key], basestring) and target[key]:
122 if target[key].lower() == constants.VALUE_FALSE:
123 target[key] = False
124 elif target[key].lower() == constants.VALUE_TRUE:
125 target[key] = True
126 else:
127 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
128 raise errors.TypeEnforcementError(msg)
129 elif target[key]:
130 target[key] = True
131 else:
132 target[key] = False
133 elif ktype == constants.VTYPE_SIZE:
134 try:
135 target[key] = ParseUnit(target[key])
136 except errors.UnitParseError, err:
137 msg = "'%s' (value %s) is not a valid size. error: %s" % \
138 (key, target[key], err)
139 raise errors.TypeEnforcementError(msg)
140 elif ktype == constants.VTYPE_INT:
141 try:
142 target[key] = int(target[key])
143 except (ValueError, TypeError):
144 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
145 raise errors.TypeEnforcementError(msg)
146 elif ktype == constants.VTYPE_FLOAT:
147 try:
148 target[key] = float(target[key])
149 except (ValueError, TypeError):
150 msg = "'%s' (value %s) is not a valid float" % (key, target[key])
151 raise errors.TypeEnforcementError(msg)
152
153
155 """Validate the given service name.
156
157 @type name: number or string
158 @param name: Service name or port specification
159
160 """
161 try:
162 numport = int(name)
163 except (ValueError, TypeError):
164
165 valid = _VALID_SERVICE_NAME_RE.match(name)
166 else:
167
168
169 valid = (numport >= 0 and numport < (1 << 16))
170
171 if not valid:
172 raise errors.OpPrereqError("Invalid service name '%s'" % name,
173 errors.ECODE_INVAL)
174
175 return name
176
177
179 """Helper functions to compute which keys a invalid.
180
181 @param key_path: The current key path (if any)
182 @param options: The user provided options
183 @param defaults: The default dictionary
184 @return: A list of invalid keys
185
186 """
187 defaults_keys = frozenset(defaults.keys())
188 invalid = []
189 for key, value in options.items():
190 if key_path:
191 new_path = "%s/%s" % (key_path, key)
192 else:
193 new_path = key
194
195 if key not in defaults_keys:
196 invalid.append(new_path)
197 elif isinstance(value, dict):
198 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
199
200 return invalid
201
202
204 """Verify a dict has only keys set which also are in the defaults dict.
205
206 @param options: The user provided options
207 @param defaults: The default dictionary
208 @raise error.OpPrereqError: If one of the keys is not supported
209
210 """
211 invalid = _ComputeMissingKeys("", options, defaults)
212
213 if invalid:
214 raise errors.OpPrereqError("Provided option keys not supported: %s" %
215 CommaJoin(invalid), errors.ECODE_INVAL)
216
217
219 """List volume groups and their size
220
221 @rtype: dict
222 @return:
223 Dictionary with keys volume name and values
224 the size of the volume
225
226 """
227 command = "vgs --noheadings --units m --nosuffix -o name,size"
228 result = RunCmd(command)
229 retval = {}
230 if result.failed:
231 return retval
232
233 for line in result.stdout.splitlines():
234 try:
235 name, size = line.split()
236 size = int(float(size))
237 except (IndexError, ValueError), err:
238 logging.error("Invalid output from vgs (%s): %s", err, line)
239 continue
240
241 retval[name] = size
242
243 return retval
244
245
247 """Check whether the given bridge exists in the system
248
249 @type bridge: str
250 @param bridge: the bridge name to check
251 @rtype: boolean
252 @return: True if it does
253
254 """
255 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
256
257
259 """Try to convert a value ignoring errors.
260
261 This function tries to apply function I{fn} to I{val}. If no
262 C{ValueError} or C{TypeError} exceptions are raised, it will return
263 the result, else it will return the original value. Any other
264 exceptions are propagated to the caller.
265
266 @type fn: callable
267 @param fn: function to apply to the value
268 @param val: the value to be converted
269 @return: The converted value if the conversion was successful,
270 otherwise the original value.
271
272 """
273 try:
274 nv = fn(val)
275 except (ValueError, TypeError):
276 nv = val
277 return nv
278
279
281 """Parse a CPU mask definition and return the list of CPU IDs.
282
283 CPU mask format: comma-separated list of CPU IDs
284 or dash-separated ID ranges
285 Example: "0-2,5" -> "0,1,2,5"
286
287 @type cpu_mask: str
288 @param cpu_mask: CPU mask definition
289 @rtype: list of int
290 @return: list of CPU IDs
291
292 """
293 if not cpu_mask:
294 return []
295 cpu_list = []
296 for range_def in cpu_mask.split(","):
297 boundaries = range_def.split("-")
298 n_elements = len(boundaries)
299 if n_elements > 2:
300 raise errors.ParseError("Invalid CPU ID range definition"
301 " (only one hyphen allowed): %s" % range_def)
302 try:
303 lower = int(boundaries[0])
304 except (ValueError, TypeError), err:
305 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
306 " CPU ID range: %s" % str(err))
307 try:
308 higher = int(boundaries[-1])
309 except (ValueError, TypeError), err:
310 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
311 " CPU ID range: %s" % str(err))
312 if lower > higher:
313 raise errors.ParseError("Invalid CPU ID range definition"
314 " (%d > %d): %s" % (lower, higher, range_def))
315 cpu_list.extend(range(lower, higher + 1))
316 return cpu_list
317
318
320 """Parse a multiple CPU mask definition and return the list of CPU IDs.
321
322 CPU mask format: colon-separated list of comma-separated list of CPU IDs
323 or dash-separated ID ranges, with optional "all" as CPU value
324 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
325
326 @type cpu_mask: str
327 @param cpu_mask: multiple CPU mask definition
328 @rtype: list of lists of int
329 @return: list of lists of CPU IDs
330
331 """
332 if not cpu_mask:
333 return []
334 cpu_list = []
335 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
336 if range_def == constants.CPU_PINNING_ALL:
337 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
338 else:
339
340 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
341
342 return cpu_list
343
344
346 """Try to get the homedir of the given user.
347
348 The user can be passed either as a string (denoting the name) or as
349 an integer (denoting the user id). If the user is not found, the
350 C{default} argument is returned, which defaults to C{None}.
351
352 """
353 try:
354 if isinstance(user, basestring):
355 result = pwd.getpwnam(user)
356 elif isinstance(user, (int, long)):
357 result = pwd.getpwuid(user)
358 else:
359 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
360 type(user))
361 except KeyError:
362 return default
363 return result.pw_dir
364
365
367 """Returns the first non-existing integer from seq.
368
369 The seq argument should be a sorted list of positive integers. The
370 first time the index of an element is smaller than the element
371 value, the index will be returned.
372
373 The base argument is used to start at a different offset,
374 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
375
376 Example: C{[0, 1, 3]} will return I{2}.
377
378 @type seq: sequence
379 @param seq: the sequence to be analyzed.
380 @type base: int
381 @param base: use this value as the base index of the sequence
382 @rtype: int
383 @return: the first non-used index in the sequence
384
385 """
386 for idx, elem in enumerate(seq):
387 assert elem >= base, "Passed element is higher than base offset"
388 if elem > idx + base:
389
390 return idx + base
391 return None
392
393
395 """Waits for a condition to occur on the socket.
396
397 Immediately returns at the first interruption.
398
399 @type fdobj: integer or object supporting a fileno() method
400 @param fdobj: entity to wait for events on
401 @type event: integer
402 @param event: ORed condition (see select module)
403 @type timeout: float or None
404 @param timeout: Timeout in seconds
405 @rtype: int or None
406 @return: None for timeout, otherwise occured conditions
407
408 """
409 check = (event | select.POLLPRI |
410 select.POLLNVAL | select.POLLHUP | select.POLLERR)
411
412 if timeout is not None:
413
414 timeout *= 1000
415
416 poller = select.poll()
417 poller.register(fdobj, event)
418 try:
419
420
421
422 io_events = poller.poll(timeout)
423 except select.error, err:
424 if err[0] != errno.EINTR:
425 raise
426 io_events = []
427 if io_events and io_events[0][1] & check:
428 return io_events[0][1]
429 else:
430 return None
431
432
434 """Retry helper for WaitForFdCondition.
435
436 This class contains the retried and wait functions that make sure
437 WaitForFdCondition can continue waiting until the timeout is actually
438 expired.
439
440 """
441
443 self.timeout = timeout
444
445 - def Poll(self, fdobj, event):
451
453 self.timeout = timeout
454
455
457 """Waits for a condition to occur on the socket.
458
459 Retries until the timeout is expired, even if interrupted.
460
461 @type fdobj: integer or object supporting a fileno() method
462 @param fdobj: entity to wait for events on
463 @type event: integer
464 @param event: ORed condition (see select module)
465 @type timeout: float or None
466 @param timeout: Timeout in seconds
467 @rtype: int or None
468 @return: None for timeout, otherwise occured conditions
469
470 """
471 if timeout is not None:
472 retrywaiter = FdConditionWaiterHelper(timeout)
473 try:
474 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
475 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
476 except RetryTimeout:
477 result = None
478 else:
479 result = None
480 while result is None:
481 result = SingleWaitForFdCondition(fdobj, event, timeout)
482 return result
483
484
486 """Check for and start daemon if not alive.
487
488 @type name: string
489 @param name: daemon name
490
491 @rtype: bool
492 @return: 'True' if daemon successfully started,
493 'False' otherwise
494
495 """
496 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
497 if result.failed:
498 logging.error("Can't start daemon '%s', failure %s, output: %s",
499 name, result.fail_reason, result.output)
500 return False
501
502 return True
503
504
506 """Stop daemon
507
508 @type name: string
509 @param name: daemon name
510
511 @rtype: bool
512 @return: 'True' if daemon successfully stopped,
513 'False' otherwise
514
515 """
516 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
517 if result.failed:
518 logging.error("Can't stop daemon '%s', failure %s, output: %s",
519 name, result.fail_reason, result.output)
520 return False
521
522 return True
523
524
526 """Splits time as floating point number into a tuple.
527
528 @param value: Time in seconds
529 @type value: int or float
530 @return: Tuple containing (seconds, microseconds)
531
532 """
533 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
534
535 assert 0 <= seconds, \
536 "Seconds must be larger than or equal to 0, but are %s" % seconds
537 assert 0 <= microseconds <= 999999, \
538 "Microseconds must be 0-999999, but are %s" % microseconds
539
540 return (int(seconds), int(microseconds))
541
542
544 """Merges a tuple into time as a floating point number.
545
546 @param timetuple: Time as tuple, (seconds, microseconds)
547 @type timetuple: tuple
548 @return: Time as a floating point number expressed in seconds
549
550 """
551 (seconds, microseconds) = timetuple
552
553 assert 0 <= seconds, \
554 "Seconds must be larger than or equal to 0, but are %s" % seconds
555 assert 0 <= microseconds <= 999999, \
556 "Microseconds must be 0-999999, but are %s" % microseconds
557
558 return float(seconds) + (float(microseconds) * 0.000001)
559
560
562 """Return the current timestamp expressed as number of nanoseconds since the
563 unix epoch
564
565 @return: nanoseconds since the Unix epoch
566
567 """
568 return int(time.time() * 1000000000)
569
570
572 """Tries to find an item in a dictionary matching a name.
573
574 Callers have to ensure the data names aren't contradictory (e.g. a regexp
575 that matches a string). If the name isn't a direct key, all regular
576 expression objects in the dictionary are matched against it.
577
578 @type data: dict
579 @param data: Dictionary containing data
580 @type name: string
581 @param name: Name to look for
582 @rtype: tuple; (value in dictionary, matched groups as list)
583
584 """
585 if name in data:
586 return (data[name], [])
587
588 for key, value in data.items():
589
590 if hasattr(key, "match"):
591 m = key.match(name)
592 if m:
593 return (value, list(m.groups()))
594
595 return None
596
597
599 """Returns the list of mounted filesystems.
600
601 This function is Linux-specific.
602
603 @param filename: path of mounts file (/proc/mounts by default)
604 @rtype: list of tuples
605 @return: list of mount entries (device, mountpoint, fstype, options)
606
607 """
608
609 data = []
610 mountlines = ReadFile(filename).splitlines()
611 for line in mountlines:
612 device, mountpoint, fstype, options, _ = line.split(None, 4)
613 data.append((device, mountpoint, fstype, options))
614
615 return data
616
617
619 """Signal Handled decoration.
620
621 This special decorator installs a signal handler and then calls the target
622 function. The function must accept a 'signal_handlers' keyword argument,
623 which will contain a dict indexed by signal number, with SignalHandler
624 objects as values.
625
626 The decorator can be safely stacked with iself, to handle multiple signals
627 with different handlers.
628
629 @type signums: list
630 @param signums: signals to intercept
631
632 """
633 def wrap(fn):
634 def sig_function(*args, **kwargs):
635 assert "signal_handlers" not in kwargs or \
636 kwargs["signal_handlers"] is None or \
637 isinstance(kwargs["signal_handlers"], dict), \
638 "Wrong signal_handlers parameter in original function call"
639 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
640 signal_handlers = kwargs["signal_handlers"]
641 else:
642 signal_handlers = {}
643 kwargs["signal_handlers"] = signal_handlers
644 sighandler = SignalHandler(signums)
645 try:
646 for sig in signums:
647 signal_handlers[sig] = sighandler
648 return fn(*args, **kwargs)
649 finally:
650 sighandler.Reset()
651 return sig_function
652 return wrap
653
654
656 """Checks whether a timeout has expired.
657
658 """
659 return _time_fn() > (epoch + timeout)
660
661
672 else:
673
676
678 """Initializes this class.
679
680 """
681 (read_fd, write_fd) = os.pipe()
682
683
684
685
686 self._read_fh = os.fdopen(read_fd, "r", 0)
687 self._write_fh = os.fdopen(write_fd, "w", 0)
688
689 self._previous = self._SetWakeupFd(self._write_fh.fileno())
690
691
692 self.fileno = self._read_fh.fileno
693 self.read = self._read_fh.read
694
696 """Restores the previous wakeup file descriptor.
697
698 """
699 if hasattr(self, "_previous") and self._previous is not None:
700 self._SetWakeupFd(self._previous)
701 self._previous = None
702
704 """Notifies the wakeup file descriptor.
705
706 """
707 self._write_fh.write(chr(0))
708
710 """Called before object deletion.
711
712 """
713 self.Reset()
714
715
717 """Generic signal handler class.
718
719 It automatically restores the original handler when deconstructed or
720 when L{Reset} is called. You can either pass your own handler
721 function in or query the L{called} attribute to detect whether the
722 signal was sent.
723
724 @type signum: list
725 @ivar signum: the signals we handle
726 @type called: boolean
727 @ivar called: tracks whether any of the signals have been raised
728
729 """
730 - def __init__(self, signum, handler_fn=None, wakeup=None):
731 """Constructs a new SignalHandler instance.
732
733 @type signum: int or list of ints
734 @param signum: Single signal number or set of signal numbers
735 @type handler_fn: callable
736 @param handler_fn: Signal handling function
737
738 """
739 assert handler_fn is None or callable(handler_fn)
740
741 self.signum = set(signum)
742 self.called = False
743
744 self._handler_fn = handler_fn
745 self._wakeup = wakeup
746
747 self._previous = {}
748 try:
749 for signum in self.signum:
750
751 prev_handler = signal.signal(signum, self._HandleSignal)
752 try:
753 self._previous[signum] = prev_handler
754 except:
755
756 signal.signal(signum, prev_handler)
757 raise
758 except:
759
760 self.Reset()
761
762
763 raise
764
767
769 """Restore previous handler.
770
771 This will reset all the signals to their previous handlers.
772
773 """
774 for signum, prev_handler in self._previous.items():
775 signal.signal(signum, prev_handler)
776
777 del self._previous[signum]
778
780 """Unsets the L{called} flag.
781
782 This function can be used in case a signal may arrive several times.
783
784 """
785 self.called = False
786
788 """Actual signal handling function.
789
790 """
791
792
793 self.called = True
794
795 if self._wakeup:
796
797 self._wakeup.Notify()
798
799 if self._handler_fn:
800 self._handler_fn(signum, frame)
801
802
804 """A simple field set.
805
806 Among the features are:
807 - checking if a string is among a list of static string or regex objects
808 - checking if a whole list of string matches
809 - returning the matching groups from a regex match
810
811 Internally, all fields are held as regular expression objects.
812
813 """
815 self.items = [re.compile("^%s$" % value) for value in items]
816
818 """Extend the field set with the items from another one"""
819 self.items.extend(other_set.items)
820
822 """Checks if a field matches the current set
823
824 @type field: str
825 @param field: the string to match
826 @return: either None or a regular expression match object
827
828 """
829 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
830 return m
831 return None
832
834 """Returns the list of fields not matching the current set
835
836 @type items: list
837 @param items: the list of fields to check
838 @rtype: list
839 @return: list of non-matching fields
840
841 """
842 return [val for val in items if not self.Matches(val)]
843
844
887