1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Ganeti utility module.
32
33 This module holds functions that can be used in both daemons (all) and
34 the command line scripts.
35
36 """
37
38
39
40 import os
41 import re
42 import errno
43 import pwd
44 import time
45 import itertools
46 import select
47 import logging
48 import signal
49
50 from ganeti import errors
51 from ganeti import constants
52 from ganeti import compat
53 from ganeti import pathutils
54
55 from ganeti.utils.algo import *
56 from ganeti.utils.filelock import *
57 from ganeti.utils.hash import *
58 from ganeti.utils.io import *
59 from ganeti.utils.log import *
60 from ganeti.utils.lvm import *
61 from ganeti.utils.mlock import *
62 from ganeti.utils.nodesetup import *
63 from ganeti.utils.process import *
64 from ganeti.utils.retry import *
65 from ganeti.utils.security import *
66 from ganeti.utils.storage import *
67 from ganeti.utils.text import *
68 from ganeti.utils.wrapper import *
69 from ganeti.utils.version import *
70 from ganeti.utils.x509 import *
71
72
73 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
74
75 UUID_RE = re.compile(constants.UUID_REGEX)
76
77
79 """Force the values of a dict to have certain types.
80
81 @type target: dict
82 @param target: the dict to update
83 @type key_types: dict
84 @param key_types: dict mapping target dict keys to types
85 in constants.ENFORCEABLE_TYPES
86 @type allowed_values: list
87 @keyword allowed_values: list of specially allowed values
88
89 """
90 if allowed_values is None:
91 allowed_values = []
92
93 if not isinstance(target, dict):
94 msg = "Expected dictionary, got '%s'" % target
95 raise errors.TypeEnforcementError(msg)
96
97 for key in target:
98 if key not in key_types:
99 msg = "Unknown parameter '%s'" % key
100 raise errors.TypeEnforcementError(msg)
101
102 if target[key] in allowed_values:
103 continue
104
105 ktype = key_types[key]
106 if ktype not in constants.ENFORCEABLE_TYPES:
107 msg = "'%s' has non-enforceable type %s" % (key, ktype)
108 raise errors.ProgrammerError(msg)
109
110 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
111 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
112 pass
113 elif not isinstance(target[key], basestring):
114 if isinstance(target[key], bool) and not target[key]:
115 target[key] = ""
116 else:
117 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
118 raise errors.TypeEnforcementError(msg)
119 elif ktype == constants.VTYPE_BOOL:
120 if isinstance(target[key], basestring) and target[key]:
121 if target[key].lower() == constants.VALUE_FALSE:
122 target[key] = False
123 elif target[key].lower() == constants.VALUE_TRUE:
124 target[key] = True
125 else:
126 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
127 raise errors.TypeEnforcementError(msg)
128 elif target[key]:
129 target[key] = True
130 else:
131 target[key] = False
132 elif ktype == constants.VTYPE_SIZE:
133 try:
134 target[key] = ParseUnit(target[key])
135 except errors.UnitParseError, err:
136 msg = "'%s' (value %s) is not a valid size. error: %s" % \
137 (key, target[key], err)
138 raise errors.TypeEnforcementError(msg)
139 elif ktype == constants.VTYPE_INT:
140 try:
141 target[key] = int(target[key])
142 except (ValueError, TypeError):
143 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
144 raise errors.TypeEnforcementError(msg)
145
146
148 """Validate the given service name.
149
150 @type name: number or string
151 @param name: Service name or port specification
152
153 """
154 try:
155 numport = int(name)
156 except (ValueError, TypeError):
157
158 valid = _VALID_SERVICE_NAME_RE.match(name)
159 else:
160
161
162 valid = (numport >= 0 and numport < (1 << 16))
163
164 if not valid:
165 raise errors.OpPrereqError("Invalid service name '%s'" % name,
166 errors.ECODE_INVAL)
167
168 return name
169
170
172 """Helper functions to compute which keys a invalid.
173
174 @param key_path: The current key path (if any)
175 @param options: The user provided options
176 @param defaults: The default dictionary
177 @return: A list of invalid keys
178
179 """
180 defaults_keys = frozenset(defaults.keys())
181 invalid = []
182 for key, value in options.items():
183 if key_path:
184 new_path = "%s/%s" % (key_path, key)
185 else:
186 new_path = key
187
188 if key not in defaults_keys:
189 invalid.append(new_path)
190 elif isinstance(value, dict):
191 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
192
193 return invalid
194
195
197 """Verify a dict has only keys set which also are in the defaults dict.
198
199 @param options: The user provided options
200 @param defaults: The default dictionary
201 @raise error.OpPrereqError: If one of the keys is not supported
202
203 """
204 invalid = _ComputeMissingKeys("", options, defaults)
205
206 if invalid:
207 raise errors.OpPrereqError("Provided option keys not supported: %s" %
208 CommaJoin(invalid), errors.ECODE_INVAL)
209
210
212 """List volume groups and their size
213
214 @rtype: dict
215 @return:
216 Dictionary with keys volume name and values
217 the size of the volume
218
219 """
220 command = "vgs --noheadings --units m --nosuffix -o name,size"
221 result = RunCmd(command)
222 retval = {}
223 if result.failed:
224 return retval
225
226 for line in result.stdout.splitlines():
227 try:
228 name, size = line.split()
229 size = int(float(size))
230 except (IndexError, ValueError), err:
231 logging.error("Invalid output from vgs (%s): %s", err, line)
232 continue
233
234 retval[name] = size
235
236 return retval
237
238
240 """Check whether the given bridge exists in the system
241
242 @type bridge: str
243 @param bridge: the bridge name to check
244 @rtype: boolean
245 @return: True if it does
246
247 """
248 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
249
250
252 """Try to convert a value ignoring errors.
253
254 This function tries to apply function I{fn} to I{val}. If no
255 C{ValueError} or C{TypeError} exceptions are raised, it will return
256 the result, else it will return the original value. Any other
257 exceptions are propagated to the caller.
258
259 @type fn: callable
260 @param fn: function to apply to the value
261 @param val: the value to be converted
262 @return: The converted value if the conversion was successful,
263 otherwise the original value.
264
265 """
266 try:
267 nv = fn(val)
268 except (ValueError, TypeError):
269 nv = val
270 return nv
271
272
274 """Parse a CPU mask definition and return the list of CPU IDs.
275
276 CPU mask format: comma-separated list of CPU IDs
277 or dash-separated ID ranges
278 Example: "0-2,5" -> "0,1,2,5"
279
280 @type cpu_mask: str
281 @param cpu_mask: CPU mask definition
282 @rtype: list of int
283 @return: list of CPU IDs
284
285 """
286 if not cpu_mask:
287 return []
288 cpu_list = []
289 for range_def in cpu_mask.split(","):
290 boundaries = range_def.split("-")
291 n_elements = len(boundaries)
292 if n_elements > 2:
293 raise errors.ParseError("Invalid CPU ID range definition"
294 " (only one hyphen allowed): %s" % range_def)
295 try:
296 lower = int(boundaries[0])
297 except (ValueError, TypeError), err:
298 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
299 " CPU ID range: %s" % str(err))
300 try:
301 higher = int(boundaries[-1])
302 except (ValueError, TypeError), err:
303 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
304 " CPU ID range: %s" % str(err))
305 if lower > higher:
306 raise errors.ParseError("Invalid CPU ID range definition"
307 " (%d > %d): %s" % (lower, higher, range_def))
308 cpu_list.extend(range(lower, higher + 1))
309 return cpu_list
310
311
313 """Parse a multiple CPU mask definition and return the list of CPU IDs.
314
315 CPU mask format: colon-separated list of comma-separated list of CPU IDs
316 or dash-separated ID ranges, with optional "all" as CPU value
317 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
318
319 @type cpu_mask: str
320 @param cpu_mask: multiple CPU mask definition
321 @rtype: list of lists of int
322 @return: list of lists of CPU IDs
323
324 """
325 if not cpu_mask:
326 return []
327 cpu_list = []
328 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
329 if range_def == constants.CPU_PINNING_ALL:
330 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
331 else:
332
333 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
334
335 return cpu_list
336
337
339 """Try to get the homedir of the given user.
340
341 The user can be passed either as a string (denoting the name) or as
342 an integer (denoting the user id). If the user is not found, the
343 C{default} argument is returned, which defaults to C{None}.
344
345 """
346 try:
347 if isinstance(user, basestring):
348 result = pwd.getpwnam(user)
349 elif isinstance(user, (int, long)):
350 result = pwd.getpwuid(user)
351 else:
352 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
353 type(user))
354 except KeyError:
355 return default
356 return result.pw_dir
357
358
360 """Returns the first non-existing integer from seq.
361
362 The seq argument should be a sorted list of positive integers. The
363 first time the index of an element is smaller than the element
364 value, the index will be returned.
365
366 The base argument is used to start at a different offset,
367 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
368
369 Example: C{[0, 1, 3]} will return I{2}.
370
371 @type seq: sequence
372 @param seq: the sequence to be analyzed.
373 @type base: int
374 @param base: use this value as the base index of the sequence
375 @rtype: int
376 @return: the first non-used index in the sequence
377
378 """
379 for idx, elem in enumerate(seq):
380 assert elem >= base, "Passed element is higher than base offset"
381 if elem > idx + base:
382
383 return idx + base
384 return None
385
386
388 """Waits for a condition to occur on the socket.
389
390 Immediately returns at the first interruption.
391
392 @type fdobj: integer or object supporting a fileno() method
393 @param fdobj: entity to wait for events on
394 @type event: integer
395 @param event: ORed condition (see select module)
396 @type timeout: float or None
397 @param timeout: Timeout in seconds
398 @rtype: int or None
399 @return: None for timeout, otherwise occured conditions
400
401 """
402 check = (event | select.POLLPRI |
403 select.POLLNVAL | select.POLLHUP | select.POLLERR)
404
405 if timeout is not None:
406
407 timeout *= 1000
408
409 poller = select.poll()
410 poller.register(fdobj, event)
411 try:
412
413
414
415 io_events = poller.poll(timeout)
416 except select.error, err:
417 if err[0] != errno.EINTR:
418 raise
419 io_events = []
420 if io_events and io_events[0][1] & check:
421 return io_events[0][1]
422 else:
423 return None
424
425
427 """Retry helper for WaitForFdCondition.
428
429 This class contains the retried and wait functions that make sure
430 WaitForFdCondition can continue waiting until the timeout is actually
431 expired.
432
433 """
434
436 self.timeout = timeout
437
438 - def Poll(self, fdobj, event):
444
446 self.timeout = timeout
447
448
450 """Waits for a condition to occur on the socket.
451
452 Retries until the timeout is expired, even if interrupted.
453
454 @type fdobj: integer or object supporting a fileno() method
455 @param fdobj: entity to wait for events on
456 @type event: integer
457 @param event: ORed condition (see select module)
458 @type timeout: float or None
459 @param timeout: Timeout in seconds
460 @rtype: int or None
461 @return: None for timeout, otherwise occured conditions
462
463 """
464 if timeout is not None:
465 retrywaiter = FdConditionWaiterHelper(timeout)
466 try:
467 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
468 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
469 except RetryTimeout:
470 result = None
471 else:
472 result = None
473 while result is None:
474 result = SingleWaitForFdCondition(fdobj, event, timeout)
475 return result
476
477
479 """Check for and start daemon if not alive.
480
481 @type name: string
482 @param name: daemon name
483
484 @rtype: bool
485 @return: 'True' if daemon successfully started,
486 'False' otherwise
487
488 """
489 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
490 if result.failed:
491 logging.error("Can't start daemon '%s', failure %s, output: %s",
492 name, result.fail_reason, result.output)
493 return False
494
495 return True
496
497
499 """Stop daemon
500
501 @type name: string
502 @param name: daemon name
503
504 @rtype: bool
505 @return: 'True' if daemon successfully stopped,
506 'False' otherwise
507
508 """
509 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
510 if result.failed:
511 logging.error("Can't stop daemon '%s', failure %s, output: %s",
512 name, result.fail_reason, result.output)
513 return False
514
515 return True
516
517
519 """Splits time as floating point number into a tuple.
520
521 @param value: Time in seconds
522 @type value: int or float
523 @return: Tuple containing (seconds, microseconds)
524
525 """
526 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
527
528 assert 0 <= seconds, \
529 "Seconds must be larger than or equal to 0, but are %s" % seconds
530 assert 0 <= microseconds <= 999999, \
531 "Microseconds must be 0-999999, but are %s" % microseconds
532
533 return (int(seconds), int(microseconds))
534
535
537 """Merges a tuple into time as a floating point number.
538
539 @param timetuple: Time as tuple, (seconds, microseconds)
540 @type timetuple: tuple
541 @return: Time as a floating point number expressed in seconds
542
543 """
544 (seconds, microseconds) = timetuple
545
546 assert 0 <= seconds, \
547 "Seconds must be larger than or equal to 0, but are %s" % seconds
548 assert 0 <= microseconds <= 999999, \
549 "Microseconds must be 0-999999, but are %s" % microseconds
550
551 return float(seconds) + (float(microseconds) * 0.000001)
552
553
555 """Return the current timestamp expressed as number of nanoseconds since the
556 unix epoch
557
558 @return: nanoseconds since the Unix epoch
559
560 """
561 return int(time.time() * 1000000000)
562
563
565 """Tries to find an item in a dictionary matching a name.
566
567 Callers have to ensure the data names aren't contradictory (e.g. a regexp
568 that matches a string). If the name isn't a direct key, all regular
569 expression objects in the dictionary are matched against it.
570
571 @type data: dict
572 @param data: Dictionary containing data
573 @type name: string
574 @param name: Name to look for
575 @rtype: tuple; (value in dictionary, matched groups as list)
576
577 """
578 if name in data:
579 return (data[name], [])
580
581 for key, value in data.items():
582
583 if hasattr(key, "match"):
584 m = key.match(name)
585 if m:
586 return (value, list(m.groups()))
587
588 return None
589
590
592 """Returns the list of mounted filesystems.
593
594 This function is Linux-specific.
595
596 @param filename: path of mounts file (/proc/mounts by default)
597 @rtype: list of tuples
598 @return: list of mount entries (device, mountpoint, fstype, options)
599
600 """
601
602 data = []
603 mountlines = ReadFile(filename).splitlines()
604 for line in mountlines:
605 device, mountpoint, fstype, options, _ = line.split(None, 4)
606 data.append((device, mountpoint, fstype, options))
607
608 return data
609
610
612 """Signal Handled decoration.
613
614 This special decorator installs a signal handler and then calls the target
615 function. The function must accept a 'signal_handlers' keyword argument,
616 which will contain a dict indexed by signal number, with SignalHandler
617 objects as values.
618
619 The decorator can be safely stacked with iself, to handle multiple signals
620 with different handlers.
621
622 @type signums: list
623 @param signums: signals to intercept
624
625 """
626 def wrap(fn):
627 def sig_function(*args, **kwargs):
628 assert "signal_handlers" not in kwargs or \
629 kwargs["signal_handlers"] is None or \
630 isinstance(kwargs["signal_handlers"], dict), \
631 "Wrong signal_handlers parameter in original function call"
632 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
633 signal_handlers = kwargs["signal_handlers"]
634 else:
635 signal_handlers = {}
636 kwargs["signal_handlers"] = signal_handlers
637 sighandler = SignalHandler(signums)
638 try:
639 for sig in signums:
640 signal_handlers[sig] = sighandler
641 return fn(*args, **kwargs)
642 finally:
643 sighandler.Reset()
644 return sig_function
645 return wrap
646
647
649 """Checks whether a timeout has expired.
650
651 """
652 return _time_fn() > (epoch + timeout)
653
654
665 else:
666
669
671 """Initializes this class.
672
673 """
674 (read_fd, write_fd) = os.pipe()
675
676
677
678
679 self._read_fh = os.fdopen(read_fd, "r", 0)
680 self._write_fh = os.fdopen(write_fd, "w", 0)
681
682 self._previous = self._SetWakeupFd(self._write_fh.fileno())
683
684
685 self.fileno = self._read_fh.fileno
686 self.read = self._read_fh.read
687
689 """Restores the previous wakeup file descriptor.
690
691 """
692 if hasattr(self, "_previous") and self._previous is not None:
693 self._SetWakeupFd(self._previous)
694 self._previous = None
695
697 """Notifies the wakeup file descriptor.
698
699 """
700 self._write_fh.write(chr(0))
701
703 """Called before object deletion.
704
705 """
706 self.Reset()
707
708
710 """Generic signal handler class.
711
712 It automatically restores the original handler when deconstructed or
713 when L{Reset} is called. You can either pass your own handler
714 function in or query the L{called} attribute to detect whether the
715 signal was sent.
716
717 @type signum: list
718 @ivar signum: the signals we handle
719 @type called: boolean
720 @ivar called: tracks whether any of the signals have been raised
721
722 """
723 - def __init__(self, signum, handler_fn=None, wakeup=None):
724 """Constructs a new SignalHandler instance.
725
726 @type signum: int or list of ints
727 @param signum: Single signal number or set of signal numbers
728 @type handler_fn: callable
729 @param handler_fn: Signal handling function
730
731 """
732 assert handler_fn is None or callable(handler_fn)
733
734 self.signum = set(signum)
735 self.called = False
736
737 self._handler_fn = handler_fn
738 self._wakeup = wakeup
739
740 self._previous = {}
741 try:
742 for signum in self.signum:
743
744 prev_handler = signal.signal(signum, self._HandleSignal)
745 try:
746 self._previous[signum] = prev_handler
747 except:
748
749 signal.signal(signum, prev_handler)
750 raise
751 except:
752
753 self.Reset()
754
755
756 raise
757
760
762 """Restore previous handler.
763
764 This will reset all the signals to their previous handlers.
765
766 """
767 for signum, prev_handler in self._previous.items():
768 signal.signal(signum, prev_handler)
769
770 del self._previous[signum]
771
773 """Unsets the L{called} flag.
774
775 This function can be used in case a signal may arrive several times.
776
777 """
778 self.called = False
779
781 """Actual signal handling function.
782
783 """
784
785
786 self.called = True
787
788 if self._wakeup:
789
790 self._wakeup.Notify()
791
792 if self._handler_fn:
793 self._handler_fn(signum, frame)
794
795
797 """A simple field set.
798
799 Among the features are:
800 - checking if a string is among a list of static string or regex objects
801 - checking if a whole list of string matches
802 - returning the matching groups from a regex match
803
804 Internally, all fields are held as regular expression objects.
805
806 """
808 self.items = [re.compile("^%s$" % value) for value in items]
809
811 """Extend the field set with the items from another one"""
812 self.items.extend(other_set.items)
813
815 """Checks if a field matches the current set
816
817 @type field: str
818 @param field: the string to match
819 @return: either None or a regular expression match object
820
821 """
822 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
823 return m
824 return None
825
827 """Returns the list of fields not matching the current set
828
829 @type items: list
830 @param items: the list of fields to check
831 @rtype: list
832 @return: list of non-matching fields
833
834 """
835 return [val for val in items if not self.Matches(val)]
836
837
880