1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Ganeti utility module.
32
33 This module holds functions that can be used in both daemons (all) and
34 the command line scripts.
35
36 """
37
38
39
40 import os
41 import re
42 import errno
43 import pwd
44 import time
45 import itertools
46 import select
47 import logging
48 import signal
49
50 from ganeti import errors
51 from ganeti import constants
52 from ganeti import compat
53 from ganeti import pathutils
54
55 from ganeti.utils.algo import *
56 from ganeti.utils.filelock import *
57 from ganeti.utils.hash import *
58 from ganeti.utils.io import *
59 from ganeti.utils.log import *
60 from ganeti.utils.lvm import *
61 from ganeti.utils.mlock import *
62 from ganeti.utils.nodesetup import *
63 from ganeti.utils.process import *
64 from ganeti.utils.retry import *
65 from ganeti.utils.storage import *
66 from ganeti.utils.text import *
67 from ganeti.utils.wrapper import *
68 from ganeti.utils.version import *
69 from ganeti.utils.x509 import *
70
71
72 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
73
74 UUID_RE = re.compile(constants.UUID_REGEX)
75
76
78 """Force the values of a dict to have certain types.
79
80 @type target: dict
81 @param target: the dict to update
82 @type key_types: dict
83 @param key_types: dict mapping target dict keys to types
84 in constants.ENFORCEABLE_TYPES
85 @type allowed_values: list
86 @keyword allowed_values: list of specially allowed values
87
88 """
89 if allowed_values is None:
90 allowed_values = []
91
92 if not isinstance(target, dict):
93 msg = "Expected dictionary, got '%s'" % target
94 raise errors.TypeEnforcementError(msg)
95
96 for key in target:
97 if key not in key_types:
98 msg = "Unknown parameter '%s'" % key
99 raise errors.TypeEnforcementError(msg)
100
101 if target[key] in allowed_values:
102 continue
103
104 ktype = key_types[key]
105 if ktype not in constants.ENFORCEABLE_TYPES:
106 msg = "'%s' has non-enforceable type %s" % (key, ktype)
107 raise errors.ProgrammerError(msg)
108
109 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
110 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
111 pass
112 elif not isinstance(target[key], basestring):
113 if isinstance(target[key], bool) and not target[key]:
114 target[key] = ""
115 else:
116 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
117 raise errors.TypeEnforcementError(msg)
118 elif ktype == constants.VTYPE_BOOL:
119 if isinstance(target[key], basestring) and target[key]:
120 if target[key].lower() == constants.VALUE_FALSE:
121 target[key] = False
122 elif target[key].lower() == constants.VALUE_TRUE:
123 target[key] = True
124 else:
125 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
126 raise errors.TypeEnforcementError(msg)
127 elif target[key]:
128 target[key] = True
129 else:
130 target[key] = False
131 elif ktype == constants.VTYPE_SIZE:
132 try:
133 target[key] = ParseUnit(target[key])
134 except errors.UnitParseError, err:
135 msg = "'%s' (value %s) is not a valid size. error: %s" % \
136 (key, target[key], err)
137 raise errors.TypeEnforcementError(msg)
138 elif ktype == constants.VTYPE_INT:
139 try:
140 target[key] = int(target[key])
141 except (ValueError, TypeError):
142 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
143 raise errors.TypeEnforcementError(msg)
144
145
147 """Validate the given service name.
148
149 @type name: number or string
150 @param name: Service name or port specification
151
152 """
153 try:
154 numport = int(name)
155 except (ValueError, TypeError):
156
157 valid = _VALID_SERVICE_NAME_RE.match(name)
158 else:
159
160
161 valid = (numport >= 0 and numport < (1 << 16))
162
163 if not valid:
164 raise errors.OpPrereqError("Invalid service name '%s'" % name,
165 errors.ECODE_INVAL)
166
167 return name
168
169
171 """Helper functions to compute which keys a invalid.
172
173 @param key_path: The current key path (if any)
174 @param options: The user provided options
175 @param defaults: The default dictionary
176 @return: A list of invalid keys
177
178 """
179 defaults_keys = frozenset(defaults.keys())
180 invalid = []
181 for key, value in options.items():
182 if key_path:
183 new_path = "%s/%s" % (key_path, key)
184 else:
185 new_path = key
186
187 if key not in defaults_keys:
188 invalid.append(new_path)
189 elif isinstance(value, dict):
190 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
191
192 return invalid
193
194
196 """Verify a dict has only keys set which also are in the defaults dict.
197
198 @param options: The user provided options
199 @param defaults: The default dictionary
200 @raise error.OpPrereqError: If one of the keys is not supported
201
202 """
203 invalid = _ComputeMissingKeys("", options, defaults)
204
205 if invalid:
206 raise errors.OpPrereqError("Provided option keys not supported: %s" %
207 CommaJoin(invalid), errors.ECODE_INVAL)
208
209
211 """List volume groups and their size
212
213 @rtype: dict
214 @return:
215 Dictionary with keys volume name and values
216 the size of the volume
217
218 """
219 command = "vgs --noheadings --units m --nosuffix -o name,size"
220 result = RunCmd(command)
221 retval = {}
222 if result.failed:
223 return retval
224
225 for line in result.stdout.splitlines():
226 try:
227 name, size = line.split()
228 size = int(float(size))
229 except (IndexError, ValueError), err:
230 logging.error("Invalid output from vgs (%s): %s", err, line)
231 continue
232
233 retval[name] = size
234
235 return retval
236
237
239 """Check whether the given bridge exists in the system
240
241 @type bridge: str
242 @param bridge: the bridge name to check
243 @rtype: boolean
244 @return: True if it does
245
246 """
247 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
248
249
251 """Try to convert a value ignoring errors.
252
253 This function tries to apply function I{fn} to I{val}. If no
254 C{ValueError} or C{TypeError} exceptions are raised, it will return
255 the result, else it will return the original value. Any other
256 exceptions are propagated to the caller.
257
258 @type fn: callable
259 @param fn: function to apply to the value
260 @param val: the value to be converted
261 @return: The converted value if the conversion was successful,
262 otherwise the original value.
263
264 """
265 try:
266 nv = fn(val)
267 except (ValueError, TypeError):
268 nv = val
269 return nv
270
271
273 """Parse a CPU mask definition and return the list of CPU IDs.
274
275 CPU mask format: comma-separated list of CPU IDs
276 or dash-separated ID ranges
277 Example: "0-2,5" -> "0,1,2,5"
278
279 @type cpu_mask: str
280 @param cpu_mask: CPU mask definition
281 @rtype: list of int
282 @return: list of CPU IDs
283
284 """
285 if not cpu_mask:
286 return []
287 cpu_list = []
288 for range_def in cpu_mask.split(","):
289 boundaries = range_def.split("-")
290 n_elements = len(boundaries)
291 if n_elements > 2:
292 raise errors.ParseError("Invalid CPU ID range definition"
293 " (only one hyphen allowed): %s" % range_def)
294 try:
295 lower = int(boundaries[0])
296 except (ValueError, TypeError), err:
297 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
298 " CPU ID range: %s" % str(err))
299 try:
300 higher = int(boundaries[-1])
301 except (ValueError, TypeError), err:
302 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
303 " CPU ID range: %s" % str(err))
304 if lower > higher:
305 raise errors.ParseError("Invalid CPU ID range definition"
306 " (%d > %d): %s" % (lower, higher, range_def))
307 cpu_list.extend(range(lower, higher + 1))
308 return cpu_list
309
310
312 """Parse a multiple CPU mask definition and return the list of CPU IDs.
313
314 CPU mask format: colon-separated list of comma-separated list of CPU IDs
315 or dash-separated ID ranges, with optional "all" as CPU value
316 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
317
318 @type cpu_mask: str
319 @param cpu_mask: multiple CPU mask definition
320 @rtype: list of lists of int
321 @return: list of lists of CPU IDs
322
323 """
324 if not cpu_mask:
325 return []
326 cpu_list = []
327 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
328 if range_def == constants.CPU_PINNING_ALL:
329 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
330 else:
331
332 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
333
334 return cpu_list
335
336
338 """Try to get the homedir of the given user.
339
340 The user can be passed either as a string (denoting the name) or as
341 an integer (denoting the user id). If the user is not found, the
342 C{default} argument is returned, which defaults to C{None}.
343
344 """
345 try:
346 if isinstance(user, basestring):
347 result = pwd.getpwnam(user)
348 elif isinstance(user, (int, long)):
349 result = pwd.getpwuid(user)
350 else:
351 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
352 type(user))
353 except KeyError:
354 return default
355 return result.pw_dir
356
357
359 """Returns the first non-existing integer from seq.
360
361 The seq argument should be a sorted list of positive integers. The
362 first time the index of an element is smaller than the element
363 value, the index will be returned.
364
365 The base argument is used to start at a different offset,
366 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
367
368 Example: C{[0, 1, 3]} will return I{2}.
369
370 @type seq: sequence
371 @param seq: the sequence to be analyzed.
372 @type base: int
373 @param base: use this value as the base index of the sequence
374 @rtype: int
375 @return: the first non-used index in the sequence
376
377 """
378 for idx, elem in enumerate(seq):
379 assert elem >= base, "Passed element is higher than base offset"
380 if elem > idx + base:
381
382 return idx + base
383 return None
384
385
387 """Waits for a condition to occur on the socket.
388
389 Immediately returns at the first interruption.
390
391 @type fdobj: integer or object supporting a fileno() method
392 @param fdobj: entity to wait for events on
393 @type event: integer
394 @param event: ORed condition (see select module)
395 @type timeout: float or None
396 @param timeout: Timeout in seconds
397 @rtype: int or None
398 @return: None for timeout, otherwise occured conditions
399
400 """
401 check = (event | select.POLLPRI |
402 select.POLLNVAL | select.POLLHUP | select.POLLERR)
403
404 if timeout is not None:
405
406 timeout *= 1000
407
408 poller = select.poll()
409 poller.register(fdobj, event)
410 try:
411
412
413
414 io_events = poller.poll(timeout)
415 except select.error, err:
416 if err[0] != errno.EINTR:
417 raise
418 io_events = []
419 if io_events and io_events[0][1] & check:
420 return io_events[0][1]
421 else:
422 return None
423
424
426 """Retry helper for WaitForFdCondition.
427
428 This class contains the retried and wait functions that make sure
429 WaitForFdCondition can continue waiting until the timeout is actually
430 expired.
431
432 """
433
435 self.timeout = timeout
436
437 - def Poll(self, fdobj, event):
443
445 self.timeout = timeout
446
447
449 """Waits for a condition to occur on the socket.
450
451 Retries until the timeout is expired, even if interrupted.
452
453 @type fdobj: integer or object supporting a fileno() method
454 @param fdobj: entity to wait for events on
455 @type event: integer
456 @param event: ORed condition (see select module)
457 @type timeout: float or None
458 @param timeout: Timeout in seconds
459 @rtype: int or None
460 @return: None for timeout, otherwise occured conditions
461
462 """
463 if timeout is not None:
464 retrywaiter = FdConditionWaiterHelper(timeout)
465 try:
466 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
467 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
468 except RetryTimeout:
469 result = None
470 else:
471 result = None
472 while result is None:
473 result = SingleWaitForFdCondition(fdobj, event, timeout)
474 return result
475
476
478 """Check for and start daemon if not alive.
479
480 """
481 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
482 if result.failed:
483 logging.error("Can't start daemon '%s', failure %s, output: %s",
484 name, result.fail_reason, result.output)
485 return False
486
487 return True
488
489
491 """Stop daemon
492
493 """
494 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
495 if result.failed:
496 logging.error("Can't stop daemon '%s', failure %s, output: %s",
497 name, result.fail_reason, result.output)
498 return False
499
500 return True
501
502
504 """Splits time as floating point number into a tuple.
505
506 @param value: Time in seconds
507 @type value: int or float
508 @return: Tuple containing (seconds, microseconds)
509
510 """
511 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
512
513 assert 0 <= seconds, \
514 "Seconds must be larger than or equal to 0, but are %s" % seconds
515 assert 0 <= microseconds <= 999999, \
516 "Microseconds must be 0-999999, but are %s" % microseconds
517
518 return (int(seconds), int(microseconds))
519
520
522 """Merges a tuple into time as a floating point number.
523
524 @param timetuple: Time as tuple, (seconds, microseconds)
525 @type timetuple: tuple
526 @return: Time as a floating point number expressed in seconds
527
528 """
529 (seconds, microseconds) = timetuple
530
531 assert 0 <= seconds, \
532 "Seconds must be larger than or equal to 0, but are %s" % seconds
533 assert 0 <= microseconds <= 999999, \
534 "Microseconds must be 0-999999, but are %s" % microseconds
535
536 return float(seconds) + (float(microseconds) * 0.000001)
537
538
540 """Return the current timestamp expressed as number of nanoseconds since the
541 unix epoch
542
543 @return: nanoseconds since the Unix epoch
544
545 """
546 return int(time.time() * 1000000000)
547
548
550 """Tries to find an item in a dictionary matching a name.
551
552 Callers have to ensure the data names aren't contradictory (e.g. a regexp
553 that matches a string). If the name isn't a direct key, all regular
554 expression objects in the dictionary are matched against it.
555
556 @type data: dict
557 @param data: Dictionary containing data
558 @type name: string
559 @param name: Name to look for
560 @rtype: tuple; (value in dictionary, matched groups as list)
561
562 """
563 if name in data:
564 return (data[name], [])
565
566 for key, value in data.items():
567
568 if hasattr(key, "match"):
569 m = key.match(name)
570 if m:
571 return (value, list(m.groups()))
572
573 return None
574
575
577 """Returns the list of mounted filesystems.
578
579 This function is Linux-specific.
580
581 @param filename: path of mounts file (/proc/mounts by default)
582 @rtype: list of tuples
583 @return: list of mount entries (device, mountpoint, fstype, options)
584
585 """
586
587 data = []
588 mountlines = ReadFile(filename).splitlines()
589 for line in mountlines:
590 device, mountpoint, fstype, options, _ = line.split(None, 4)
591 data.append((device, mountpoint, fstype, options))
592
593 return data
594
595
597 """Signal Handled decoration.
598
599 This special decorator installs a signal handler and then calls the target
600 function. The function must accept a 'signal_handlers' keyword argument,
601 which will contain a dict indexed by signal number, with SignalHandler
602 objects as values.
603
604 The decorator can be safely stacked with iself, to handle multiple signals
605 with different handlers.
606
607 @type signums: list
608 @param signums: signals to intercept
609
610 """
611 def wrap(fn):
612 def sig_function(*args, **kwargs):
613 assert "signal_handlers" not in kwargs or \
614 kwargs["signal_handlers"] is None or \
615 isinstance(kwargs["signal_handlers"], dict), \
616 "Wrong signal_handlers parameter in original function call"
617 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
618 signal_handlers = kwargs["signal_handlers"]
619 else:
620 signal_handlers = {}
621 kwargs["signal_handlers"] = signal_handlers
622 sighandler = SignalHandler(signums)
623 try:
624 for sig in signums:
625 signal_handlers[sig] = sighandler
626 return fn(*args, **kwargs)
627 finally:
628 sighandler.Reset()
629 return sig_function
630 return wrap
631
632
634 """Checks whether a timeout has expired.
635
636 """
637 return _time_fn() > (epoch + timeout)
638
639
650 else:
651
654
656 """Initializes this class.
657
658 """
659 (read_fd, write_fd) = os.pipe()
660
661
662
663
664 self._read_fh = os.fdopen(read_fd, "r", 0)
665 self._write_fh = os.fdopen(write_fd, "w", 0)
666
667 self._previous = self._SetWakeupFd(self._write_fh.fileno())
668
669
670 self.fileno = self._read_fh.fileno
671 self.read = self._read_fh.read
672
674 """Restores the previous wakeup file descriptor.
675
676 """
677 if hasattr(self, "_previous") and self._previous is not None:
678 self._SetWakeupFd(self._previous)
679 self._previous = None
680
682 """Notifies the wakeup file descriptor.
683
684 """
685 self._write_fh.write(chr(0))
686
688 """Called before object deletion.
689
690 """
691 self.Reset()
692
693
695 """Generic signal handler class.
696
697 It automatically restores the original handler when deconstructed or
698 when L{Reset} is called. You can either pass your own handler
699 function in or query the L{called} attribute to detect whether the
700 signal was sent.
701
702 @type signum: list
703 @ivar signum: the signals we handle
704 @type called: boolean
705 @ivar called: tracks whether any of the signals have been raised
706
707 """
708 - def __init__(self, signum, handler_fn=None, wakeup=None):
709 """Constructs a new SignalHandler instance.
710
711 @type signum: int or list of ints
712 @param signum: Single signal number or set of signal numbers
713 @type handler_fn: callable
714 @param handler_fn: Signal handling function
715
716 """
717 assert handler_fn is None or callable(handler_fn)
718
719 self.signum = set(signum)
720 self.called = False
721
722 self._handler_fn = handler_fn
723 self._wakeup = wakeup
724
725 self._previous = {}
726 try:
727 for signum in self.signum:
728
729 prev_handler = signal.signal(signum, self._HandleSignal)
730 try:
731 self._previous[signum] = prev_handler
732 except:
733
734 signal.signal(signum, prev_handler)
735 raise
736 except:
737
738 self.Reset()
739
740
741 raise
742
745
747 """Restore previous handler.
748
749 This will reset all the signals to their previous handlers.
750
751 """
752 for signum, prev_handler in self._previous.items():
753 signal.signal(signum, prev_handler)
754
755 del self._previous[signum]
756
758 """Unsets the L{called} flag.
759
760 This function can be used in case a signal may arrive several times.
761
762 """
763 self.called = False
764
766 """Actual signal handling function.
767
768 """
769
770
771 self.called = True
772
773 if self._wakeup:
774
775 self._wakeup.Notify()
776
777 if self._handler_fn:
778 self._handler_fn(signum, frame)
779
780
782 """A simple field set.
783
784 Among the features are:
785 - checking if a string is among a list of static string or regex objects
786 - checking if a whole list of string matches
787 - returning the matching groups from a regex match
788
789 Internally, all fields are held as regular expression objects.
790
791 """
793 self.items = [re.compile("^%s$" % value) for value in items]
794
796 """Extend the field set with the items from another one"""
797 self.items.extend(other_set.items)
798
800 """Checks if a field matches the current set
801
802 @type field: str
803 @param field: the string to match
804 @return: either None or a regular expression match object
805
806 """
807 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
808 return m
809 return None
810
812 """Returns the list of fields not matching the current set
813
814 @type items: list
815 @param items: the list of fields to check
816 @rtype: list
817 @return: list of non-matching fields
818
819 """
820 return [val for val in items if not self.Matches(val)]
821
822
865