1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Ganeti utility module.
23
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
26
27 """
28
29
30
31 import os
32 import re
33 import errno
34 import pwd
35 import time
36 import itertools
37 import select
38 import logging
39 import signal
40
41 from ganeti import errors
42 from ganeti import constants
43 from ganeti import compat
44 from ganeti import pathutils
45
46 from ganeti.utils.algo import *
47 from ganeti.utils.filelock import *
48 from ganeti.utils.hash import *
49 from ganeti.utils.io import *
50 from ganeti.utils.log import *
51 from ganeti.utils.lvm import *
52 from ganeti.utils.mlock import *
53 from ganeti.utils.nodesetup import *
54 from ganeti.utils.process import *
55 from ganeti.utils.retry import *
56 from ganeti.utils.text import *
57 from ganeti.utils.wrapper import *
58 from ganeti.utils.x509 import *
59
60
61 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
62
63 UUID_RE = re.compile(constants.UUID_REGEX)
64
65
67 """Force the values of a dict to have certain types.
68
69 @type target: dict
70 @param target: the dict to update
71 @type key_types: dict
72 @param key_types: dict mapping target dict keys to types
73 in constants.ENFORCEABLE_TYPES
74 @type allowed_values: list
75 @keyword allowed_values: list of specially allowed values
76
77 """
78 if allowed_values is None:
79 allowed_values = []
80
81 if not isinstance(target, dict):
82 msg = "Expected dictionary, got '%s'" % target
83 raise errors.TypeEnforcementError(msg)
84
85 for key in target:
86 if key not in key_types:
87 msg = "Unknown parameter '%s'" % key
88 raise errors.TypeEnforcementError(msg)
89
90 if target[key] in allowed_values:
91 continue
92
93 ktype = key_types[key]
94 if ktype not in constants.ENFORCEABLE_TYPES:
95 msg = "'%s' has non-enforceable type %s" % (key, ktype)
96 raise errors.ProgrammerError(msg)
97
98 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
99 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
100 pass
101 elif not isinstance(target[key], basestring):
102 if isinstance(target[key], bool) and not target[key]:
103 target[key] = ""
104 else:
105 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
106 raise errors.TypeEnforcementError(msg)
107 elif ktype == constants.VTYPE_BOOL:
108 if isinstance(target[key], basestring) and target[key]:
109 if target[key].lower() == constants.VALUE_FALSE:
110 target[key] = False
111 elif target[key].lower() == constants.VALUE_TRUE:
112 target[key] = True
113 else:
114 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
115 raise errors.TypeEnforcementError(msg)
116 elif target[key]:
117 target[key] = True
118 else:
119 target[key] = False
120 elif ktype == constants.VTYPE_SIZE:
121 try:
122 target[key] = ParseUnit(target[key])
123 except errors.UnitParseError, err:
124 msg = "'%s' (value %s) is not a valid size. error: %s" % \
125 (key, target[key], err)
126 raise errors.TypeEnforcementError(msg)
127 elif ktype == constants.VTYPE_INT:
128 try:
129 target[key] = int(target[key])
130 except (ValueError, TypeError):
131 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
132 raise errors.TypeEnforcementError(msg)
133
134
136 """Validate the given service name.
137
138 @type name: number or string
139 @param name: Service name or port specification
140
141 """
142 try:
143 numport = int(name)
144 except (ValueError, TypeError):
145
146 valid = _VALID_SERVICE_NAME_RE.match(name)
147 else:
148
149
150 valid = (numport >= 0 and numport < (1 << 16))
151
152 if not valid:
153 raise errors.OpPrereqError("Invalid service name '%s'" % name,
154 errors.ECODE_INVAL)
155
156 return name
157
158
160 """Helper functions to compute which keys a invalid.
161
162 @param key_path: The current key path (if any)
163 @param options: The user provided options
164 @param defaults: The default dictionary
165 @return: A list of invalid keys
166
167 """
168 defaults_keys = frozenset(defaults.keys())
169 invalid = []
170 for key, value in options.items():
171 if key_path:
172 new_path = "%s/%s" % (key_path, key)
173 else:
174 new_path = key
175
176 if key not in defaults_keys:
177 invalid.append(new_path)
178 elif isinstance(value, dict):
179 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
180
181 return invalid
182
183
185 """Verify a dict has only keys set which also are in the defaults dict.
186
187 @param options: The user provided options
188 @param defaults: The default dictionary
189 @raise error.OpPrereqError: If one of the keys is not supported
190
191 """
192 invalid = _ComputeMissingKeys("", options, defaults)
193
194 if invalid:
195 raise errors.OpPrereqError("Provided option keys not supported: %s" %
196 CommaJoin(invalid), errors.ECODE_INVAL)
197
198
200 """List volume groups and their size
201
202 @rtype: dict
203 @return:
204 Dictionary with keys volume name and values
205 the size of the volume
206
207 """
208 command = "vgs --noheadings --units m --nosuffix -o name,size"
209 result = RunCmd(command)
210 retval = {}
211 if result.failed:
212 return retval
213
214 for line in result.stdout.splitlines():
215 try:
216 name, size = line.split()
217 size = int(float(size))
218 except (IndexError, ValueError), err:
219 logging.error("Invalid output from vgs (%s): %s", err, line)
220 continue
221
222 retval[name] = size
223
224 return retval
225
226
228 """Check whether the given bridge exists in the system
229
230 @type bridge: str
231 @param bridge: the bridge name to check
232 @rtype: boolean
233 @return: True if it does
234
235 """
236 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
237
238
240 """Try to convert a value ignoring errors.
241
242 This function tries to apply function I{fn} to I{val}. If no
243 C{ValueError} or C{TypeError} exceptions are raised, it will return
244 the result, else it will return the original value. Any other
245 exceptions are propagated to the caller.
246
247 @type fn: callable
248 @param fn: function to apply to the value
249 @param val: the value to be converted
250 @return: The converted value if the conversion was successful,
251 otherwise the original value.
252
253 """
254 try:
255 nv = fn(val)
256 except (ValueError, TypeError):
257 nv = val
258 return nv
259
260
262 """Parse a CPU mask definition and return the list of CPU IDs.
263
264 CPU mask format: comma-separated list of CPU IDs
265 or dash-separated ID ranges
266 Example: "0-2,5" -> "0,1,2,5"
267
268 @type cpu_mask: str
269 @param cpu_mask: CPU mask definition
270 @rtype: list of int
271 @return: list of CPU IDs
272
273 """
274 if not cpu_mask:
275 return []
276 cpu_list = []
277 for range_def in cpu_mask.split(","):
278 boundaries = range_def.split("-")
279 n_elements = len(boundaries)
280 if n_elements > 2:
281 raise errors.ParseError("Invalid CPU ID range definition"
282 " (only one hyphen allowed): %s" % range_def)
283 try:
284 lower = int(boundaries[0])
285 except (ValueError, TypeError), err:
286 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
287 " CPU ID range: %s" % str(err))
288 try:
289 higher = int(boundaries[-1])
290 except (ValueError, TypeError), err:
291 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
292 " CPU ID range: %s" % str(err))
293 if lower > higher:
294 raise errors.ParseError("Invalid CPU ID range definition"
295 " (%d > %d): %s" % (lower, higher, range_def))
296 cpu_list.extend(range(lower, higher + 1))
297 return cpu_list
298
299
301 """Parse a multiple CPU mask definition and return the list of CPU IDs.
302
303 CPU mask format: colon-separated list of comma-separated list of CPU IDs
304 or dash-separated ID ranges, with optional "all" as CPU value
305 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
306
307 @type cpu_mask: str
308 @param cpu_mask: multiple CPU mask definition
309 @rtype: list of lists of int
310 @return: list of lists of CPU IDs
311
312 """
313 if not cpu_mask:
314 return []
315 cpu_list = []
316 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
317 if range_def == constants.CPU_PINNING_ALL:
318 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
319 else:
320
321 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
322
323 return cpu_list
324
325
327 """Try to get the homedir of the given user.
328
329 The user can be passed either as a string (denoting the name) or as
330 an integer (denoting the user id). If the user is not found, the
331 C{default} argument is returned, which defaults to C{None}.
332
333 """
334 try:
335 if isinstance(user, basestring):
336 result = pwd.getpwnam(user)
337 elif isinstance(user, (int, long)):
338 result = pwd.getpwuid(user)
339 else:
340 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
341 type(user))
342 except KeyError:
343 return default
344 return result.pw_dir
345
346
348 """Returns the first non-existing integer from seq.
349
350 The seq argument should be a sorted list of positive integers. The
351 first time the index of an element is smaller than the element
352 value, the index will be returned.
353
354 The base argument is used to start at a different offset,
355 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
356
357 Example: C{[0, 1, 3]} will return I{2}.
358
359 @type seq: sequence
360 @param seq: the sequence to be analyzed.
361 @type base: int
362 @param base: use this value as the base index of the sequence
363 @rtype: int
364 @return: the first non-used index in the sequence
365
366 """
367 for idx, elem in enumerate(seq):
368 assert elem >= base, "Passed element is higher than base offset"
369 if elem > idx + base:
370
371 return idx + base
372 return None
373
374
376 """Waits for a condition to occur on the socket.
377
378 Immediately returns at the first interruption.
379
380 @type fdobj: integer or object supporting a fileno() method
381 @param fdobj: entity to wait for events on
382 @type event: integer
383 @param event: ORed condition (see select module)
384 @type timeout: float or None
385 @param timeout: Timeout in seconds
386 @rtype: int or None
387 @return: None for timeout, otherwise occured conditions
388
389 """
390 check = (event | select.POLLPRI |
391 select.POLLNVAL | select.POLLHUP | select.POLLERR)
392
393 if timeout is not None:
394
395 timeout *= 1000
396
397 poller = select.poll()
398 poller.register(fdobj, event)
399 try:
400
401
402
403 io_events = poller.poll(timeout)
404 except select.error, err:
405 if err[0] != errno.EINTR:
406 raise
407 io_events = []
408 if io_events and io_events[0][1] & check:
409 return io_events[0][1]
410 else:
411 return None
412
413
415 """Retry helper for WaitForFdCondition.
416
417 This class contains the retried and wait functions that make sure
418 WaitForFdCondition can continue waiting until the timeout is actually
419 expired.
420
421 """
422
424 self.timeout = timeout
425
426 - def Poll(self, fdobj, event):
432
434 self.timeout = timeout
435
436
438 """Waits for a condition to occur on the socket.
439
440 Retries until the timeout is expired, even if interrupted.
441
442 @type fdobj: integer or object supporting a fileno() method
443 @param fdobj: entity to wait for events on
444 @type event: integer
445 @param event: ORed condition (see select module)
446 @type timeout: float or None
447 @param timeout: Timeout in seconds
448 @rtype: int or None
449 @return: None for timeout, otherwise occured conditions
450
451 """
452 if timeout is not None:
453 retrywaiter = FdConditionWaiterHelper(timeout)
454 try:
455 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
456 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
457 except RetryTimeout:
458 result = None
459 else:
460 result = None
461 while result is None:
462 result = SingleWaitForFdCondition(fdobj, event, timeout)
463 return result
464
465
467 """Check for and start daemon if not alive.
468
469 """
470 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
471 if result.failed:
472 logging.error("Can't start daemon '%s', failure %s, output: %s",
473 name, result.fail_reason, result.output)
474 return False
475
476 return True
477
478
480 """Stop daemon
481
482 """
483 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
484 if result.failed:
485 logging.error("Can't stop daemon '%s', failure %s, output: %s",
486 name, result.fail_reason, result.output)
487 return False
488
489 return True
490
491
493 """Splits time as floating point number into a tuple.
494
495 @param value: Time in seconds
496 @type value: int or float
497 @return: Tuple containing (seconds, microseconds)
498
499 """
500 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
501
502 assert 0 <= seconds, \
503 "Seconds must be larger than or equal to 0, but are %s" % seconds
504 assert 0 <= microseconds <= 999999, \
505 "Microseconds must be 0-999999, but are %s" % microseconds
506
507 return (int(seconds), int(microseconds))
508
509
511 """Merges a tuple into time as a floating point number.
512
513 @param timetuple: Time as tuple, (seconds, microseconds)
514 @type timetuple: tuple
515 @return: Time as a floating point number expressed in seconds
516
517 """
518 (seconds, microseconds) = timetuple
519
520 assert 0 <= seconds, \
521 "Seconds must be larger than or equal to 0, but are %s" % seconds
522 assert 0 <= microseconds <= 999999, \
523 "Microseconds must be 0-999999, but are %s" % microseconds
524
525 return float(seconds) + (float(microseconds) * 0.000001)
526
527
529 """Tries to find an item in a dictionary matching a name.
530
531 Callers have to ensure the data names aren't contradictory (e.g. a regexp
532 that matches a string). If the name isn't a direct key, all regular
533 expression objects in the dictionary are matched against it.
534
535 @type data: dict
536 @param data: Dictionary containing data
537 @type name: string
538 @param name: Name to look for
539 @rtype: tuple; (value in dictionary, matched groups as list)
540
541 """
542 if name in data:
543 return (data[name], [])
544
545 for key, value in data.items():
546
547 if hasattr(key, "match"):
548 m = key.match(name)
549 if m:
550 return (value, list(m.groups()))
551
552 return None
553
554
556 """Returns the list of mounted filesystems.
557
558 This function is Linux-specific.
559
560 @param filename: path of mounts file (/proc/mounts by default)
561 @rtype: list of tuples
562 @return: list of mount entries (device, mountpoint, fstype, options)
563
564 """
565
566 data = []
567 mountlines = ReadFile(filename).splitlines()
568 for line in mountlines:
569 device, mountpoint, fstype, options, _ = line.split(None, 4)
570 data.append((device, mountpoint, fstype, options))
571
572 return data
573
574
576 """Signal Handled decoration.
577
578 This special decorator installs a signal handler and then calls the target
579 function. The function must accept a 'signal_handlers' keyword argument,
580 which will contain a dict indexed by signal number, with SignalHandler
581 objects as values.
582
583 The decorator can be safely stacked with iself, to handle multiple signals
584 with different handlers.
585
586 @type signums: list
587 @param signums: signals to intercept
588
589 """
590 def wrap(fn):
591 def sig_function(*args, **kwargs):
592 assert "signal_handlers" not in kwargs or \
593 kwargs["signal_handlers"] is None or \
594 isinstance(kwargs["signal_handlers"], dict), \
595 "Wrong signal_handlers parameter in original function call"
596 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
597 signal_handlers = kwargs["signal_handlers"]
598 else:
599 signal_handlers = {}
600 kwargs["signal_handlers"] = signal_handlers
601 sighandler = SignalHandler(signums)
602 try:
603 for sig in signums:
604 signal_handlers[sig] = sighandler
605 return fn(*args, **kwargs)
606 finally:
607 sighandler.Reset()
608 return sig_function
609 return wrap
610
611
613 """Checks whether a timeout has expired.
614
615 """
616 return _time_fn() > (epoch + timeout)
617
618
629 else:
630
633
635 """Initializes this class.
636
637 """
638 (read_fd, write_fd) = os.pipe()
639
640
641
642
643 self._read_fh = os.fdopen(read_fd, "r", 0)
644 self._write_fh = os.fdopen(write_fd, "w", 0)
645
646 self._previous = self._SetWakeupFd(self._write_fh.fileno())
647
648
649 self.fileno = self._read_fh.fileno
650 self.read = self._read_fh.read
651
653 """Restores the previous wakeup file descriptor.
654
655 """
656 if hasattr(self, "_previous") and self._previous is not None:
657 self._SetWakeupFd(self._previous)
658 self._previous = None
659
661 """Notifies the wakeup file descriptor.
662
663 """
664 self._write_fh.write("\0")
665
667 """Called before object deletion.
668
669 """
670 self.Reset()
671
672
674 """Generic signal handler class.
675
676 It automatically restores the original handler when deconstructed or
677 when L{Reset} is called. You can either pass your own handler
678 function in or query the L{called} attribute to detect whether the
679 signal was sent.
680
681 @type signum: list
682 @ivar signum: the signals we handle
683 @type called: boolean
684 @ivar called: tracks whether any of the signals have been raised
685
686 """
687 - def __init__(self, signum, handler_fn=None, wakeup=None):
688 """Constructs a new SignalHandler instance.
689
690 @type signum: int or list of ints
691 @param signum: Single signal number or set of signal numbers
692 @type handler_fn: callable
693 @param handler_fn: Signal handling function
694
695 """
696 assert handler_fn is None or callable(handler_fn)
697
698 self.signum = set(signum)
699 self.called = False
700
701 self._handler_fn = handler_fn
702 self._wakeup = wakeup
703
704 self._previous = {}
705 try:
706 for signum in self.signum:
707
708 prev_handler = signal.signal(signum, self._HandleSignal)
709 try:
710 self._previous[signum] = prev_handler
711 except:
712
713 signal.signal(signum, prev_handler)
714 raise
715 except:
716
717 self.Reset()
718
719
720 raise
721
724
726 """Restore previous handler.
727
728 This will reset all the signals to their previous handlers.
729
730 """
731 for signum, prev_handler in self._previous.items():
732 signal.signal(signum, prev_handler)
733
734 del self._previous[signum]
735
737 """Unsets the L{called} flag.
738
739 This function can be used in case a signal may arrive several times.
740
741 """
742 self.called = False
743
745 """Actual signal handling function.
746
747 """
748
749
750 self.called = True
751
752 if self._wakeup:
753
754 self._wakeup.Notify()
755
756 if self._handler_fn:
757 self._handler_fn(signum, frame)
758
759
761 """A simple field set.
762
763 Among the features are:
764 - checking if a string is among a list of static string or regex objects
765 - checking if a whole list of string matches
766 - returning the matching groups from a regex match
767
768 Internally, all fields are held as regular expression objects.
769
770 """
772 self.items = [re.compile("^%s$" % value) for value in items]
773
775 """Extend the field set with the items from another one"""
776 self.items.extend(other_set.items)
777
779 """Checks if a field matches the current set
780
781 @type field: str
782 @param field: the string to match
783 @return: either None or a regular expression match object
784
785 """
786 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
787 return m
788 return None
789
791 """Returns the list of fields not matching the current set
792
793 @type items: list
794 @param items: the list of fields to check
795 @rtype: list
796 @return: list of non-matching fields
797
798 """
799 return [val for val in items if not self.Matches(val)]
800