1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Ganeti utility module.
23
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
26
27 """
28
29
30
31 import os
32 import re
33 import errno
34 import pwd
35 import time
36 import itertools
37 import select
38 import logging
39 import signal
40
41 from ganeti import errors
42 from ganeti import constants
43 from ganeti import compat
44
45 from ganeti.utils.algo import *
46 from ganeti.utils.filelock import *
47 from ganeti.utils.hash import *
48 from ganeti.utils.io import *
49 from ganeti.utils.log import *
50 from ganeti.utils.mlock import *
51 from ganeti.utils.nodesetup import *
52 from ganeti.utils.process import *
53 from ganeti.utils.retry import *
54 from ganeti.utils.text import *
55 from ganeti.utils.wrapper import *
56 from ganeti.utils.x509 import *
57
58
59 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
60
61 UUID_RE = re.compile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-"
62 "[a-f0-9]{4}-[a-f0-9]{12}$")
63
64
66 """Force the values of a dict to have certain types.
67
68 @type target: dict
69 @param target: the dict to update
70 @type key_types: dict
71 @param key_types: dict mapping target dict keys to types
72 in constants.ENFORCEABLE_TYPES
73 @type allowed_values: list
74 @keyword allowed_values: list of specially allowed values
75
76 """
77 if allowed_values is None:
78 allowed_values = []
79
80 if not isinstance(target, dict):
81 msg = "Expected dictionary, got '%s'" % target
82 raise errors.TypeEnforcementError(msg)
83
84 for key in target:
85 if key not in key_types:
86 msg = "Unknown parameter '%s'" % key
87 raise errors.TypeEnforcementError(msg)
88
89 if target[key] in allowed_values:
90 continue
91
92 ktype = key_types[key]
93 if ktype not in constants.ENFORCEABLE_TYPES:
94 msg = "'%s' has non-enforceable type %s" % (key, ktype)
95 raise errors.ProgrammerError(msg)
96
97 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
98 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
99 pass
100 elif not isinstance(target[key], basestring):
101 if isinstance(target[key], bool) and not target[key]:
102 target[key] = ""
103 else:
104 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
105 raise errors.TypeEnforcementError(msg)
106 elif ktype == constants.VTYPE_BOOL:
107 if isinstance(target[key], basestring) and target[key]:
108 if target[key].lower() == constants.VALUE_FALSE:
109 target[key] = False
110 elif target[key].lower() == constants.VALUE_TRUE:
111 target[key] = True
112 else:
113 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
114 raise errors.TypeEnforcementError(msg)
115 elif target[key]:
116 target[key] = True
117 else:
118 target[key] = False
119 elif ktype == constants.VTYPE_SIZE:
120 try:
121 target[key] = ParseUnit(target[key])
122 except errors.UnitParseError, err:
123 msg = "'%s' (value %s) is not a valid size. error: %s" % \
124 (key, target[key], err)
125 raise errors.TypeEnforcementError(msg)
126 elif ktype == constants.VTYPE_INT:
127 try:
128 target[key] = int(target[key])
129 except (ValueError, TypeError):
130 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
131 raise errors.TypeEnforcementError(msg)
132
133
135 """Validate the given service name.
136
137 @type name: number or string
138 @param name: Service name or port specification
139
140 """
141 try:
142 numport = int(name)
143 except (ValueError, TypeError):
144
145 valid = _VALID_SERVICE_NAME_RE.match(name)
146 else:
147
148
149 valid = (numport >= 0 and numport < (1 << 16))
150
151 if not valid:
152 raise errors.OpPrereqError("Invalid service name '%s'" % name,
153 errors.ECODE_INVAL)
154
155 return name
156
157
159 """Helper functions to compute which keys a invalid.
160
161 @param key_path: The current key path (if any)
162 @param options: The user provided options
163 @param defaults: The default dictionary
164 @return: A list of invalid keys
165
166 """
167 defaults_keys = frozenset(defaults.keys())
168 invalid = []
169 for key, value in options.items():
170 if key_path:
171 new_path = "%s/%s" % (key_path, key)
172 else:
173 new_path = key
174
175 if key not in defaults_keys:
176 invalid.append(new_path)
177 elif isinstance(value, dict):
178 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
179
180 return invalid
181
182
184 """Verify a dict has only keys set which also are in the defaults dict.
185
186 @param options: The user provided options
187 @param defaults: The default dictionary
188 @raise error.OpPrereqError: If one of the keys is not supported
189
190 """
191 invalid = _ComputeMissingKeys("", options, defaults)
192
193 if invalid:
194 raise errors.OpPrereqError("Provided option keys not supported: %s" %
195 CommaJoin(invalid), errors.ECODE_INVAL)
196
197
199 """List volume groups and their size
200
201 @rtype: dict
202 @return:
203 Dictionary with keys volume name and values
204 the size of the volume
205
206 """
207 command = "vgs --noheadings --units m --nosuffix -o name,size"
208 result = RunCmd(command)
209 retval = {}
210 if result.failed:
211 return retval
212
213 for line in result.stdout.splitlines():
214 try:
215 name, size = line.split()
216 size = int(float(size))
217 except (IndexError, ValueError), err:
218 logging.error("Invalid output from vgs (%s): %s", err, line)
219 continue
220
221 retval[name] = size
222
223 return retval
224
225
227 """Check whether the given bridge exists in the system
228
229 @type bridge: str
230 @param bridge: the bridge name to check
231 @rtype: boolean
232 @return: True if it does
233
234 """
235 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
236
237
239 """Try to convert a value ignoring errors.
240
241 This function tries to apply function I{fn} to I{val}. If no
242 C{ValueError} or C{TypeError} exceptions are raised, it will return
243 the result, else it will return the original value. Any other
244 exceptions are propagated to the caller.
245
246 @type fn: callable
247 @param fn: function to apply to the value
248 @param val: the value to be converted
249 @return: The converted value if the conversion was successful,
250 otherwise the original value.
251
252 """
253 try:
254 nv = fn(val)
255 except (ValueError, TypeError):
256 nv = val
257 return nv
258
259
261 """Parse a CPU mask definition and return the list of CPU IDs.
262
263 CPU mask format: comma-separated list of CPU IDs
264 or dash-separated ID ranges
265 Example: "0-2,5" -> "0,1,2,5"
266
267 @type cpu_mask: str
268 @param cpu_mask: CPU mask definition
269 @rtype: list of int
270 @return: list of CPU IDs
271
272 """
273 if not cpu_mask:
274 return []
275 cpu_list = []
276 for range_def in cpu_mask.split(","):
277 boundaries = range_def.split("-")
278 n_elements = len(boundaries)
279 if n_elements > 2:
280 raise errors.ParseError("Invalid CPU ID range definition"
281 " (only one hyphen allowed): %s" % range_def)
282 try:
283 lower = int(boundaries[0])
284 except (ValueError, TypeError), err:
285 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
286 " CPU ID range: %s" % str(err))
287 try:
288 higher = int(boundaries[-1])
289 except (ValueError, TypeError), err:
290 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
291 " CPU ID range: %s" % str(err))
292 if lower > higher:
293 raise errors.ParseError("Invalid CPU ID range definition"
294 " (%d > %d): %s" % (lower, higher, range_def))
295 cpu_list.extend(range(lower, higher + 1))
296 return cpu_list
297
298
300 """Parse a multiple CPU mask definition and return the list of CPU IDs.
301
302 CPU mask format: colon-separated list of comma-separated list of CPU IDs
303 or dash-separated ID ranges, with optional "all" as CPU value
304 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
305
306 @type cpu_mask: str
307 @param cpu_mask: multiple CPU mask definition
308 @rtype: list of lists of int
309 @return: list of lists of CPU IDs
310
311 """
312 if not cpu_mask:
313 return []
314 cpu_list = []
315 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
316 if range_def == constants.CPU_PINNING_ALL:
317 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
318 else:
319
320 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
321
322 return cpu_list
323
324
326 """Try to get the homedir of the given user.
327
328 The user can be passed either as a string (denoting the name) or as
329 an integer (denoting the user id). If the user is not found, the
330 C{default} argument is returned, which defaults to C{None}.
331
332 """
333 try:
334 if isinstance(user, basestring):
335 result = pwd.getpwnam(user)
336 elif isinstance(user, (int, long)):
337 result = pwd.getpwuid(user)
338 else:
339 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
340 type(user))
341 except KeyError:
342 return default
343 return result.pw_dir
344
345
347 """Returns the first non-existing integer from seq.
348
349 The seq argument should be a sorted list of positive integers. The
350 first time the index of an element is smaller than the element
351 value, the index will be returned.
352
353 The base argument is used to start at a different offset,
354 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
355
356 Example: C{[0, 1, 3]} will return I{2}.
357
358 @type seq: sequence
359 @param seq: the sequence to be analyzed.
360 @type base: int
361 @param base: use this value as the base index of the sequence
362 @rtype: int
363 @return: the first non-used index in the sequence
364
365 """
366 for idx, elem in enumerate(seq):
367 assert elem >= base, "Passed element is higher than base offset"
368 if elem > idx + base:
369
370 return idx + base
371 return None
372
373
375 """Waits for a condition to occur on the socket.
376
377 Immediately returns at the first interruption.
378
379 @type fdobj: integer or object supporting a fileno() method
380 @param fdobj: entity to wait for events on
381 @type event: integer
382 @param event: ORed condition (see select module)
383 @type timeout: float or None
384 @param timeout: Timeout in seconds
385 @rtype: int or None
386 @return: None for timeout, otherwise occured conditions
387
388 """
389 check = (event | select.POLLPRI |
390 select.POLLNVAL | select.POLLHUP | select.POLLERR)
391
392 if timeout is not None:
393
394 timeout *= 1000
395
396 poller = select.poll()
397 poller.register(fdobj, event)
398 try:
399
400
401
402 io_events = poller.poll(timeout)
403 except select.error, err:
404 if err[0] != errno.EINTR:
405 raise
406 io_events = []
407 if io_events and io_events[0][1] & check:
408 return io_events[0][1]
409 else:
410 return None
411
412
414 """Retry helper for WaitForFdCondition.
415
416 This class contains the retried and wait functions that make sure
417 WaitForFdCondition can continue waiting until the timeout is actually
418 expired.
419
420 """
421
423 self.timeout = timeout
424
425 - def Poll(self, fdobj, event):
431
433 self.timeout = timeout
434
435
437 """Waits for a condition to occur on the socket.
438
439 Retries until the timeout is expired, even if interrupted.
440
441 @type fdobj: integer or object supporting a fileno() method
442 @param fdobj: entity to wait for events on
443 @type event: integer
444 @param event: ORed condition (see select module)
445 @type timeout: float or None
446 @param timeout: Timeout in seconds
447 @rtype: int or None
448 @return: None for timeout, otherwise occured conditions
449
450 """
451 if timeout is not None:
452 retrywaiter = FdConditionWaiterHelper(timeout)
453 try:
454 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
455 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
456 except RetryTimeout:
457 result = None
458 else:
459 result = None
460 while result is None:
461 result = SingleWaitForFdCondition(fdobj, event, timeout)
462 return result
463
464
466 """Check for and start daemon if not alive.
467
468 """
469 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
470 if result.failed:
471 logging.error("Can't start daemon '%s', failure %s, output: %s",
472 name, result.fail_reason, result.output)
473 return False
474
475 return True
476
477
479 """Stop daemon
480
481 """
482 result = RunCmd([constants.DAEMON_UTIL, "stop", name])
483 if result.failed:
484 logging.error("Can't stop daemon '%s', failure %s, output: %s",
485 name, result.fail_reason, result.output)
486 return False
487
488 return True
489
490
492 """Checks if the volume group list is valid.
493
494 The function will check if a given volume group is in the list of
495 volume groups and has a minimum size.
496
497 @type vglist: dict
498 @param vglist: dictionary of volume group names and their size
499 @type vgname: str
500 @param vgname: the volume group we should check
501 @type minsize: int
502 @param minsize: the minimum size we accept
503 @rtype: None or str
504 @return: None for success, otherwise the error message
505
506 """
507 vgsize = vglist.get(vgname, None)
508 if vgsize is None:
509 return "volume group '%s' missing" % vgname
510 elif vgsize < minsize:
511 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
512 (vgname, minsize, vgsize))
513 return None
514
515
517 """Splits time as floating point number into a tuple.
518
519 @param value: Time in seconds
520 @type value: int or float
521 @return: Tuple containing (seconds, microseconds)
522
523 """
524 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
525
526 assert 0 <= seconds, \
527 "Seconds must be larger than or equal to 0, but are %s" % seconds
528 assert 0 <= microseconds <= 999999, \
529 "Microseconds must be 0-999999, but are %s" % microseconds
530
531 return (int(seconds), int(microseconds))
532
533
535 """Merges a tuple into time as a floating point number.
536
537 @param timetuple: Time as tuple, (seconds, microseconds)
538 @type timetuple: tuple
539 @return: Time as a floating point number expressed in seconds
540
541 """
542 (seconds, microseconds) = timetuple
543
544 assert 0 <= seconds, \
545 "Seconds must be larger than or equal to 0, but are %s" % seconds
546 assert 0 <= microseconds <= 999999, \
547 "Microseconds must be 0-999999, but are %s" % microseconds
548
549 return float(seconds) + (float(microseconds) * 0.000001)
550
551
553 """Tries to find an item in a dictionary matching a name.
554
555 Callers have to ensure the data names aren't contradictory (e.g. a regexp
556 that matches a string). If the name isn't a direct key, all regular
557 expression objects in the dictionary are matched against it.
558
559 @type data: dict
560 @param data: Dictionary containing data
561 @type name: string
562 @param name: Name to look for
563 @rtype: tuple; (value in dictionary, matched groups as list)
564
565 """
566 if name in data:
567 return (data[name], [])
568
569 for key, value in data.items():
570
571 if hasattr(key, "match"):
572 m = key.match(name)
573 if m:
574 return (value, list(m.groups()))
575
576 return None
577
578
580 """Returns the list of mounted filesystems.
581
582 This function is Linux-specific.
583
584 @param filename: path of mounts file (/proc/mounts by default)
585 @rtype: list of tuples
586 @return: list of mount entries (device, mountpoint, fstype, options)
587
588 """
589
590 data = []
591 mountlines = ReadFile(filename).splitlines()
592 for line in mountlines:
593 device, mountpoint, fstype, options, _ = line.split(None, 4)
594 data.append((device, mountpoint, fstype, options))
595
596 return data
597
598
600 """Signal Handled decoration.
601
602 This special decorator installs a signal handler and then calls the target
603 function. The function must accept a 'signal_handlers' keyword argument,
604 which will contain a dict indexed by signal number, with SignalHandler
605 objects as values.
606
607 The decorator can be safely stacked with iself, to handle multiple signals
608 with different handlers.
609
610 @type signums: list
611 @param signums: signals to intercept
612
613 """
614 def wrap(fn):
615 def sig_function(*args, **kwargs):
616 assert "signal_handlers" not in kwargs or \
617 kwargs["signal_handlers"] is None or \
618 isinstance(kwargs["signal_handlers"], dict), \
619 "Wrong signal_handlers parameter in original function call"
620 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
621 signal_handlers = kwargs["signal_handlers"]
622 else:
623 signal_handlers = {}
624 kwargs["signal_handlers"] = signal_handlers
625 sighandler = SignalHandler(signums)
626 try:
627 for sig in signums:
628 signal_handlers[sig] = sighandler
629 return fn(*args, **kwargs)
630 finally:
631 sighandler.Reset()
632 return sig_function
633 return wrap
634
635
637 """Checks whether a timeout has expired.
638
639 """
640 return _time_fn() > (epoch + timeout)
641
642
652 else:
655
657 """Initializes this class.
658
659 """
660 (read_fd, write_fd) = os.pipe()
661
662
663
664
665 self._read_fh = os.fdopen(read_fd, "r", 0)
666 self._write_fh = os.fdopen(write_fd, "w", 0)
667
668 self._previous = self._SetWakeupFd(self._write_fh.fileno())
669
670
671 self.fileno = self._read_fh.fileno
672 self.read = self._read_fh.read
673
675 """Restores the previous wakeup file descriptor.
676
677 """
678 if hasattr(self, "_previous") and self._previous is not None:
679 self._SetWakeupFd(self._previous)
680 self._previous = None
681
683 """Notifies the wakeup file descriptor.
684
685 """
686 self._write_fh.write("\0")
687
689 """Called before object deletion.
690
691 """
692 self.Reset()
693
694
696 """Generic signal handler class.
697
698 It automatically restores the original handler when deconstructed or
699 when L{Reset} is called. You can either pass your own handler
700 function in or query the L{called} attribute to detect whether the
701 signal was sent.
702
703 @type signum: list
704 @ivar signum: the signals we handle
705 @type called: boolean
706 @ivar called: tracks whether any of the signals have been raised
707
708 """
709 - def __init__(self, signum, handler_fn=None, wakeup=None):
710 """Constructs a new SignalHandler instance.
711
712 @type signum: int or list of ints
713 @param signum: Single signal number or set of signal numbers
714 @type handler_fn: callable
715 @param handler_fn: Signal handling function
716
717 """
718 assert handler_fn is None or callable(handler_fn)
719
720 self.signum = set(signum)
721 self.called = False
722
723 self._handler_fn = handler_fn
724 self._wakeup = wakeup
725
726 self._previous = {}
727 try:
728 for signum in self.signum:
729
730 prev_handler = signal.signal(signum, self._HandleSignal)
731 try:
732 self._previous[signum] = prev_handler
733 except:
734
735 signal.signal(signum, prev_handler)
736 raise
737 except:
738
739 self.Reset()
740
741
742 raise
743
746
748 """Restore previous handler.
749
750 This will reset all the signals to their previous handlers.
751
752 """
753 for signum, prev_handler in self._previous.items():
754 signal.signal(signum, prev_handler)
755
756 del self._previous[signum]
757
759 """Unsets the L{called} flag.
760
761 This function can be used in case a signal may arrive several times.
762
763 """
764 self.called = False
765
767 """Actual signal handling function.
768
769 """
770
771
772 self.called = True
773
774 if self._wakeup:
775
776 self._wakeup.Notify()
777
778 if self._handler_fn:
779 self._handler_fn(signum, frame)
780
781
783 """A simple field set.
784
785 Among the features are:
786 - checking if a string is among a list of static string or regex objects
787 - checking if a whole list of string matches
788 - returning the matching groups from a regex match
789
790 Internally, all fields are held as regular expression objects.
791
792 """
794 self.items = [re.compile("^%s$" % value) for value in items]
795
797 """Extend the field set with the items from another one"""
798 self.items.extend(other_set.items)
799
801 """Checks if a field matches the current set
802
803 @type field: str
804 @param field: the string to match
805 @return: either None or a regular expression match object
806
807 """
808 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
809 return m
810 return None
811
813 """Returns the list of fields not matching the current set
814
815 @type items: list
816 @param items: the list of fields to check
817 @rtype: list
818 @return: list of non-matching fields
819
820 """
821 return [val for val in items if not self.Matches(val)]
822