1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Ganeti utility module.
23
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
26
27 """
28
29
30
31 import os
32 import re
33 import errno
34 import pwd
35 import time
36 import itertools
37 import select
38 import logging
39 import signal
40
41 from ganeti import errors
42 from ganeti import constants
43 from ganeti import compat
44 from ganeti import pathutils
45
46 from ganeti.utils.algo import *
47 from ganeti.utils.filelock import *
48 from ganeti.utils.hash import *
49 from ganeti.utils.io import *
50 from ganeti.utils.log import *
51 from ganeti.utils.lvm import *
52 from ganeti.utils.mlock import *
53 from ganeti.utils.nodesetup import *
54 from ganeti.utils.process import *
55 from ganeti.utils.retry import *
56 from ganeti.utils.text import *
57 from ganeti.utils.wrapper import *
58 from ganeti.utils.x509 import *
59
60
61 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
62
63 UUID_RE = re.compile(constants.UUID_REGEX)
64
65
67 """Force the values of a dict to have certain types.
68
69 @type target: dict
70 @param target: the dict to update
71 @type key_types: dict
72 @param key_types: dict mapping target dict keys to types
73 in constants.ENFORCEABLE_TYPES
74 @type allowed_values: list
75 @keyword allowed_values: list of specially allowed values
76
77 """
78 if allowed_values is None:
79 allowed_values = []
80
81 if not isinstance(target, dict):
82 msg = "Expected dictionary, got '%s'" % target
83 raise errors.TypeEnforcementError(msg)
84
85 for key in target:
86 if key not in key_types:
87 msg = "Unknown parameter '%s'" % key
88 raise errors.TypeEnforcementError(msg)
89
90 if target[key] in allowed_values:
91 continue
92
93 ktype = key_types[key]
94 if ktype not in constants.ENFORCEABLE_TYPES:
95 msg = "'%s' has non-enforceable type %s" % (key, ktype)
96 raise errors.ProgrammerError(msg)
97
98 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
99 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
100 pass
101 elif not isinstance(target[key], basestring):
102 if isinstance(target[key], bool) and not target[key]:
103 target[key] = ""
104 else:
105 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
106 raise errors.TypeEnforcementError(msg)
107 elif ktype == constants.VTYPE_BOOL:
108 if isinstance(target[key], basestring) and target[key]:
109 if target[key].lower() == constants.VALUE_FALSE:
110 target[key] = False
111 elif target[key].lower() == constants.VALUE_TRUE:
112 target[key] = True
113 else:
114 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
115 raise errors.TypeEnforcementError(msg)
116 elif target[key]:
117 target[key] = True
118 else:
119 target[key] = False
120 elif ktype == constants.VTYPE_SIZE:
121 try:
122 target[key] = ParseUnit(target[key])
123 except errors.UnitParseError, err:
124 msg = "'%s' (value %s) is not a valid size. error: %s" % \
125 (key, target[key], err)
126 raise errors.TypeEnforcementError(msg)
127 elif ktype == constants.VTYPE_INT:
128 try:
129 target[key] = int(target[key])
130 except (ValueError, TypeError):
131 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
132 raise errors.TypeEnforcementError(msg)
133
134
136 """Validate the given service name.
137
138 @type name: number or string
139 @param name: Service name or port specification
140
141 """
142 try:
143 numport = int(name)
144 except (ValueError, TypeError):
145
146 valid = _VALID_SERVICE_NAME_RE.match(name)
147 else:
148
149
150 valid = (numport >= 0 and numport < (1 << 16))
151
152 if not valid:
153 raise errors.OpPrereqError("Invalid service name '%s'" % name,
154 errors.ECODE_INVAL)
155
156 return name
157
158
160 """Helper functions to compute which keys a invalid.
161
162 @param key_path: The current key path (if any)
163 @param options: The user provided options
164 @param defaults: The default dictionary
165 @return: A list of invalid keys
166
167 """
168 defaults_keys = frozenset(defaults.keys())
169 invalid = []
170 for key, value in options.items():
171 if key_path:
172 new_path = "%s/%s" % (key_path, key)
173 else:
174 new_path = key
175
176 if key not in defaults_keys:
177 invalid.append(new_path)
178 elif isinstance(value, dict):
179 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
180
181 return invalid
182
183
185 """Verify a dict has only keys set which also are in the defaults dict.
186
187 @param options: The user provided options
188 @param defaults: The default dictionary
189 @raise error.OpPrereqError: If one of the keys is not supported
190
191 """
192 invalid = _ComputeMissingKeys("", options, defaults)
193
194 if invalid:
195 raise errors.OpPrereqError("Provided option keys not supported: %s" %
196 CommaJoin(invalid), errors.ECODE_INVAL)
197
198
200 """List volume groups and their size
201
202 @rtype: dict
203 @return:
204 Dictionary with keys volume name and values
205 the size of the volume
206
207 """
208 command = "vgs --noheadings --units m --nosuffix -o name,size"
209 result = RunCmd(command)
210 retval = {}
211 if result.failed:
212 return retval
213
214 for line in result.stdout.splitlines():
215 try:
216 name, size = line.split()
217 size = int(float(size))
218 except (IndexError, ValueError), err:
219 logging.error("Invalid output from vgs (%s): %s", err, line)
220 continue
221
222 retval[name] = size
223
224 return retval
225
226
228 """Check whether the given bridge exists in the system
229
230 @type bridge: str
231 @param bridge: the bridge name to check
232 @rtype: boolean
233 @return: True if it does
234
235 """
236 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
237
238
240 """Try to convert a value ignoring errors.
241
242 This function tries to apply function I{fn} to I{val}. If no
243 C{ValueError} or C{TypeError} exceptions are raised, it will return
244 the result, else it will return the original value. Any other
245 exceptions are propagated to the caller.
246
247 @type fn: callable
248 @param fn: function to apply to the value
249 @param val: the value to be converted
250 @return: The converted value if the conversion was successful,
251 otherwise the original value.
252
253 """
254 try:
255 nv = fn(val)
256 except (ValueError, TypeError):
257 nv = val
258 return nv
259
260
262 """Parse a CPU mask definition and return the list of CPU IDs.
263
264 CPU mask format: comma-separated list of CPU IDs
265 or dash-separated ID ranges
266 Example: "0-2,5" -> "0,1,2,5"
267
268 @type cpu_mask: str
269 @param cpu_mask: CPU mask definition
270 @rtype: list of int
271 @return: list of CPU IDs
272
273 """
274 if not cpu_mask:
275 return []
276 cpu_list = []
277 for range_def in cpu_mask.split(","):
278 boundaries = range_def.split("-")
279 n_elements = len(boundaries)
280 if n_elements > 2:
281 raise errors.ParseError("Invalid CPU ID range definition"
282 " (only one hyphen allowed): %s" % range_def)
283 try:
284 lower = int(boundaries[0])
285 except (ValueError, TypeError), err:
286 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
287 " CPU ID range: %s" % str(err))
288 try:
289 higher = int(boundaries[-1])
290 except (ValueError, TypeError), err:
291 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
292 " CPU ID range: %s" % str(err))
293 if lower > higher:
294 raise errors.ParseError("Invalid CPU ID range definition"
295 " (%d > %d): %s" % (lower, higher, range_def))
296 cpu_list.extend(range(lower, higher + 1))
297 return cpu_list
298
299
301 """Parse a multiple CPU mask definition and return the list of CPU IDs.
302
303 CPU mask format: colon-separated list of comma-separated list of CPU IDs
304 or dash-separated ID ranges, with optional "all" as CPU value
305 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
306
307 @type cpu_mask: str
308 @param cpu_mask: multiple CPU mask definition
309 @rtype: list of lists of int
310 @return: list of lists of CPU IDs
311
312 """
313 if not cpu_mask:
314 return []
315 cpu_list = []
316 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
317 if range_def == constants.CPU_PINNING_ALL:
318 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
319 else:
320
321 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
322
323 return cpu_list
324
325
327 """Try to get the homedir of the given user.
328
329 The user can be passed either as a string (denoting the name) or as
330 an integer (denoting the user id). If the user is not found, the
331 C{default} argument is returned, which defaults to C{None}.
332
333 """
334 try:
335 if isinstance(user, basestring):
336 result = pwd.getpwnam(user)
337 elif isinstance(user, (int, long)):
338 result = pwd.getpwuid(user)
339 else:
340 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
341 type(user))
342 except KeyError:
343 return default
344 return result.pw_dir
345
346
348 """Returns the first non-existing integer from seq.
349
350 The seq argument should be a sorted list of positive integers. The
351 first time the index of an element is smaller than the element
352 value, the index will be returned.
353
354 The base argument is used to start at a different offset,
355 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
356
357 Example: C{[0, 1, 3]} will return I{2}.
358
359 @type seq: sequence
360 @param seq: the sequence to be analyzed.
361 @type base: int
362 @param base: use this value as the base index of the sequence
363 @rtype: int
364 @return: the first non-used index in the sequence
365
366 """
367 for idx, elem in enumerate(seq):
368 assert elem >= base, "Passed element is higher than base offset"
369 if elem > idx + base:
370
371 return idx + base
372 return None
373
374
376 """Waits for a condition to occur on the socket.
377
378 Immediately returns at the first interruption.
379
380 @type fdobj: integer or object supporting a fileno() method
381 @param fdobj: entity to wait for events on
382 @type event: integer
383 @param event: ORed condition (see select module)
384 @type timeout: float or None
385 @param timeout: Timeout in seconds
386 @rtype: int or None
387 @return: None for timeout, otherwise occured conditions
388
389 """
390 check = (event | select.POLLPRI |
391 select.POLLNVAL | select.POLLHUP | select.POLLERR)
392
393 if timeout is not None:
394
395 timeout *= 1000
396
397 poller = select.poll()
398 poller.register(fdobj, event)
399 try:
400
401
402
403 io_events = poller.poll(timeout)
404 except select.error, err:
405 if err[0] != errno.EINTR:
406 raise
407 io_events = []
408 if io_events and io_events[0][1] & check:
409 return io_events[0][1]
410 else:
411 return None
412
413
415 """Retry helper for WaitForFdCondition.
416
417 This class contains the retried and wait functions that make sure
418 WaitForFdCondition can continue waiting until the timeout is actually
419 expired.
420
421 """
422
424 self.timeout = timeout
425
426 - def Poll(self, fdobj, event):
432
434 self.timeout = timeout
435
436
438 """Waits for a condition to occur on the socket.
439
440 Retries until the timeout is expired, even if interrupted.
441
442 @type fdobj: integer or object supporting a fileno() method
443 @param fdobj: entity to wait for events on
444 @type event: integer
445 @param event: ORed condition (see select module)
446 @type timeout: float or None
447 @param timeout: Timeout in seconds
448 @rtype: int or None
449 @return: None for timeout, otherwise occured conditions
450
451 """
452 if timeout is not None:
453 retrywaiter = FdConditionWaiterHelper(timeout)
454 try:
455 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
456 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
457 except RetryTimeout:
458 result = None
459 else:
460 result = None
461 while result is None:
462 result = SingleWaitForFdCondition(fdobj, event, timeout)
463 return result
464
465
467 """Check for and start daemon if not alive.
468
469 """
470 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
471 if result.failed:
472 logging.error("Can't start daemon '%s', failure %s, output: %s",
473 name, result.fail_reason, result.output)
474 return False
475
476 return True
477
478
480 """Stop daemon
481
482 """
483 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
484 if result.failed:
485 logging.error("Can't stop daemon '%s', failure %s, output: %s",
486 name, result.fail_reason, result.output)
487 return False
488
489 return True
490
491
493 """Splits time as floating point number into a tuple.
494
495 @param value: Time in seconds
496 @type value: int or float
497 @return: Tuple containing (seconds, microseconds)
498
499 """
500 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
501
502 assert 0 <= seconds, \
503 "Seconds must be larger than or equal to 0, but are %s" % seconds
504 assert 0 <= microseconds <= 999999, \
505 "Microseconds must be 0-999999, but are %s" % microseconds
506
507 return (int(seconds), int(microseconds))
508
509
511 """Merges a tuple into time as a floating point number.
512
513 @param timetuple: Time as tuple, (seconds, microseconds)
514 @type timetuple: tuple
515 @return: Time as a floating point number expressed in seconds
516
517 """
518 (seconds, microseconds) = timetuple
519
520 assert 0 <= seconds, \
521 "Seconds must be larger than or equal to 0, but are %s" % seconds
522 assert 0 <= microseconds <= 999999, \
523 "Microseconds must be 0-999999, but are %s" % microseconds
524
525 return float(seconds) + (float(microseconds) * 0.000001)
526
527
529 """Return the current timestamp expressed as number of nanoseconds since the
530 unix epoch
531
532 @return: nanoseconds since the Unix epoch
533
534 """
535 return int(time.time() * 1000000000)
536
537
539 """Tries to find an item in a dictionary matching a name.
540
541 Callers have to ensure the data names aren't contradictory (e.g. a regexp
542 that matches a string). If the name isn't a direct key, all regular
543 expression objects in the dictionary are matched against it.
544
545 @type data: dict
546 @param data: Dictionary containing data
547 @type name: string
548 @param name: Name to look for
549 @rtype: tuple; (value in dictionary, matched groups as list)
550
551 """
552 if name in data:
553 return (data[name], [])
554
555 for key, value in data.items():
556
557 if hasattr(key, "match"):
558 m = key.match(name)
559 if m:
560 return (value, list(m.groups()))
561
562 return None
563
564
566 """Returns the list of mounted filesystems.
567
568 This function is Linux-specific.
569
570 @param filename: path of mounts file (/proc/mounts by default)
571 @rtype: list of tuples
572 @return: list of mount entries (device, mountpoint, fstype, options)
573
574 """
575
576 data = []
577 mountlines = ReadFile(filename).splitlines()
578 for line in mountlines:
579 device, mountpoint, fstype, options, _ = line.split(None, 4)
580 data.append((device, mountpoint, fstype, options))
581
582 return data
583
584
586 """Signal Handled decoration.
587
588 This special decorator installs a signal handler and then calls the target
589 function. The function must accept a 'signal_handlers' keyword argument,
590 which will contain a dict indexed by signal number, with SignalHandler
591 objects as values.
592
593 The decorator can be safely stacked with iself, to handle multiple signals
594 with different handlers.
595
596 @type signums: list
597 @param signums: signals to intercept
598
599 """
600 def wrap(fn):
601 def sig_function(*args, **kwargs):
602 assert "signal_handlers" not in kwargs or \
603 kwargs["signal_handlers"] is None or \
604 isinstance(kwargs["signal_handlers"], dict), \
605 "Wrong signal_handlers parameter in original function call"
606 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
607 signal_handlers = kwargs["signal_handlers"]
608 else:
609 signal_handlers = {}
610 kwargs["signal_handlers"] = signal_handlers
611 sighandler = SignalHandler(signums)
612 try:
613 for sig in signums:
614 signal_handlers[sig] = sighandler
615 return fn(*args, **kwargs)
616 finally:
617 sighandler.Reset()
618 return sig_function
619 return wrap
620
621
623 """Checks whether a timeout has expired.
624
625 """
626 return _time_fn() > (epoch + timeout)
627
628
639 else:
640
643
645 """Initializes this class.
646
647 """
648 (read_fd, write_fd) = os.pipe()
649
650
651
652
653 self._read_fh = os.fdopen(read_fd, "r", 0)
654 self._write_fh = os.fdopen(write_fd, "w", 0)
655
656 self._previous = self._SetWakeupFd(self._write_fh.fileno())
657
658
659 self.fileno = self._read_fh.fileno
660 self.read = self._read_fh.read
661
663 """Restores the previous wakeup file descriptor.
664
665 """
666 if hasattr(self, "_previous") and self._previous is not None:
667 self._SetWakeupFd(self._previous)
668 self._previous = None
669
671 """Notifies the wakeup file descriptor.
672
673 """
674 self._write_fh.write("\0")
675
677 """Called before object deletion.
678
679 """
680 self.Reset()
681
682
684 """Generic signal handler class.
685
686 It automatically restores the original handler when deconstructed or
687 when L{Reset} is called. You can either pass your own handler
688 function in or query the L{called} attribute to detect whether the
689 signal was sent.
690
691 @type signum: list
692 @ivar signum: the signals we handle
693 @type called: boolean
694 @ivar called: tracks whether any of the signals have been raised
695
696 """
697 - def __init__(self, signum, handler_fn=None, wakeup=None):
698 """Constructs a new SignalHandler instance.
699
700 @type signum: int or list of ints
701 @param signum: Single signal number or set of signal numbers
702 @type handler_fn: callable
703 @param handler_fn: Signal handling function
704
705 """
706 assert handler_fn is None or callable(handler_fn)
707
708 self.signum = set(signum)
709 self.called = False
710
711 self._handler_fn = handler_fn
712 self._wakeup = wakeup
713
714 self._previous = {}
715 try:
716 for signum in self.signum:
717
718 prev_handler = signal.signal(signum, self._HandleSignal)
719 try:
720 self._previous[signum] = prev_handler
721 except:
722
723 signal.signal(signum, prev_handler)
724 raise
725 except:
726
727 self.Reset()
728
729
730 raise
731
734
736 """Restore previous handler.
737
738 This will reset all the signals to their previous handlers.
739
740 """
741 for signum, prev_handler in self._previous.items():
742 signal.signal(signum, prev_handler)
743
744 del self._previous[signum]
745
747 """Unsets the L{called} flag.
748
749 This function can be used in case a signal may arrive several times.
750
751 """
752 self.called = False
753
755 """Actual signal handling function.
756
757 """
758
759
760 self.called = True
761
762 if self._wakeup:
763
764 self._wakeup.Notify()
765
766 if self._handler_fn:
767 self._handler_fn(signum, frame)
768
769
771 """A simple field set.
772
773 Among the features are:
774 - checking if a string is among a list of static string or regex objects
775 - checking if a whole list of string matches
776 - returning the matching groups from a regex match
777
778 Internally, all fields are held as regular expression objects.
779
780 """
782 self.items = [re.compile("^%s$" % value) for value in items]
783
785 """Extend the field set with the items from another one"""
786 self.items.extend(other_set.items)
787
789 """Checks if a field matches the current set
790
791 @type field: str
792 @param field: the string to match
793 @return: either None or a regular expression match object
794
795 """
796 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
797 return m
798 return None
799
801 """Returns the list of fields not matching the current set
802
803 @type items: list
804 @param items: the list of fields to check
805 @rtype: list
806 @return: list of non-matching fields
807
808 """
809 return [val for val in items if not self.Matches(val)]
810
811
813 """Validate instance device names.
814
815 Check that a device container contains only unique and valid names.
816
817 @type kind: string
818 @param kind: One-word item description
819 @type container: list
820 @param container: Container containing the devices
821
822 """
823
824 valid = []
825 for device in container:
826 if isinstance(device, dict):
827 if kind == "NIC":
828 name = device.get(constants.INIC_NAME, None)
829 elif kind == "disk":
830 name = device.get(constants.IDISK_NAME, None)
831 else:
832 raise errors.OpPrereqError("Invalid container kind '%s'" % kind,
833 errors.ECODE_INVAL)
834 else:
835 name = device.name
836
837 valid.append(device.uuid)
838
839 try:
840 int(name)
841 except (ValueError, TypeError):
842 pass
843 else:
844 raise errors.OpPrereqError("Invalid name '%s'. Purely numeric %s names"
845 " are not allowed" % (name, kind),
846 errors.ECODE_INVAL)
847
848 if name is not None and name.lower() != constants.VALUE_NONE:
849 if name in valid:
850 raise errors.OpPrereqError("%s name '%s' already used" % (kind, name),
851 errors.ECODE_NOTUNIQUE)
852 else:
853 valid.append(name)
854