1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Ganeti utility module.
32
33 This module holds functions that can be used in both daemons (all) and
34 the command line scripts.
35
36 """
37
38
39
40 import os
41 import re
42 import errno
43 import pwd
44 import time
45 import itertools
46 import select
47 import logging
48 import signal
49
50 from ganeti import errors
51 from ganeti import constants
52 from ganeti import compat
53 from ganeti import pathutils
54
55 from ganeti.utils.algo import *
56 from ganeti.utils.filelock import *
57 from ganeti.utils.hash import *
58 from ganeti.utils.io import *
59 from ganeti.utils.livelock import *
60 from ganeti.utils.log import *
61 from ganeti.utils.lvm import *
62 from ganeti.utils.mlock import *
63 from ganeti.utils.nodesetup import *
64 from ganeti.utils.process import *
65 from ganeti.utils.retry import *
66 from ganeti.utils.security import *
67 from ganeti.utils.storage import *
68 from ganeti.utils.tags import *
69 from ganeti.utils.text import *
70 from ganeti.utils.wrapper import *
71 from ganeti.utils.version import *
72 from ganeti.utils.x509 import *
73 from ganeti.utils.bitarrays import *
74
75
76 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
77
78 UUID_RE = re.compile(constants.UUID_REGEX)
79
80
82 """Force the values of a dict to have certain types.
83
84 @type target: dict
85 @param target: the dict to update
86 @type key_types: dict
87 @param key_types: dict mapping target dict keys to types
88 in constants.ENFORCEABLE_TYPES
89 @type allowed_values: list
90 @keyword allowed_values: list of specially allowed values
91
92 """
93 if allowed_values is None:
94 allowed_values = []
95
96 if not isinstance(target, dict):
97 msg = "Expected dictionary, got '%s'" % target
98 raise errors.TypeEnforcementError(msg)
99
100 for key in target:
101 if key not in key_types:
102 msg = "Unknown parameter '%s'" % key
103 raise errors.TypeEnforcementError(msg)
104
105 if target[key] in allowed_values:
106 continue
107
108 ktype = key_types[key]
109 if ktype not in constants.ENFORCEABLE_TYPES:
110 msg = "'%s' has non-enforceable type %s" % (key, ktype)
111 raise errors.ProgrammerError(msg)
112
113 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
114 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
115 pass
116 elif not isinstance(target[key], basestring):
117 if isinstance(target[key], bool) and not target[key]:
118 target[key] = ""
119 else:
120 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
121 raise errors.TypeEnforcementError(msg)
122 elif ktype == constants.VTYPE_BOOL:
123 if isinstance(target[key], basestring) and target[key]:
124 if target[key].lower() == constants.VALUE_FALSE:
125 target[key] = False
126 elif target[key].lower() == constants.VALUE_TRUE:
127 target[key] = True
128 else:
129 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
130 raise errors.TypeEnforcementError(msg)
131 elif target[key]:
132 target[key] = True
133 else:
134 target[key] = False
135 elif ktype == constants.VTYPE_SIZE:
136 try:
137 target[key] = ParseUnit(target[key])
138 except errors.UnitParseError, err:
139 msg = "'%s' (value %s) is not a valid size. error: %s" % \
140 (key, target[key], err)
141 raise errors.TypeEnforcementError(msg)
142 elif ktype == constants.VTYPE_INT:
143 try:
144 target[key] = int(target[key])
145 except (ValueError, TypeError):
146 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
147 raise errors.TypeEnforcementError(msg)
148 elif ktype == constants.VTYPE_FLOAT:
149 try:
150 target[key] = float(target[key])
151 except (ValueError, TypeError):
152 msg = "'%s' (value %s) is not a valid float" % (key, target[key])
153 raise errors.TypeEnforcementError(msg)
154
155
157 """Validate the given service name.
158
159 @type name: number or string
160 @param name: Service name or port specification
161
162 """
163 try:
164 numport = int(name)
165 except (ValueError, TypeError):
166
167 valid = _VALID_SERVICE_NAME_RE.match(name)
168 else:
169
170
171 valid = (numport >= 0 and numport < (1 << 16))
172
173 if not valid:
174 raise errors.OpPrereqError("Invalid service name '%s'" % name,
175 errors.ECODE_INVAL)
176
177 return name
178
179
181 """Helper functions to compute which keys a invalid.
182
183 @param key_path: The current key path (if any)
184 @param options: The user provided options
185 @param defaults: The default dictionary
186 @return: A list of invalid keys
187
188 """
189 defaults_keys = frozenset(defaults.keys())
190 invalid = []
191 for key, value in options.items():
192 if key_path:
193 new_path = "%s/%s" % (key_path, key)
194 else:
195 new_path = key
196
197 if key not in defaults_keys:
198 invalid.append(new_path)
199 elif isinstance(value, dict):
200 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
201
202 return invalid
203
204
206 """Verify a dict has only keys set which also are in the defaults dict.
207
208 @param options: The user provided options
209 @param defaults: The default dictionary
210 @raise error.OpPrereqError: If one of the keys is not supported
211
212 """
213 invalid = _ComputeMissingKeys("", options, defaults)
214
215 if invalid:
216 raise errors.OpPrereqError("Provided option keys not supported: %s" %
217 CommaJoin(invalid), errors.ECODE_INVAL)
218
219
221 """List volume groups and their size
222
223 @rtype: dict
224 @return:
225 Dictionary with keys volume name and values
226 the size of the volume
227
228 """
229 command = "vgs --noheadings --units m --nosuffix -o name,size"
230 result = RunCmd(command)
231 retval = {}
232 if result.failed:
233 return retval
234
235 for line in result.stdout.splitlines():
236 try:
237 name, size = line.split()
238 size = int(float(size))
239 except (IndexError, ValueError), err:
240 logging.error("Invalid output from vgs (%s): %s", err, line)
241 continue
242
243 retval[name] = size
244
245 return retval
246
247
249 """Check whether the given bridge exists in the system
250
251 @type bridge: str
252 @param bridge: the bridge name to check
253 @rtype: boolean
254 @return: True if it does
255
256 """
257 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
258
259
261 """Try to convert a value ignoring errors.
262
263 This function tries to apply function I{fn} to I{val}. If no
264 C{ValueError} or C{TypeError} exceptions are raised, it will return
265 the result, else it will return the original value. Any other
266 exceptions are propagated to the caller.
267
268 @type fn: callable
269 @param fn: function to apply to the value
270 @param val: the value to be converted
271 @return: The converted value if the conversion was successful,
272 otherwise the original value.
273
274 """
275 try:
276 nv = fn(val)
277 except (ValueError, TypeError):
278 nv = val
279 return nv
280
281
283 """Parse a CPU mask definition and return the list of CPU IDs.
284
285 CPU mask format: comma-separated list of CPU IDs
286 or dash-separated ID ranges
287 Example: "0-2,5" -> "0,1,2,5"
288
289 @type cpu_mask: str
290 @param cpu_mask: CPU mask definition
291 @rtype: list of int
292 @return: list of CPU IDs
293
294 """
295 if not cpu_mask:
296 return []
297 cpu_list = []
298 for range_def in cpu_mask.split(","):
299 boundaries = range_def.split("-")
300 n_elements = len(boundaries)
301 if n_elements > 2:
302 raise errors.ParseError("Invalid CPU ID range definition"
303 " (only one hyphen allowed): %s" % range_def)
304 try:
305 lower = int(boundaries[0])
306 except (ValueError, TypeError), err:
307 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
308 " CPU ID range: %s" % str(err))
309 try:
310 higher = int(boundaries[-1])
311 except (ValueError, TypeError), err:
312 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
313 " CPU ID range: %s" % str(err))
314 if lower > higher:
315 raise errors.ParseError("Invalid CPU ID range definition"
316 " (%d > %d): %s" % (lower, higher, range_def))
317 cpu_list.extend(range(lower, higher + 1))
318 return cpu_list
319
320
322 """Parse a multiple CPU mask definition and return the list of CPU IDs.
323
324 CPU mask format: colon-separated list of comma-separated list of CPU IDs
325 or dash-separated ID ranges, with optional "all" as CPU value
326 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
327
328 @type cpu_mask: str
329 @param cpu_mask: multiple CPU mask definition
330 @rtype: list of lists of int
331 @return: list of lists of CPU IDs
332
333 """
334 if not cpu_mask:
335 return []
336 cpu_list = []
337 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
338 if range_def == constants.CPU_PINNING_ALL:
339 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
340 else:
341
342 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
343
344 return cpu_list
345
346
348 """Try to get the homedir of the given user.
349
350 The user can be passed either as a string (denoting the name) or as
351 an integer (denoting the user id). If the user is not found, the
352 C{default} argument is returned, which defaults to C{None}.
353
354 """
355 try:
356 if isinstance(user, basestring):
357 result = pwd.getpwnam(user)
358 elif isinstance(user, (int, long)):
359 result = pwd.getpwuid(user)
360 else:
361 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
362 type(user))
363 except KeyError:
364 return default
365 return result.pw_dir
366
367
369 """Returns the first non-existing integer from seq.
370
371 The seq argument should be a sorted list of positive integers. The
372 first time the index of an element is smaller than the element
373 value, the index will be returned.
374
375 The base argument is used to start at a different offset,
376 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
377
378 Example: C{[0, 1, 3]} will return I{2}.
379
380 @type seq: sequence
381 @param seq: the sequence to be analyzed.
382 @type base: int
383 @param base: use this value as the base index of the sequence
384 @rtype: int
385 @return: the first non-used index in the sequence
386
387 """
388 for idx, elem in enumerate(seq):
389 assert elem >= base, "Passed element is higher than base offset"
390 if elem > idx + base:
391
392 return idx + base
393 return None
394
395
397 """Waits for a condition to occur on the socket.
398
399 Immediately returns at the first interruption.
400
401 @type fdobj: integer or object supporting a fileno() method
402 @param fdobj: entity to wait for events on
403 @type event: integer
404 @param event: ORed condition (see select module)
405 @type timeout: float or None
406 @param timeout: Timeout in seconds
407 @rtype: int or None
408 @return: None for timeout, otherwise occured conditions
409
410 """
411 check = (event | select.POLLPRI |
412 select.POLLNVAL | select.POLLHUP | select.POLLERR)
413
414 if timeout is not None:
415
416 timeout *= 1000
417
418 poller = select.poll()
419 poller.register(fdobj, event)
420 try:
421
422
423
424 io_events = poller.poll(timeout)
425 except select.error, err:
426 if err[0] != errno.EINTR:
427 raise
428 io_events = []
429 if io_events and io_events[0][1] & check:
430 return io_events[0][1]
431 else:
432 return None
433
434
436 """Retry helper for WaitForFdCondition.
437
438 This class contains the retried and wait functions that make sure
439 WaitForFdCondition can continue waiting until the timeout is actually
440 expired.
441
442 """
443
445 self.timeout = timeout
446
447 - def Poll(self, fdobj, event):
453
455 self.timeout = timeout
456
457
459 """Waits for a condition to occur on the socket.
460
461 Retries until the timeout is expired, even if interrupted.
462
463 @type fdobj: integer or object supporting a fileno() method
464 @param fdobj: entity to wait for events on
465 @type event: integer
466 @param event: ORed condition (see select module)
467 @type timeout: float or None
468 @param timeout: Timeout in seconds
469 @rtype: int or None
470 @return: None for timeout, otherwise occured conditions
471
472 """
473 if timeout is not None:
474 retrywaiter = FdConditionWaiterHelper(timeout)
475 try:
476 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
477 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
478 except RetryTimeout:
479 result = None
480 else:
481 result = None
482 while result is None:
483 result = SingleWaitForFdCondition(fdobj, event, timeout)
484 return result
485
486
488 """Check for and start daemon if not alive.
489
490 @type name: string
491 @param name: daemon name
492
493 @rtype: bool
494 @return: 'True' if daemon successfully started,
495 'False' otherwise
496
497 """
498 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
499 if result.failed:
500 logging.error("Can't start daemon '%s', failure %s, output: %s",
501 name, result.fail_reason, result.output)
502 return False
503
504 return True
505
506
508 """Stop daemon
509
510 @type name: string
511 @param name: daemon name
512
513 @rtype: bool
514 @return: 'True' if daemon successfully stopped,
515 'False' otherwise
516
517 """
518 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
519 if result.failed:
520 logging.error("Can't stop daemon '%s', failure %s, output: %s",
521 name, result.fail_reason, result.output)
522 return False
523
524 return True
525
526
528 """Splits time as floating point number into a tuple.
529
530 @param value: Time in seconds
531 @type value: int or float
532 @return: Tuple containing (seconds, microseconds)
533
534 """
535 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
536
537 assert 0 <= seconds, \
538 "Seconds must be larger than or equal to 0, but are %s" % seconds
539 assert 0 <= microseconds <= 999999, \
540 "Microseconds must be 0-999999, but are %s" % microseconds
541
542 return (int(seconds), int(microseconds))
543
544
546 """Merges a tuple into time as a floating point number.
547
548 @param timetuple: Time as tuple, (seconds, microseconds)
549 @type timetuple: tuple
550 @return: Time as a floating point number expressed in seconds
551
552 """
553 (seconds, microseconds) = timetuple
554
555 assert 0 <= seconds, \
556 "Seconds must be larger than or equal to 0, but are %s" % seconds
557 assert 0 <= microseconds <= 999999, \
558 "Microseconds must be 0-999999, but are %s" % microseconds
559
560 return float(seconds) + (float(microseconds) * 0.000001)
561
562
564 """Return the current timestamp expressed as number of nanoseconds since the
565 unix epoch
566
567 @return: nanoseconds since the Unix epoch
568
569 """
570 return int(time.time() * 1000000000)
571
572
574 """Tries to find an item in a dictionary matching a name.
575
576 Callers have to ensure the data names aren't contradictory (e.g. a regexp
577 that matches a string). If the name isn't a direct key, all regular
578 expression objects in the dictionary are matched against it.
579
580 @type data: dict
581 @param data: Dictionary containing data
582 @type name: string
583 @param name: Name to look for
584 @rtype: tuple; (value in dictionary, matched groups as list)
585
586 """
587 if name in data:
588 return (data[name], [])
589
590 for key, value in data.items():
591
592 if hasattr(key, "match"):
593 m = key.match(name)
594 if m:
595 return (value, list(m.groups()))
596
597 return None
598
599
601 """Returns the list of mounted filesystems.
602
603 This function is Linux-specific.
604
605 @param filename: path of mounts file (/proc/mounts by default)
606 @rtype: list of tuples
607 @return: list of mount entries (device, mountpoint, fstype, options)
608
609 """
610
611 data = []
612 mountlines = ReadFile(filename).splitlines()
613 for line in mountlines:
614 device, mountpoint, fstype, options, _ = line.split(None, 4)
615 data.append((device, mountpoint, fstype, options))
616
617 return data
618
619
621 """Signal Handled decoration.
622
623 This special decorator installs a signal handler and then calls the target
624 function. The function must accept a 'signal_handlers' keyword argument,
625 which will contain a dict indexed by signal number, with SignalHandler
626 objects as values.
627
628 The decorator can be safely stacked with iself, to handle multiple signals
629 with different handlers.
630
631 @type signums: list
632 @param signums: signals to intercept
633
634 """
635 def wrap(fn):
636 def sig_function(*args, **kwargs):
637 assert "signal_handlers" not in kwargs or \
638 kwargs["signal_handlers"] is None or \
639 isinstance(kwargs["signal_handlers"], dict), \
640 "Wrong signal_handlers parameter in original function call"
641 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
642 signal_handlers = kwargs["signal_handlers"]
643 else:
644 signal_handlers = {}
645 kwargs["signal_handlers"] = signal_handlers
646 sighandler = SignalHandler(signums)
647 try:
648 for sig in signums:
649 signal_handlers[sig] = sighandler
650 return fn(*args, **kwargs)
651 finally:
652 sighandler.Reset()
653 return sig_function
654 return wrap
655
656
658 """Checks whether a timeout has expired.
659
660 """
661 return _time_fn() > (epoch + timeout)
662
663
674 else:
675
678
680 """Initializes this class.
681
682 """
683 (read_fd, write_fd) = os.pipe()
684
685
686
687
688 self._read_fh = os.fdopen(read_fd, "r", 0)
689 self._write_fh = os.fdopen(write_fd, "w", 0)
690
691 self._previous = self._SetWakeupFd(self._write_fh.fileno())
692
693
694 self.fileno = self._read_fh.fileno
695 self.read = self._read_fh.read
696
698 """Restores the previous wakeup file descriptor.
699
700 """
701 if hasattr(self, "_previous") and self._previous is not None:
702 self._SetWakeupFd(self._previous)
703 self._previous = None
704
706 """Notifies the wakeup file descriptor.
707
708 """
709 self._write_fh.write(chr(0))
710
712 """Called before object deletion.
713
714 """
715 self.Reset()
716
717
719 """Generic signal handler class.
720
721 It automatically restores the original handler when deconstructed or
722 when L{Reset} is called. You can either pass your own handler
723 function in or query the L{called} attribute to detect whether the
724 signal was sent.
725
726 @type signum: list
727 @ivar signum: the signals we handle
728 @type called: boolean
729 @ivar called: tracks whether any of the signals have been raised
730
731 """
732 - def __init__(self, signum, handler_fn=None, wakeup=None):
733 """Constructs a new SignalHandler instance.
734
735 @type signum: int or list of ints
736 @param signum: Single signal number or set of signal numbers
737 @type handler_fn: callable
738 @param handler_fn: Signal handling function
739
740 """
741 assert handler_fn is None or callable(handler_fn)
742
743 self.signum = set(signum)
744 self.called = False
745
746 self._handler_fn = handler_fn
747 self._wakeup = wakeup
748
749 self._previous = {}
750 try:
751 for signum in self.signum:
752
753 prev_handler = signal.signal(signum, self._HandleSignal)
754 try:
755 self._previous[signum] = prev_handler
756 except:
757
758 signal.signal(signum, prev_handler)
759 raise
760 except:
761
762 self.Reset()
763
764
765 raise
766
769
771 """Restore previous handler.
772
773 This will reset all the signals to their previous handlers.
774
775 """
776 for signum, prev_handler in self._previous.items():
777 signal.signal(signum, prev_handler)
778
779 del self._previous[signum]
780
782 """Unsets the L{called} flag.
783
784 This function can be used in case a signal may arrive several times.
785
786 """
787 self.called = False
788
790 """Actual signal handling function.
791
792 """
793
794
795 self.called = True
796
797 if self._wakeup:
798
799 self._wakeup.Notify()
800
801 if self._handler_fn:
802 self._handler_fn(signum, frame)
803
804
806 """A simple field set.
807
808 Among the features are:
809 - checking if a string is among a list of static string or regex objects
810 - checking if a whole list of string matches
811 - returning the matching groups from a regex match
812
813 Internally, all fields are held as regular expression objects.
814
815 """
817 self.items = [re.compile("^%s$" % value) for value in items]
818
820 """Extend the field set with the items from another one"""
821 self.items.extend(other_set.items)
822
824 """Checks if a field matches the current set
825
826 @type field: str
827 @param field: the string to match
828 @return: either None or a regular expression match object
829
830 """
831 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
832 return m
833 return None
834
836 """Returns the list of fields not matching the current set
837
838 @type items: list
839 @param items: the list of fields to check
840 @rtype: list
841 @return: list of non-matching fields
842
843 """
844 return [val for val in items if not self.Matches(val)]
845
846
889
890
892 """Checks if the instance has only disks of any of the dev_types.
893
894 @type disks_info: list of L{Disk}
895 @param disks_info: all the disks of the instance.
896 @type dev_types: list of disk templates
897 @param dev_types: the disk type required.
898
899 @rtype: bool
900 @return: True iff the instance only has disks of type dev_type.
901 """
902
903 assert not isinstance(dev_types, str)
904
905 if not disks_info and constants.DT_DISKLESS not in dev_types:
906 return False
907
908 for disk in disks_info:
909 if disk.dev_type not in dev_types:
910 return False
911
912 return True
913
914
916 """Checks if the instance has some disks of any types in dev_types.
917
918 @type disks_info: list of L{Disk}
919 @param disks_info: all the disks of the instance.
920 @type dev_types: list of disk template
921 @param dev_types: the disk type required.
922
923 @rtype: bool
924 @return: True if the instance has disks of type dev_types or the instance has
925 no disks and the dev_types allow DT_DISKLESS.
926 """
927
928 assert not isinstance(dev_types, str)
929
930 if not disks_info and constants.DT_DISKLESS in dev_types:
931 return True
932
933 for disk in disks_info:
934 if disk.dev_type in dev_types:
935 return True
936
937 return False
938
939
941 """Gives a summary disk template from disk devtypes.
942
943 @type disk_types: list of string
944 @param disk_types: all the dev_types of the instance.
945 @rtype disk template
946 @returns the summarized disk template of the disk types.
947
948 """
949 disk_types = set(dev_type for dev_type in disk_types)
950 if not disk_types:
951 return constants.DT_DISKLESS
952 elif len(disk_types) > 1:
953 return constants.DT_MIXED
954 else:
955 return disk_types.pop()
956
957
959 """Gives a summary disk template from disks.
960
961 @type disks_info: list of L{Disk}
962 @param disks_info: all the disks of the instance.
963 @rtype disk template
964 @returns the summarized disk template of the disk types.
965
966 """
967 return GetDiskTemplateString(d.dev_type for d in disks_info)
968