1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Ganeti utility module.
23
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
26
27 """
28
29
30
31 import os
32 import re
33 import errno
34 import pwd
35 import itertools
36 import select
37 import logging
38 import signal
39
40 from ganeti import errors
41 from ganeti import constants
42 from ganeti import compat
43
44 from ganeti.utils.algo import *
45 from ganeti.utils.filelock import *
46 from ganeti.utils.hash import *
47 from ganeti.utils.io import *
48 from ganeti.utils.log import *
49 from ganeti.utils.mlock import *
50 from ganeti.utils.nodesetup import *
51 from ganeti.utils.process import *
52 from ganeti.utils.retry import *
53 from ganeti.utils.text import *
54 from ganeti.utils.wrapper import *
55 from ganeti.utils.x509 import *
56
57
58 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
59
60 UUID_RE = re.compile('^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-'
61 '[a-f0-9]{4}-[a-f0-9]{12}$')
62
63
65 """Force the values of a dict to have certain types.
66
67 @type target: dict
68 @param target: the dict to update
69 @type key_types: dict
70 @param key_types: dict mapping target dict keys to types
71 in constants.ENFORCEABLE_TYPES
72 @type allowed_values: list
73 @keyword allowed_values: list of specially allowed values
74
75 """
76 if allowed_values is None:
77 allowed_values = []
78
79 if not isinstance(target, dict):
80 msg = "Expected dictionary, got '%s'" % target
81 raise errors.TypeEnforcementError(msg)
82
83 for key in target:
84 if key not in key_types:
85 msg = "Unknown parameter '%s'" % key
86 raise errors.TypeEnforcementError(msg)
87
88 if target[key] in allowed_values:
89 continue
90
91 ktype = key_types[key]
92 if ktype not in constants.ENFORCEABLE_TYPES:
93 msg = "'%s' has non-enforceable type %s" % (key, ktype)
94 raise errors.ProgrammerError(msg)
95
96 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
97 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
98 pass
99 elif not isinstance(target[key], basestring):
100 if isinstance(target[key], bool) and not target[key]:
101 target[key] = ''
102 else:
103 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
104 raise errors.TypeEnforcementError(msg)
105 elif ktype == constants.VTYPE_BOOL:
106 if isinstance(target[key], basestring) and target[key]:
107 if target[key].lower() == constants.VALUE_FALSE:
108 target[key] = False
109 elif target[key].lower() == constants.VALUE_TRUE:
110 target[key] = True
111 else:
112 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
113 raise errors.TypeEnforcementError(msg)
114 elif target[key]:
115 target[key] = True
116 else:
117 target[key] = False
118 elif ktype == constants.VTYPE_SIZE:
119 try:
120 target[key] = ParseUnit(target[key])
121 except errors.UnitParseError, err:
122 msg = "'%s' (value %s) is not a valid size. error: %s" % \
123 (key, target[key], err)
124 raise errors.TypeEnforcementError(msg)
125 elif ktype == constants.VTYPE_INT:
126 try:
127 target[key] = int(target[key])
128 except (ValueError, TypeError):
129 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
130 raise errors.TypeEnforcementError(msg)
131
132
134 """Validate the given service name.
135
136 @type name: number or string
137 @param name: Service name or port specification
138
139 """
140 try:
141 numport = int(name)
142 except (ValueError, TypeError):
143
144 valid = _VALID_SERVICE_NAME_RE.match(name)
145 else:
146
147
148 valid = (numport >= 0 and numport < (1 << 16))
149
150 if not valid:
151 raise errors.OpPrereqError("Invalid service name '%s'" % name,
152 errors.ECODE_INVAL)
153
154 return name
155
156
158 """List volume groups and their size
159
160 @rtype: dict
161 @return:
162 Dictionary with keys volume name and values
163 the size of the volume
164
165 """
166 command = "vgs --noheadings --units m --nosuffix -o name,size"
167 result = RunCmd(command)
168 retval = {}
169 if result.failed:
170 return retval
171
172 for line in result.stdout.splitlines():
173 try:
174 name, size = line.split()
175 size = int(float(size))
176 except (IndexError, ValueError), err:
177 logging.error("Invalid output from vgs (%s): %s", err, line)
178 continue
179
180 retval[name] = size
181
182 return retval
183
184
186 """Check whether the given bridge exists in the system
187
188 @type bridge: str
189 @param bridge: the bridge name to check
190 @rtype: boolean
191 @return: True if it does
192
193 """
194 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
195
196
198 """Try to convert a value ignoring errors.
199
200 This function tries to apply function I{fn} to I{val}. If no
201 C{ValueError} or C{TypeError} exceptions are raised, it will return
202 the result, else it will return the original value. Any other
203 exceptions are propagated to the caller.
204
205 @type fn: callable
206 @param fn: function to apply to the value
207 @param val: the value to be converted
208 @return: The converted value if the conversion was successful,
209 otherwise the original value.
210
211 """
212 try:
213 nv = fn(val)
214 except (ValueError, TypeError):
215 nv = val
216 return nv
217
218
220 """Parse a CPU mask definition and return the list of CPU IDs.
221
222 CPU mask format: comma-separated list of CPU IDs
223 or dash-separated ID ranges
224 Example: "0-2,5" -> "0,1,2,5"
225
226 @type cpu_mask: str
227 @param cpu_mask: CPU mask definition
228 @rtype: list of int
229 @return: list of CPU IDs
230
231 """
232 if not cpu_mask:
233 return []
234 cpu_list = []
235 for range_def in cpu_mask.split(","):
236 boundaries = range_def.split("-")
237 n_elements = len(boundaries)
238 if n_elements > 2:
239 raise errors.ParseError("Invalid CPU ID range definition"
240 " (only one hyphen allowed): %s" % range_def)
241 try:
242 lower = int(boundaries[0])
243 except (ValueError, TypeError), err:
244 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
245 " CPU ID range: %s" % str(err))
246 try:
247 higher = int(boundaries[-1])
248 except (ValueError, TypeError), err:
249 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
250 " CPU ID range: %s" % str(err))
251 if lower > higher:
252 raise errors.ParseError("Invalid CPU ID range definition"
253 " (%d > %d): %s" % (lower, higher, range_def))
254 cpu_list.extend(range(lower, higher + 1))
255 return cpu_list
256
257
259 """Try to get the homedir of the given user.
260
261 The user can be passed either as a string (denoting the name) or as
262 an integer (denoting the user id). If the user is not found, the
263 'default' argument is returned, which defaults to None.
264
265 """
266 try:
267 if isinstance(user, basestring):
268 result = pwd.getpwnam(user)
269 elif isinstance(user, (int, long)):
270 result = pwd.getpwuid(user)
271 else:
272 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
273 type(user))
274 except KeyError:
275 return default
276 return result.pw_dir
277
278
280 """Returns the first non-existing integer from seq.
281
282 The seq argument should be a sorted list of positive integers. The
283 first time the index of an element is smaller than the element
284 value, the index will be returned.
285
286 The base argument is used to start at a different offset,
287 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
288
289 Example: C{[0, 1, 3]} will return I{2}.
290
291 @type seq: sequence
292 @param seq: the sequence to be analyzed.
293 @type base: int
294 @param base: use this value as the base index of the sequence
295 @rtype: int
296 @return: the first non-used index in the sequence
297
298 """
299 for idx, elem in enumerate(seq):
300 assert elem >= base, "Passed element is higher than base offset"
301 if elem > idx + base:
302
303 return idx + base
304 return None
305
306
308 """Waits for a condition to occur on the socket.
309
310 Immediately returns at the first interruption.
311
312 @type fdobj: integer or object supporting a fileno() method
313 @param fdobj: entity to wait for events on
314 @type event: integer
315 @param event: ORed condition (see select module)
316 @type timeout: float or None
317 @param timeout: Timeout in seconds
318 @rtype: int or None
319 @return: None for timeout, otherwise occured conditions
320
321 """
322 check = (event | select.POLLPRI |
323 select.POLLNVAL | select.POLLHUP | select.POLLERR)
324
325 if timeout is not None:
326
327 timeout *= 1000
328
329 poller = select.poll()
330 poller.register(fdobj, event)
331 try:
332
333
334
335 io_events = poller.poll(timeout)
336 except select.error, err:
337 if err[0] != errno.EINTR:
338 raise
339 io_events = []
340 if io_events and io_events[0][1] & check:
341 return io_events[0][1]
342 else:
343 return None
344
345
347 """Retry helper for WaitForFdCondition.
348
349 This class contains the retried and wait functions that make sure
350 WaitForFdCondition can continue waiting until the timeout is actually
351 expired.
352
353 """
354
356 self.timeout = timeout
357
358 - def Poll(self, fdobj, event):
364
366 self.timeout = timeout
367
368
370 """Waits for a condition to occur on the socket.
371
372 Retries until the timeout is expired, even if interrupted.
373
374 @type fdobj: integer or object supporting a fileno() method
375 @param fdobj: entity to wait for events on
376 @type event: integer
377 @param event: ORed condition (see select module)
378 @type timeout: float or None
379 @param timeout: Timeout in seconds
380 @rtype: int or None
381 @return: None for timeout, otherwise occured conditions
382
383 """
384 if timeout is not None:
385 retrywaiter = FdConditionWaiterHelper(timeout)
386 try:
387 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
388 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
389 except RetryTimeout:
390 result = None
391 else:
392 result = None
393 while result is None:
394 result = SingleWaitForFdCondition(fdobj, event, timeout)
395 return result
396
397
399 """Check for and start daemon if not alive.
400
401 """
402 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
403 if result.failed:
404 logging.error("Can't start daemon '%s', failure %s, output: %s",
405 name, result.fail_reason, result.output)
406 return False
407
408 return True
409
410
412 """Stop daemon
413
414 """
415 result = RunCmd([constants.DAEMON_UTIL, "stop", name])
416 if result.failed:
417 logging.error("Can't stop daemon '%s', failure %s, output: %s",
418 name, result.fail_reason, result.output)
419 return False
420
421 return True
422
423
425 """Checks if the volume group list is valid.
426
427 The function will check if a given volume group is in the list of
428 volume groups and has a minimum size.
429
430 @type vglist: dict
431 @param vglist: dictionary of volume group names and their size
432 @type vgname: str
433 @param vgname: the volume group we should check
434 @type minsize: int
435 @param minsize: the minimum size we accept
436 @rtype: None or str
437 @return: None for success, otherwise the error message
438
439 """
440 vgsize = vglist.get(vgname, None)
441 if vgsize is None:
442 return "volume group '%s' missing" % vgname
443 elif vgsize < minsize:
444 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
445 (vgname, minsize, vgsize))
446 return None
447
448
450 """Splits time as floating point number into a tuple.
451
452 @param value: Time in seconds
453 @type value: int or float
454 @return: Tuple containing (seconds, microseconds)
455
456 """
457 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
458
459 assert 0 <= seconds, \
460 "Seconds must be larger than or equal to 0, but are %s" % seconds
461 assert 0 <= microseconds <= 999999, \
462 "Microseconds must be 0-999999, but are %s" % microseconds
463
464 return (int(seconds), int(microseconds))
465
466
468 """Merges a tuple into time as a floating point number.
469
470 @param timetuple: Time as tuple, (seconds, microseconds)
471 @type timetuple: tuple
472 @return: Time as a floating point number expressed in seconds
473
474 """
475 (seconds, microseconds) = timetuple
476
477 assert 0 <= seconds, \
478 "Seconds must be larger than or equal to 0, but are %s" % seconds
479 assert 0 <= microseconds <= 999999, \
480 "Microseconds must be 0-999999, but are %s" % microseconds
481
482 return float(seconds) + (float(microseconds) * 0.000001)
483
484
486 """Tries to find an item in a dictionary matching a name.
487
488 Callers have to ensure the data names aren't contradictory (e.g. a regexp
489 that matches a string). If the name isn't a direct key, all regular
490 expression objects in the dictionary are matched against it.
491
492 @type data: dict
493 @param data: Dictionary containing data
494 @type name: string
495 @param name: Name to look for
496 @rtype: tuple; (value in dictionary, matched groups as list)
497
498 """
499 if name in data:
500 return (data[name], [])
501
502 for key, value in data.items():
503
504 if hasattr(key, "match"):
505 m = key.match(name)
506 if m:
507 return (value, list(m.groups()))
508
509 return None
510
511
513 """Returns the list of mounted filesystems.
514
515 This function is Linux-specific.
516
517 @param filename: path of mounts file (/proc/mounts by default)
518 @rtype: list of tuples
519 @return: list of mount entries (device, mountpoint, fstype, options)
520
521 """
522
523 data = []
524 mountlines = ReadFile(filename).splitlines()
525 for line in mountlines:
526 device, mountpoint, fstype, options, _ = line.split(None, 4)
527 data.append((device, mountpoint, fstype, options))
528
529 return data
530
531
533 """Signal Handled decoration.
534
535 This special decorator installs a signal handler and then calls the target
536 function. The function must accept a 'signal_handlers' keyword argument,
537 which will contain a dict indexed by signal number, with SignalHandler
538 objects as values.
539
540 The decorator can be safely stacked with iself, to handle multiple signals
541 with different handlers.
542
543 @type signums: list
544 @param signums: signals to intercept
545
546 """
547 def wrap(fn):
548 def sig_function(*args, **kwargs):
549 assert 'signal_handlers' not in kwargs or \
550 kwargs['signal_handlers'] is None or \
551 isinstance(kwargs['signal_handlers'], dict), \
552 "Wrong signal_handlers parameter in original function call"
553 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
554 signal_handlers = kwargs['signal_handlers']
555 else:
556 signal_handlers = {}
557 kwargs['signal_handlers'] = signal_handlers
558 sighandler = SignalHandler(signums)
559 try:
560 for sig in signums:
561 signal_handlers[sig] = sighandler
562 return fn(*args, **kwargs)
563 finally:
564 sighandler.Reset()
565 return sig_function
566 return wrap
567
568
578 else:
581
583 """Initializes this class.
584
585 """
586 (read_fd, write_fd) = os.pipe()
587
588
589
590
591 self._read_fh = os.fdopen(read_fd, "r", 0)
592 self._write_fh = os.fdopen(write_fd, "w", 0)
593
594 self._previous = self._SetWakeupFd(self._write_fh.fileno())
595
596
597 self.fileno = self._read_fh.fileno
598 self.read = self._read_fh.read
599
601 """Restores the previous wakeup file descriptor.
602
603 """
604 if hasattr(self, "_previous") and self._previous is not None:
605 self._SetWakeupFd(self._previous)
606 self._previous = None
607
609 """Notifies the wakeup file descriptor.
610
611 """
612 self._write_fh.write("\0")
613
615 """Called before object deletion.
616
617 """
618 self.Reset()
619
620
622 """Generic signal handler class.
623
624 It automatically restores the original handler when deconstructed or
625 when L{Reset} is called. You can either pass your own handler
626 function in or query the L{called} attribute to detect whether the
627 signal was sent.
628
629 @type signum: list
630 @ivar signum: the signals we handle
631 @type called: boolean
632 @ivar called: tracks whether any of the signals have been raised
633
634 """
635 - def __init__(self, signum, handler_fn=None, wakeup=None):
636 """Constructs a new SignalHandler instance.
637
638 @type signum: int or list of ints
639 @param signum: Single signal number or set of signal numbers
640 @type handler_fn: callable
641 @param handler_fn: Signal handling function
642
643 """
644 assert handler_fn is None or callable(handler_fn)
645
646 self.signum = set(signum)
647 self.called = False
648
649 self._handler_fn = handler_fn
650 self._wakeup = wakeup
651
652 self._previous = {}
653 try:
654 for signum in self.signum:
655
656 prev_handler = signal.signal(signum, self._HandleSignal)
657 try:
658 self._previous[signum] = prev_handler
659 except:
660
661 signal.signal(signum, prev_handler)
662 raise
663 except:
664
665 self.Reset()
666
667
668 raise
669
672
674 """Restore previous handler.
675
676 This will reset all the signals to their previous handlers.
677
678 """
679 for signum, prev_handler in self._previous.items():
680 signal.signal(signum, prev_handler)
681
682 del self._previous[signum]
683
685 """Unsets the L{called} flag.
686
687 This function can be used in case a signal may arrive several times.
688
689 """
690 self.called = False
691
693 """Actual signal handling function.
694
695 """
696
697
698 self.called = True
699
700 if self._wakeup:
701
702 self._wakeup.Notify()
703
704 if self._handler_fn:
705 self._handler_fn(signum, frame)
706
707
709 """A simple field set.
710
711 Among the features are:
712 - checking if a string is among a list of static string or regex objects
713 - checking if a whole list of string matches
714 - returning the matching groups from a regex match
715
716 Internally, all fields are held as regular expression objects.
717
718 """
720 self.items = [re.compile("^%s$" % value) for value in items]
721
723 """Extend the field set with the items from another one"""
724 self.items.extend(other_set.items)
725
727 """Checks if a field matches the current set
728
729 @type field: str
730 @param field: the string to match
731 @return: either None or a regular expression match object
732
733 """
734 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
735 return m
736 return None
737
739 """Returns the list of fields not matching the current set
740
741 @type items: list
742 @param items: the list of fields to check
743 @rtype: list
744 @return: list of non-matching fields
745
746 """
747 return [val for val in items if not self.Matches(val)]
748