Package ganeti :: Package utils :: Module retry
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.retry

  1  # 
  2  # 
  3   
  4  # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, but 
 12  # WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
 14  # General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the Free Software 
 18  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
 19  # 02110-1301, USA. 
 20   
 21  """Utility functions for retrying function calls with a timeout. 
 22   
 23  """ 
 24   
 25   
 26  import time 
 27   
 28  from ganeti import errors 
 29   
 30   
 31  #: Special delay to specify whole remaining timeout 
 32  RETRY_REMAINING_TIME = object() 
 33   
 34   
35 -class RetryTimeout(Exception):
36 """Retry loop timed out. 37 38 Any arguments which was passed by the retried function to RetryAgain will be 39 preserved in RetryTimeout, if it is raised. If such argument was an exception 40 the RaiseInner helper method will reraise it. 41 42 """
43 - def RaiseInner(self):
44 if self.args and isinstance(self.args[0], Exception): 45 raise self.args[0] 46 else: 47 raise RetryTimeout(*self.args)
48 49
50 -class RetryAgain(Exception):
51 """Retry again. 52 53 Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as 54 arguments to RetryTimeout. If an exception is passed, the RaiseInner() method 55 of the RetryTimeout() method can be used to reraise it. 56 57 """
58 59
60 -class _RetryDelayCalculator(object):
61 """Calculator for increasing delays. 62 63 """ 64 __slots__ = [ 65 "_factor", 66 "_limit", 67 "_next", 68 "_start", 69 ] 70
71 - def __init__(self, start, factor, limit):
72 """Initializes this class. 73 74 @type start: float 75 @param start: Initial delay 76 @type factor: float 77 @param factor: Factor for delay increase 78 @type limit: float or None 79 @param limit: Upper limit for delay or None for no limit 80 81 """ 82 assert start > 0.0 83 assert factor >= 1.0 84 assert limit is None or limit >= 0.0 85 86 self._start = start 87 self._factor = factor 88 self._limit = limit 89 90 self._next = start
91
92 - def __call__(self):
93 """Returns current delay and calculates the next one. 94 95 """ 96 current = self._next 97 98 # Update for next run 99 if self._limit is None or self._next < self._limit: 100 self._next = min(self._limit, self._next * self._factor) 101 102 return current
103 104
105 -def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, 106 _time_fn=time.time):
107 """Call a function repeatedly until it succeeds. 108 109 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain} 110 anymore. Between calls a delay, specified by C{delay}, is inserted. After a 111 total of C{timeout} seconds, this function throws L{RetryTimeout}. 112 113 C{delay} can be one of the following: 114 - callable returning the delay length as a float 115 - Tuple of (start, factor, limit) 116 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is 117 useful when overriding L{wait_fn} to wait for an external event) 118 - A static delay as a number (int or float) 119 120 @type fn: callable 121 @param fn: Function to be called 122 @param delay: Either a callable (returning the delay), a tuple of (start, 123 factor, limit) (see L{_RetryDelayCalculator}), 124 L{RETRY_REMAINING_TIME} or a number (int or float) 125 @type timeout: float 126 @param timeout: Total timeout 127 @type wait_fn: callable 128 @param wait_fn: Waiting function 129 @return: Return value of function 130 131 """ 132 assert callable(fn) 133 assert callable(wait_fn) 134 assert callable(_time_fn) 135 136 if args is None: 137 args = [] 138 139 end_time = _time_fn() + timeout 140 141 if callable(delay): 142 # External function to calculate delay 143 calc_delay = delay 144 145 elif isinstance(delay, (tuple, list)): 146 # Increasing delay with optional upper boundary 147 (start, factor, limit) = delay 148 calc_delay = _RetryDelayCalculator(start, factor, limit) 149 150 elif delay is RETRY_REMAINING_TIME: 151 # Always use the remaining time 152 calc_delay = None 153 154 else: 155 # Static delay 156 calc_delay = lambda: delay 157 158 assert calc_delay is None or callable(calc_delay) 159 160 while True: 161 retry_args = [] 162 try: 163 # pylint: disable=W0142 164 return fn(*args) 165 except RetryAgain, err: 166 retry_args = err.args 167 except RetryTimeout: 168 raise errors.ProgrammerError("Nested retry loop detected that didn't" 169 " handle RetryTimeout") 170 171 remaining_time = end_time - _time_fn() 172 173 if remaining_time < 0.0: 174 # pylint: disable=W0142 175 raise RetryTimeout(*retry_args) 176 177 assert remaining_time >= 0.0 178 179 if calc_delay is None: 180 wait_fn(remaining_time) 181 else: 182 current_delay = calc_delay() 183 if current_delay > 0.0: 184 wait_fn(current_delay)
185 186
187 -def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, 188 _time_fn=time.time):
189 """A wrapper over L{Retry} implementing a simpler interface. 190 191 All the parameters are the same as for L{Retry}, except it has one 192 extra argument: expected, which can be either a value (will be 193 compared with the result of the function, or a callable (which will 194 get the result passed and has to return a boolean). If the test is 195 false, we will retry until either the timeout has passed or the 196 tests succeeds. In both cases, the last result from calling the 197 function will be returned. 198 199 Note that this function is not expected to raise any retry-related 200 exceptions, always simply returning values. As such, the function is 201 designed to allow easy wrapping of code that doesn't use retry at 202 all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn, 203 ...)". 204 205 @see: L{Retry} 206 207 """ 208 rdict = {} 209 210 def helper(*innerargs): 211 # pylint: disable=W0142 212 result = rdict["result"] = fn(*innerargs) 213 if not ((callable(expected) and expected(result)) or result == expected): 214 raise RetryAgain() 215 return result
216 217 try: 218 result = Retry(helper, delay, timeout, args=args, 219 wait_fn=wait_fn, _time_fn=_time_fn) 220 except RetryTimeout: 221 assert "result" in rdict 222 result = rdict["result"] 223 return result 224