Package ganeti :: Package utils :: Module retry
[hide private]
[frames] | no frames]

Source Code for Module ganeti.utils.retry

  1  # 
  2  # 
  3   
  4  # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. 
  5  # All rights reserved. 
  6  # 
  7  # Redistribution and use in source and binary forms, with or without 
  8  # modification, are permitted provided that the following conditions are 
  9  # met: 
 10  # 
 11  # 1. Redistributions of source code must retain the above copyright notice, 
 12  # this list of conditions and the following disclaimer. 
 13  # 
 14  # 2. Redistributions in binary form must reproduce the above copyright 
 15  # notice, this list of conditions and the following disclaimer in the 
 16  # documentation and/or other materials provided with the distribution. 
 17  # 
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 
 19  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
 20  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
 21  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
 22  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 23  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 24  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
 25  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
 26  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
 27  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
 28  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 29   
 30  """Utility functions for retrying function calls with a timeout. 
 31   
 32  """ 
 33   
 34   
 35  import time 
 36   
 37  from ganeti import errors 
 38   
 39   
 40  #: Special delay to specify whole remaining timeout 
 41  RETRY_REMAINING_TIME = object() 
 42   
 43   
44 -class RetryTimeout(Exception):
45 """Retry loop timed out. 46 47 Any arguments which was passed by the retried function to RetryAgain will be 48 preserved in RetryTimeout, if it is raised. If such argument was an exception 49 the RaiseInner helper method will reraise it. 50 51 """
52 - def RaiseInner(self):
53 if self.args and isinstance(self.args[0], Exception): 54 raise self.args[0] 55 else: 56 raise RetryTimeout(*self.args)
57 58
59 -class RetryAgain(Exception):
60 """Retry again. 61 62 Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as 63 arguments to RetryTimeout. If an exception is passed, the RaiseInner() method 64 of the RetryTimeout() method can be used to reraise it. 65 66 """
67 68
69 -class _RetryDelayCalculator(object):
70 """Calculator for increasing delays. 71 72 """ 73 __slots__ = [ 74 "_factor", 75 "_limit", 76 "_next", 77 "_start", 78 ] 79
80 - def __init__(self, start, factor, limit):
81 """Initializes this class. 82 83 @type start: float 84 @param start: Initial delay 85 @type factor: float 86 @param factor: Factor for delay increase 87 @type limit: float or None 88 @param limit: Upper limit for delay or None for no limit 89 90 """ 91 assert start > 0.0 92 assert factor >= 1.0 93 assert limit is None or limit >= 0.0 94 95 self._start = start 96 self._factor = factor 97 self._limit = limit 98 99 self._next = start
100
101 - def __call__(self):
102 """Returns current delay and calculates the next one. 103 104 """ 105 current = self._next 106 107 # Update for next run 108 if self._limit is None or self._next < self._limit: 109 self._next = min(self._limit, self._next * self._factor) 110 111 return current
112 113
114 -def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, 115 _time_fn=time.time):
116 """Call a function repeatedly until it succeeds. 117 118 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain} 119 anymore. Between calls a delay, specified by C{delay}, is inserted. After a 120 total of C{timeout} seconds, this function throws L{RetryTimeout}. 121 122 C{delay} can be one of the following: 123 - callable returning the delay length as a float 124 - Tuple of (start, factor, limit) 125 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is 126 useful when overriding L{wait_fn} to wait for an external event) 127 - A static delay as a number (int or float) 128 129 @type fn: callable 130 @param fn: Function to be called 131 @param delay: Either a callable (returning the delay), a tuple of (start, 132 factor, limit) (see L{_RetryDelayCalculator}), 133 L{RETRY_REMAINING_TIME} or a number (int or float) 134 @type timeout: float 135 @param timeout: Total timeout 136 @type wait_fn: callable 137 @param wait_fn: Waiting function 138 @return: Return value of function 139 140 """ 141 assert callable(fn) 142 assert callable(wait_fn) 143 assert callable(_time_fn) 144 145 if args is None: 146 args = [] 147 148 end_time = _time_fn() + timeout 149 150 if callable(delay): 151 # External function to calculate delay 152 calc_delay = delay 153 154 elif isinstance(delay, (tuple, list)): 155 # Increasing delay with optional upper boundary 156 (start, factor, limit) = delay 157 calc_delay = _RetryDelayCalculator(start, factor, limit) 158 159 elif delay is RETRY_REMAINING_TIME: 160 # Always use the remaining time 161 calc_delay = None 162 163 else: 164 # Static delay 165 calc_delay = lambda: delay 166 167 assert calc_delay is None or callable(calc_delay) 168 169 while True: 170 retry_args = [] 171 try: 172 # pylint: disable=W0142 173 return fn(*args) 174 except RetryAgain, err: 175 retry_args = err.args 176 except RetryTimeout: 177 raise errors.ProgrammerError("Nested retry loop detected that didn't" 178 " handle RetryTimeout") 179 180 remaining_time = end_time - _time_fn() 181 182 if remaining_time <= 0.0: 183 # pylint: disable=W0142 184 raise RetryTimeout(*retry_args) 185 186 assert remaining_time > 0.0 187 188 if calc_delay is None: 189 wait_fn(remaining_time) 190 else: 191 current_delay = calc_delay() 192 if current_delay > 0.0: 193 wait_fn(current_delay)
194 195
196 -def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, 197 _time_fn=time.time):
198 """A wrapper over L{Retry} implementing a simpler interface. 199 200 All the parameters are the same as for L{Retry}, except it has one 201 extra argument: expected, which can be either a value (will be 202 compared with the result of the function, or a callable (which will 203 get the result passed and has to return a boolean). If the test is 204 false, we will retry until either the timeout has passed or the 205 tests succeeds. In both cases, the last result from calling the 206 function will be returned. 207 208 Note that this function is not expected to raise any retry-related 209 exceptions, always simply returning values. As such, the function is 210 designed to allow easy wrapping of code that doesn't use retry at 211 all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn, 212 ...)". 213 214 @see: L{Retry} 215 216 """ 217 rdict = {} 218 219 def helper(*innerargs): 220 # pylint: disable=W0142 221 result = rdict["result"] = fn(*innerargs) 222 if not ((callable(expected) and expected(result)) or result == expected): 223 raise RetryAgain() 224 return result
225 226 try: 227 result = Retry(helper, delay, timeout, args=args, 228 wait_fn=wait_fn, _time_fn=_time_fn) 229 except RetryTimeout: 230 assert "result" in rdict 231 result = rdict["result"] 232 return result 233