1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 """Utility functions for retrying function calls with a timeout.
31
32 """
33
34
35 import logging
36 import time
37
38 from ganeti import errors
39
40
41
42 RETRY_REMAINING_TIME = object()
43
44
46 """Retry loop timed out.
47
48 Any arguments which was passed by the retried function to RetryAgain will be
49 preserved in RetryTimeout, if it is raised. If such argument was an exception
50 the RaiseInner helper method will reraise it.
51
52 """
58
59
61 """Retry again.
62
63 Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
64 arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
65 of the RetryTimeout() method can be used to reraise it.
66
67 """
68
69
71 """Calculator for increasing delays.
72
73 """
74 __slots__ = [
75 "_factor",
76 "_limit",
77 "_next",
78 "_start",
79 ]
80
81 - def __init__(self, start, factor, limit):
82 """Initializes this class.
83
84 @type start: float
85 @param start: Initial delay
86 @type factor: float
87 @param factor: Factor for delay increase
88 @type limit: float or None
89 @param limit: Upper limit for delay or None for no limit
90
91 """
92 assert start > 0.0
93 assert factor >= 1.0
94 assert limit is None or limit >= 0.0
95
96 self._start = start
97 self._factor = factor
98 self._limit = limit
99
100 self._next = start
101
103 """Returns current delay and calculates the next one.
104
105 """
106 current = self._next
107
108
109 if self._limit is None or self._next < self._limit:
110 self._next = min(self._limit, self._next * self._factor)
111
112 return current
113
114
115 -def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
116 _time_fn=time.time):
117 """Call a function repeatedly until it succeeds.
118
119 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
120 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
121 total of C{timeout} seconds, this function throws L{RetryTimeout}.
122
123 C{delay} can be one of the following:
124 - callable returning the delay length as a float
125 - Tuple of (start, factor, limit)
126 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
127 useful when overriding L{wait_fn} to wait for an external event)
128 - A static delay as a number (int or float)
129
130 @type fn: callable
131 @param fn: Function to be called
132 @param delay: Either a callable (returning the delay), a tuple of (start,
133 factor, limit) (see L{_RetryDelayCalculator}),
134 L{RETRY_REMAINING_TIME} or a number (int or float)
135 @type timeout: float
136 @param timeout: Total timeout
137 @type wait_fn: callable
138 @param wait_fn: Waiting function
139 @return: Return value of function
140
141 """
142 assert callable(fn)
143 assert callable(wait_fn)
144 assert callable(_time_fn)
145
146 if args is None:
147 args = []
148
149 end_time = _time_fn() + timeout
150
151 if callable(delay):
152
153 calc_delay = delay
154
155 elif isinstance(delay, (tuple, list)):
156
157 (start, factor, limit) = delay
158 calc_delay = _RetryDelayCalculator(start, factor, limit)
159
160 elif delay is RETRY_REMAINING_TIME:
161
162 calc_delay = None
163
164 else:
165
166 calc_delay = lambda: delay
167
168 assert calc_delay is None or callable(calc_delay)
169
170 while True:
171 retry_args = []
172 try:
173
174 return fn(*args)
175 except RetryAgain, err:
176 retry_args = err.args
177 except RetryTimeout:
178 raise errors.ProgrammerError("Nested retry loop detected that didn't"
179 " handle RetryTimeout")
180
181 remaining_time = end_time - _time_fn()
182
183 if remaining_time <= 0.0:
184
185 raise RetryTimeout(*retry_args)
186
187 assert remaining_time > 0.0
188
189 if calc_delay is None:
190 wait_fn(remaining_time)
191 else:
192 current_delay = calc_delay()
193 if current_delay > 0.0:
194 wait_fn(current_delay)
195
196
197 -def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep,
198 _time_fn=time.time):
199 """A wrapper over L{Retry} implementing a simpler interface.
200
201 All the parameters are the same as for L{Retry}, except it has one
202 extra argument: expected, which can be either a value (will be
203 compared with the result of the function, or a callable (which will
204 get the result passed and has to return a boolean). If the test is
205 false, we will retry until either the timeout has passed or the
206 tests succeeds. In both cases, the last result from calling the
207 function will be returned.
208
209 Note that this function is not expected to raise any retry-related
210 exceptions, always simply returning values. As such, the function is
211 designed to allow easy wrapping of code that doesn't use retry at
212 all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
213 ...)".
214
215 @see: L{Retry}
216
217 """
218 rdict = {}
219
220 def helper(*innerargs):
221
222 result = rdict["result"] = fn(*innerargs)
223 if not ((callable(expected) and expected(result)) or result == expected):
224 raise RetryAgain()
225 return result
226
227 try:
228 result = Retry(helper, delay, timeout, args=args,
229 wait_fn=wait_fn, _time_fn=_time_fn)
230 except RetryTimeout:
231 assert "result" in rdict
232 result = rdict["result"]
233 return result
234
235
237 """A wrapper over L{SimpleRetry} implementing a count down.
238
239 Where L{Retry} fixes the time, after which the command is assumed to be
240 failing, this function assumes the total number of tries.
241
242 @see: L{Retry}
243 """
244
245 rdict = {"tries": 0}
246
247 get_tries = lambda: rdict["tries"]
248
249 def inc_tries(t):
250 rdict["tries"] += t
251
252 return SimpleRetry(expected, fn, 1, count, args=args,
253 wait_fn=inc_tries, _time_fn=get_tries)
254
255
258 """Retries calling a function up to the specified number of times.
259
260 @type max_retries: integer
261 @param max_retries: Maximum number of retries.
262 @type exception_class: class
263 @param exception_class: Exception class which is used for throwing the
264 final exception.
265 @type fn: callable
266 @param fn: Function to be called (up to the specified maximum number of
267 retries.
268 @type backoff: int
269 @param backoff: this enables and configures the back off behavior after
270 failed tries. If value is '0', there will be no delay between failed
271 tries. If the value is a positive integer, it is interpreted as the
272 base length of the back off delay (in seconds). That means there will be a
273 delay between failed tries of the length specified in this paramter. With
274 each next retry, the delay is increased by the factor of two. For example,
275 if the value is '2', the first delay is 2 seconds, the second 4 seconds,
276 the third 8 seconds (until the max_retries) are hit or the function call
277 succeeds.
278
279 """
280 if backoff < 0:
281 raise exception_class("Backoff must be a non-negative integer.")
282
283 last_exception = None
284 delay = backoff
285 for i in range(max_retries):
286 try:
287 fn(*args, **kwargs)
288 break
289 except errors.OpExecError as e:
290 logging.error("Error after retry no. %s: %s.", i, e)
291 last_exception = e
292 time.sleep(delay)
293 delay *= 2
294 else:
295 if last_exception:
296 raise exception_class("Error after %s retries. Last exception: %s."
297 % (max_retries, last_exception))
298