1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 """Utility functions for retrying function calls with a timeout.
31
32 """
33
34
35 import time
36
37 from ganeti import errors
38
39
40
41 RETRY_REMAINING_TIME = object()
42
43
45 """Retry loop timed out.
46
47 Any arguments which was passed by the retried function to RetryAgain will be
48 preserved in RetryTimeout, if it is raised. If such argument was an exception
49 the RaiseInner helper method will reraise it.
50
51 """
57
58
60 """Retry again.
61
62 Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
63 arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
64 of the RetryTimeout() method can be used to reraise it.
65
66 """
67
68
70 """Calculator for increasing delays.
71
72 """
73 __slots__ = [
74 "_factor",
75 "_limit",
76 "_next",
77 "_start",
78 ]
79
80 - def __init__(self, start, factor, limit):
81 """Initializes this class.
82
83 @type start: float
84 @param start: Initial delay
85 @type factor: float
86 @param factor: Factor for delay increase
87 @type limit: float or None
88 @param limit: Upper limit for delay or None for no limit
89
90 """
91 assert start > 0.0
92 assert factor >= 1.0
93 assert limit is None or limit >= 0.0
94
95 self._start = start
96 self._factor = factor
97 self._limit = limit
98
99 self._next = start
100
102 """Returns current delay and calculates the next one.
103
104 """
105 current = self._next
106
107
108 if self._limit is None or self._next < self._limit:
109 self._next = min(self._limit, self._next * self._factor)
110
111 return current
112
113
114 -def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
115 _time_fn=time.time):
116 """Call a function repeatedly until it succeeds.
117
118 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
119 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
120 total of C{timeout} seconds, this function throws L{RetryTimeout}.
121
122 C{delay} can be one of the following:
123 - callable returning the delay length as a float
124 - Tuple of (start, factor, limit)
125 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
126 useful when overriding L{wait_fn} to wait for an external event)
127 - A static delay as a number (int or float)
128
129 @type fn: callable
130 @param fn: Function to be called
131 @param delay: Either a callable (returning the delay), a tuple of (start,
132 factor, limit) (see L{_RetryDelayCalculator}),
133 L{RETRY_REMAINING_TIME} or a number (int or float)
134 @type timeout: float
135 @param timeout: Total timeout
136 @type wait_fn: callable
137 @param wait_fn: Waiting function
138 @return: Return value of function
139
140 """
141 assert callable(fn)
142 assert callable(wait_fn)
143 assert callable(_time_fn)
144
145 if args is None:
146 args = []
147
148 end_time = _time_fn() + timeout
149
150 if callable(delay):
151
152 calc_delay = delay
153
154 elif isinstance(delay, (tuple, list)):
155
156 (start, factor, limit) = delay
157 calc_delay = _RetryDelayCalculator(start, factor, limit)
158
159 elif delay is RETRY_REMAINING_TIME:
160
161 calc_delay = None
162
163 else:
164
165 calc_delay = lambda: delay
166
167 assert calc_delay is None or callable(calc_delay)
168
169 while True:
170 retry_args = []
171 try:
172
173 return fn(*args)
174 except RetryAgain, err:
175 retry_args = err.args
176 except RetryTimeout:
177 raise errors.ProgrammerError("Nested retry loop detected that didn't"
178 " handle RetryTimeout")
179
180 remaining_time = end_time - _time_fn()
181
182 if remaining_time <= 0.0:
183
184 raise RetryTimeout(*retry_args)
185
186 assert remaining_time > 0.0
187
188 if calc_delay is None:
189 wait_fn(remaining_time)
190 else:
191 current_delay = calc_delay()
192 if current_delay > 0.0:
193 wait_fn(current_delay)
194
195
196 -def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep,
197 _time_fn=time.time):
198 """A wrapper over L{Retry} implementing a simpler interface.
199
200 All the parameters are the same as for L{Retry}, except it has one
201 extra argument: expected, which can be either a value (will be
202 compared with the result of the function, or a callable (which will
203 get the result passed and has to return a boolean). If the test is
204 false, we will retry until either the timeout has passed or the
205 tests succeeds. In both cases, the last result from calling the
206 function will be returned.
207
208 Note that this function is not expected to raise any retry-related
209 exceptions, always simply returning values. As such, the function is
210 designed to allow easy wrapping of code that doesn't use retry at
211 all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
212 ...)".
213
214 @see: L{Retry}
215
216 """
217 rdict = {}
218
219 def helper(*innerargs):
220
221 result = rdict["result"] = fn(*innerargs)
222 if not ((callable(expected) and expected(result)) or result == expected):
223 raise RetryAgain()
224 return result
225
226 try:
227 result = Retry(helper, delay, timeout, args=args,
228 wait_fn=wait_fn, _time_fn=_time_fn)
229 except RetryTimeout:
230 assert "result" in rdict
231 result = rdict["result"]
232 return result
233