Source Code for Module ganeti.utils.retry

  1  # 
  2  # 
  3   
  4  # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, but 
 12  # WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
 14  # General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the Free Software 
 18  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
 19  # 02110-1301, USA. 
 20   
 21  """Utility functions for retrying function calls with a timeout. 
 22   
 23  """ 
 24   
 25   
 26  import time 
 27   
 28  from ganeti import errors 
 29   
 30   
 31  #: Special delay to specify whole remaining timeout 
 32  RETRY_REMAINING_TIME = object() 
 33   
 34   
 35 -class RetryTimeout(Exception): 
 36    """Retry loop timed out. 
 37   
 38    Any arguments which was passed by the retried function to RetryAgain will be 
 39    preserved in RetryTimeout, if it is raised. If such argument was an exception 
 40    the RaiseInner helper method will reraise it. 
 41   
 42    """ 
 43 -  def RaiseInner(self): 
 44      if self.args and isinstance(self.args[0], Exception): 
 45        raise self.args[0] 
 46      else: 
 47        raise RetryTimeout(*self.args) 
 48   
 49   
 50 -class RetryAgain(Exception): 
 51    """Retry again. 
 52   
 53    Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as 
 54    arguments to RetryTimeout. If an exception is passed, the RaiseInner() method 
 55    of the RetryTimeout() method can be used to reraise it. 
 56   
 57    """ 
 58   
 59   
 60 -class _RetryDelayCalculator(object): 
 61    """Calculator for increasing delays. 
 62   
 63    """ 
 64    __slots__ = [ 
 65      "_factor", 
 66      "_limit", 
 67      "_next", 
 68      "_start", 
 69      ] 
 70   
 71 -  def __init__(self, start, factor, limit): 
 72      """Initializes this class. 
 73   
 74      @type start: float 
 75      @param start: Initial delay 
 76      @type factor: float 
 77      @param factor: Factor for delay increase 
 78      @type limit: float or None 
 79      @param limit: Upper limit for delay or None for no limit 
 80   
 81      """ 
 82      assert start > 0.0 
 83      assert factor >= 1.0 
 84      assert limit is None or limit >= 0.0 
 85   
 86      self._start = start 
 87      self._factor = factor 
 88      self._limit = limit 
 89   
 90      self._next = start 
 91   
 92 -  def __call__(self): 
 93      """Returns current delay and calculates the next one. 
 94   
 95      """ 
 96      current = self._next 
 97   
 98      # Update for next run 
 99      if self._limit is None or self._next < self._limit: 
100        self._next = min(self._limit, self._next * self._factor) 
101   
102      return current 
103   
104   
105 -def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, 
106            _time_fn=time.time): 
107    """Call a function repeatedly until it succeeds. 
108   
109    The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain} 
110    anymore. Between calls a delay, specified by C{delay}, is inserted. After a 
111    total of C{timeout} seconds, this function throws L{RetryTimeout}. 
112   
113    C{delay} can be one of the following: 
114      - callable returning the delay length as a float 
115      - Tuple of (start, factor, limit) 
116      - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is 
117        useful when overriding L{wait_fn} to wait for an external event) 
118      - A static delay as a number (int or float) 
119   
120    @type fn: callable 
121    @param fn: Function to be called 
122    @param delay: Either a callable (returning the delay), a tuple of (start, 
123                  factor, limit) (see L{_RetryDelayCalculator}), 
124                  L{RETRY_REMAINING_TIME} or a number (int or float) 
125    @type timeout: float 
126    @param timeout: Total timeout 
127    @type wait_fn: callable 
128    @param wait_fn: Waiting function 
129    @return: Return value of function 
130   
131    """ 
132    assert callable(fn) 
133    assert callable(wait_fn) 
134    assert callable(_time_fn) 
135   
136    if args is None: 
137      args = [] 
138   
139    end_time = _time_fn() + timeout 
140   
141    if callable(delay): 
142      # External function to calculate delay 
143      calc_delay = delay 
144   
145    elif isinstance(delay, (tuple, list)): 
146      # Increasing delay with optional upper boundary 
147      (start, factor, limit) = delay 
148      calc_delay = _RetryDelayCalculator(start, factor, limit) 
149   
150    elif delay is RETRY_REMAINING_TIME: 
151      # Always use the remaining time 
152      calc_delay = None 
153   
154    else: 
155      # Static delay 
156      calc_delay = lambda: delay 
157   
158    assert calc_delay is None or callable(calc_delay) 
159   
160    while True: 
161      retry_args = [] 
162      try: 
163        # pylint: disable=W0142 
164        return fn(*args) 
165      except RetryAgain, err: 
166        retry_args = err.args 
167      except RetryTimeout: 
168        raise errors.ProgrammerError("Nested retry loop detected that didn't" 
169                                     " handle RetryTimeout") 
170   
171      remaining_time = end_time - _time_fn() 
172   
173      if remaining_time < 0.0: 
174        # pylint: disable=W0142 
175        raise RetryTimeout(*retry_args) 
176   
177      assert remaining_time >= 0.0 
178   
179      if calc_delay is None: 
180        wait_fn(remaining_time) 
181      else: 
182        current_delay = calc_delay() 
183        if current_delay > 0.0: 
184          wait_fn(current_delay) 
185   
186   
187 -def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, 
188                  _time_fn=time.time): 
189    """A wrapper over L{Retry} implementing a simpler interface. 
190   
191    All the parameters are the same as for L{Retry}, except it has one 
192    extra argument: expected, which can be either a value (will be 
193    compared with the result of the function, or a callable (which will 
194    get the result passed and has to return a boolean). If the test is 
195    false, we will retry until either the timeout has passed or the 
196    tests succeeds. In both cases, the last result from calling the 
197    function will be returned. 
198   
199    Note that this function is not expected to raise any retry-related 
200    exceptions, always simply returning values. As such, the function is 
201    designed to allow easy wrapping of code that doesn't use retry at 
202    all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn, 
203    ...)". 
204   
205    @see: L{Retry} 
206   
207    """ 
208    rdict = {} 
209   
210    def helper(*innerargs): 
211      # pylint: disable=W0142 
212      result = rdict["result"] = fn(*innerargs) 
213      if not ((callable(expected) and expected(result)) or result == expected): 
214        raise RetryAgain() 
215      return result 
216   
217    try: 
218      result = Retry(helper, delay, timeout, args=args, 
219                     wait_fn=wait_fn, _time_fn=_time_fn) 
220    except RetryTimeout: 
221      assert "result" in rdict 
222      result = rdict["result"] 
223    return result 
224