OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """A utility to run functions with timeouts and retries.""" | 5 """A utility to run functions with timeouts and retries.""" |
6 # pylint: disable=W0702 | 6 # pylint: disable=W0702 |
7 | 7 |
8 import logging | 8 import logging |
9 import threading | 9 import threading |
10 import time | 10 import time |
(...skipping 10 matching lines...) Expand all Loading... |
21 super(TimeoutRetryThreadGroup, self).__init__(threads) | 21 super(TimeoutRetryThreadGroup, self).__init__(threads) |
22 self._watcher = watchdog_timer.WatchdogTimer(timeout) | 22 self._watcher = watchdog_timer.WatchdogTimer(timeout) |
23 | 23 |
24 def GetWatcher(self): | 24 def GetWatcher(self): |
25 """Returns the watchdog keeping track of this thread's time.""" | 25 """Returns the watchdog keeping track of this thread's time.""" |
26 return self._watcher | 26 return self._watcher |
27 | 27 |
28 def GetElapsedTime(self): | 28 def GetElapsedTime(self): |
29 return self._watcher.GetElapsed() | 29 return self._watcher.GetElapsed() |
30 | 30 |
31 def GetRemainingTime(self, required=0, msg=None): | 31 def GetRemainingTime(self, required=0, suffix=None): |
32 """Get the remaining time before the thread times out. | 32 """Get the remaining time before the thread times out. |
33 | 33 |
34 Useful to send as the |timeout| parameter of async IO operations. | 34 Useful to send as the |timeout| parameter of async IO operations. |
35 | 35 |
36 Args: | 36 Args: |
37 required: minimum amount of time that will be required to complete, e.g., | 37 required: minimum amount of time that will be required to complete, e.g., |
38 some sleep or IO operation. | 38 some sleep or IO operation. |
39 msg: error message to show if timing out. | 39 msg: error message to show if timing out. |
40 | 40 |
41 Returns: | 41 Returns: |
42 The number of seconds remaining before the thread times out, or None | 42 The number of seconds remaining before the thread times out, or None |
43 if the thread never times out. | 43 if the thread never times out. |
44 | 44 |
45 Raises: | 45 Raises: |
46 reraiser_thread.TimeoutError if the remaining time is less than the | 46 reraiser_thread.TimeoutError if the remaining time is less than the |
47 required time. | 47 required time. |
48 """ | 48 """ |
49 remaining = self._watcher.GetRemaining() | 49 remaining = self._watcher.GetRemaining() |
50 if remaining is not None and remaining < required: | 50 if remaining is not None and remaining < required: |
51 if msg is None: | 51 msg = 'Timeout of %.1f secs expired' % self._watcher.GetTimeout() |
52 msg = 'Timeout expired' | 52 if suffix: |
53 if remaining > 0: | 53 msg += suffix |
54 msg += (', wait of %.1f secs required but only %.1f secs left' | |
55 % (required, remaining)) | |
56 raise reraiser_thread.TimeoutError(msg) | 54 raise reraiser_thread.TimeoutError(msg) |
57 return remaining | 55 return remaining |
58 | 56 |
59 | 57 |
60 def CurrentTimeoutThreadGroup(): | 58 def CurrentTimeoutThreadGroup(): |
61 """Returns the thread group that owns or is blocked on the active thread. | 59 """Returns the thread group that owns or is blocked on the active thread. |
62 | 60 |
63 Returns: | 61 Returns: |
64 Returns None if no TimeoutRetryThreadGroup is tracking the current thread. | 62 Returns None if no TimeoutRetryThreadGroup is tracking the current thread. |
65 """ | 63 """ |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
103 msg = ['condition', repr(condition_name), 'met' if result else 'not met'] | 101 msg = ['condition', repr(condition_name), 'met' if result else 'not met'] |
104 if timeout_thread_group: | 102 if timeout_thread_group: |
105 # pylint: disable=no-member | 103 # pylint: disable=no-member |
106 msg.append('(%.1fs)' % timeout_thread_group.GetElapsedTime()) | 104 msg.append('(%.1fs)' % timeout_thread_group.GetElapsedTime()) |
107 logger.info(' '.join(msg)) | 105 logger.info(' '.join(msg)) |
108 if result: | 106 if result: |
109 return result | 107 return result |
110 if timeout_thread_group: | 108 if timeout_thread_group: |
111 # pylint: disable=no-member | 109 # pylint: disable=no-member |
112 timeout_thread_group.GetRemainingTime(wait_period, | 110 timeout_thread_group.GetRemainingTime(wait_period, |
113 msg='Timed out waiting for %r' % condition_name) | 111 suffix=' waiting for condition %r' % condition_name) |
114 time.sleep(wait_period) | 112 time.sleep(wait_period) |
115 return None | 113 return None |
116 | 114 |
117 | 115 |
118 def AlwaysRetry(_exception): | 116 def AlwaysRetry(_exception): |
119 return True | 117 return True |
120 | 118 |
121 | 119 |
122 def Run(func, timeout, retries, args=None, kwargs=None, desc=None, | 120 def Run(func, timeout, retries, args=None, kwargs=None, desc=None, |
123 error_log_func=logging.critical, retry_if_func=AlwaysRetry): | 121 error_log_func=logging.critical, retry_if_func=AlwaysRetry): |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
166 if num_try > retries or not retry_if_func(e): | 164 if num_try > retries or not retry_if_func(e): |
167 raise | 165 raise |
168 # Do not catch KeyboardInterrupt. | 166 # Do not catch KeyboardInterrupt. |
169 except Exception as e: # pylint: disable=broad-except | 167 except Exception as e: # pylint: disable=broad-except |
170 if num_try > retries or not retry_if_func(e): | 168 if num_try > retries or not retry_if_func(e): |
171 raise | 169 raise |
172 error_log_func( | 170 error_log_func( |
173 '(%s) Exception on %s, attempt %d of %d: %r', | 171 '(%s) Exception on %s, attempt %d of %d: %r', |
174 thread_name, desc, num_try, retries + 1, e) | 172 thread_name, desc, num_try, retries + 1, e) |
175 num_try += 1 | 173 num_try += 1 |
OLD | NEW |