_sync.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. from __future__ import annotations
  2. import time
  3. from http import HTTPStatus
  4. from typing import TYPE_CHECKING, final
  5. from pyqwest import HTTPHeaderName, ReadError, SyncTransport
  6. from pyqwest._pyqwest import SyncRequest, SyncResponse, _Backoff
  7. from ._shared import (
  8. default_should_retry_request,
  9. default_should_retry_response,
  10. parse_retry_after,
  11. )
  12. if TYPE_CHECKING:
  13. from collections.abc import Callable, Iterator
  14. class SyncRetryTransport(SyncTransport):
  15. """Retry middleware for sync clients.
  16. Wrap a SyncTransport with this class to allow requests to be automatically retried.
  17. By default, known-safe errors are retried, meaning connection errors for any request,
  18. and I/O errors or 429/5xx responses for idempotent methods.
  19. The default behavior can be overridden by subclassing this class and overriding the
  20. `should_retry_request` and `should_retry_response` methods to suit any need.
  21. Examples:
  22. ```python
  23. from pyqwest import SyncClient, SyncHTTPTransport, SyncRequest
  24. from pyqwest.middleware.retry import SyncRetryTransport
  25. class MyRetryTransport(SyncRetryTransport):
  26. def should_retry_request(self, request: SyncRequest) -> bool:
  27. return not request.url.endswith("/unsafe-method")
  28. client = SyncClient(transport=MyRetryTransport(SyncHTTPTransport()))
  29. client.get("http://localhost/safe-method") # will retry on transient errors
  30. client.get("http://localhost/unsafe-method") # will not retry
  31. ```
  32. """
  33. _transport: SyncTransport
  34. _initial_interval: float
  35. _randomization_factor: float
  36. _multiplier: float
  37. _max_interval: float
  38. _max_retries: int
  39. def __init__(
  40. self,
  41. transport: SyncTransport,
  42. initial_interval: float = 0.5,
  43. randomization_factor: float = 0.5,
  44. multiplier: float = 1.5,
  45. max_interval: float = 60.0,
  46. max_retries: int = 4,
  47. ) -> None:
  48. self._transport = transport
  49. self._initial_interval = initial_interval
  50. self._randomization_factor = randomization_factor
  51. self._multiplier = multiplier
  52. self._max_interval = max_interval
  53. self._max_retries = max_retries
  54. @final
  55. def execute_sync(self, request: SyncRequest) -> SyncResponse:
  56. if not self.should_retry_request(request):
  57. return self._transport.execute_sync(request)
  58. backoff = _Backoff(
  59. self._initial_interval,
  60. self._randomization_factor,
  61. self._multiplier,
  62. self._max_interval,
  63. )
  64. get_content: Callable[[], bytes | Iterator[bytes]]
  65. content = request.content
  66. if isinstance(content, bytes):
  67. def _get_content() -> bytes:
  68. return content
  69. get_content = _get_content
  70. else:
  71. retrying_content = RetryingRequestContent(content)
  72. get_content = retrying_content.get
  73. resp: SyncResponse | Exception
  74. try:
  75. resp = self._transport.execute_sync(
  76. SyncRequest(
  77. method=request.method,
  78. url=request.url,
  79. headers=request.headers,
  80. content=get_content(),
  81. )
  82. )
  83. except Exception as e:
  84. resp = e
  85. retries = 0
  86. while True:
  87. if not self.should_retry_response(request, resp):
  88. break
  89. if isinstance(resp, SyncResponse):
  90. resp.close()
  91. retries += 1
  92. if retries > self._max_retries:
  93. if isinstance(resp, ConnectionError):
  94. # Connection errors that don't resolve with retries are better
  95. # surfaced as-is since they are network issues rather than backend.
  96. raise resp
  97. msg = f"Maximum retry attempts exceeded: {self._max_retries}"
  98. if isinstance(resp, Exception):
  99. raise ReadError(msg) from resp
  100. raise ReadError(msg)
  101. if (
  102. isinstance(resp, SyncResponse)
  103. and resp.status == HTTPStatus.TOO_MANY_REQUESTS
  104. and (
  105. wt := parse_retry_after(
  106. resp.headers.get(HTTPHeaderName.RETRY_AFTER)
  107. )
  108. )
  109. is not None
  110. ):
  111. wait_time = wt
  112. else:
  113. wait_time = backoff.next_backoff()
  114. if wait_time is None:
  115. break
  116. time.sleep(wait_time)
  117. try:
  118. resp = self._transport.execute_sync(
  119. SyncRequest(
  120. method=request.method,
  121. url=request.url,
  122. headers=request.headers,
  123. content=get_content(),
  124. )
  125. )
  126. except Exception as e:
  127. resp = e
  128. if isinstance(resp, Exception):
  129. raise resp
  130. return resp
  131. def should_retry_request(self, request: SyncRequest) -> bool:
  132. return default_should_retry_request(request.method)
  133. def should_retry_response(
  134. self, request: SyncRequest, response: SyncResponse | Exception
  135. ) -> bool:
  136. return default_should_retry_response(
  137. request.method,
  138. response.status if isinstance(response, SyncResponse) else response,
  139. )
  140. class RetryingRequestContent:
  141. def __init__(self, content: Iterator[bytes]) -> None:
  142. self._content = content
  143. self._buffer = bytearray()
  144. def get(self) -> Iterator[bytes]:
  145. if self._buffer:
  146. yield bytes(self._buffer)
  147. for chunk in self._content:
  148. self._buffer.extend(chunk)
  149. yield chunk