Skip to content

Commit eec4c98

Browse files
authored
Merge pull request #549 from gaupon/app-token-refresh-on-retry
Resolve the app token again when retrying after a rate-limit wait
2 parents ceb50ee + e14305e commit eec4c98

2 files changed

Lines changed: 128 additions & 14 deletions

File tree

‎github_backup/github_backup.py‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -901,23 +901,31 @@ def _extract_legal_url(response_body_bytes):
901901

902902
next_url = None
903903

904+
def build_request(auth):
905+
return _construct_request(
906+
per_page=per_page if paginated else None,
907+
query_args=query_args,
908+
template=next_url or template,
909+
auth=auth,
910+
as_app=args.as_app,
911+
fine=args.token_fine is not None,
912+
)
913+
904914
while True:
905915
# FIRST: Fetch response
906916

907917
for attempt in range(args.max_retries + 1):
908-
# Resolved per request so long runs pick up refreshed app tokens
918+
# Resolved per request (and again before each retry) so long
919+
# runs pick up refreshed app tokens
909920
auth = get_auth(args, encode=not args.as_app)
910-
request = _construct_request(
911-
per_page=per_page if paginated else None,
912-
query_args=query_args,
913-
template=next_url or template,
914-
auth=auth,
915-
as_app=args.as_app,
916-
fine=args.token_fine is not None,
917-
)
918921
try:
919922
http_response = make_request_with_retry(
920-
request, auth, args.max_retries
923+
build_request(auth),
924+
auth,
925+
args.max_retries,
926+
rebuild_request=lambda: build_request(
927+
get_auth(args, encode=not args.as_app)
928+
),
921929
)
922930
except HTTPError as exc:
923931
if exc.code == 451:
@@ -1028,17 +1036,29 @@ def retrieve_graphql_data(args, query, variables=None, log_context=None):
10281036
).encode("utf-8")
10291037
endpoint = get_github_graphql_url(args)
10301038

1031-
for attempt in range(args.max_retries + 1):
1039+
def build_request(auth):
10321040
request = Request(endpoint, data=payload, method="POST")
10331041
request.add_header("Accept", "application/json")
10341042
request.add_header("Content-Type", "application/json")
10351043
request.add_header("Authorization", "bearer " + auth)
1044+
return request
1045+
1046+
for attempt in range(args.max_retries + 1):
1047+
if attempt:
1048+
# Resolved again after a retry wait so long runs pick up refreshed
1049+
# app tokens
1050+
auth = get_graphql_auth(args)
10361051
log_url = endpoint
10371052
if log_context:
10381053
log_url = "{0} ({1})".format(log_url, log_context)
10391054
logger.info("Requesting {0}".format(log_url))
10401055

1041-
http_response = make_request_with_retry(request, auth, args.max_retries)
1056+
http_response = make_request_with_retry(
1057+
build_request(auth),
1058+
auth,
1059+
args.max_retries,
1060+
rebuild_request=lambda: build_request(get_graphql_auth(args)),
1061+
)
10421062

10431063
status = http_response.getcode()
10441064
if status != 200:
@@ -1097,8 +1117,14 @@ def retrieve_graphql_data(args, query, variables=None, log_context=None):
10971117
) # pragma: no cover
10981118

10991119

1100-
def make_request_with_retry(request, auth, max_retries=5):
1101-
"""Make HTTP request with automatic retry for transient errors."""
1120+
def make_request_with_retry(request, auth, max_retries=5, rebuild_request=None):
1121+
"""Make HTTP request with automatic retry for transient errors.
1122+
1123+
If given, rebuild_request is called to build the request again before each
1124+
retry. A rate-limit wait can last longer than a GitHub App installation
1125+
token, so callers that resolve credentials per request use it to resend
1126+
with the token that is current after the wait.
1127+
"""
11021128

11031129
def is_retryable_status(status_code, headers):
11041130
# Server errors are always retryable
@@ -1110,6 +1136,8 @@ def is_retryable_status(status_code, headers):
11101136
return False
11111137

11121138
for attempt in range(max_retries + 1):
1139+
if attempt and rebuild_request is not None:
1140+
request = rebuild_request()
11131141
try:
11141142
return urlopen(request, context=https_ctx)
11151143

‎tests/test_app_auth.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,3 +337,89 @@ def mock_urlopen(request, *args, **kwargs):
337337

338338
assert results == [{"id": 1}, {"id": 2}]
339339
assert page_auth == [b"token ghs_1", b"token ghs_2"]
340+
341+
342+
def rate_limited_403():
343+
reset = int(time.time()) + 900
344+
return HTTPError(
345+
url="https://api.github.com/test",
346+
code=403,
347+
msg="rate limit exceeded",
348+
hdrs={"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(reset)},
349+
fp=None,
350+
)
351+
352+
353+
@pytest.fixture
354+
def token_rotates_during_wait():
355+
"""The installation token is replaced while a retry is waiting, as happens
356+
when the rate limit resets after the token has expired."""
357+
current = {"token": "ghs_1"}
358+
359+
def sleep(seconds):
360+
current["token"] = "ghs_2"
361+
362+
with patch(
363+
"github_backup.github_backup.get_app_installation_token",
364+
side_effect=lambda args: current["token"],
365+
), patch("github_backup.github_backup.time.sleep", side_effect=sleep):
366+
yield
367+
368+
369+
def test_rate_limit_retry_picks_up_refreshed_token(app_args, token_rotates_during_wait):
370+
"""Retrying after a rate-limit wait sends the token that is current after the
371+
wait, not the one the request was first built with."""
372+
sent_auth = []
373+
374+
def mock_urlopen(request, *args, **kwargs):
375+
sent_auth.append(request.get_header("Authorization"))
376+
if len(sent_auth) == 1:
377+
raise rate_limited_403()
378+
return MockHTTPResponse([{"id": 1}])
379+
380+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
381+
results = github_backup.retrieve_data(
382+
app_args, "https://api.github.com/orgs/testorg/repos"
383+
)
384+
385+
assert results == [{"id": 1}]
386+
assert sent_auth == [b"token ghs_1", b"token ghs_2"]
387+
388+
389+
def test_graphql_rate_limit_retry_picks_up_refreshed_token(
390+
app_args, token_rotates_during_wait
391+
):
392+
sent_auth = []
393+
394+
def mock_urlopen(request, *args, **kwargs):
395+
sent_auth.append(request.get_header("Authorization"))
396+
if len(sent_auth) == 1:
397+
raise rate_limited_403()
398+
return MockHTTPResponse({"data": {"ok": True}})
399+
400+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
401+
data = github_backup.retrieve_graphql_data(app_args, "query { ok }")
402+
403+
assert data == {"ok": True}
404+
assert sent_auth == ["bearer ghs_1", "bearer ghs_2"]
405+
406+
407+
def test_graphql_rate_limited_error_retry_picks_up_refreshed_token(
408+
app_args, token_rotates_during_wait
409+
):
410+
"""GraphQL can also report the rate limit as a RATE_LIMITED error in a 200."""
411+
sent_auth = []
412+
413+
def mock_urlopen(request, *args, **kwargs):
414+
sent_auth.append(request.get_header("Authorization"))
415+
if len(sent_auth) == 1:
416+
return MockHTTPResponse(
417+
{"errors": [{"type": "RATE_LIMITED", "message": "rate limited"}]}
418+
)
419+
return MockHTTPResponse({"data": {"ok": True}})
420+
421+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
422+
data = github_backup.retrieve_graphql_data(app_args, "query { ok }")
423+
424+
assert data == {"ok": True}
425+
assert sent_auth == ["bearer ghs_1", "bearer ghs_2"]

0 commit comments

Comments
 (0)