Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions xtuner/v1/rl/utils/misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,19 @@ def get_eos_token(model_path: str) -> int | List[int]:
return eos_token_id


def _post_proxy_admin(url: str, payload: dict, headers: dict, *, read_timeout: float):
# Retry only connect timeouts: no request reached the admin server yet.
# Read timeouts have an ambiguous mutation outcome and must propagate.
for attempt in range(5):
try:
return requests.post(url, json=payload, headers=headers, timeout=(5, read_timeout))
except requests.exceptions.ConnectTimeout:
if attempt == 4:
raise
logger.warning(f"Proxy admin connect timeout; retry {attempt + 1}/4: {url}")
time.sleep(1)


def register_to_routedapiproxy(admin_url: str, model_name: str, api_server_url: str) -> dict:
url = f"{admin_url.rstrip('/')}/v1/models/new"
payload = {
Expand All @@ -256,7 +269,7 @@ def register_to_routedapiproxy(admin_url: str, model_name: str, api_server_url:
"accept": "application/json",
"Content-Type": "application/json",
}
resp = requests.post(url, json=payload, headers=headers, timeout=180)
resp = _post_proxy_admin(url, payload, headers, read_timeout=180)
resp.raise_for_status()
return resp.json()

Expand All @@ -272,7 +285,7 @@ def delete_from_routedapiproxy(admin_url: str, model_name: str, api_server_url:
"accept": "application/json",
"Content-Type": "application/json",
}
resp = requests.post(url, json=payload, headers=headers, timeout=30)
resp = _post_proxy_admin(url, payload, headers, read_timeout=30)
resp.raise_for_status()


Expand Down
Loading