Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[codespell]
ignore-words-list = nd, ba, warmup, ans, dout, ptd
# "te" is the HTTP hop-by-hop header, not a typo.
ignore-words-list = nd, ba, warmup, ans, dout, ptd, te

[flake8]
max-line-length = 119
Expand Down
138 changes: 138 additions & 0 deletions tests/rl/test_affinity_router.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
"""SessionAffinityRouter 的真实 HTTP 路由测试。

Good Tests:
- 启动真实 aiohttp 后端和真实 router 线程,只通过 HTTP 观察路由行为。
- 用上游请求计数验证固定绑定、失败关闭和不重放。

Bad Tests:
- 不读取 bindings/retired 等内部计数。
- 不 mock router 的 HTTP 处理路径。
"""

import json
import unittest

from aiohttp import web
from aiohttp.test_utils import TestServer

from xtuner.v1.rl.rollout.affinity_router import SessionAffinityRouter


class _Backend:
def __init__(self, name: str):
self.name = name
self.calls: list[dict] = []
self.status = 200

async def handle(self, request: web.Request) -> web.Response:
body = await request.read()
self.calls.append({"method": request.method, "path": request.path, "body": body})
if self.status != 200:
return web.json_response({"error": {"code": "backend_error"}}, status=self.status)
return web.json_response({"backend": self.name, "path": request.path})


class TestSessionAffinityRouter(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
self.backends = [_Backend("a"), _Backend("b")]
self.servers: list[TestServer] = []
for backend in self.backends:
app = web.Application()
app.router.add_route("*", "/{path:.*}", backend.handle)
server = TestServer(app)
await server.start_server()
self.servers.append(server)
self.urls = [str(server.make_url("")).rstrip("/") for server in self.servers]
self.router = SessionAffinityRouter("127.0.0.1")
self.router.replace_backends(self.urls)

async def asyncTearDown(self):
self.router.close()
for server in self.servers:
await server.close()

async def _post(self, session_id: str | None, *, path: str = "/v1/chat/completions"):
import aiohttp

payload = {"model": "m", "messages": [{"role": "user", "content": "hi"}]}
if session_id is not None:
payload["session_id"] = session_id
async with aiohttp.ClientSession() as client:
async with client.post(f"{self.router.url}{path}", json=payload) as response:
return response.status, await response.json()

async def test_session_stays_on_one_backend_across_requests(self):
_, first = await self._post("session-0")
_, second = await self._post("session-0")
_, other = await self._post("session-1")

self.assertEqual(first["backend"], second["backend"])
self.assertEqual(len(self.backends[0].calls) + len(self.backends[1].calls), 3)
self.assertIn(other["backend"], {"a", "b"})

async def test_replacing_backends_keeps_existing_bindings(self):
_, first = await self._post("session-0")
bound = self.backends[0] if first["backend"] == "a" else self.backends[1]

self.router.replace_backends(list(reversed(self.urls)))
_, second = await self._post("session-0")

self.assertEqual(second["backend"], first["backend"])
self.assertEqual(len(bound.calls), 2)

async def test_dead_bound_backend_fails_closed_without_rebinding(self):
_, first = await self._post("session-0")
bound_index = 0 if first["backend"] == "a" else 1
other_backend = self.backends[1 - bound_index]

self.router.set_backend(self.urls[bound_index], False)
status, payload = await self._post("session-0")

self.assertEqual(status, 503)
self.assertEqual(payload["error"]["code"], "session_backend_unavailable")
self.assertEqual(other_backend.calls, [])

async def test_retired_session_gets_410_and_never_reaches_backend(self):
await self._post("session-0")
self.router.retire_sessions(["session-0"])
status, payload = await self._post("session-0")

self.assertEqual(status, 410)
self.assertEqual(payload["error"]["code"], "session_retired")
self.assertEqual(sum(len(backend.calls) for backend in self.backends), 1)

async def test_missing_session_id_is_rejected(self):
status, payload = await self._post(None)

self.assertEqual(status, 400)
self.assertEqual(payload["error"]["code"], "valid_session_id_required")

async def test_upstream_error_is_forwarded_without_replay(self):
self.backends[0].status = 500
self.backends[1].status = 500
status, _ = await self._post("session-0")
await self._post("session-0")

self.assertEqual(status, 500)
self.assertEqual(sum(len(backend.calls) for backend in self.backends), 2)

async def test_request_body_is_forwarded_unchanged(self):
await self._post("session-0")
backend = self.backends[0] if self.backends[0].calls else self.backends[1]
forwarded = json.loads(backend.calls[0]["body"])
self.assertEqual(forwarded["session_id"], "session-0")
self.assertEqual(forwarded["messages"], [{"role": "user", "content": "hi"}])

async def test_model_listing_is_forwarded_without_a_session(self):
import aiohttp

async with aiohttp.ClientSession() as client:
async with client.get(f"{self.router.url}/v1/models") as response:
self.assertEqual(response.status, 200)
payload = await response.json()
self.assertIn(payload["backend"], {"a", "b"})
self.assertEqual(payload["path"], "/v1/models")


if __name__ == "__main__":
unittest.main()
138 changes: 135 additions & 3 deletions tests/rl/test_agent_loop_manager_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,16 @@
import asyncio
import inspect
import json
import logging
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

from tokenizers import Tokenizer, models, pre_tokenizers

from transformers import PreTrainedTokenizerFast, Qwen3Config
from xtuner.v1.data_proto.rl_data import RolloutState, SampleParams, Status
from xtuner.v1.datasets.config import DataloaderConfig, DatasetConfig
from xtuner.v1.datasets.rl_tokenize_fn import RLTextTokenizeFnConfig
Expand All @@ -41,7 +45,9 @@
SamplerConfig,
SyncProduceStrategyConfig,
)
from xtuner.v1.rl.agent_loop_manager.produce_utils import manager_state_path
from xtuner.v1.rl.replay_buffer import AsyncReplayBufferConfig, SyncReplayBufferConfig
from xtuner.v1.rl.rollout.cross_weight_update_rollout import CrossWeightUpdateRolloutConfig


QWEN3_4B_PATH = os.environ.get("QWEN3_4B_PATH")
Expand Down Expand Up @@ -83,6 +89,7 @@ def __init__(self):
self.generate = _RemoteMethod(self._generate)
self.pause_generation = _RemoteMethod()
self.continue_generation = _RemoteMethod()
self.assert_request_gates_healthy = _RemoteMethod()
self.set_enable_partial_rollout = _SyncRemoteMethod()

def _generate(self, rollout_state: RolloutState) -> RolloutState:
Expand Down Expand Up @@ -135,8 +142,12 @@ def _build_manager(
rollout_controller=None,
produce_strategy_config=None,
mode="colocate",
hf_checkpoint=None,
cross_weight_update_rollout=None,
logger=None,
):
assert QWEN3_4B_PATH is not None
hf_checkpoint = hf_checkpoint or QWEN3_4B_PATH
assert hf_checkpoint is not None
rollout_controller = rollout_controller or _FakeRolloutController()
produce_strategy_config = produce_strategy_config or (
DisaggAsyncProduceStrategyConfig() if mode == "disaggregated" else SyncProduceStrategyConfig()
Expand Down Expand Up @@ -166,7 +177,7 @@ def _build_manager(
{
"task_name": "unit_task",
"agent_loop_config": SingleTurnAgentLoopConfig(
hf_checkpoint=QWEN3_4B_PATH,
hf_checkpoint=hf_checkpoint,
sample_params=SampleParams(max_tokens=2, temperature=0.0, top_k=1),
),
"produce_strategy_config": produce_strategy_config,
Expand All @@ -176,9 +187,108 @@ def _build_manager(
)
return manager_cfg.build(
rollout_controller=rollout_controller,
tokenizer=QWEN3_4B_PATH,
tokenizer=hf_checkpoint,
replay_buffer=replay_buffer_config.build(),
cross_weight_update_rollout=cross_weight_update_rollout,
logger=logger,
)

async def test_cross_weight_update_checkpoint_retains_live_groups_only_in_the_running_process(self):
# 自带最小 HF 资产,使用真实 AgentLoop/Sampler/Dataloader/Strategy/Buffer,无需模型权重或 GPU。
tokenizer_path = Path(self.temp_dir.name) / "tokenizer"
backend = Tokenizer(models.WordLevel({"[UNK]": 0, "[PAD]": 1, "[EOS]": 2, "question": 3}, unk_token="[UNK]"))
backend.pre_tokenizer = pre_tokenizers.Whitespace()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=backend,
unk_token="[UNK]",
pad_token="[PAD]",
eos_token="[EOS]",
chat_template="{% for message in messages %}{{ message['content'] }} {% endfor %}{{ eos_token }}",
)
tokenizer.save_pretrained(tokenizer_path)
Qwen3Config(
vocab_size=4,
hidden_size=8,
intermediate_size=16,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=2,
).save_pretrained(tokenizer_path)

for mode in ("colocate", "disaggregated"):
with self.subTest(mode=mode):
controller = _BlockingRolloutController()

async def generate(state):
# group_id 表达采样顺序;真实 dataloader 会 shuffle,JSON 行号不代表产出顺序。
if state.group_id >= 1:
controller.started.set()
await controller.release.wait()
return controller._generate(state)

controller.generate = _RemoteMethod(generate)
logger = logging.getLogger(f"{__name__}.{mode}")

def build(rollout_controller):
strategy_cls = (
AsyncProduceStrategyConfig if mode == "colocate" else DisaggAsyncProduceStrategyConfig
)
return self._build_manager(
AsyncReplayBufferConfig(),
rollout_controller=rollout_controller,
produce_strategy_config=strategy_cls(over_sample_threshold=1.0, max_staleness=10),
mode=mode,
hf_checkpoint=str(tokenizer_path),
cross_weight_update_rollout=CrossWeightUpdateRolloutConfig(),
logger=logger,
)

with patch("ray.get", side_effect=lambda ref, *_, **__: ref):
manager = build(controller)
restored = build(_FakeRolloutController())
producer = restored_producer = None
try:
if mode == "colocate":
first = await asyncio.wait_for(manager.produce_batch(1, 1, model_step=0), timeout=5)
else:
producer = asyncio.create_task(manager.produce_loop(1))
first = await asyncio.wait_for(manager.get_batch(1, 1), timeout=5)
await asyncio.wait_for(manager.pause_produce(), timeout=5)
self.assertEqual(first.rollout_states[0][0].group_id, 0)
await asyncio.wait_for(controller.started.wait(), timeout=5)
checkpoint = Path(self.temp_dir.name) / mode
with self.assertLogs(logger, level="WARNING") as logs:
await asyncio.wait_for(manager.save(checkpoint, model_step=1), timeout=5)
self.assertIn("Live Agent/tool state is not saved", "\n".join(logs.output))
self.assertFalse(controller.release.is_set())

# 冷恢复从已保存的游标继续,不能重建正在执行的组 1。
with self.assertLogs(logger, level="WARNING"):
self.assertEqual(await restored.resume(checkpoint), 1)
if mode == "colocate":
cold = await asyncio.wait_for(restored.produce_batch(1, 2, model_step=1), timeout=5)
else:
restored_producer = asyncio.create_task(restored.produce_loop(1))
await restored.continue_produce(model_step=1)
cold = await asyncio.wait_for(restored.get_batch(1, 2), timeout=5)
self.assertGreater(cold.rollout_states[0][0].group_id, 1)

# 保存动作不取消同进程中的组 1;恢复生产后它仍能正常交付。
controller.release.set()
if mode == "colocate":
live = await asyncio.wait_for(manager.produce_batch(1, 2, model_step=1), timeout=5)
else:
await manager.continue_produce(model_step=1)
live = await asyncio.wait_for(manager.get_batch(1, 2), timeout=5)
self.assertEqual(live.rollout_states[0][0].group_id, 1)
finally:
controller.release.set()
for current, task in ((manager, producer), (restored, restored_producer)):
if task is not None:
current.shutdown()
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await current.close()

def _build_async_manager(self, *, rollout_controller=None):
with patch("ray.get", side_effect=lambda ref, *_, **__: ref):
Expand Down Expand Up @@ -397,6 +507,28 @@ async def test_resume_requires_continue_before_async_producer_generates(self):
self.assertEqual(len(restored_rollout_controller.continue_generation.calls), 1)
self.assertGreaterEqual(len(restored_rollout_controller.generate.calls), 1)

@unittest.skipUnless(QWEN3_4B_PATH, "QWEN3_4B_PATH is required for AgentLoopManager checkpoint tests")
async def test_resume_rejects_checkpoint_without_complete_group_version(self):
# 两种 manager 都在恢复 sampler/buffer 前拒绝缺少格式标识的旧 checkpoint。
for mode, produce_strategy_config in (
("colocate", SyncProduceStrategyConfig()),
("disaggregated", DisaggAsyncProduceStrategyConfig()),
):
with tempfile.TemporaryDirectory() as tmp_dir:
checkpoint_path = Path(tmp_dir) / "ckpt"
checkpoint_path.mkdir()
manager_state_path(checkpoint_path).write_text(
json.dumps({"model_step": 1, "replay_buffer_saved": True}), encoding="utf-8"
)
with patch("ray.get", side_effect=lambda ref, *_, **__: ref):
manager = self._build_manager(
SyncReplayBufferConfig(),
produce_strategy_config=produce_strategy_config,
mode=mode,
)
with self.assertRaisesRegex(ValueError, "complete-group checkpoint version 1"):
await manager.resume(checkpoint_path)


if __name__ == "__main__":
unittest.main()
Loading
Loading