Round 3 fixes: cancelled polling, aggregator invalid_count, filter state, scheduler atomicity, HTTP exception handler, tests
This commit is contained in:
@@ -74,21 +74,36 @@ class AsyncWorkerPool:
|
||||
await self._queue.join()
|
||||
|
||||
async def resize(self, new_worker_count: int) -> None:
|
||||
"""动态调整 Worker 数量:先全部停止,再按新数量启动"""
|
||||
"""动态调整 Worker 数量,不丢失队列中的任务"""
|
||||
if new_worker_count == self.worker_count:
|
||||
return
|
||||
logger.info(f"{self.name} resizing from {self.worker_count} to {new_worker_count}")
|
||||
# 安全做法:先 stop 再 start,避免新旧 Worker 竞争 sentinel 导致死锁
|
||||
await self.stop()
|
||||
if new_worker_count > self.worker_count:
|
||||
for i in range(self.worker_count, new_worker_count):
|
||||
self._workers.append(
|
||||
asyncio.create_task(self._worker_loop(i), name=f"{self.name}-worker-{i}")
|
||||
)
|
||||
elif new_worker_count < self.worker_count:
|
||||
for _ in range(self.worker_count - new_worker_count):
|
||||
await self._queue.put(None)
|
||||
await asyncio.sleep(0)
|
||||
still_running = []
|
||||
for w in self._workers:
|
||||
if w.done():
|
||||
try:
|
||||
await w
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
else:
|
||||
still_running.append(w)
|
||||
self._workers = still_running
|
||||
self.worker_count = new_worker_count
|
||||
await self.start()
|
||||
|
||||
async def _worker_loop(self, worker_id: int) -> None:
|
||||
while True:
|
||||
item = await self._queue.get()
|
||||
try:
|
||||
if item is None or not self._running:
|
||||
self._queue.task_done()
|
||||
break
|
||||
await self.handler(item)
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user