From e1d9a63e3b92f6084dd55c41d34ba0f9f8da20b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A5=80=E6=A2=A6?= <3501646051@qq.com> Date: Tue, 27 Jan 2026 21:56:23 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=88=AC=E8=99=AB=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E5=99=A8session=E7=94=9F=E5=91=BD=E5=91=A8=E6=9C=9F?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E5=B0=86validator=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E7=AE=A1=E7=90=86=E5=99=A8=E7=A7=BB=E8=87=B3tasks=5Fm?= =?UTF-8?q?anager=E5=86=85=E9=83=A8=EF=BC=8C=E7=A1=AE=E4=BF=9Dsession?= =?UTF-8?q?=E5=9C=A8=E6=95=B4=E4=B8=AA=E9=AA=8C=E8=AF=81=E8=BF=87=E7=A8=8B?= =?UTF-8?q?=E4=B8=AD=E4=BF=9D=E6=8C=81=E6=89=93=E5=BC=80=E7=8A=B6=E6=80=81?= =?UTF-8?q?=EF=BC=9B=E5=90=8C=E6=97=B6=E4=BF=AE=E6=94=B9CORS=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=94=AF=E6=8C=81=E6=89=80=E6=9C=89=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E8=AE=BF=E9=97=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api_server.py | 3 +-- tasks_manager.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api_server.py b/api_server.py index 9883a6b..579fe27 100644 --- a/api_server.py +++ b/api_server.py @@ -393,8 +393,7 @@ async def start_crawler(request: CrawlerRequest, _permission: str = Depends(requ tasks_manager.set_callbacks(progress_callback, status_callback) db = SQLiteManager() - async with ProxyValidator(max_concurrency=200) as validator: - asyncio.create_task(tasks_manager.start_task(db, validator, request.num_validators)) + asyncio.create_task(tasks_manager.start_task(db, request.num_validators)) return {"code": 200, "message": "爬虫任务开始啦~", "data": None} except Exception as e: diff --git a/tasks_manager.py b/tasks_manager.py index 0929dcd..c4f7c34 100644 --- a/tasks_manager.py +++ b/tasks_manager.py @@ -111,7 +111,7 @@ class TasksManager: await self._notify_status('validating_done', f'验证完成啦,入库 {verified_count} 个代理~') logger.info(f"验证协程完成,入库 {verified_count} 个代理。") - async def start_task(self, db: SQLiteManager, validator: ProxyValidator, num_validators: int = 50): + async def start_task(self, db: SQLiteManager, num_validators: int = 50): if self.is_running: await self._notify_status('error', '任务正在运行中呢~') return False @@ -128,16 +128,17 @@ class TasksManager: await self._notify_status('running', '任务开始啦~') - crawler_task = asyncio.create_task(self.run_crawler()) - self.validator_tasks = [asyncio.create_task(self.run_validator(db, validator)) for _ in range(num_validators)] - - await crawler_task - - for _ in range(num_validators): - await self.proxy_queue.put(None) + async with ProxyValidator(max_concurrency=200) as validator: + crawler_task = asyncio.create_task(self.run_crawler()) + self.validator_tasks = [asyncio.create_task(self.run_validator(db, validator)) for _ in range(num_validators)] - await self.proxy_queue.join() - await asyncio.gather(*self.validator_tasks, return_exceptions=True) + await crawler_task + + for _ in range(num_validators): + await self.proxy_queue.put(None) + + await self.proxy_queue.join() + await asyncio.gather(*self.validator_tasks, return_exceptions=True) total = await db.count_proxies() self.is_running = False