任务管理页面后端优化:提升进度更新频率和状态详细程度

1. 提高爬取阶段进度更新频率:从每10个改为每5个代理更新一次
2. 提高验证阶段进度更新频率:从每5个改为每验证1个代理就更新一次
3. 添加进度百分比计算所需字段:在progress消息中添加current和total字段
4. 增强状态信息详细程度:
   - 添加connecting状态:正在连接插件源
   - 添加starting状态:正在启动爬虫
   - 添加crawling_start状态:开始爬取代理
   - 添加validating_start状态:开始验证代理
   - 在进度消息中添加message字段,显示更详细的进度描述

这些改进可以让前端显示更实时、更详细的任务进度和状态信息
This commit is contained in:
祀梦
2026-01-27 23:15:43 +08:00
parent 466c77b28d
commit b5932a95b2
6 changed files with 844 additions and 51 deletions

View File

@@ -22,6 +22,7 @@ class TasksManager:
'current_url': None,
'plugins': []
}
self.estimated_total = 1000
def set_callbacks(self, progress_callback: Optional[Callable] = None, status_callback: Optional[Callable] = None):
self.progress_callback = progress_callback
@@ -34,6 +35,10 @@ class TasksManager:
if 'found' in data and 'verified' in data:
data['success_rate'] = round((data['verified'] / data['found'] * 100), 2) if data['found'] > 0 else 0
if 'found' in data:
data['current'] = data['found'] + self.stats['total_verified']
data['total'] = self.estimated_total
await self.progress_callback(data)
async def _notify_status(self, status: str, message: str):
@@ -45,7 +50,7 @@ class TasksManager:
})
async def run_crawler(self):
await self._notify_status('crawling', '开始爬取代理啦~')
await self._notify_status('crawling_start', '开始爬取代理啦~')
manager = PluginManager()
count = 0
@@ -59,11 +64,13 @@ class TasksManager:
count += 1
self.stats['total_found'] = count
if count % 10 == 0:
if count % 5 == 0:
await self._notify_progress({
'type': 'crawling',
'found': count,
'verified': self.stats['total_verified']
'verified': self.stats['total_verified'],
'current_proxy': f"{ip}:{port}",
'message': f'正在爬取:已发现 {count} 个代理'
})
if self.stop_requested:
@@ -73,7 +80,7 @@ class TasksManager:
logger.info(f"爬虫抓取阶段完成,共发现 {count} 个潜在代理。")
async def run_validator(self, db: SQLiteManager, validator: ProxyValidator):
await self._notify_status('validating', '开始验证代理啦~')
await self._notify_status('validating_start', '开始验证代理啦~')
verified_count = 0
while True:
@@ -91,13 +98,13 @@ class TasksManager:
verified_count += 1
self.stats['total_verified'] = verified_count
if verified_count % 5 == 0:
await self._notify_progress({
'type': 'validating',
'found': self.stats['total_found'],
'verified': verified_count,
'current_proxy': f"{ip}:{port}"
})
await self._notify_progress({
'type': 'validating',
'found': self.stats['total_found'],
'verified': verified_count,
'current_proxy': f"{ip}:{port}",
'message': f'正在验证:已验证 {verified_count} 个代理'
})
else:
logger.info(f"验证失败: {ip}:{port} ({protocol})")
except Exception as e:
@@ -126,6 +133,8 @@ class TasksManager:
'plugins': []
}
await self._notify_status('connecting', '正在连接插件源...')
await self._notify_status('starting', '正在启动爬虫...')
await self._notify_status('running', '任务开始啦~')
async with ProxyValidator(max_concurrency=200) as validator: