任务管理页面后端优化:提升进度更新频率和状态详细程度
1. 提高爬取阶段进度更新频率:从每10个改为每5个代理更新一次 2. 提高验证阶段进度更新频率:从每5个改为每验证1个代理就更新一次 3. 添加进度百分比计算所需字段:在progress消息中添加current和total字段 4. 增强状态信息详细程度: - 添加connecting状态:正在连接插件源 - 添加starting状态:正在启动爬虫 - 添加crawling_start状态:开始爬取代理 - 添加validating_start状态:开始验证代理 - 在进度消息中添加message字段,显示更详细的进度描述 这些改进可以让前端显示更实时、更详细的任务进度和状态信息
This commit is contained in:
@@ -22,6 +22,7 @@ class TasksManager:
|
||||
'current_url': None,
|
||||
'plugins': []
|
||||
}
|
||||
self.estimated_total = 1000
|
||||
|
||||
def set_callbacks(self, progress_callback: Optional[Callable] = None, status_callback: Optional[Callable] = None):
|
||||
self.progress_callback = progress_callback
|
||||
@@ -34,6 +35,10 @@ class TasksManager:
|
||||
if 'found' in data and 'verified' in data:
|
||||
data['success_rate'] = round((data['verified'] / data['found'] * 100), 2) if data['found'] > 0 else 0
|
||||
|
||||
if 'found' in data:
|
||||
data['current'] = data['found'] + self.stats['total_verified']
|
||||
data['total'] = self.estimated_total
|
||||
|
||||
await self.progress_callback(data)
|
||||
|
||||
async def _notify_status(self, status: str, message: str):
|
||||
@@ -45,7 +50,7 @@ class TasksManager:
|
||||
})
|
||||
|
||||
async def run_crawler(self):
|
||||
await self._notify_status('crawling', '开始爬取代理啦~')
|
||||
await self._notify_status('crawling_start', '开始爬取代理啦~')
|
||||
manager = PluginManager()
|
||||
|
||||
count = 0
|
||||
@@ -59,11 +64,13 @@ class TasksManager:
|
||||
count += 1
|
||||
self.stats['total_found'] = count
|
||||
|
||||
if count % 10 == 0:
|
||||
if count % 5 == 0:
|
||||
await self._notify_progress({
|
||||
'type': 'crawling',
|
||||
'found': count,
|
||||
'verified': self.stats['total_verified']
|
||||
'verified': self.stats['total_verified'],
|
||||
'current_proxy': f"{ip}:{port}",
|
||||
'message': f'正在爬取:已发现 {count} 个代理'
|
||||
})
|
||||
|
||||
if self.stop_requested:
|
||||
@@ -73,7 +80,7 @@ class TasksManager:
|
||||
logger.info(f"爬虫抓取阶段完成,共发现 {count} 个潜在代理。")
|
||||
|
||||
async def run_validator(self, db: SQLiteManager, validator: ProxyValidator):
|
||||
await self._notify_status('validating', '开始验证代理啦~')
|
||||
await self._notify_status('validating_start', '开始验证代理啦~')
|
||||
verified_count = 0
|
||||
|
||||
while True:
|
||||
@@ -91,13 +98,13 @@ class TasksManager:
|
||||
verified_count += 1
|
||||
self.stats['total_verified'] = verified_count
|
||||
|
||||
if verified_count % 5 == 0:
|
||||
await self._notify_progress({
|
||||
'type': 'validating',
|
||||
'found': self.stats['total_found'],
|
||||
'verified': verified_count,
|
||||
'current_proxy': f"{ip}:{port}"
|
||||
})
|
||||
await self._notify_progress({
|
||||
'type': 'validating',
|
||||
'found': self.stats['total_found'],
|
||||
'verified': verified_count,
|
||||
'current_proxy': f"{ip}:{port}",
|
||||
'message': f'正在验证:已验证 {verified_count} 个代理'
|
||||
})
|
||||
else:
|
||||
logger.info(f"验证失败: {ip}:{port} ({protocol})")
|
||||
except Exception as e:
|
||||
@@ -126,6 +133,8 @@ class TasksManager:
|
||||
'plugins': []
|
||||
}
|
||||
|
||||
await self._notify_status('connecting', '正在连接插件源...')
|
||||
await self._notify_status('starting', '正在启动爬虫...')
|
||||
await self._notify_status('running', '任务开始啦~')
|
||||
|
||||
async with ProxyValidator(max_concurrency=200) as validator:
|
||||
|
||||
Reference in New Issue
Block a user