refactor(backend): optimize database safety, validator performance, and scheduler concurrency
- Fix SQL injection risks in proxy_repo and task_repo - Atomic acquire_pending with UPDATE ... RETURNING - Reuse aiohttp ClientSession in ValidatorService - Replace polling with asyncio.Event in SchedulerService - Optimize ValidationQueue.drain with asyncio.Condition - Concurrent plugin crawling with asyncio.gather - Unify ProxyRaw model import path - Fix test baseline and remove tracked __pycache__ files
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""插件业务服务"""
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from app.core.db import get_db
|
||||
@@ -108,14 +109,13 @@ class PluginService:
|
||||
async def run_all_plugins(self) -> List[ProxyRaw]:
|
||||
"""执行所有启用插件的爬取"""
|
||||
all_results: List[ProxyRaw] = []
|
||||
for plugin in registry.list_plugins():
|
||||
if not plugin.enabled:
|
||||
tasks = [self.run_plugin(plugin.name) for plugin in registry.list_plugins() if plugin.enabled]
|
||||
results_list = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for results in results_list:
|
||||
if isinstance(results, Exception):
|
||||
logger.error(f"Run all plugins error: {results}")
|
||||
continue
|
||||
try:
|
||||
results = await self.run_plugin(plugin.name)
|
||||
all_results.extend(results)
|
||||
except Exception as e:
|
||||
logger.error(f"Run all plugins error at {plugin.name}: {e}")
|
||||
all_results.extend(results)
|
||||
# 去重
|
||||
seen = set()
|
||||
unique = []
|
||||
|
||||
Reference in New Issue
Block a user