fix: 全面修复代码问题并优化架构
修复问题: - 添加缺失的 httpx 依赖到 requirements.txt - 修复前端批量删除参数格式与后端不匹配(数组->对象数组) - 移除 app/api/main.py 中重复创建 app 的冗余代码 - 修复 Plugins.vue v-model 直接修改 store 状态的 Vue 警告 - 修复 README 端口/启动命令文档与实际配置不一致 - 修正 pytest.ini 过时配置 (asyncio_default_fixture_loop_scope) - 修复 WebUI index.html 语言设置为 zh-CN - 修复 .gitignore 错误忽略 tests/ 目录 后端优化: - 修复调度器默认间隔从 5 秒改为 30 分钟,避免无节制验证 - 修复 validate_all_now 在调度器停止时无法执行的 bug - 设置保存后热更新运行中调度器的验证间隔 - 将 update_score 优化为原子单事务 SQL,消除并发竞态 - 导出功能改为真正的流式分批读取(iter_batches),降低大导出内存占用 - ProxyResponse Schema 补齐 response_time_ms 字段 - 日志级别改为从配置动态读取,不再硬编码 INFO - 清理 validator_service 中的冗余 try/finally 代码 插件健壮性: - 修复 ip3366/ip89/kuaidaili/proxylist_download/speedx/yundaili/proxyscrape 的端口范围检查和 IPv6 地址解析问题(改用 rsplit + 1-65535 校验) - 修复 PluginService.list_plugins 并发竞争条件 - 修复 run_all_plugins 去重逻辑与数据库 UNIQUE 约束保持一致 - 修复 proxyscrape 异常时错误跳过 fallback 的 bug 测试: - 新增 7 个插件解析单元测试 - 新增 update_score 自动删除和 iter_batches 流式读取测试 - 全部 74 个测试通过
This commit is contained in:
@@ -24,10 +24,9 @@ class PluginService:
|
||||
|
||||
result = []
|
||||
for plugin in registry.list_plugins():
|
||||
# 合并持久化状态
|
||||
# 合并持久化状态(不修改全局实例,避免并发竞争)
|
||||
state = db_states.get(plugin.name, {})
|
||||
if "enabled" in state:
|
||||
plugin.enabled = state["enabled"]
|
||||
enabled = state.get("enabled", plugin.enabled)
|
||||
if "config" in state and isinstance(state["config"], dict):
|
||||
plugin.update_config(state["config"])
|
||||
|
||||
@@ -50,7 +49,7 @@ class PluginService:
|
||||
name=plugin.name,
|
||||
display_name=plugin.display_name or plugin.name,
|
||||
description=plugin.description or f"从 {plugin.name} 爬取代理",
|
||||
enabled=plugin.enabled,
|
||||
enabled=enabled,
|
||||
last_run=stat.get("last_run"),
|
||||
success_count=stat.get("success_count", 0),
|
||||
failure_count=stat.get("failure_count", 0),
|
||||
@@ -133,11 +132,11 @@ class PluginService:
|
||||
logger.error(f"Run all plugins error: {results}")
|
||||
continue
|
||||
all_results.extend(results)
|
||||
# 去重
|
||||
# 去重(与数据库 UNIQUE(ip, port) 约束保持一致)
|
||||
seen = set()
|
||||
unique = []
|
||||
for p in all_results:
|
||||
key = (p.ip, p.port, p.protocol)
|
||||
key = (p.ip, p.port)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(p)
|
||||
|
||||
@@ -61,28 +61,41 @@ class ProxyService:
|
||||
protocol: Optional[str] = None,
|
||||
limit: int = 10000,
|
||||
) -> AsyncIterator[str]:
|
||||
async with get_db() as db:
|
||||
proxies = await self.proxy_repo.list_all(db, protocol=protocol, limit=limit)
|
||||
|
||||
if fmt == "csv":
|
||||
yield "IP,Port,Protocol,Score,Last Check\n"
|
||||
for p in proxies:
|
||||
yield f"{p.ip},{p.port},{p.protocol},{p.score},{self._fmt_time(p.last_check)}\n"
|
||||
yield "\ufeffIP,Port,Protocol,Score,Last Check\n"
|
||||
elif fmt == "txt":
|
||||
for p in proxies:
|
||||
yield f"{p.ip}:{p.port}\n"
|
||||
pass
|
||||
elif fmt == "json":
|
||||
data = [
|
||||
{
|
||||
"ip": p.ip,
|
||||
"port": p.port,
|
||||
"protocol": p.protocol,
|
||||
"score": p.score,
|
||||
"last_check": self._fmt_time(p.last_check),
|
||||
}
|
||||
for p in proxies
|
||||
]
|
||||
yield json.dumps(data, ensure_ascii=False, indent=2)
|
||||
yield "["
|
||||
first = True
|
||||
|
||||
exported = 0
|
||||
async with get_db() as db:
|
||||
async for batch in self.proxy_repo.iter_batches(db, protocol=protocol, batch_size=1000):
|
||||
for p in batch:
|
||||
if exported >= limit:
|
||||
break
|
||||
if fmt == "csv":
|
||||
yield f"{p.ip},{p.port},{p.protocol},{p.score},{self._fmt_time(p.last_check)}\n"
|
||||
elif fmt == "txt":
|
||||
yield f"{p.ip}:{p.port}\n"
|
||||
elif fmt == "json":
|
||||
item = {
|
||||
"ip": p.ip,
|
||||
"port": p.port,
|
||||
"protocol": p.protocol,
|
||||
"score": p.score,
|
||||
"last_check": self._fmt_time(p.last_check),
|
||||
}
|
||||
prefix = "" if first else ","
|
||||
yield prefix + json.dumps(item, ensure_ascii=False)
|
||||
first = False
|
||||
exported += 1
|
||||
if exported >= limit:
|
||||
break
|
||||
|
||||
if fmt == "json":
|
||||
yield "]"
|
||||
|
||||
@staticmethod
|
||||
def _fmt_time(dt: Optional[datetime]) -> str:
|
||||
|
||||
@@ -8,6 +8,7 @@ from app.core.tasks.queue import ValidationQueue
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.log import logger
|
||||
from app.models.domain import ProxyRaw
|
||||
from app.services.task_service import task_service
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
@@ -58,16 +59,15 @@ class SchedulerService:
|
||||
"""立即执行一次全量验证(后台运行,不阻塞)"""
|
||||
if self._validate_task and not self._validate_task.done():
|
||||
return
|
||||
self._validate_task = asyncio.create_task(self._do_validate_all())
|
||||
self._validate_task = asyncio.create_task(self._do_validate_all(from_loop=False))
|
||||
|
||||
async def _run_loop(self):
|
||||
"""定时循环"""
|
||||
while self.running:
|
||||
try:
|
||||
# 清理过期任务,防止内存无限增长
|
||||
from app.services.task_service import task_service
|
||||
task_service.cleanup_old_tasks()
|
||||
await self._do_validate_all()
|
||||
await self._do_validate_all(from_loop=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Scheduler loop error: {e}", exc_info=True)
|
||||
# 等待下一次
|
||||
@@ -76,10 +76,17 @@ class SchedulerService:
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def _do_validate_all(self):
|
||||
async def _do_validate_all(self, from_loop: bool = True):
|
||||
"""验证数据库中所有存量代理"""
|
||||
queue_started_here = False
|
||||
try:
|
||||
logger.info("Starting scheduled validation for all proxies")
|
||||
|
||||
# 如果队列未运行,临时启动它(适用于 validate_all_now 在调度器停止时调用)
|
||||
if not self.validation_queue._running:
|
||||
await self.validation_queue.start()
|
||||
queue_started_here = True
|
||||
|
||||
async with get_db() as db:
|
||||
# 清理 7 天前的验证任务记录,防止表无限增长
|
||||
cleaned = await ValidationTaskRepository.cleanup_old(db, days=7)
|
||||
@@ -95,7 +102,7 @@ class SchedulerService:
|
||||
batch_size = 100
|
||||
total_batches = (len(proxies) - 1) // batch_size + 1
|
||||
for i in range(0, len(proxies), batch_size):
|
||||
if not self.running:
|
||||
if from_loop and not self.running:
|
||||
break
|
||||
batch = proxies[i : i + batch_size]
|
||||
await self.validation_queue.submit([
|
||||
@@ -106,3 +113,6 @@ class SchedulerService:
|
||||
logger.info("Scheduled validation batches submitted")
|
||||
except Exception as e:
|
||||
logger.error(f"Scheduled validation error: {e}", exc_info=True)
|
||||
finally:
|
||||
if queue_started_here:
|
||||
await self.validation_queue.stop()
|
||||
|
||||
@@ -95,17 +95,13 @@ class ValidatorService:
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
||||
test_url = self._get_test_url("http")
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
async with session.get(test_url, allow_redirects=True) as response:
|
||||
if response.status in (200, 301, 302):
|
||||
latency = round((time.time() - start) * 1000, 2)
|
||||
logger.info(f"SOCKS valid: {ip}:{port} ({protocol}) {latency}ms")
|
||||
return True, latency
|
||||
return False, 0.0
|
||||
finally:
|
||||
# ClientSession 的 async with 退出时会自动关闭 connector,无需手动重复关闭
|
||||
pass
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
async with session.get(test_url, allow_redirects=True) as response:
|
||||
if response.status in (200, 301, 302):
|
||||
latency = round((time.time() - start) * 1000, 2)
|
||||
logger.info(f"SOCKS valid: {ip}:{port} ({protocol}) {latency}ms")
|
||||
return True, latency
|
||||
return False, 0.0
|
||||
|
||||
async def close(self):
|
||||
"""关闭共享的 HTTP ClientSession"""
|
||||
|
||||
Reference in New Issue
Block a user