4ef7931941
修复问题： - 添加缺失的 httpx 依赖到 requirements.txt - 修复前端批量删除参数格式与后端不匹配（数组->对象数组） - 移除 app/api/main.py 中重复创建 app 的冗余代码 - 修复 Plugins.vue v-model 直接修改 store 状态的 Vue 警告 - 修复 README 端口/启动命令文档与实际配置不一致 - 修正 pytest.ini 过时配置 (asyncio_default_fixture_loop_scope) - 修复 WebUI index.html 语言设置为 zh-CN - 修复 .gitignore 错误忽略 tests/ 目录 后端优化： - 修复调度器默认间隔从 5 秒改为 30 分钟，避免无节制验证 - 修复 validate_all_now 在调度器停止时无法执行的 bug - 设置保存后热更新运行中调度器的验证间隔 - 将 update_score 优化为原子单事务 SQL，消除并发竞态 - 导出功能改为真正的流式分批读取（iter_batches），降低大导出内存占用 - ProxyResponse Schema 补齐 response_time_ms 字段 - 日志级别改为从配置动态读取，不再硬编码 INFO - 清理 validator_service 中的冗余 try/finally 代码 插件健壮性： - 修复 ip3366/ip89/kuaidaili/proxylist_download/speedx/yundaili/proxyscrape 的端口范围检查和 IPv6 地址解析问题（改用 rsplit + 1-65535 校验） - 修复 PluginService.list_plugins 并发竞争条件 - 修复 run_all_plugins 去重逻辑与数据库 UNIQUE 约束保持一致 - 修复 proxyscrape 异常时错误跳过 fallback 的 bug 测试： - 新增 7 个插件解析单元测试 - 新增 update_score 自动删除和 iter_batches 流式读取测试 - 全部 74 个测试通过
119 lines
4.5 KiB
Python
119 lines
4.5 KiB
Python
"""调度器服务 - 定时验证存量代理"""
|
|
import asyncio
|
|
from datetime import datetime
|
|
from app.core.db import get_db
|
|
from app.repositories.proxy_repo import ProxyRepository
|
|
from app.repositories.task_repo import ValidationTaskRepository
|
|
from app.core.tasks.queue import ValidationQueue
|
|
from app.core.config import settings as app_settings
|
|
from app.core.log import logger
|
|
from app.models.domain import ProxyRaw
|
|
from app.services.task_service import task_service
|
|
|
|
|
|
class SchedulerService:
|
|
"""代理验证调度器"""
|
|
|
|
def __init__(
|
|
self,
|
|
validation_queue: ValidationQueue,
|
|
proxy_repo: ProxyRepository = ProxyRepository(),
|
|
):
|
|
self.validation_queue = validation_queue
|
|
self.proxy_repo = proxy_repo
|
|
self.interval_minutes = 30
|
|
self.running = False
|
|
self._stop_event = asyncio.Event()
|
|
self._task: asyncio.Task | None = None
|
|
self._validate_task: asyncio.Task | None = None
|
|
|
|
async def start(self):
|
|
if self.running:
|
|
logger.warning("Scheduler already running")
|
|
return
|
|
self._stop_event.clear()
|
|
self.running = True
|
|
await self.validation_queue.start()
|
|
self._task = asyncio.create_task(self._run_loop())
|
|
logger.info("Scheduler started")
|
|
|
|
async def stop(self):
|
|
self.running = False
|
|
self._stop_event.set()
|
|
if self._task:
|
|
self._task.cancel()
|
|
try:
|
|
await self._task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
self._task = None
|
|
await self.validation_queue.stop()
|
|
logger.info("Scheduler stopped")
|
|
|
|
def cancel_validate_task(self):
|
|
"""取消正在执行的全量验证后台任务"""
|
|
if self._validate_task and not self._validate_task.done():
|
|
self._validate_task.cancel()
|
|
|
|
async def validate_all_now(self):
|
|
"""立即执行一次全量验证（后台运行，不阻塞）"""
|
|
if self._validate_task and not self._validate_task.done():
|
|
return
|
|
self._validate_task = asyncio.create_task(self._do_validate_all(from_loop=False))
|
|
|
|
async def _run_loop(self):
|
|
"""定时循环"""
|
|
while self.running:
|
|
try:
|
|
# 清理过期任务，防止内存无限增长
|
|
task_service.cleanup_old_tasks()
|
|
await self._do_validate_all(from_loop=True)
|
|
except Exception as e:
|
|
logger.error(f"Scheduler loop error: {e}", exc_info=True)
|
|
# 等待下一次
|
|
try:
|
|
await asyncio.wait_for(self._stop_event.wait(), timeout=self.interval_minutes * 60)
|
|
except asyncio.TimeoutError:
|
|
pass
|
|
|
|
async def _do_validate_all(self, from_loop: bool = True):
|
|
"""验证数据库中所有存量代理"""
|
|
queue_started_here = False
|
|
try:
|
|
logger.info("Starting scheduled validation for all proxies")
|
|
|
|
# 如果队列未运行，临时启动它（适用于 validate_all_now 在调度器停止时调用）
|
|
if not self.validation_queue._running:
|
|
await self.validation_queue.start()
|
|
queue_started_here = True
|
|
|
|
async with get_db() as db:
|
|
# 清理 7 天前的验证任务记录，防止表无限增长
|
|
cleaned = await ValidationTaskRepository.cleanup_old(db, days=7)
|
|
if cleaned:
|
|
logger.info(f"Cleaned up {cleaned} old validation tasks")
|
|
proxies = await self.proxy_repo.list_all(db)
|
|
if not proxies:
|
|
logger.info("No proxies to validate")
|
|
return
|
|
|
|
logger.info(f"Validating {len(proxies)} proxies from database")
|
|
# 批量提交到验证队列，不再阻塞等待 drain
|
|
batch_size = 100
|
|
total_batches = (len(proxies) - 1) // batch_size + 1
|
|
for i in range(0, len(proxies), batch_size):
|
|
if from_loop and not self.running:
|
|
break
|
|
batch = proxies[i : i + batch_size]
|
|
await self.validation_queue.submit([
|
|
ProxyRaw(p.ip, p.port, p.protocol) for p in batch
|
|
])
|
|
logger.info(f"Submitted batch {i // batch_size + 1}/{total_batches}")
|
|
|
|
logger.info("Scheduled validation batches submitted")
|
|
except Exception as e:
|
|
logger.error(f"Scheduled validation error: {e}", exc_info=True)
|
|
finally:
|
|
if queue_started_here:
|
|
await self.validation_queue.stop()
|