fix: 修复设置系统脱节、队列计数漂移、资源泄露等全量问题

- 统一设置系统:create_scheduler_service 读取 DB 设置覆盖默认值
- 修复 ProxyRepository.update_score 误删所有无效代理的 SQL
- ValidationQueue:修复 Worker 计数漂移与启动恢复任务饿死
- SchedulerService:移除 drain() 阻塞,主循环可正常响应 stop
- TaskService:在调度器周期内自动清理过期任务,防止内存泄漏
- lifespan/conftest:规范关闭顺序,消除 Event loop closed 警告
- Repository:异常日志增加 exc_info,今日新增按 created_at 统计
- ValidatorService:防止 HTTP session 重复关闭,移除 SOCKS 多余 close
- 前端:补全 pluginsStore.isEmpty,ProxyList 最低分数上限改为 100
- 删除 config.py 中冗余的 cors_origins_list property
This commit is contained in:
祀梦
2026-04-04 20:31:52 +08:00
parent 0788a13c8a
commit 875e61f17e
26 changed files with 568 additions and 355 deletions

View File

@@ -3,9 +3,11 @@ import asyncio
from datetime import datetime
from app.core.db import get_db
from app.repositories.proxy_repo import ProxyRepository
from app.repositories.task_repo import ValidationTaskRepository
from app.core.tasks.queue import ValidationQueue
from app.core.config import settings as app_settings
from app.core.log import logger
from app.models.domain import ProxyRaw
class SchedulerService:
@@ -47,6 +49,11 @@ class SchedulerService:
await self.validation_queue.stop()
logger.info("Scheduler stopped")
def cancel_validate_task(self):
"""取消正在执行的全量验证后台任务"""
if self._validate_task and not self._validate_task.done():
self._validate_task.cancel()
async def validate_all_now(self):
"""立即执行一次全量验证(后台运行,不阻塞)"""
if self._validate_task and not self._validate_task.done():
@@ -57,9 +64,12 @@ class SchedulerService:
"""定时循环"""
while self.running:
try:
# 清理过期任务,防止内存无限增长
from app.services.task_service import task_service
task_service.cleanup_old_tasks()
await self._do_validate_all()
except Exception as e:
logger.error(f"Scheduler loop error: {e}")
logger.error(f"Scheduler loop error: {e}", exc_info=True)
# 等待下一次
try:
await asyncio.wait_for(self._stop_event.wait(), timeout=self.interval_minutes * 60)
@@ -71,16 +81,19 @@ class SchedulerService:
try:
logger.info("Starting scheduled validation for all proxies")
async with get_db() as db:
# 清理 7 天前的验证任务记录,防止表无限增长
cleaned = await ValidationTaskRepository.cleanup_old(db, days=7)
if cleaned:
logger.info(f"Cleaned up {cleaned} old validation tasks")
proxies = await self.proxy_repo.list_all(db)
if not proxies:
logger.info("No proxies to validate")
return
logger.info(f"Validating {len(proxies)} proxies from database")
from app.models.domain import ProxyRaw
# 批量提交到验证队列
# 批量提交到验证队列,不再阻塞等待 drain
batch_size = 100
total_batches = (len(proxies) - 1) // batch_size + 1
for i in range(0, len(proxies), batch_size):
if not self.running:
break
@@ -88,10 +101,8 @@ class SchedulerService:
await self.validation_queue.submit([
ProxyRaw(p.ip, p.port, p.protocol) for p in batch
])
# 等待当前批次处理完
await self.validation_queue.drain()
logger.info(f"Validated batch {i//batch_size + 1}/{(len(proxies)-1)//batch_size + 1}")
logger.info(f"Submitted batch {i // batch_size + 1}/{total_batches}")
logger.info("Scheduled validation completed")
logger.info("Scheduled validation batches submitted")
except Exception as e:
logger.error(f"Scheduled validation error: {e}")
logger.error(f"Scheduled validation error: {e}", exc_info=True)

View File

@@ -0,0 +1,62 @@
"""轻量级异步任务服务 - 内存存储,用于跟踪后台爬取任务"""
import uuid
from datetime import datetime, timedelta
from typing import Optional, Dict, Any
from app.core.log import logger
class TaskService:
"""异步任务状态管理器"""
def __init__(self, max_age_seconds: int = 3600):
self._tasks: Dict[str, dict] = {}
self._max_age_seconds = max_age_seconds
def create_task(self, task_type: str) -> str:
task_id = str(uuid.uuid4())
now = datetime.now()
self._tasks[task_id] = {
"id": task_id,
"type": task_type,
"status": "pending",
"message": "",
"data": {},
"created_at": now,
"updated_at": now,
}
return task_id
def update_task(
self,
task_id: str,
status: Optional[str] = None,
message: Optional[str] = None,
data: Optional[dict] = None,
) -> bool:
task = self._tasks.get(task_id)
if not task:
return False
if status is not None:
task["status"] = status
if message is not None:
task["message"] = message
if data is not None:
task["data"].update(data)
task["updated_at"] = datetime.now()
return True
def get_task(self, task_id: str) -> Optional[dict]:
return self._tasks.get(task_id)
def cleanup_old_tasks(self) -> int:
cutoff = datetime.now() - timedelta(seconds=self._max_age_seconds)
to_remove = [tid for tid, task in self._tasks.items() if task["created_at"] < cutoff]
for tid in to_remove:
del self._tasks[tid]
if to_remove:
logger.info(f"TaskService cleaned up {len(to_remove)} old tasks")
return len(to_remove)
# 全局任务服务实例
task_service = TaskService()

View File

@@ -104,8 +104,10 @@ class ValidatorService:
return True, latency
return False, 0.0
finally:
await connector.close()
# ClientSession 的 async with 退出时会自动关闭 connector无需手动重复关闭
pass
async def close(self):
"""关闭共享的 HTTP ClientSession"""
await self._http_session.close()
if self._http_session and not self._http_session.closed:
await self._http_session.close()