refactor: 全面重构核心架构，消除反复修改的根因
- 删除 ValidationQueue 双轨持久化队列，替换为纯内存 AsyncWorkerPool - 引入统一后台任务框架 JobExecutor（Job/CrawlJob/ValidateAllJob） - 新增 PluginRunner 统一插件执行（超时、重试、健康检查、统计） - 重构 SchedulerService 职责收敛为仅定时触发 ValidateAllJob - 使用 AsyncExitStack 重构 lifespan，安全管理长生命周期资源 - 路由层瘦身 50%+，业务异常上抛由全局中间件统一处理 - 实现设置全热更新（WorkerPool 并发、Validator 超时即时生效） - 前端 Store 强制写后重新拉取，消除乐观更新数据不同步 - 删除 queue.py / task_repo.py / task_service.py - 新增 execution 单元测试，全部 85 个测试通过
This commit is contained in:
祀梦
@@ -1,56 +1,118 @@
|
||||
"""应用生命周期管理"""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.core.db import init_db, get_db
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.log import logger
|
||||
from app.api.deps import create_scheduler_service
|
||||
from app.core.execution import AsyncWorkerPool, JobExecutor
|
||||
from app.core.plugin_system.registry import registry
|
||||
from app.repositories.proxy_repo import ProxyRepository
|
||||
from app.repositories.settings_repo import SettingsRepository, DEFAULT_SETTINGS
|
||||
from app.services.validator_service import ValidatorService
|
||||
from app.services.plugin_runner import PluginRunner
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
|
||||
settings_repo = SettingsRepository()
|
||||
proxy_repo = ProxyRepository()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用启动和关闭时的生命周期管理"""
|
||||
# 初始化数据库
|
||||
await init_db()
|
||||
|
||||
# 加载设置并决定是否启动调度器
|
||||
async def _load_settings() -> dict:
|
||||
db_settings = DEFAULT_SETTINGS.copy()
|
||||
try:
|
||||
async with get_db() as db:
|
||||
db_settings = await settings_repo.get_all(db)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load settings on startup: {e}")
|
||||
return db_settings
|
||||
|
||||
# 创建调度器并挂载到 app.state（使用 DB 设置覆盖默认值）
|
||||
scheduler_service = create_scheduler_service(db_settings)
|
||||
app.state.scheduler_service = scheduler_service
|
||||
app.state.validation_queue = scheduler_service.validation_queue
|
||||
|
||||
if db_settings.get("auto_validate", True):
|
||||
try:
|
||||
await scheduler_service.start()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler on startup: {e}")
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用启动和关闭时的生命周期管理"""
|
||||
await init_db()
|
||||
db_settings = await _load_settings()
|
||||
|
||||
logger.info("API server started")
|
||||
yield
|
||||
async with AsyncExitStack() as stack:
|
||||
# 验证器
|
||||
validator = ValidatorService(
|
||||
timeout=db_settings.get("validation_timeout", app_settings.validator_timeout),
|
||||
connect_timeout=app_settings.validator_connect_timeout,
|
||||
max_concurrency=db_settings.get("default_concurrency", app_settings.validator_max_concurrency),
|
||||
)
|
||||
|
||||
# 关闭调度器
|
||||
scheduler_service.cancel_validate_task()
|
||||
await scheduler_service.stop()
|
||||
# 验证 WorkerPool
|
||||
async def validation_handler(proxy):
|
||||
from app.models.domain import ProxyRaw
|
||||
is_valid, latency = await validator.validate(
|
||||
proxy.ip, proxy.port, proxy.protocol
|
||||
)
|
||||
async with get_db() as db:
|
||||
if is_valid:
|
||||
await proxy_repo.insert_or_update(
|
||||
db, proxy.ip, proxy.port, proxy.protocol, score=app_settings.score_valid
|
||||
)
|
||||
if latency:
|
||||
await proxy_repo.update_response_time(db, proxy.ip, proxy.port, latency)
|
||||
else:
|
||||
await proxy_repo.update_score(
|
||||
db, proxy.ip, proxy.port, app_settings.score_invalid,
|
||||
app_settings.score_min, app_settings.score_max
|
||||
)
|
||||
|
||||
# 关闭验证器 HTTP session
|
||||
worker_pool = AsyncWorkerPool(
|
||||
worker_count=db_settings.get("default_concurrency", app_settings.validator_max_concurrency),
|
||||
handler=validation_handler,
|
||||
name="ValidationPool",
|
||||
)
|
||||
await stack.enter_async_context(worker_pool)
|
||||
|
||||
# Job 执行器
|
||||
executor = JobExecutor(worker_pool=worker_pool, max_concurrent_jobs=10)
|
||||
await stack.enter_async_context(executor)
|
||||
|
||||
# 插件运行器
|
||||
plugin_runner = PluginRunner(timeout=db_settings.get("crawl_timeout", 30))
|
||||
|
||||
# 调度器
|
||||
scheduler = SchedulerService(
|
||||
executor=executor,
|
||||
interval_minutes=db_settings.get("validate_interval_minutes", 30),
|
||||
)
|
||||
|
||||
# 挂载到 app.state
|
||||
app.state.validator = validator
|
||||
app.state.worker_pool = worker_pool
|
||||
app.state.executor = executor
|
||||
app.state.plugin_runner = plugin_runner
|
||||
app.state.scheduler = scheduler
|
||||
|
||||
# 启动调度器
|
||||
if db_settings.get("auto_validate", True):
|
||||
try:
|
||||
await scheduler.start()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler on startup: {e}")
|
||||
|
||||
logger.info("API server started")
|
||||
yield
|
||||
|
||||
# 停止调度器
|
||||
await scheduler.stop()
|
||||
|
||||
# 取消所有运行中的 Job
|
||||
await executor.cancel_all()
|
||||
|
||||
# AsyncExitStack 会自动关闭 executor 和 worker_pool
|
||||
|
||||
# 关闭验证器 session
|
||||
try:
|
||||
await scheduler_service.validation_queue.validator.close()
|
||||
await validator.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 关闭所有插件的 HTTP 客户端
|
||||
from app.core.plugin_system.registry import registry
|
||||
for plugin in registry.list_plugins():
|
||||
if hasattr(plugin, "close"):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user