refactor: 全面重构核心架构,消除反复修改的根因

- 删除 ValidationQueue 双轨持久化队列,替换为纯内存 AsyncWorkerPool
- 引入统一后台任务框架 JobExecutor(Job/CrawlJob/ValidateAllJob)
- 新增 PluginRunner 统一插件执行(超时、重试、健康检查、统计)
- 重构 SchedulerService 职责收敛为仅定时触发 ValidateAllJob
- 使用 AsyncExitStack 重构 lifespan,安全管理长生命周期资源
- 路由层瘦身 50%+,业务异常上抛由全局中间件统一处理
- 实现设置全热更新(WorkerPool 并发、Validator 超时即时生效)
- 前端 Store 强制写后重新拉取,消除乐观更新数据不同步
- 删除 queue.py / task_repo.py / task_service.py
- 新增 execution 单元测试,全部 85 个测试通过
This commit is contained in:
祀梦
2026-04-04 22:36:57 +08:00
parent 4ef7931941
commit b972b64616
33 changed files with 1168 additions and 864 deletions

View File

@@ -7,6 +7,17 @@ from httpx import AsyncClient, ASGITransport
from app.api import create_app
from app.core.db import init_db, get_db
from app.core.plugin_system.registry import registry
from app.plugins import (
Fate0Plugin,
ProxyListDownloadPlugin,
Ip3366Plugin,
Ip89Plugin,
KuaiDaiLiPlugin,
SpeedXPlugin,
YunDaiLiPlugin,
ProxyScrapePlugin,
)
from app.repositories.proxy_repo import ProxyRepository
from app.models.domain import ProxyRaw
@@ -14,23 +25,32 @@ from app.models.domain import ProxyRaw
@pytest_asyncio.fixture(scope="function")
async def app():
"""创建应用实例"""
# 初始化测试数据库并清空历史数据,避免任务残留或设置状态导致 drain() 卡住
# 初始化测试数据库并清空历史数据
await init_db()
async with get_db() as db:
await db.execute("DELETE FROM validation_tasks")
await db.execute("DELETE FROM proxies")
await db.execute("DELETE FROM settings")
await db.commit()
# 清理全局内存状态,防止跨测试污染
from app.services.task_service import task_service
task_service._tasks.clear()
# 清理并重新注册插件,防止跨测试污染
registry.clear()
for plugin_cls in [
Fate0Plugin,
ProxyListDownloadPlugin,
Ip3366Plugin,
Ip89Plugin,
KuaiDaiLiPlugin,
SpeedXPlugin,
YunDaiLiPlugin,
ProxyScrapePlugin,
]:
registry.register(plugin_cls)
test_app = create_app()
async with test_app.router.lifespan_context(test_app):
yield test_app
# 给 aiosqlite / aiohttp 后台线程留出收尾时间,降低 Event loop closed 警告概率
# 给 aiosqlite / aiohttp 后台线程留出收尾时间
await asyncio.sleep(0.1)
@@ -71,18 +91,19 @@ async def mock_external_requests(monkeypatch):
1. 插件爬取返回固定测试代理,避免真实 HTTP 请求
2. 代理验证瞬间成功,避免连接超时等待
"""
from app.services.plugin_service import PluginService
from app.services.plugin_runner import PluginRunner
from app.services.validator_service import ValidatorService
async def _mock_run_plugin(self, plugin_id: str):
return [ProxyRaw("192.168.100.10", 8080, "http")]
async def _mock_run_all_plugins(self):
return [ProxyRaw("192.168.100.10", 8080, "http")]
async def _mock_run(self, plugin):
from app.models.domain import CrawlResult
return CrawlResult(
plugin_name=plugin.name,
proxies=[ProxyRaw("192.168.100.10", 8080, "http")],
success_count=1,
)
async def _mock_validate(self, ip: str, port: int, protocol: str = "http"):
return True, 1.23
monkeypatch.setattr(PluginService, "run_plugin", _mock_run_plugin)
monkeypatch.setattr(PluginService, "run_all_plugins", _mock_run_all_plugins)
monkeypatch.setattr(PluginRunner, "run", _mock_run)
monkeypatch.setattr(ValidatorService, "validate", _mock_validate)