fix: 修复设置系统脱节、队列计数漂移、资源泄露等全量问题

- 统一设置系统:create_scheduler_service 读取 DB 设置覆盖默认值
- 修复 ProxyRepository.update_score 误删所有无效代理的 SQL
- ValidationQueue:修复 Worker 计数漂移与启动恢复任务饿死
- SchedulerService:移除 drain() 阻塞,主循环可正常响应 stop
- TaskService:在调度器周期内自动清理过期任务,防止内存泄漏
- lifespan/conftest:规范关闭顺序,消除 Event loop closed 警告
- Repository:异常日志增加 exc_info,今日新增按 created_at 统计
- ValidatorService:防止 HTTP session 重复关闭,移除 SOCKS 多余 close
- 前端:补全 pluginsStore.isEmpty,ProxyList 最低分数上限改为 100
- 删除 config.py 中冗余的 cors_origins_list property
This commit is contained in:
祀梦
2026-04-04 20:31:52 +08:00
parent 0788a13c8a
commit 875e61f17e
26 changed files with 568 additions and 355 deletions

View File

@@ -1,4 +1,5 @@
"""pytest 配置文件和 fixtures"""
import asyncio
import pytest
import pytest_asyncio
from typing import AsyncGenerator
@@ -7,16 +8,30 @@ from httpx import AsyncClient, ASGITransport
from app.api import create_app
from app.core.db import init_db, get_db
from app.repositories.proxy_repo import ProxyRepository
from app.models.domain import ProxyRaw
@pytest_asyncio.fixture(scope="function")
async def app():
"""创建应用实例"""
# 初始化测试数据库
# 初始化测试数据库并清空历史数据,避免任务残留或设置状态导致 drain() 卡住
await init_db()
app = create_app()
async with app.router.lifespan_context(app):
yield app
async with get_db() as db:
await db.execute("DELETE FROM validation_tasks")
await db.execute("DELETE FROM proxies")
await db.execute("DELETE FROM settings")
await db.commit()
# 清理全局内存状态,防止跨测试污染
from app.services.task_service import task_service
task_service._tasks.clear()
test_app = create_app()
async with test_app.router.lifespan_context(test_app):
yield test_app
# 给 aiosqlite / aiohttp 后台线程留出收尾时间,降低 Event loop closed 警告概率
await asyncio.sleep(0.1)
@pytest_asyncio.fixture
@@ -47,3 +62,27 @@ async def sample_proxy(db, proxy_repo):
yield {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
# 清理
await proxy_repo.delete(db, "192.168.1.1", 8080)
@pytest_asyncio.fixture(autouse=True)
async def mock_external_requests(monkeypatch):
"""
自动在所有测试中 mock 外部网络请求:
1. 插件爬取返回固定测试代理,避免真实 HTTP 请求
2. 代理验证瞬间成功,避免连接超时等待
"""
from app.services.plugin_service import PluginService
from app.services.validator_service import ValidatorService
async def _mock_run_plugin(self, plugin_id: str):
return [ProxyRaw("192.168.100.10", 8080, "http")]
async def _mock_run_all_plugins(self):
return [ProxyRaw("192.168.100.10", 8080, "http")]
async def _mock_validate(self, ip: str, port: int, protocol: str = "http"):
return True, 1.23
monkeypatch.setattr(PluginService, "run_plugin", _mock_run_plugin)
monkeypatch.setattr(PluginService, "run_all_plugins", _mock_run_all_plugins)
monkeypatch.setattr(ValidatorService, "validate", _mock_validate)

View File

@@ -117,19 +117,33 @@ class TestPluginsAPI:
@pytest.mark.asyncio
async def test_crawl_plugin(self, client):
"""测试 POST /api/plugins/{id}/crawl"""
"""测试 POST /api/plugins/{id}/crawl - 异步任务模式"""
import asyncio
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
# 这个测试可能需要较长时间,设置较短的超时
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "proxy_count" in data["data"]
assert "task_id" in data["data"]
task_id = data["data"]["task_id"]
# 轮询任务状态
task_data = None
for _ in range(10):
await asyncio.sleep(0.3)
res = await client.get(f"/api/tasks/{task_id}")
assert res.status_code == 200
task_data = res.json()["data"]
if task_data["status"] in ("completed", "failed"):
break
assert task_data is not None
assert task_data["status"] == "completed"
@pytest.mark.asyncio
async def test_crawl_nonexistent_plugin(self, client):
@@ -139,9 +153,24 @@ class TestPluginsAPI:
@pytest.mark.asyncio
async def test_crawl_all_plugins(self, client):
"""测试 POST /api/plugins/crawl-all"""
"""测试 POST /api/plugins/crawl-all - 异步任务模式"""
import asyncio
response = await client.post("/api/plugins/crawl-all")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "total_crawled" in data["data"]
assert "task_id" in data["data"]
task_id = data["data"]["task_id"]
# 轮询任务状态
task_data = None
for _ in range(10):
await asyncio.sleep(0.3)
res = await client.get(f"/api/tasks/{task_id}")
assert res.status_code == 200
task_data = res.json()["data"]
if task_data["status"] in ("completed", "failed"):
break
assert task_data is not None
assert task_data["status"] == "completed"