refactor: 全面重构核心架构,消除反复修改的根因

- 删除 ValidationQueue 双轨持久化队列,替换为纯内存 AsyncWorkerPool
- 引入统一后台任务框架 JobExecutor(Job/CrawlJob/ValidateAllJob)
- 新增 PluginRunner 统一插件执行(超时、重试、健康检查、统计)
- 重构 SchedulerService 职责收敛为仅定时触发 ValidateAllJob
- 使用 AsyncExitStack 重构 lifespan,安全管理长生命周期资源
- 路由层瘦身 50%+,业务异常上抛由全局中间件统一处理
- 实现设置全热更新(WorkerPool 并发、Validator 超时即时生效)
- 前端 Store 强制写后重新拉取,消除乐观更新数据不同步
- 删除 queue.py / task_repo.py / task_service.py
- 新增 execution 单元测试,全部 85 个测试通过
This commit is contained in:
祀梦
2026-04-04 22:36:57 +08:00
parent 4ef7931941
commit b972b64616
33 changed files with 1168 additions and 864 deletions

View File

@@ -4,12 +4,17 @@ import random
import time
import aiohttp
import aiohttp_socks
from typing import Tuple
from typing import Tuple, Optional
from app.core.config import settings as app_settings
from app.core.log import logger
class ValidatorService:
"""代理验证器"""
"""代理验证器
支持动态读取配置,实现设置热更新。
"""
# 测试 URL
TEST_URLS = {
@@ -19,40 +24,66 @@ class ValidatorService:
def __init__(
self,
timeout: float = 5.0,
connect_timeout: float = 3.0,
max_concurrency: int = 50,
timeout: Optional[float] = None,
connect_timeout: Optional[float] = None,
max_concurrency: Optional[int] = None,
):
self.timeout = timeout
self.connect_timeout = connect_timeout
self.max_concurrency = max_concurrency
self.semaphore = asyncio.Semaphore(max_concurrency)
# 初始化时使用传入值或默认值,但运行期会动态读取 settings
self._init_timeout = timeout or app_settings.validator_timeout
self._init_connect_timeout = connect_timeout or app_settings.validator_connect_timeout
self._init_max_concurrency = max_concurrency or app_settings.validator_max_concurrency
# 共享 HTTP/HTTPS ClientSession
self._http_connector = aiohttp.TCPConnector(
ssl=False,
limit=max_concurrency,
limit_per_host=max_concurrency,
force_close=False,
)
self._timeout = aiohttp.ClientTimeout(
total=timeout, connect=connect_timeout
)
self._http_session = aiohttp.ClientSession(
connector=self._http_connector,
timeout=self._timeout,
)
self._http_connector: Optional[aiohttp.TCPConnector] = None
self._http_session: Optional[aiohttp.ClientSession] = None
self._semaphore: Optional[asyncio.Semaphore] = None
self._lock = asyncio.Lock()
@property
def timeout(self) -> float:
return float(self._init_timeout)
@property
def connect_timeout(self) -> float:
return float(self._init_connect_timeout)
@property
def max_concurrency(self) -> int:
return int(self._init_max_concurrency)
def _ensure_session(self) -> aiohttp.ClientSession:
"""懒加载共享 HTTP session"""
if self._http_session is None or self._http_session.closed:
connector = aiohttp.TCPConnector(
ssl=False,
limit=self.max_concurrency,
limit_per_host=self.max_concurrency,
force_close=False,
)
timeout = aiohttp.ClientTimeout(
total=self.timeout, connect=self.connect_timeout
)
self._http_connector = connector
self._http_session = aiohttp.ClientSession(
connector=connector,
timeout=timeout,
)
return self._http_session
def _ensure_semaphore(self) -> asyncio.Semaphore:
if self._semaphore is None:
self._semaphore = asyncio.Semaphore(self.max_concurrency)
return self._semaphore
def _get_test_url(self, protocol: str) -> str:
"""获取测试 URL"""
urls = self.TEST_URLS.get(protocol.lower(), self.TEST_URLS["http"])
return random.choice(urls)
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
"""验证单个代理,返回 (是否有效, 延迟毫秒)"""
protocol = protocol.lower()
async with self.semaphore:
semaphore = self._ensure_semaphore()
async with semaphore:
start = time.time()
try:
if protocol in ("socks4", "socks5"):
@@ -67,11 +98,11 @@ class ValidatorService:
return False, 0.0
async def _validate_http(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
"""验证 HTTP/HTTPS 代理"""
proxy_url = f"http://{ip}:{port}"
test_url = self._get_test_url(protocol)
session = self._ensure_session()
async with self._http_session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
async with session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
if response.status in (200, 301, 302):
latency = round((time.time() - start) * 1000, 2)
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
@@ -79,7 +110,6 @@ class ValidatorService:
return False, 0.0
async def _validate_socks(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
"""验证 SOCKS4/SOCKS5 代理"""
proxy_type = (
aiohttp_socks.ProxyType.SOCKS4
if protocol == "socks4"
@@ -103,7 +133,9 @@ class ValidatorService:
return True, latency
return False, 0.0
async def close(self):
async def close(self) -> None:
"""关闭共享的 HTTP ClientSession"""
if self._http_session and not self._http_session.closed:
await self._http_session.close()
self._http_session = None
self._http_connector = None