refactor: 全面重构核心架构,消除反复修改的根因
- 删除 ValidationQueue 双轨持久化队列,替换为纯内存 AsyncWorkerPool - 引入统一后台任务框架 JobExecutor(Job/CrawlJob/ValidateAllJob) - 新增 PluginRunner 统一插件执行(超时、重试、健康检查、统计) - 重构 SchedulerService 职责收敛为仅定时触发 ValidateAllJob - 使用 AsyncExitStack 重构 lifespan,安全管理长生命周期资源 - 路由层瘦身 50%+,业务异常上抛由全局中间件统一处理 - 实现设置全热更新(WorkerPool 并发、Validator 超时即时生效) - 前端 Store 强制写后重新拉取,消除乐观更新数据不同步 - 删除 queue.py / task_repo.py / task_service.py - 新增 execution 单元测试,全部 85 个测试通过
This commit is contained in:
@@ -4,12 +4,17 @@ import random
|
||||
import time
|
||||
import aiohttp
|
||||
import aiohttp_socks
|
||||
from typing import Tuple
|
||||
from typing import Tuple, Optional
|
||||
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ValidatorService:
|
||||
"""代理验证器"""
|
||||
"""代理验证器
|
||||
|
||||
支持动态读取配置,实现设置热更新。
|
||||
"""
|
||||
|
||||
# 测试 URL
|
||||
TEST_URLS = {
|
||||
@@ -19,40 +24,66 @@ class ValidatorService:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timeout: float = 5.0,
|
||||
connect_timeout: float = 3.0,
|
||||
max_concurrency: int = 50,
|
||||
timeout: Optional[float] = None,
|
||||
connect_timeout: Optional[float] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
):
|
||||
self.timeout = timeout
|
||||
self.connect_timeout = connect_timeout
|
||||
self.max_concurrency = max_concurrency
|
||||
self.semaphore = asyncio.Semaphore(max_concurrency)
|
||||
# 初始化时使用传入值或默认值,但运行期会动态读取 settings
|
||||
self._init_timeout = timeout or app_settings.validator_timeout
|
||||
self._init_connect_timeout = connect_timeout or app_settings.validator_connect_timeout
|
||||
self._init_max_concurrency = max_concurrency or app_settings.validator_max_concurrency
|
||||
|
||||
# 共享 HTTP/HTTPS ClientSession
|
||||
self._http_connector = aiohttp.TCPConnector(
|
||||
ssl=False,
|
||||
limit=max_concurrency,
|
||||
limit_per_host=max_concurrency,
|
||||
force_close=False,
|
||||
)
|
||||
self._timeout = aiohttp.ClientTimeout(
|
||||
total=timeout, connect=connect_timeout
|
||||
)
|
||||
self._http_session = aiohttp.ClientSession(
|
||||
connector=self._http_connector,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
self._http_connector: Optional[aiohttp.TCPConnector] = None
|
||||
self._http_session: Optional[aiohttp.ClientSession] = None
|
||||
self._semaphore: Optional[asyncio.Semaphore] = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
return float(self._init_timeout)
|
||||
|
||||
@property
|
||||
def connect_timeout(self) -> float:
|
||||
return float(self._init_connect_timeout)
|
||||
|
||||
@property
|
||||
def max_concurrency(self) -> int:
|
||||
return int(self._init_max_concurrency)
|
||||
|
||||
def _ensure_session(self) -> aiohttp.ClientSession:
|
||||
"""懒加载共享 HTTP session"""
|
||||
if self._http_session is None or self._http_session.closed:
|
||||
connector = aiohttp.TCPConnector(
|
||||
ssl=False,
|
||||
limit=self.max_concurrency,
|
||||
limit_per_host=self.max_concurrency,
|
||||
force_close=False,
|
||||
)
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=self.timeout, connect=self.connect_timeout
|
||||
)
|
||||
self._http_connector = connector
|
||||
self._http_session = aiohttp.ClientSession(
|
||||
connector=connector,
|
||||
timeout=timeout,
|
||||
)
|
||||
return self._http_session
|
||||
|
||||
def _ensure_semaphore(self) -> asyncio.Semaphore:
|
||||
if self._semaphore is None:
|
||||
self._semaphore = asyncio.Semaphore(self.max_concurrency)
|
||||
return self._semaphore
|
||||
|
||||
def _get_test_url(self, protocol: str) -> str:
|
||||
"""获取测试 URL"""
|
||||
urls = self.TEST_URLS.get(protocol.lower(), self.TEST_URLS["http"])
|
||||
return random.choice(urls)
|
||||
|
||||
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
|
||||
"""验证单个代理,返回 (是否有效, 延迟毫秒)"""
|
||||
protocol = protocol.lower()
|
||||
|
||||
async with self.semaphore:
|
||||
semaphore = self._ensure_semaphore()
|
||||
|
||||
async with semaphore:
|
||||
start = time.time()
|
||||
try:
|
||||
if protocol in ("socks4", "socks5"):
|
||||
@@ -67,11 +98,11 @@ class ValidatorService:
|
||||
return False, 0.0
|
||||
|
||||
async def _validate_http(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||
"""验证 HTTP/HTTPS 代理"""
|
||||
proxy_url = f"http://{ip}:{port}"
|
||||
test_url = self._get_test_url(protocol)
|
||||
session = self._ensure_session()
|
||||
|
||||
async with self._http_session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
|
||||
async with session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
|
||||
if response.status in (200, 301, 302):
|
||||
latency = round((time.time() - start) * 1000, 2)
|
||||
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
|
||||
@@ -79,7 +110,6 @@ class ValidatorService:
|
||||
return False, 0.0
|
||||
|
||||
async def _validate_socks(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||
"""验证 SOCKS4/SOCKS5 代理"""
|
||||
proxy_type = (
|
||||
aiohttp_socks.ProxyType.SOCKS4
|
||||
if protocol == "socks4"
|
||||
@@ -103,7 +133,9 @@ class ValidatorService:
|
||||
return True, latency
|
||||
return False, 0.0
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> None:
|
||||
"""关闭共享的 HTTP ClientSession"""
|
||||
if self._http_session and not self._http_session.closed:
|
||||
await self._http_session.close()
|
||||
self._http_session = None
|
||||
self._http_connector = None
|
||||
|
||||
Reference in New Issue
Block a user