重构代理池系统：简化架构并增强核心功能
后端变更： - 移除 tasks_manager.py 和 core/auth.py，简化架构 - 新增 core/scheduler.py 验证调度器，替代原有任务管理 - 大幅优化 api_server.py：统一错误处理、增强参数验证、支持调度器控制 - validator.py 增强 SOCKS4/SOCKS5 代理验证支持 - config.py 清理废弃配置（WebSocket、API Key、认证开关） - SQLite 数据库操作性能优化 前端变更： - 移除任务管理页面 (CrawlerTasks) 和 WebSocket 相关代码 - 路由简化为 4 个核心页面：总览、代理列表、插件管理、设置 - 提取前端工具函数（clipboard、confirm、format）和 API 类型定义 - 优化 CSS 架构：完善 variables、utilities、element-plus 样式 - Dashboard、Plugins、ProxyList、Settings 页面 UI/UX 优化 - App.vue 响应式侧边栏和页面过渡动画优化 其他： - 移除 PowerShell 启动脚本，简化 Windows 批处理脚本 - 新增 README_SOCKS.md SOCKS 代理支持文档 - .env.example 和 .gitignore 更新
This commit is contained in:
祀梦
114
core/auth.py
114
core/auth.py
@@ -1,114 +0,0 @@
|
||||
from fastapi import HTTPException, Depends, Header, status
|
||||
from typing import Optional
|
||||
from config import Config
|
||||
from core.log import logger
|
||||
|
||||
class PermissionLevel:
|
||||
READ_ONLY = "read_only"
|
||||
ADMIN = "admin"
|
||||
|
||||
def verify_api_key(
|
||||
x_api_key: Optional[str] = Header(None, alias="X-API-Key"),
|
||||
authorization: Optional[str] = Header(None)
|
||||
) -> str:
|
||||
"""
|
||||
验证API Key并返回权限级别
|
||||
|
||||
Args:
|
||||
x_api_key: X-API-Key header中的API Key
|
||||
authorization: Authorization header中的Bearer token
|
||||
|
||||
Returns:
|
||||
str: 权限级别
|
||||
|
||||
Raises:
|
||||
HTTPException: 认证失败时抛出401错误
|
||||
"""
|
||||
api_key = x_api_key
|
||||
|
||||
if authorization and authorization.startswith("Bearer "):
|
||||
api_key = authorization.replace("Bearer ", "")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("API请求缺少API Key")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="缺少API Key，请在请求头中添加 X-API-Key 或 Authorization: Bearer <key>",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
if api_key == Config.ADMIN_API_KEY:
|
||||
logger.info(f"管理员API认证成功: {api_key[:8]}...")
|
||||
return PermissionLevel.ADMIN
|
||||
elif api_key == Config.API_KEY:
|
||||
logger.info(f"普通用户API认证成功: {api_key[:8]}...")
|
||||
return PermissionLevel.READ_ONLY
|
||||
else:
|
||||
logger.warning(f"无效的API Key尝试: {api_key[:8]}...")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="无效的API Key",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
def require_admin(
|
||||
x_api_key: Optional[str] = Header(None, alias="X-API-Key"),
|
||||
authorization: Optional[str] = Header(None)
|
||||
) -> str:
|
||||
"""
|
||||
要求管理员权限的依赖函数
|
||||
|
||||
Args:
|
||||
x_api_key: X-API-Key header中的API Key
|
||||
authorization: Authorization header中的Bearer token
|
||||
|
||||
Returns:
|
||||
str: 权限级别
|
||||
|
||||
Raises:
|
||||
HTTPException: 权限不足时抛出403错误
|
||||
"""
|
||||
# 如果未启用认证，直接返回管理员权限
|
||||
if not Config.REQUIRE_AUTH:
|
||||
logger.info("开发模式：跳过管理员权限检查")
|
||||
return PermissionLevel.ADMIN
|
||||
|
||||
# 验证API Key
|
||||
api_key = x_api_key
|
||||
|
||||
if authorization and authorization.startswith("Bearer "):
|
||||
api_key = authorization.replace("Bearer ", "")
|
||||
|
||||
if not api_key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="缺少API Key，请在请求头中添加 X-API-Key 或 Authorization: Bearer <key>",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
# 检查权限级别
|
||||
if api_key == Config.ADMIN_API_KEY:
|
||||
logger.info(f"管理员API认证成功: {api_key[:8]}...")
|
||||
return PermissionLevel.ADMIN
|
||||
else:
|
||||
logger.warning(f"非管理员用户尝试访问管理接口")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="需要管理员权限才能执行此操作"
|
||||
)
|
||||
|
||||
def skip_auth_for_dev() -> Optional[str]:
|
||||
"""
|
||||
开发环境跳过认证（仅在开发模式下使用）
|
||||
|
||||
Returns:
|
||||
Optional[str]: 返回管理员权限级别
|
||||
|
||||
Warning:
|
||||
仅用于开发环境，生产环境务必使用真实认证
|
||||
"""
|
||||
import os
|
||||
if os.getenv("SKIP_AUTH", "false").lower() == "true":
|
||||
logger.warning("开发模式：跳过API Key认证")
|
||||
return PermissionLevel.ADMIN
|
||||
return None
|
||||
206
core/scheduler.py
Normal file
206
core/scheduler.py
Normal file
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
代理验证调度器
|
||||
负责定期验证数据库中的代理，并更新分数
|
||||
"""
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
from core.sqlite import SQLiteManager
|
||||
from core.validator import ProxyValidator
|
||||
from core.log import logger
|
||||
from config import config
|
||||
|
||||
|
||||
class ValidationScheduler:
|
||||
"""代理验证调度器"""
|
||||
|
||||
def __init__(self):
|
||||
self.db = SQLiteManager()
|
||||
self.validator: Optional[ProxyValidator] = None
|
||||
self.running = False
|
||||
self.task: Optional[asyncio.Task] = None
|
||||
self.interval_minutes = 30 # 默认每30分钟验证一次
|
||||
self.batch_size = 100 # 每批验证数量
|
||||
|
||||
async def start(self):
|
||||
"""启动验证调度器"""
|
||||
if self.running:
|
||||
logger.warning("验证调度器已在运行")
|
||||
return
|
||||
|
||||
self.running = True
|
||||
self.validator = ProxyValidator(
|
||||
max_concurrency=config.VALIDATOR_MAX_CONCURRENCY,
|
||||
timeout=config.VALIDATOR_TIMEOUT
|
||||
)
|
||||
self.task = asyncio.create_task(self._run_loop())
|
||||
logger.info("代理验证调度器已启动")
|
||||
|
||||
async def stop(self):
|
||||
"""停止验证调度器"""
|
||||
self.running = False
|
||||
if self.task:
|
||||
self.task.cancel()
|
||||
try:
|
||||
await self.task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if self.validator:
|
||||
await self.validator.__aexit__(None, None, None)
|
||||
logger.info("代理验证调度器已停止")
|
||||
|
||||
async def _run_loop(self):
|
||||
"""运行循环"""
|
||||
while self.running:
|
||||
try:
|
||||
await self.validate_all_proxies()
|
||||
except Exception as e:
|
||||
logger.error(f"验证循环出错: {e}")
|
||||
|
||||
# 等待下一次验证
|
||||
await asyncio.sleep(self.interval_minutes * 60)
|
||||
|
||||
async def validate_all_proxies(self):
|
||||
"""验证所有代理"""
|
||||
logger.info("开始批量验证代理...")
|
||||
|
||||
try:
|
||||
# 获取所有代理
|
||||
proxies = await self.db.get_all_proxies()
|
||||
if not proxies:
|
||||
logger.info("数据库中没有代理需要验证")
|
||||
return
|
||||
|
||||
logger.info(f"需要验证 {len(proxies)} 个代理")
|
||||
|
||||
# 分批验证
|
||||
validated_count = 0
|
||||
valid_count = 0
|
||||
invalid_count = 0
|
||||
|
||||
async with self.validator:
|
||||
for i in range(0, len(proxies), self.batch_size):
|
||||
if not self.running:
|
||||
break
|
||||
|
||||
batch = proxies[i:i + self.batch_size]
|
||||
tasks = []
|
||||
|
||||
for proxy in batch:
|
||||
ip, port, protocol, score, last_check = proxy
|
||||
task = self._validate_and_update(ip, port, protocol)
|
||||
tasks.append(task)
|
||||
|
||||
# 并发验证一批
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for result in results:
|
||||
validated_count += 1
|
||||
if isinstance(result, Exception):
|
||||
logger.error(f"验证过程出错: {result}")
|
||||
continue
|
||||
if result:
|
||||
valid_count += 1
|
||||
else:
|
||||
invalid_count += 1
|
||||
|
||||
logger.info(f"已验证 {validated_count}/{len(proxies)} 个代理")
|
||||
|
||||
# 批次间短暂延迟，避免过载
|
||||
if i + self.batch_size < len(proxies):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info(f"验证完成: 总计 {validated_count}, 有效 {valid_count}, 无效 {invalid_count}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"批量验证代理失败: {e}", exc_info=True)
|
||||
|
||||
async def _validate_and_update(self, ip: str, port: int, protocol: str) -> bool:
|
||||
"""验证单个代理并更新分数"""
|
||||
try:
|
||||
is_valid, latency = await self.validator.validate(ip, port, protocol)
|
||||
|
||||
if is_valid:
|
||||
# 验证成功，增加分数
|
||||
await self.db.update_score(
|
||||
ip, port,
|
||||
config.SCORE_VALID,
|
||||
min_score=config.SCORE_MIN,
|
||||
max_score=config.SCORE_MAX
|
||||
)
|
||||
logger.debug(f"代理验证成功 {ip}:{port} ({protocol}) - 延迟 {latency}ms")
|
||||
return True
|
||||
else:
|
||||
# 验证失败，减少分数
|
||||
await self.db.update_score(
|
||||
ip, port,
|
||||
config.SCORE_INVALID,
|
||||
min_score=config.SCORE_MIN,
|
||||
max_score=config.SCORE_MAX
|
||||
)
|
||||
logger.debug(f"代理验证失败 {ip}:{port} ({protocol})")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"验证代理 {ip}:{port} 时出错: {e}")
|
||||
# 出错也视为失败
|
||||
await self.db.update_score(
|
||||
ip, port,
|
||||
config.SCORE_INVALID,
|
||||
min_score=config.SCORE_MIN,
|
||||
max_score=config.SCORE_MAX
|
||||
)
|
||||
return False
|
||||
|
||||
async def validate_proxies_batch(self, proxies: list) -> tuple:
|
||||
"""
|
||||
验证一批新抓取的代理
|
||||
|
||||
Args:
|
||||
proxies: [(ip, port, protocol), ...]
|
||||
|
||||
Returns:
|
||||
(有效代理列表, 无效代理列表)
|
||||
"""
|
||||
if not proxies:
|
||||
return [], []
|
||||
|
||||
valid_proxies = []
|
||||
invalid_proxies = []
|
||||
|
||||
logger.info(f"开始验证 {len(proxies)} 个新抓取代理...")
|
||||
|
||||
try:
|
||||
validator = ProxyValidator(
|
||||
max_concurrency=min(config.VALIDATOR_MAX_CONCURRENCY, 50),
|
||||
timeout=config.VALIDATOR_TIMEOUT
|
||||
)
|
||||
|
||||
async with validator:
|
||||
tasks = []
|
||||
for ip, port, protocol in proxies:
|
||||
task = validator.validate(ip, port, protocol)
|
||||
tasks.append((ip, port, protocol, task))
|
||||
|
||||
for ip, port, protocol, task in tasks:
|
||||
try:
|
||||
is_valid, latency = await task
|
||||
if is_valid:
|
||||
valid_proxies.append((ip, port, protocol))
|
||||
logger.debug(f"新代理有效: {ip}:{port} ({protocol}) - {latency}ms")
|
||||
else:
|
||||
invalid_proxies.append((ip, port, protocol))
|
||||
except Exception as e:
|
||||
logger.warning(f"验证新代理 {ip}:{port} 失败: {e}")
|
||||
invalid_proxies.append((ip, port, protocol))
|
||||
|
||||
logger.info(f"新代理验证完成: 有效 {len(valid_proxies)}, 无效 {len(invalid_proxies)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"批量验证新代理失败: {e}")
|
||||
|
||||
return valid_proxies, invalid_proxies
|
||||
|
||||
|
||||
# 全局调度器实例
|
||||
scheduler = ValidationScheduler()
|
||||
@@ -1,12 +1,16 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import aiohttp_socks
|
||||
import random
|
||||
import time
|
||||
from core.log import logger
|
||||
|
||||
|
||||
class ProxyValidator:
|
||||
"""代理验证器 - 支持 HTTP/HTTPS/SOCKS4/SOCKS5"""
|
||||
|
||||
def __init__(self, max_concurrency=50, timeout=5):
|
||||
# 验证目标源（使用更适合代理验证的源）
|
||||
# 验证目标源
|
||||
self.http_sources = [
|
||||
"http://httpbin.org/ip",
|
||||
"http://api.ipify.org"
|
||||
@@ -20,57 +24,169 @@ class ProxyValidator:
|
||||
self.session = None
|
||||
|
||||
async def __aenter__(self):
|
||||
# 允许通过 async with 管理 session
|
||||
if not self.session:
|
||||
self.session = aiohttp.ClientSession(
|
||||
connector=aiohttp.TCPConnector(ssl=False, limit=0, force_close=True),
|
||||
timeout=aiohttp.ClientTimeout(total=self.timeout, connect=3)
|
||||
)
|
||||
"""异步上下文管理器入口"""
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
"""异步上下文管理器出口"""
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
self.session = None
|
||||
|
||||
async def validate(self, ip, port, protocol='http'):
|
||||
def _get_test_url(self, protocol: str) -> str:
|
||||
"""根据协议获取测试 URL"""
|
||||
protocol = protocol.lower()
|
||||
if protocol == 'https':
|
||||
return random.choice(self.https_sources)
|
||||
return random.choice(self.http_sources)
|
||||
|
||||
def _create_connector(self, ip: str, port: int, protocol: str):
|
||||
"""创建代理连接器"""
|
||||
protocol = protocol.lower()
|
||||
|
||||
if protocol == 'socks4':
|
||||
return aiohttp_socks.ProxyConnector(
|
||||
proxy_type=aiohttp_socks.ProxyType.SOCKS4,
|
||||
host=ip,
|
||||
port=port,
|
||||
rdns=True
|
||||
)
|
||||
elif protocol == 'socks5':
|
||||
return aiohttp_socks.ProxyConnector(
|
||||
proxy_type=aiohttp_socks.ProxyType.SOCKS5,
|
||||
host=ip,
|
||||
port=port,
|
||||
rdns=True
|
||||
)
|
||||
elif protocol in ('http', 'https'):
|
||||
# HTTP/HTTPS 使用普通 connector，在请求时指定 proxy 参数
|
||||
return aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
||||
else:
|
||||
# 未知协议默认使用 HTTP
|
||||
return aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
||||
|
||||
async def validate(self, ip: str, port: int, protocol: str = 'http'):
|
||||
"""
|
||||
验证单个代理是否可用
|
||||
|
||||
Args:
|
||||
ip: 代理 IP
|
||||
port: 代理端口
|
||||
protocol: 协议类型 (http/https/socks4/socks5)
|
||||
|
||||
Returns:
|
||||
(is_valid: bool, latency_ms: float)
|
||||
"""
|
||||
protocol = protocol.lower()
|
||||
sources = self.https_sources if protocol == 'https' else self.http_sources
|
||||
test_url = random.choice(sources)
|
||||
|
||||
# aiohttp 代理 URL 格式
|
||||
proxy_url = f"http://{ip}:{port}"
|
||||
test_url = self._get_test_url(protocol)
|
||||
|
||||
async with self.semaphore:
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# 复用 session
|
||||
async with self.session.get(
|
||||
test_url,
|
||||
proxy=proxy_url,
|
||||
allow_redirects=True,
|
||||
timeout=aiohttp.ClientTimeout(total=self.timeout, connect=3)
|
||||
) as response:
|
||||
# 检查状态码和响应内容
|
||||
if response.status in [200, 301, 302]:
|
||||
try:
|
||||
content = await response.text()
|
||||
# 确保返回了有效的JSON响应
|
||||
if 'ip' in content.lower() or 'origin' in content.lower():
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
except:
|
||||
# 即使无法解析内容，如果状态码正常也认为可用
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
return False, 0
|
||||
if protocol in ('socks4', 'socks5'):
|
||||
return await self._validate_socks(ip, port, protocol, test_url, start_time)
|
||||
else:
|
||||
return await self._validate_http(ip, port, protocol, test_url, start_time)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f"验证超时: {ip}:{port} ({protocol})")
|
||||
return False, 0
|
||||
except Exception as e:
|
||||
logger.warning(f"验证失败: {ip}:{port} ({protocol}) - {e}")
|
||||
return False, 0
|
||||
|
||||
async def _validate_http(self, ip: str, port: int, protocol: str, test_url: str, start_time: float):
|
||||
"""验证 HTTP/HTTPS 代理"""
|
||||
proxy_url = f"http://{ip}:{port}"
|
||||
|
||||
connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=3)
|
||||
|
||||
async with aiohttp.ClientSession(
|
||||
connector=connector,
|
||||
timeout=timeout
|
||||
) as session:
|
||||
async with session.get(
|
||||
test_url,
|
||||
proxy=proxy_url,
|
||||
allow_redirects=True
|
||||
) as response:
|
||||
if response.status in [200, 301, 302]:
|
||||
try:
|
||||
content = await response.text()
|
||||
if 'ip' in content.lower() or 'origin' in content.lower():
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
except:
|
||||
pass
|
||||
|
||||
# 内容解析失败但状态码正常，也算可用
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
|
||||
return False, 0
|
||||
|
||||
async def _validate_socks(self, ip: str, port: int, protocol: str, test_url: str, start_time: float):
|
||||
"""验证 SOCKS4/SOCKS5 代理"""
|
||||
proxy_type = (
|
||||
aiohttp_socks.ProxyType.SOCKS4
|
||||
if protocol == 'socks4'
|
||||
else aiohttp_socks.ProxyType.SOCKS5
|
||||
)
|
||||
|
||||
connector = aiohttp_socks.ProxyConnector(
|
||||
proxy_type=proxy_type,
|
||||
host=ip,
|
||||
port=port,
|
||||
rdns=True, # 远程 DNS 解析，避免 DNS 泄漏
|
||||
ssl=False
|
||||
)
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=3)
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(
|
||||
connector=connector,
|
||||
timeout=timeout
|
||||
) as session:
|
||||
async with session.get(test_url, allow_redirects=True) as response:
|
||||
if response.status in [200, 301, 302]:
|
||||
try:
|
||||
content = await response.text()
|
||||
if 'ip' in content.lower() or 'origin' in content.lower():
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
except:
|
||||
pass
|
||||
|
||||
# 内容解析失败但状态码正常
|
||||
latency = round((time.time() - start_time) * 1000, 2)
|
||||
logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms")
|
||||
return True, latency
|
||||
|
||||
return False, 0
|
||||
finally:
|
||||
await connector.close()
|
||||
|
||||
|
||||
class ProxyValidatorLegacy:
|
||||
"""
|
||||
兼容旧版本的验证器
|
||||
保持原有接口不变
|
||||
"""
|
||||
def __init__(self, max_concurrency=50, timeout=5):
|
||||
self.validator = ProxyValidator(max_concurrency, timeout)
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.validator.__aenter__()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
await self.validator.__aexit__(exc_type, exc_val, exc_tb)
|
||||
|
||||
async def validate(self, ip, port, protocol='http'):
|
||||
return await self.validator.validate(ip, port, protocol)
|
||||
|
||||
Reference in New Issue
Block a user