fix: 全面修复代码问题并优化架构
修复问题: - 添加缺失的 httpx 依赖到 requirements.txt - 修复前端批量删除参数格式与后端不匹配(数组->对象数组) - 移除 app/api/main.py 中重复创建 app 的冗余代码 - 修复 Plugins.vue v-model 直接修改 store 状态的 Vue 警告 - 修复 README 端口/启动命令文档与实际配置不一致 - 修正 pytest.ini 过时配置 (asyncio_default_fixture_loop_scope) - 修复 WebUI index.html 语言设置为 zh-CN - 修复 .gitignore 错误忽略 tests/ 目录 后端优化: - 修复调度器默认间隔从 5 秒改为 30 分钟,避免无节制验证 - 修复 validate_all_now 在调度器停止时无法执行的 bug - 设置保存后热更新运行中调度器的验证间隔 - 将 update_score 优化为原子单事务 SQL,消除并发竞态 - 导出功能改为真正的流式分批读取(iter_batches),降低大导出内存占用 - ProxyResponse Schema 补齐 response_time_ms 字段 - 日志级别改为从配置动态读取,不再硬编码 INFO - 清理 validator_service 中的冗余 try/finally 代码 插件健壮性: - 修复 ip3366/ip89/kuaidaili/proxylist_download/speedx/yundaili/proxyscrape 的端口范围检查和 IPv6 地址解析问题(改用 rsplit + 1-65535 校验) - 修复 PluginService.list_plugins 并发竞争条件 - 修复 run_all_plugins 去重逻辑与数据库 UNIQUE 约束保持一致 - 修复 proxyscrape 异常时错误跳过 fallback 的 bug 测试: - 新增 7 个插件解析单元测试 - 新增 update_score 自动删除和 iter_batches 流式读取测试 - 全部 74 个测试通过
This commit is contained in:
@@ -76,23 +76,23 @@ class ProxyRepository:
|
||||
max_score: int = 100,
|
||||
) -> bool:
|
||||
try:
|
||||
async with db.execute(
|
||||
"SELECT score FROM proxies WHERE ip = ? AND port = ?", (ip, port)
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
current_score = row[0]
|
||||
new_score = max(min_score, min(max_score, current_score + delta))
|
||||
await db.execute(
|
||||
"UPDATE proxies SET score = ?, last_check = CURRENT_TIMESTAMP WHERE ip = ? AND port = ?",
|
||||
(new_score, ip, port),
|
||||
)
|
||||
if new_score <= 0:
|
||||
# 只删除当前代理,避免误删其他无效代理
|
||||
await db.execute("DELETE FROM proxies WHERE ip = ? AND port = ?", (ip, port))
|
||||
await db.commit()
|
||||
return True
|
||||
# 原子更新:计算新分数并直接更新
|
||||
await db.execute(
|
||||
"""
|
||||
UPDATE proxies
|
||||
SET score = MAX(?, MIN(?, score + ?)),
|
||||
last_check = CURRENT_TIMESTAMP
|
||||
WHERE ip = ? AND port = ?
|
||||
""",
|
||||
(min_score, max_score, delta, ip, port),
|
||||
)
|
||||
# 删除分数已降至 0 及以下的代理
|
||||
await db.execute(
|
||||
"DELETE FROM proxies WHERE ip = ? AND port = ? AND score <= ?",
|
||||
(ip, port, min_score),
|
||||
)
|
||||
await db.commit()
|
||||
return db.total_changes > 0
|
||||
except Exception as e:
|
||||
logger.error(f"update_score failed: {e}", exc_info=True)
|
||||
return False
|
||||
@@ -156,19 +156,35 @@ class ProxyRepository:
|
||||
db: aiosqlite.Connection,
|
||||
protocol: Optional[str] = None,
|
||||
limit: int = 100000,
|
||||
offset: int = 0,
|
||||
) -> List[Proxy]:
|
||||
query = "SELECT ip, port, protocol, score, response_time_ms, last_check, created_at FROM proxies"
|
||||
params: List = []
|
||||
if protocol:
|
||||
query += " WHERE protocol = ?"
|
||||
params.append(protocol.lower())
|
||||
query += " LIMIT ?"
|
||||
params.append(limit)
|
||||
query += " LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
async with db.execute(query, params) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [_row_to_proxy(row) for row in rows]
|
||||
|
||||
@staticmethod
|
||||
async def iter_batches(
|
||||
db: aiosqlite.Connection,
|
||||
protocol: Optional[str] = None,
|
||||
batch_size: int = 1000,
|
||||
):
|
||||
"""流式分批读取代理,避免一次性加载大量数据到内存"""
|
||||
offset = 0
|
||||
while True:
|
||||
batch = await ProxyRepository.list_all(db, protocol, batch_size, offset)
|
||||
if not batch:
|
||||
break
|
||||
yield batch
|
||||
offset += batch_size
|
||||
|
||||
@staticmethod
|
||||
async def list_paginated(
|
||||
db: aiosqlite.Connection,
|
||||
|
||||
Reference in New Issue
Block a user