Files
ProxyPool/repositories/task_repo.py
祀梦 66943df864 实现插件配置持久化与任务队列持久化
插件配置持久化:
- plugin_settings 表新增 config_json 字段,支持存储每个插件的自定义配置
- BaseCrawlerPlugin 新增 default_config 属性和 update_config 方法
- PluginSettingsRepository 新增 get_config / set_config 方法
- PluginService 新增 get_plugin_config 和 update_plugin_config
- api/routes/plugins.py 新增 GET /{id}/config 和 POST /{id}/config 接口
- 前端 Plugins.vue 增加配置编辑对话框,支持动态渲染数字/布尔/字符串类型配置
- ip3366 插件示例化:增加 max_pages 配置项,验证配置生效后会动态更新爬取 URL

任务队列持久化:
- 新建 validation_tasks 表:id, ip, port, protocol, status, result, response_time_ms, created_at, updated_at
- 新建 ValidationTaskRepository,提供 insert_batch / acquire_pending / complete_task / reset_processing 等方法
- ValidationQueue 重构:
  - submit() 时把任务写入数据库并唤醒 Worker
  - Worker 通过 acquire_pending 原子取任务并验证
  - 验证完成后更新任务状态并入库有效代理
  - 启动时自动恢复之前中断的 processing 任务为 pending
  - 支持 drain() 等待所有 pending 完成
- 调度器验证流程同样自动持久化到任务表

其他适配:
- 更新 api/deps.py 和 api/lifespan.py,移除对已删除 settings_service 的残留引用
- 更新前端 pluginService.js 和 api/index.js 增加配置相关 API
2026-04-02 12:35:06 +08:00

136 lines
4.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""验证任务队列持久化层"""
import aiosqlite
from typing import List, Optional
from models.domain import ProxyRaw
from core.log import logger
class ValidationTaskRepository:
"""验证任务 Repository —— 支持队列持久化"""
@staticmethod
async def insert_batch(db: aiosqlite.Connection, proxies: List[ProxyRaw]) -> int:
if not proxies:
return 0
try:
rows = [(p.ip, p.port, p.protocol) for p in proxies]
await db.executemany(
"""
INSERT INTO validation_tasks (ip, port, protocol, status, created_at)
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
""",
rows,
)
await db.commit()
return len(rows)
except Exception as e:
logger.error(f"insert_batch validation tasks failed: {e}")
return 0
@staticmethod
async def acquire_pending(db: aiosqlite.Connection) -> Optional[dict]:
"""原子性地获取一个 pending 任务并将其标记为 processing"""
try:
async with db.execute(
"""
SELECT id, ip, port, protocol FROM validation_tasks
WHERE status = 'pending'
ORDER BY id ASC
LIMIT 1
"""
) as cursor:
row = await cursor.fetchone()
if not row:
return None
task_id = row[0]
await db.execute(
"UPDATE validation_tasks SET status = 'processing', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
(task_id,),
)
await db.commit()
return {"id": task_id, "ip": row[1], "port": row[2], "protocol": row[3]}
except Exception as e:
logger.error(f"acquire_pending failed: {e}")
return None
@staticmethod
async def complete_task(
db: aiosqlite.Connection,
task_id: int,
is_valid: bool,
response_time_ms: Optional[float] = None,
) -> bool:
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'completed',
result = ?,
response_time_ms = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
("valid" if is_valid else "invalid", response_time_ms, task_id),
)
await db.commit()
return True
except Exception as e:
logger.error(f"complete_task failed: {e}")
return False
@staticmethod
async def fail_task(db: aiosqlite.Connection, task_id: int) -> bool:
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'failed',
result = 'invalid',
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(task_id,),
)
await db.commit()
return True
except Exception as e:
logger.error(f"fail_task failed: {e}")
return False
@staticmethod
async def get_pending_count(db: aiosqlite.Connection) -> int:
async with db.execute(
"SELECT COUNT(*) FROM validation_tasks WHERE status = 'pending'"
) as cursor:
row = await cursor.fetchone()
return row[0] if row else 0
@staticmethod
async def reset_processing(db: aiosqlite.Connection) -> int:
"""将异常中断的 processing 任务重置为 pending用于启动恢复"""
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'pending', updated_at = CURRENT_TIMESTAMP
WHERE status = 'processing'
"""
)
await db.commit()
return db.total_changes
except Exception as e:
logger.error(f"reset_processing failed: {e}")
return 0
@staticmethod
async def cleanup_old(db: aiosqlite.Connection, days: int = 7) -> int:
try:
await db.execute(
"DELETE FROM validation_tasks WHERE updated_at < datetime('now', '-{} days')".format(days)
)
await db.commit()
return db.total_changes
except Exception as e:
logger.error(f"cleanup_old tasks failed: {e}")
return 0