66943df864
插件配置持久化：
- plugin_settings 表新增 config_json 字段，支持存储每个插件的自定义配置
- BaseCrawlerPlugin 新增 default_config 属性和 update_config 方法
- PluginSettingsRepository 新增 get_config / set_config 方法
- PluginService 新增 get_plugin_config 和 update_plugin_config
- api/routes/plugins.py 新增 GET /{id}/config 和 POST /{id}/config 接口
- 前端 Plugins.vue 增加配置编辑对话框，支持动态渲染数字/布尔/字符串类型配置
- ip3366 插件示例化：增加 max_pages 配置项，验证配置生效后会动态更新爬取 URL
任务队列持久化：
- 新建 validation_tasks 表：id, ip, port, protocol, status, result, response_time_ms, created_at, updated_at
- 新建 ValidationTaskRepository，提供 insert_batch / acquire_pending / complete_task / reset_processing 等方法
- ValidationQueue 重构：
- submit() 时把任务写入数据库并唤醒 Worker
- Worker 通过 acquire_pending 原子取任务并验证
- 验证完成后更新任务状态并入库有效代理
- 启动时自动恢复之前中断的 processing 任务为 pending
- 支持 drain() 等待所有 pending 完成
- 调度器验证流程同样自动持久化到任务表
其他适配：
- 更新 api/deps.py 和 api/lifespan.py，移除对已删除 settings_service 的残留引用
- 更新前端 pluginService.js 和 api/index.js 增加配置相关 API
121 lines
4.6 KiB
Python
121 lines
4.6 KiB
Python
"""数据库连接管理 - 使用上下文管理器，避免全局单例连接泄漏"""
|
|
import os
|
|
import aiosqlite
|
|
from contextlib import asynccontextmanager
|
|
from typing import AsyncIterator
|
|
from core.config import settings
|
|
from core.log import logger
|
|
|
|
|
|
DB_PATH = os.path.join(settings.base_dir, settings.db_path)
|
|
|
|
|
|
def ensure_db_dir():
|
|
db_dir = os.path.dirname(DB_PATH)
|
|
if db_dir and not os.path.exists(db_dir):
|
|
os.makedirs(db_dir, exist_ok=True)
|
|
|
|
|
|
async def init_db():
|
|
"""初始化数据库表结构（支持迁移）"""
|
|
ensure_db_dir()
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
await db.execute("PRAGMA journal_mode=WAL")
|
|
await db.execute("PRAGMA synchronous=NORMAL")
|
|
await db.execute("PRAGMA cache_size=-64000")
|
|
await db.execute("PRAGMA temp_store=MEMORY")
|
|
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS proxies (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
ip TEXT NOT NULL,
|
|
port INTEGER NOT NULL,
|
|
protocol TEXT DEFAULT 'http',
|
|
score INTEGER DEFAULT 10,
|
|
response_time_ms REAL,
|
|
last_check TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(ip, port)
|
|
)
|
|
""")
|
|
|
|
# 迁移：如果旧表缺少 response_time_ms 列，则添加
|
|
try:
|
|
await db.execute("SELECT response_time_ms FROM proxies LIMIT 1")
|
|
except Exception:
|
|
await db.execute("ALTER TABLE proxies ADD COLUMN response_time_ms REAL")
|
|
logger.info("Migrated: added response_time_ms column")
|
|
|
|
# 迁移：如果旧表缺少 created_at 列，则添加
|
|
try:
|
|
await db.execute("SELECT created_at FROM proxies LIMIT 1")
|
|
except Exception:
|
|
await db.execute("ALTER TABLE proxies ADD COLUMN created_at TIMESTAMP")
|
|
await db.execute("UPDATE proxies SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL")
|
|
logger.info("Migrated: added created_at column")
|
|
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_score ON proxies(score)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_protocol ON proxies(protocol)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_last_check ON proxies(last_check)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_ip_port ON proxies(ip, port)")
|
|
|
|
# 插件设置表
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS plugin_settings (
|
|
plugin_id TEXT PRIMARY KEY,
|
|
enabled INTEGER DEFAULT 1,
|
|
config_json TEXT DEFAULT '{}',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# 迁移：为旧版 plugin_settings 表增加 config_json 列
|
|
try:
|
|
await db.execute("SELECT config_json FROM plugin_settings LIMIT 1")
|
|
except Exception:
|
|
await db.execute("ALTER TABLE plugin_settings ADD COLUMN config_json TEXT DEFAULT '{}'")
|
|
logger.info("Migrated: added config_json column to plugin_settings")
|
|
|
|
# 验证任务队列表
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS validation_tasks (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
ip TEXT NOT NULL,
|
|
port INTEGER NOT NULL,
|
|
protocol TEXT DEFAULT 'http',
|
|
status TEXT DEFAULT 'pending',
|
|
result TEXT,
|
|
response_time_ms REAL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_validation_status ON validation_tasks(status)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_validation_created ON validation_tasks(created_at)")
|
|
|
|
# 系统设置表
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
await db.commit()
|
|
logger.info("Database initialized")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def get_db() -> AsyncIterator[aiosqlite.Connection]:
|
|
"""获取数据库连接的异步上下文管理器"""
|
|
ensure_db_dir()
|
|
db = await aiosqlite.connect(DB_PATH)
|
|
try:
|
|
await db.execute("PRAGMA journal_mode=WAL")
|
|
await db.execute("PRAGMA synchronous=NORMAL")
|
|
yield db
|
|
finally:
|
|
await db.close()
|