实现插件配置持久化与任务队列持久化

插件配置持久化:
- plugin_settings 表新增 config_json 字段,支持存储每个插件的自定义配置
- BaseCrawlerPlugin 新增 default_config 属性和 update_config 方法
- PluginSettingsRepository 新增 get_config / set_config 方法
- PluginService 新增 get_plugin_config 和 update_plugin_config
- api/routes/plugins.py 新增 GET /{id}/config 和 POST /{id}/config 接口
- 前端 Plugins.vue 增加配置编辑对话框,支持动态渲染数字/布尔/字符串类型配置
- ip3366 插件示例化:增加 max_pages 配置项,验证配置生效后会动态更新爬取 URL

任务队列持久化:
- 新建 validation_tasks 表:id, ip, port, protocol, status, result, response_time_ms, created_at, updated_at
- 新建 ValidationTaskRepository,提供 insert_batch / acquire_pending / complete_task / reset_processing 等方法
- ValidationQueue 重构:
  - submit() 时把任务写入数据库并唤醒 Worker
  - Worker 通过 acquire_pending 原子取任务并验证
  - 验证完成后更新任务状态并入库有效代理
  - 启动时自动恢复之前中断的 processing 任务为 pending
  - 支持 drain() 等待所有 pending 完成
- 调度器验证流程同样自动持久化到任务表

其他适配:
- 更新 api/deps.py 和 api/lifespan.py,移除对已删除 settings_service 的残留引用
- 更新前端 pluginService.js 和 api/index.js 增加配置相关 API
This commit is contained in:
祀梦
2026-04-02 12:35:06 +08:00
parent b77641f059
commit 66943df864
13 changed files with 472 additions and 73 deletions

View File

@@ -36,7 +36,6 @@ def create_scheduler_service() -> SchedulerService:
queue = ValidationQueue( queue = ValidationQueue(
validator=validator, validator=validator,
proxy_repo=proxy_repo, proxy_repo=proxy_repo,
db_ctx=get_db,
worker_count=app_settings.validator_max_concurrency, worker_count=app_settings.validator_max_concurrency,
score_valid=app_settings.score_valid, score_valid=app_settings.score_valid,
score_invalid=app_settings.score_invalid, score_invalid=app_settings.score_invalid,
@@ -44,7 +43,3 @@ def create_scheduler_service() -> SchedulerService:
score_max=app_settings.score_max, score_max=app_settings.score_max,
) )
return SchedulerService(validation_queue=queue, proxy_repo=proxy_repo) return SchedulerService(validation_queue=queue, proxy_repo=proxy_repo)
# 避免循环导入
from core.db import get_db

View File

@@ -2,7 +2,6 @@
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends
from services.plugin_service import PluginService from services.plugin_service import PluginService
from services.scheduler_service import SchedulerService from services.scheduler_service import SchedulerService
from models.schemas import PluginToggleRequest
from api.deps import get_plugin_service, get_scheduler_service from api.deps import get_plugin_service, get_scheduler_service
from core.log import logger from core.log import logger
@@ -26,7 +25,7 @@ async def list_plugins(service: PluginService = Depends(get_plugin_service)):
"plugins": [ "plugins": [
{ {
"id": p.id, "id": p.id,
"name": p.display_name, # 保持旧版本兼容name 用于展示 "name": p.display_name,
"display_name": p.display_name, "display_name": p.display_name,
"description": p.description, "description": p.description,
"enabled": p.enabled, "enabled": p.enabled,
@@ -43,18 +42,47 @@ async def list_plugins(service: PluginService = Depends(get_plugin_service)):
@router.put("/{plugin_id}/toggle") @router.put("/{plugin_id}/toggle")
async def toggle_plugin( async def toggle_plugin(
plugin_id: str, plugin_id: str,
request: PluginToggleRequest, request: dict,
service: PluginService = Depends(get_plugin_service), service: PluginService = Depends(get_plugin_service),
): ):
success = await service.toggle_plugin(plugin_id, request.enabled) enabled = request.get("enabled")
if enabled is None:
return error_response("缺少 enabled 参数", 400)
success = await service.toggle_plugin(plugin_id, enabled)
if not success: if not success:
return error_response("插件不存在", 404) return error_response("插件不存在", 404)
return success_response( return success_response(
f"插件 {plugin_id}{'启用' if request.enabled else '禁用'}", f"插件 {plugin_id}{'启用' if enabled else '禁用'}",
{"plugin_id": plugin_id, "enabled": request.enabled}, {"plugin_id": plugin_id, "enabled": enabled},
) )
@router.get("/{plugin_id}/config")
async def get_plugin_config(
plugin_id: str,
service: PluginService = Depends(get_plugin_service),
):
config = await service.get_plugin_config(plugin_id)
if config is None:
return error_response("插件不存在", 404)
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
@router.post("/{plugin_id}/config")
async def update_plugin_config(
plugin_id: str,
request: dict,
service: PluginService = Depends(get_plugin_service),
):
config = request.get("config", {})
if not isinstance(config, dict):
return error_response("config 必须是对象", 400)
success = await service.update_plugin_config(plugin_id, config)
if not success:
return error_response("插件不存在或配置无效", 404)
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
@router.post("/{plugin_id}/crawl") @router.post("/{plugin_id}/crawl")
async def crawl_plugin( async def crawl_plugin(
plugin_id: str, plugin_id: str,

View File

@@ -64,11 +64,36 @@ async def init_db():
CREATE TABLE IF NOT EXISTS plugin_settings ( CREATE TABLE IF NOT EXISTS plugin_settings (
plugin_id TEXT PRIMARY KEY, plugin_id TEXT PRIMARY KEY,
enabled INTEGER DEFAULT 1, enabled INTEGER DEFAULT 1,
config_json TEXT DEFAULT '{}',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) )
""") """)
# 迁移:为旧版 plugin_settings 表增加 config_json 列
try:
await db.execute("SELECT config_json FROM plugin_settings LIMIT 1")
except Exception:
await db.execute("ALTER TABLE plugin_settings ADD COLUMN config_json TEXT DEFAULT '{}'")
logger.info("Migrated: added config_json column to plugin_settings")
# 验证任务队列表
await db.execute("""
CREATE TABLE IF NOT EXISTS validation_tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ip TEXT NOT NULL,
port INTEGER NOT NULL,
protocol TEXT DEFAULT 'http',
status TEXT DEFAULT 'pending',
result TEXT,
response_time_ms REAL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
await db.execute("CREATE INDEX IF NOT EXISTS idx_validation_status ON validation_tasks(status)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_validation_created ON validation_tasks(created_at)")
# 系统设置表 # 系统设置表
await db.execute(""" await db.execute("""
CREATE TABLE IF NOT EXISTS settings ( CREATE TABLE IF NOT EXISTS settings (

View File

@@ -1,7 +1,7 @@
"""插件基类 - 所有爬虫插件必须继承此基类""" """插件基类 - 所有爬虫插件必须继承此基类"""
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
from typing import List from typing import List, Dict, Any
@dataclass @dataclass
@@ -30,6 +30,20 @@ class BaseCrawlerPlugin(ABC):
display_name: str = "" display_name: str = ""
description: str = "" description: str = ""
enabled: bool = True enabled: bool = True
default_config: Dict[str, Any] = {}
def __init__(self):
self._config: Dict[str, Any] = dict(self.default_config or {})
@property
def config(self) -> Dict[str, Any]:
return self._config
def update_config(self, updates: Dict[str, Any]) -> None:
"""更新插件配置,只覆盖存在的键"""
for key, value in updates.items():
if key in self._config:
self._config[key] = value
@abstractmethod @abstractmethod
async def crawl(self) -> List[ProxyRaw]: async def crawl(self) -> List[ProxyRaw]:

View File

@@ -1,24 +1,26 @@
"""验证任务队列 - 解耦爬取与验证,支持背压控制""" """验证任务队列 - 解耦爬取与验证,支持背压控制和持久化"""
import asyncio import asyncio
from typing import Optional from typing import Optional
from models.domain import ProxyRaw from models.domain import ProxyRaw
from repositories.task_repo import ValidationTaskRepository
from core.db import get_db
from core.log import logger from core.log import logger
class ValidationQueue: class ValidationQueue:
"""代理验证队列 """代理验证队列(支持持久化到 SQLite
工作流程: 工作流程:
1. 爬虫将原始代理 submit() 到队列 1. 爬虫将原始代理 submit() 到队列(写入数据库 + 内存信号)
2. Worker 池从队列消费并验证 2. Worker 池从数据库消费并验证
3. 验证通过的代理写入数据库 3. 验证通过的代理写入数据库
4. 服务重启时自动恢复未完成的 pending 任务
""" """
def __init__( def __init__(
self, self,
validator, validator,
proxy_repo, proxy_repo,
db_ctx,
worker_count: int = 50, worker_count: int = 50,
score_valid: int = 10, score_valid: int = 10,
score_invalid: int = -5, score_invalid: int = -5,
@@ -27,16 +29,17 @@ class ValidationQueue:
): ):
self.validator = validator self.validator = validator
self.proxy_repo = proxy_repo self.proxy_repo = proxy_repo
self.db_ctx = db_ctx self.task_repo = ValidationTaskRepository()
self.worker_count = worker_count self.worker_count = worker_count
self.score_valid = score_valid self.score_valid = score_valid
self.score_invalid = score_invalid self.score_invalid = score_invalid
self.score_min = score_min self.score_min = score_min
self.score_max = score_max self.score_max = score_max
self._queue: asyncio.Queue[Optional[ProxyRaw]] = asyncio.Queue() self._signal: asyncio.Queue[None] = asyncio.Queue()
self._workers: list[asyncio.Task] = [] self._workers: list[asyncio.Task] = []
self._running = False self._running = False
self._db_lock = asyncio.Lock()
# 统计 # 统计
self.valid_count = 0 self.valid_count = 0
@@ -46,6 +49,16 @@ class ValidationQueue:
if self._running: if self._running:
return return
self._running = True self._running = True
# 恢复之前中断的 processing 任务
async with get_db() as db:
recovered = await self.task_repo.reset_processing(db)
pending = await self.task_repo.get_pending_count(db)
if recovered:
logger.info(f"ValidationQueue recovered {recovered} interrupted tasks")
if pending:
logger.info(f"ValidationQueue has {pending} pending tasks to process")
for i in range(self.worker_count): for i in range(self.worker_count):
self._workers.append(asyncio.create_task(self._worker_loop(i))) self._workers.append(asyncio.create_task(self._worker_loop(i)))
logger.info(f"ValidationQueue started with {self.worker_count} workers") logger.info(f"ValidationQueue started with {self.worker_count} workers")
@@ -55,42 +68,60 @@ class ValidationQueue:
return return
self._running = False self._running = False
for _ in self._workers: for _ in self._workers:
self._queue.put_nowait(None) # sentinel self._signal.put_nowait(None) # sentinel
if self._workers: if self._workers:
await asyncio.gather(*self._workers, return_exceptions=True) await asyncio.gather(*self._workers, return_exceptions=True)
self._workers.clear() self._workers.clear()
logger.info("ValidationQueue stopped") logger.info("ValidationQueue stopped")
async def submit(self, proxies: list[ProxyRaw]): async def submit(self, proxies: list[ProxyRaw]):
"""提交代理到验证队列""" """提交代理到验证队列(持久化 + 唤醒 Worker"""
for p in proxies: async with self._db_lock:
await self._queue.put(p) async with get_db() as db:
inserted = await self.task_repo.insert_batch(db, proxies)
if inserted:
for _ in range(min(inserted, self.worker_count)):
self._signal.put_nowait(None)
async def submit_one(self, proxy: ProxyRaw): async def submit_one(self, proxy: ProxyRaw):
await self._queue.put(proxy) await self.submit([proxy])
async def drain(self): async def drain(self):
"""等待队列中当前所有任务处理完毕""" """等待队列中当前所有 pending 任务处理完毕"""
await self._queue.join() while True:
async with get_db() as db:
count = await self.task_repo.get_pending_count(db)
if count == 0:
break
await asyncio.sleep(0.5)
async def _worker_loop(self, worker_id: int): async def _worker_loop(self, worker_id: int):
while True: while True:
item = await self._queue.get() await self._signal.get()
if item is None: self._signal.task_done()
self._queue.task_done() if not self._running:
break break
try: await self._process_one_task(worker_id)
await self._validate_and_save(item)
except Exception as e:
logger.error(f"Worker {worker_id} validation error: {e}")
finally:
self._queue.task_done()
async def _validate_and_save(self, proxy: ProxyRaw): async def _process_one_task(self, worker_id: int):
"""从数据库取一个任务并验证"""
async with self._db_lock:
async with get_db() as db:
task = await self.task_repo.acquire_pending(db)
if not task:
return
proxy = ProxyRaw(task["ip"], task["port"], task["protocol"])
try:
is_valid, latency = await self.validator.validate( is_valid, latency = await self.validator.validate(
proxy.ip, proxy.port, proxy.protocol proxy.ip, proxy.port, proxy.protocol
) )
async with self.db_ctx() as db: except Exception as e:
logger.error(f"Worker {worker_id} validation error: {e}")
is_valid, latency = False, 0.0
async with self._db_lock:
async with get_db() as db:
if is_valid: if is_valid:
await self.proxy_repo.insert_or_update( await self.proxy_repo.insert_or_update(
db, proxy.ip, proxy.port, proxy.protocol, score=self.score_valid db, proxy.ip, proxy.port, proxy.protocol, score=self.score_valid
@@ -99,10 +130,11 @@ class ValidationQueue:
await self.proxy_repo.update_response_time( await self.proxy_repo.update_response_time(
db, proxy.ip, proxy.port, latency db, proxy.ip, proxy.port, latency
) )
await self.task_repo.complete_task(db, task["id"], True, latency)
self.valid_count += 1 self.valid_count += 1
logger.debug(f"ValidationQueue: valid {proxy.ip}:{proxy.port}") logger.debug(f"ValidationQueue: valid {proxy.ip}:{proxy.port}")
else: else:
# 对于新爬取的无效代理,不需要入库,直接丢弃 await self.task_repo.complete_task(db, task["id"], False, 0.0)
self.invalid_count += 1 self.invalid_count += 1
logger.debug(f"ValidationQueue: invalid {proxy.ip}:{proxy.port}") logger.debug(f"ValidationQueue: invalid {proxy.ip}:{proxy.port}")

View File

@@ -79,6 +79,8 @@ export const proxiesAPI = {
export const pluginsAPI = { export const pluginsAPI = {
getPlugins: () => api.get('/api/plugins'), getPlugins: () => api.get('/api/plugins'),
togglePlugin: (pluginId, enabled) => api.put(`/api/plugins/${pluginId}/toggle`, { enabled }), togglePlugin: (pluginId, enabled) => api.put(`/api/plugins/${pluginId}/toggle`, { enabled }),
getPluginConfig: (pluginId) => api.get(`/api/plugins/${pluginId}/config`),
updatePluginConfig: (pluginId, config) => api.post(`/api/plugins/${pluginId}/config`, { config }),
crawlPlugin: (pluginId) => api.post(`/api/plugins/${pluginId}/crawl`), crawlPlugin: (pluginId) => api.post(`/api/plugins/${pluginId}/crawl`),
crawlAll: () => api.post('/api/plugins/crawl-all') crawlAll: () => api.post('/api/plugins/crawl-all')
} }

View File

@@ -9,6 +9,14 @@ export const pluginService = {
return pluginsAPI.togglePlugin(pluginId, enabled) return pluginsAPI.togglePlugin(pluginId, enabled)
}, },
async getPluginConfig(pluginId) {
return pluginsAPI.getPluginConfig(pluginId)
},
async updatePluginConfig(pluginId, config) {
return pluginsAPI.updatePluginConfig(pluginId, config)
},
async crawlPlugin(pluginId) { async crawlPlugin(pluginId) {
return pluginsAPI.crawlPlugin(pluginId) return pluginsAPI.crawlPlugin(pluginId)
}, },

View File

@@ -74,11 +74,19 @@
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="操作" width="150" fixed="right" align="center"> <el-table-column label="操作" width="200" fixed="right" align="center">
<template #default="{ row }"> <template #default="{ row }">
<el-button <el-button
type="primary" type="primary"
size="small" size="small"
@click="handleOpenConfig(row)"
>
<el-icon class="btn-icon"><Setting /></el-icon>
配置
</el-button>
<el-button
type="success"
size="small"
@click="handleCrawl(row.id)" @click="handleCrawl(row.id)"
:loading="crawlingPlugin === row.id" :loading="crawlingPlugin === row.id"
:disabled="!row.enabled" :disabled="!row.enabled"
@@ -123,11 +131,50 @@
</template> </template>
</el-alert> </el-alert>
</el-card> </el-card>
<!-- 配置编辑对话框 -->
<el-dialog
v-model="configDialogVisible"
title="插件配置"
width="400px"
:close-on-click-modal="false"
>
<div v-if="currentPlugin">
<div class="config-plugin-name">{{ currentPlugin.name }}</div>
<el-form label-width="120px">
<el-form-item
v-for="(value, key) in configForm"
:key="key"
:label="String(key)"
>
<el-input-number
v-if="typeof value === 'number'"
v-model="configForm[key]"
:min="0"
style="width: 180px"
/>
<el-switch
v-else-if="typeof value === 'boolean'"
v-model="configForm[key]"
/>
<el-input
v-else
v-model="configForm[key]"
style="width: 180px"
/>
</el-form-item>
</el-form>
</div>
<template #footer>
<el-button @click="configDialogVisible = false">取消</el-button>
<el-button type="primary" @click="handleSaveConfig" :loading="savingConfig">保存</el-button>
</template>
</el-dialog>
</div> </div>
</template> </template>
<script setup> <script setup>
import { ref, onMounted } from 'vue' import { ref, reactive, onMounted } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus' import { ElMessage, ElMessageBox } from 'element-plus'
import { import {
Connection, Connection,
@@ -135,7 +182,8 @@ import {
Promotion, Promotion,
CircleCheck, CircleCheck,
CircleClose, CircleClose,
Box Box,
Setting
} from '@element-plus/icons-vue' } from '@element-plus/icons-vue'
import { usePluginsStore } from '../stores/plugins' import { usePluginsStore } from '../stores/plugins'
import { pluginService } from '../services/pluginService' import { pluginService } from '../services/pluginService'
@@ -147,6 +195,12 @@ const crawlingPlugin = ref(null)
const crawlingAll = ref(false) const crawlingAll = ref(false)
const lastCrawlResult = ref(null) const lastCrawlResult = ref(null)
// 配置对话框
const configDialogVisible = ref(false)
const currentPlugin = ref(null)
const configForm = reactive({})
const savingConfig = ref(false)
// ==================== 事件处理 ==================== // ==================== 事件处理 ====================
async function handleRefresh() { async function handleRefresh() {
await pluginsStore.fetchPlugins() await pluginsStore.fetchPlugins()
@@ -158,11 +212,40 @@ async function handleToggle(pluginId, enabled) {
if (success) { if (success) {
ElMessage.success(enabled ? '插件已启用' : '插件已禁用') ElMessage.success(enabled ? '插件已启用' : '插件已禁用')
} else { } else {
// 失败时刷新列表恢复状态
await pluginsStore.fetchPlugins() await pluginsStore.fetchPlugins()
} }
} }
async function handleOpenConfig(row) {
currentPlugin.value = row
const response = await pluginService.getPluginConfig(row.id)
if (response.code === 200) {
Object.keys(configForm).forEach(key => delete configForm[key])
Object.assign(configForm, response.data.config || {})
configDialogVisible.value = true
} else {
ElMessage.error('获取插件配置失败')
}
}
async function handleSaveConfig() {
if (!currentPlugin.value) return
savingConfig.value = true
try {
const response = await pluginService.updatePluginConfig(currentPlugin.value.id, { ...configForm })
if (response.code === 200) {
ElMessage.success('配置保存成功')
configDialogVisible.value = false
} else {
ElMessage.error(response.message || '保存失败')
}
} catch (error) {
ElMessage.error('保存配置出错')
} finally {
savingConfig.value = false
}
}
async function handleCrawl(pluginId) { async function handleCrawl(pluginId) {
try { try {
crawlingPlugin.value = pluginId crawlingPlugin.value = pluginId
@@ -176,7 +259,6 @@ async function handleCrawl(pluginId) {
message: response.message, message: response.message,
data: response.data data: response.data
} }
// 刷新插件统计
await pluginsStore.fetchPlugins() await pluginsStore.fetchPlugins()
} else { } else {
lastCrawlResult.value = { lastCrawlResult.value = {
@@ -196,7 +278,6 @@ async function handleCrawl(pluginId) {
async function handleCrawlAll() { async function handleCrawlAll() {
try { try {
// 确认是否爬取所有插件
const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled) const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled)
if (enabledPlugins.length === 0) { if (enabledPlugins.length === 0) {
ElMessage.warning('没有启用的插件') ElMessage.warning('没有启用的插件')
@@ -225,7 +306,6 @@ async function handleCrawlAll() {
data: response.data data: response.data
} }
ElMessage.success('批量爬取完成') ElMessage.success('批量爬取完成')
// 刷新插件统计
await pluginsStore.fetchPlugins() await pluginsStore.fetchPlugins()
} else { } else {
lastCrawlResult.value = { lastCrawlResult.value = {
@@ -373,4 +453,11 @@ onMounted(async () => {
.invalid-count { .invalid-count {
color: var(--danger); color: var(--danger);
} }
.config-plugin-name {
font-size: 16px;
font-weight: 600;
margin-bottom: 16px;
color: var(--text-primary);
}
</style> </style>

View File

@@ -12,13 +12,18 @@ class Ip3366Plugin(BaseHTTPPlugin):
name = "ip3366" name = "ip3366"
display_name = "IP3366" display_name = "IP3366"
description = "从 IP3366 网站爬取免费代理" description = "从 IP3366 网站爬取免费代理"
default_config = {"max_pages": 5}
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self._update_urls()
def _update_urls(self):
max_pages = self.config.get("max_pages", 5)
self.urls = [ self.urls = [
f"http://www.ip3366.net/free/?stype=1&page={i}" for i in range(1, 6) f"http://www.ip3366.net/free/?stype=1&page={i}" for i in range(1, max_pages + 1)
] + [ ] + [
f"http://www.ip3366.net/free/?stype=2&page={i}" for i in range(1, 6) f"http://www.ip3366.net/free/?stype=2&page={i}" for i in range(1, max_pages + 1)
] ]
async def crawl(self) -> List[ProxyRaw]: async def crawl(self) -> List[ProxyRaw]:

View File

@@ -1,3 +1,5 @@
from .proxy_repo import ProxyRepository from .proxy_repo import ProxyRepository
from .settings_repo import SettingsRepository, PluginSettingsRepository
from .task_repo import ValidationTaskRepository
__all__ = ["ProxyRepository"] __all__ = ["ProxyRepository", "SettingsRepository", "PluginSettingsRepository", "ValidationTaskRepository"]

View File

@@ -93,10 +93,48 @@ class PluginSettingsRepository:
return False return False
@staticmethod @staticmethod
async def list_all(db: aiosqlite.Connection) -> Dict[str, bool]: async def get_config(db: aiosqlite.Connection, plugin_id: str) -> Optional[Dict[str, Any]]:
async with db.execute(
"SELECT config_json FROM plugin_settings WHERE plugin_id = ?", (plugin_id,)
) as cursor:
row = await cursor.fetchone()
if row and row[0]:
try:
return json.loads(row[0])
except json.JSONDecodeError:
return None
return None
@staticmethod
async def set_config(db: aiosqlite.Connection, plugin_id: str, config: Dict[str, Any]) -> bool:
try:
await db.execute(
"""
INSERT INTO plugin_settings (plugin_id, config_json, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(plugin_id) DO UPDATE SET
config_json = excluded.config_json,
updated_at = CURRENT_TIMESTAMP
""",
(plugin_id, json.dumps(config, ensure_ascii=False)),
)
await db.commit()
return True
except Exception as e:
logger.error(f"set_config failed for {plugin_id}: {e}")
return False
@staticmethod
async def list_all(db: aiosqlite.Connection) -> Dict[str, Dict[str, Any]]:
result = {} result = {}
async with db.execute("SELECT plugin_id, enabled FROM plugin_settings") as cursor: async with db.execute("SELECT plugin_id, enabled, config_json FROM plugin_settings") as cursor:
rows = await cursor.fetchall() rows = await cursor.fetchall()
for plugin_id, enabled in rows: for plugin_id, enabled, config_json in rows:
result[plugin_id] = bool(enabled) config = {}
if config_json:
try:
config = json.loads(config_json)
except json.JSONDecodeError:
pass
result[plugin_id] = {"enabled": bool(enabled), "config": config}
return result return result

135
repositories/task_repo.py Normal file
View File

@@ -0,0 +1,135 @@
"""验证任务队列持久化层"""
import aiosqlite
from typing import List, Optional
from models.domain import ProxyRaw
from core.log import logger
class ValidationTaskRepository:
"""验证任务 Repository —— 支持队列持久化"""
@staticmethod
async def insert_batch(db: aiosqlite.Connection, proxies: List[ProxyRaw]) -> int:
if not proxies:
return 0
try:
rows = [(p.ip, p.port, p.protocol) for p in proxies]
await db.executemany(
"""
INSERT INTO validation_tasks (ip, port, protocol, status, created_at)
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
""",
rows,
)
await db.commit()
return len(rows)
except Exception as e:
logger.error(f"insert_batch validation tasks failed: {e}")
return 0
@staticmethod
async def acquire_pending(db: aiosqlite.Connection) -> Optional[dict]:
"""原子性地获取一个 pending 任务并将其标记为 processing"""
try:
async with db.execute(
"""
SELECT id, ip, port, protocol FROM validation_tasks
WHERE status = 'pending'
ORDER BY id ASC
LIMIT 1
"""
) as cursor:
row = await cursor.fetchone()
if not row:
return None
task_id = row[0]
await db.execute(
"UPDATE validation_tasks SET status = 'processing', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
(task_id,),
)
await db.commit()
return {"id": task_id, "ip": row[1], "port": row[2], "protocol": row[3]}
except Exception as e:
logger.error(f"acquire_pending failed: {e}")
return None
@staticmethod
async def complete_task(
db: aiosqlite.Connection,
task_id: int,
is_valid: bool,
response_time_ms: Optional[float] = None,
) -> bool:
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'completed',
result = ?,
response_time_ms = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
("valid" if is_valid else "invalid", response_time_ms, task_id),
)
await db.commit()
return True
except Exception as e:
logger.error(f"complete_task failed: {e}")
return False
@staticmethod
async def fail_task(db: aiosqlite.Connection, task_id: int) -> bool:
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'failed',
result = 'invalid',
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(task_id,),
)
await db.commit()
return True
except Exception as e:
logger.error(f"fail_task failed: {e}")
return False
@staticmethod
async def get_pending_count(db: aiosqlite.Connection) -> int:
async with db.execute(
"SELECT COUNT(*) FROM validation_tasks WHERE status = 'pending'"
) as cursor:
row = await cursor.fetchone()
return row[0] if row else 0
@staticmethod
async def reset_processing(db: aiosqlite.Connection) -> int:
"""将异常中断的 processing 任务重置为 pending用于启动恢复"""
try:
await db.execute(
"""
UPDATE validation_tasks
SET status = 'pending', updated_at = CURRENT_TIMESTAMP
WHERE status = 'processing'
"""
)
await db.commit()
return db.total_changes
except Exception as e:
logger.error(f"reset_processing failed: {e}")
return 0
@staticmethod
async def cleanup_old(db: aiosqlite.Connection, days: int = 7) -> int:
try:
await db.execute(
"DELETE FROM validation_tasks WHERE updated_at < datetime('now', '-{} days')".format(days)
)
await db.commit()
return db.total_changes
except Exception as e:
logger.error(f"cleanup_old tasks failed: {e}")
return 0

View File

@@ -10,22 +10,25 @@ from core.log import logger
class PluginService: class PluginService:
"""插件业务服务:管理插件生命周期、执行爬取""" """插件业务服务:管理插件生命周期、执行爬取、配置管理"""
def __init__(self): def __init__(self):
self.plugin_settings_repo = PluginSettingsRepository() self.plugin_settings_repo = PluginSettingsRepository()
self._stats: dict[str, dict] = {} self._stats: dict[str, dict] = {}
async def list_plugins(self) -> List[PluginInfo]: async def list_plugins(self) -> List[PluginInfo]:
"""获取所有插件信息(合并持久化状态)""" """获取所有插件信息(合并持久化状态和配置"""
async with get_db() as db: async with get_db() as db:
db_states = await self.plugin_settings_repo.list_all(db) db_states = await self.plugin_settings_repo.list_all(db)
result = [] result = []
for plugin in registry.list_plugins(): for plugin in registry.list_plugins():
# 如果有持久化状态,覆盖内存状态 # 合并持久化状态
if plugin.name in db_states: state = db_states.get(plugin.name, {})
plugin.enabled = db_states[plugin.name] if "enabled" in state:
plugin.enabled = state["enabled"]
if "config" in state and isinstance(state["config"], dict):
plugin.update_config(state["config"])
stat = self._stats.get(plugin.name, { stat = self._stats.get(plugin.name, {
"success_count": 0, "success_count": 0,
@@ -55,6 +58,31 @@ class PluginService:
logger.info(f"Plugin {plugin_id} toggled to {enabled}") logger.info(f"Plugin {plugin_id} toggled to {enabled}")
return success return success
async def get_plugin_config(self, plugin_id: str) -> Optional[dict]:
"""获取插件当前配置(合并默认值和持久化值)"""
plugin = registry.get(plugin_id)
if not plugin:
return None
async with get_db() as db:
saved = await self.plugin_settings_repo.get_config(db, plugin_id)
config = dict(plugin.default_config)
if saved:
config.update(saved)
return config
async def update_plugin_config(self, plugin_id: str, config: dict) -> bool:
"""更新插件配置(只保存已存在于 default_config 中的键)"""
plugin = registry.get(plugin_id)
if not plugin:
return False
# 过滤非法键
safe_config = {k: v for k, v in config.items() if k in plugin.default_config}
if not safe_config:
return False
plugin.update_config(safe_config)
async with get_db() as db:
return await self.plugin_settings_repo.set_config(db, plugin_id, plugin.config)
def get_plugin(self, plugin_id: str) -> Optional[BaseCrawlerPlugin]: def get_plugin(self, plugin_id: str) -> Optional[BaseCrawlerPlugin]:
return registry.get(plugin_id) return registry.get(plugin_id)