Files
ProxyPool/api/routes/plugins.py
祀梦 209a744d94 全面架构重构:建立分层架构与高度可扩展的插件系统
后端重构:
- 新增分层架构:API Routes -> Services -> Repositories -> Infrastructure
- 彻底移除全局单例,全面采用 FastAPI 依赖注入
- 新增 api/ 目录拆分路由(proxies, plugins, scheduler, settings, stats)
- 新增 services/ 业务逻辑层:ProxyService, PluginService, SchedulerService, ValidatorService, SettingsService
- 新增 repositories/ 数据访问层:ProxyRepository, SettingsRepository, PluginSettingsRepository
- 新增 models/ 层:Pydantic Schemas + Domain Models
- 重写 core/config.py:采用 Pydantic Settings 管理配置
- 新增 core/db.py:基于 asynccontextmanager 的连接管理,支持数据库迁移
- 新增 core/exceptions.py:统一业务异常体系

插件系统重构(核心):
- 新增 core/plugin_system/:BaseCrawlerPlugin + PluginRegistry
- 采用显式注册模式(装饰器 + plugins/__init__.py),类型安全、测试友好
- 新增 plugins/base.py:BaseHTTPPlugin 通用 HTTP 爬虫基类
- 迁移全部 7 个插件到新架构(fate0, proxylist_download, ip3366, ip89, kuaidaili, speedx, yundaili)
- 插件状态持久化到 plugin_settings 表

任务调度重构:
- 新增 core/tasks/queue.py:ValidationQueue + WorkerPool
- 解耦爬取与验证:爬虫只负责爬取,代理提交队列后由 Worker 异步验证
- 调度器定时从数据库拉取存量代理并分批投入验证队列

前端调整:
- 新增 frontend/src/services/ 层拆分 API 调用逻辑
- 调整 stores/ 和 views/ 使用 Service 层
- 保持 API 兼容性,页面无需大幅修改

其他:
- 新增 main.py 作为新入口
- 新增 DESIGN.md 架构设计文档
- 更新 requirements.txt 增加 pydantic-settings
2026-04-02 11:55:05 +08:00

140 lines
4.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""插件相关路由"""
from fastapi import APIRouter, Depends
from services.plugin_service import PluginService
from services.scheduler_service import SchedulerService
from models.schemas import PluginToggleRequest
from api.deps import get_plugin_service, get_scheduler_service
from core.log import logger
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
def success_response(message: str, data=None):
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500):
return {"code": code, "message": message, "data": None}
@router.get("")
async def list_plugins(service: PluginService = Depends(get_plugin_service)):
plugins = await service.list_plugins()
return success_response(
"获取插件列表成功",
{
"plugins": [
{
"id": p.id,
"name": p.display_name, # 保持旧版本兼容name 用于展示
"display_name": p.display_name,
"description": p.description,
"enabled": p.enabled,
"last_run": p.last_run.isoformat() if p.last_run else None,
"success_count": p.success_count,
"failure_count": p.failure_count,
}
for p in plugins
]
},
)
@router.put("/{plugin_id}/toggle")
async def toggle_plugin(
plugin_id: str,
request: PluginToggleRequest,
service: PluginService = Depends(get_plugin_service),
):
success = await service.toggle_plugin(plugin_id, request.enabled)
if not success:
return error_response("插件不存在", 404)
return success_response(
f"插件 {plugin_id}{'启用' if request.enabled else '禁用'}",
{"plugin_id": plugin_id, "enabled": request.enabled},
)
@router.post("/{plugin_id}/crawl")
async def crawl_plugin(
plugin_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
plugin = plugin_service.get_plugin(plugin_id)
if not plugin:
return error_response("插件不存在", 404)
try:
results = await plugin_service.run_plugin(plugin_id)
if not results:
return success_response(
f"插件 {plugin_id} 爬取完成,未获取到代理",
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
)
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies, sending to validation queue")
scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results)
# 等待队列排空(最多等 30 秒,避免前端超时)
try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
except asyncio.TimeoutError:
pass
valid_count = scheduler_service.validation_queue.valid_count
invalid_count = scheduler_service.validation_queue.invalid_count
return success_response(
f"插件 {plugin_id} 爬取并验证完成",
{
"plugin_id": plugin_id,
"proxy_count": len(results),
"valid_count": valid_count,
"invalid_count": invalid_count,
},
)
except Exception as e:
logger.error(f"Crawl plugin {plugin_id} failed: {e}")
return error_response(f"插件爬取失败: {str(e)}")
@router.post("/crawl-all")
async def crawl_all(
plugin_service: PluginService = Depends(get_plugin_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
try:
results = await plugin_service.run_all_plugins()
if not results:
return success_response(
"所有插件爬取完成,未获取到代理",
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
)
logger.info(f"All plugins crawled {len(results)} unique proxies, sending to validation queue")
scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results)
try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
except asyncio.TimeoutError:
pass
valid_count = scheduler_service.validation_queue.valid_count
invalid_count = scheduler_service.validation_queue.invalid_count
return success_response(
"所有插件爬取并验证完成",
{
"total_crawled": len(results),
"valid_count": valid_count,
"invalid_count": invalid_count,
},
)
except Exception as e:
logger.error(f"Crawl all failed: {e}")
return error_response(f"批量爬取失败: {str(e)}")
import asyncio