重构: 迁移后端代码到 app 目录,前端移动到 WebUI,添加完整测试套件

主要变更:
- 后端代码从根目录迁移到 app/ 目录
- 前端代码从 frontend/ 重命名为 WebUI/
- 更新所有导入路径以适配新结构
- 提取公共 API 响应函数到 app/api/common.py
- 精简验证器服务代码
- 更新启动脚本和文档

测试:
- 新增完整测试套件 (tests/)
- 单元测试: 模型、仓库层
- 集成测试: 覆盖所有 22+ API 端点
- E2E 测试: 4个完整工作流场景
- 添加 pytest 配置和测试运行脚本
This commit is contained in:
祀梦
2026-04-04 13:32:36 +08:00
parent df3cc87f88
commit 38bd66128b
109 changed files with 2017 additions and 548 deletions

142
DESIGN.md
View File

@@ -309,77 +309,87 @@ Store 只负责:
``` ```
ProxyPool/ ProxyPool/
├── api/ # FastAPI 入口和路由 ├── main.py # 项目入口
│ ├── __init__.py ├── requirements.txt # Python 依赖
│ ├── main.py # 应用工厂 ├── .env.example # 环境变量示例
│ ├── lifespan.py # 生命周期管理
│ ├── deps.py # 依赖注入
│ ├── errors.py # 统一异常
│ └── routes/
│ ├── __init__.py
│ ├── proxies.py
│ ├── plugins.py
│ ├── scheduler.py
│ └── settings.py
├── core/ # 基础设施 ├── app/ # 后端代码
│ ├── __init__.py │ ├── api/ # FastAPI 入口和路由
│ ├── config.py # Pydantic Settings
│ ├── log.py # 日志
│ ├── db.py # 数据库连接池/上下文
│ └── exceptions.py # 业务异常
├── models/ # 数据模型
│ ├── __init__.py
│ ├── schemas.py # Pydantic 模型
│ └── domain.py # 领域模型ProxyRaw, PluginInfo 等)
├── repositories/ # 数据访问层
│ ├── __init__.py
│ └── proxy_repo.py # ProxyRepository
├── services/ # 业务逻辑层
│ ├── __init__.py
│ ├── proxy_service.py
│ ├── plugin_service.py
│ ├── scheduler_service.py
│ └── validator_service.py
├── core/ # 任务与插件系统
│ ├── plugin_system/
│ │ ├── __init__.py │ │ ├── __init__.py
│ │ ├── base.py # BaseCrawlerPlugin │ │ ├── main.py # 应用工厂
│ │ ── registry.py # 插件注册中心 │ │ ── lifespan.py # 生命周期管理
└── tasks/ │ ├── deps.py # 依赖注入
│ │ ├── errors.py # 统一异常
│ │ └── routes/
│ │ ├── __init__.py
│ │ ├── proxies.py
│ │ ├── plugins.py
│ │ ├── scheduler.py
│ │ └── settings.py
│ │
│ ├── core/ # 基础设施
│ │ ├── __init__.py
│ │ ├── config.py # Pydantic Settings
│ │ ├── log.py # 日志
│ │ ├── db.py # 数据库连接池/上下文
│ │ ├── exceptions.py # 业务异常
│ │ ├── plugin_system/ # 插件系统
│ │ │ ├── __init__.py
│ │ │ ├── base.py # BaseCrawlerPlugin
│ │ │ └── registry.py # 插件注册中心
│ │ └── tasks/ # 任务队列
│ │ ├── __init__.py
│ │ └── queue.py # ValidationQueue
│ │
│ ├── models/ # 数据模型
│ │ ├── __init__.py
│ │ ├── schemas.py # Pydantic 模型
│ │ └── domain.py # 领域模型
│ │
│ ├── repositories/ # 数据访问层
│ │ ├── __init__.py
│ │ ├── proxy_repo.py
│ │ ├── settings_repo.py
│ │ └── task_repo.py
│ │
│ ├── services/ # 业务逻辑层
│ │ ├── __init__.py
│ │ ├── proxy_service.py
│ │ ├── plugin_service.py
│ │ ├── scheduler_service.py
│ │ └── validator_service.py
│ │
│ └── plugins/ # 爬虫插件
│ ├── __init__.py │ ├── __init__.py
│ ├── queue.py # ValidationQueue │ ├── base.py # 通用抓取基类
── workers.py # Worker Pool ── fate0.py
│ ├── kuaidaili.py
│ ├── ip3366.py
│ ├── ip89.py
│ ├── speedx.py
│ ├── yundaili.py
│ ├── proxylist_download.py
│ └── proxyscrape.py
├── plugins/ # 爬虫插件 ├── WebUI/ # Vue3 前端
│ ├── __init__.py │ ├── src/
│ ├── base.py # 通用抓取基类HTTP 请求封装 │ ├── api/ # API 封装
│ ├── fate0.py │ ├── stores/ # Pinia 状态管理
│ ├── proxylist_download.py │ ├── views/ # 页面组件
└── ... │ ├── router/ # 路由配置
│ ├── components/ # 通用组件
├── frontend/ # Vue3 前端 └── style.css # 全局样式
── src/ ── index.html
├── services/ # 新增 └── package.json
│ ├── stores/
│ ├── api/
│ └── ...
├── tests/ # 测试目录 ├── tests/ # 测试目录
│ ├── conftest.py │ ├── conftest.py
│ ├── unit/ │ ├── unit/
│ └── integration/ │ └── integration/
├── script/ ├── script/ # 启动脚本
├── data/ ├── db/ # 数据存储
├── db/ ├── logs/ # 日志文件
├── logs/
├── requirements.txt
├── .env.example
└── DESIGN.md # 本文档 └── DESIGN.md # 本文档
``` ```
@@ -426,11 +436,11 @@ ProxyPool/
假设要添加一个名为 `mynewsource` 的爬虫: 假设要添加一个名为 `mynewsource` 的爬虫:
**Step 1**: 创建文件 `plugins/mynewsource.py` **Step 1**: 创建文件 `app/plugins/mynewsource.py`
```python ```python
from core.plugin_system import BaseCrawlerPlugin, ProxyRaw from app.core.plugin_system import BaseCrawlerPlugin, ProxyRaw
from plugins.base import BaseHTTPPlugin # 可选:如果基于 HTTP 爬取 from app.plugins.base import BaseHTTPPlugin # 可选:如果基于 HTTP 爬取
class MyNewSourcePlugin(BaseHTTPPlugin): class MyNewSourcePlugin(BaseHTTPPlugin):
name = "mynewsource" name = "mynewsource"
@@ -450,11 +460,11 @@ class MyNewSourcePlugin(BaseHTTPPlugin):
return results return results
``` ```
**Step 2**: 在 `plugins/__init__.py` 中注册 **Step 2**: 在 `app/plugins/__init__.py` 中注册
```python ```python
from .mynewsource import MyNewSourcePlugin from .mynewsource import MyNewSourcePlugin
from core.plugin_system import registry from app.core.plugin_system import registry
registry.register(MyNewSourcePlugin) registry.register(MyNewSourcePlugin)
``` ```

View File

@@ -36,7 +36,7 @@ pip install -r requirements.txt
### 2. 安装前端依赖 ### 2. 安装前端依赖
```bash ```bash
cd frontend cd WebUI
npm install npm install
``` ```
@@ -58,7 +58,7 @@ python api_server.py
**启动前端服务**(终端 2 **启动前端服务**(终端 2
```bash ```bash
cd frontend cd WebUI
npm run dev npm run dev
``` ```
@@ -77,32 +77,35 @@ stop.bat
``` ```
ProxyPool/ ProxyPool/
├── api_server.py # FastAPI 后端服务器 ├── main.py # 项目入口
├── config.py # 配置文件
├── requirements.txt # Python 依赖 ├── requirements.txt # Python 依赖
├── .env.example # 环境变量示例 ├── .env.example # 环境变量示例
├── script/ # 启动脚本 ├── app/ # 后端代码
│ ├── start.bat # Windows 启动脚本 │ ├── api/ # FastAPI 路由
└── stop.bat # Windows 停止脚本 │ ├── main.py # 应用工厂
│ │ ├── routes/ # API 路由
│ │ ├── deps.py # 依赖注入
│ │ └── ...
│ ├── core/ # 核心模块
│ │ ├── config.py # 配置管理
│ │ ├── db.py # 数据库连接
│ │ ├── log.py # 日志配置
│ │ ├── plugin_system/ # 插件系统
│ │ └── tasks/ # 任务队列
│ ├── models/ # 数据模型
│ ├── repositories/ # 数据访问层
│ ├── services/ # 业务逻辑层
│ └── plugins/ # 代理源插件
│ ├── fate0.py # Fate0 代理源
│ ├── ip3366.py # IP3366 代理源
│ ├── ip89.py # IP89 代理源
│ ├── kuaidaili.py # 快代理源
│ ├── yundaili.py # 云代理源
│ ├── speedx.py # SpeedX 代理源
│ └── proxylist_download.py # ProxyList 代理源
├── core/ # 核心模块 ├── WebUI/ # Vue3 前端
│ ├── crawler.py # 爬虫基类
│ ├── validator.py # 代理验证器
│ ├── sqlite.py # 数据库管理
│ ├── plugin_manager.py # 插件管理器
│ └── log.py # 日志配置
├── plugins/ # 代理源插件
│ ├── fate0.py # Fate0 代理源
│ ├── ip3366.py # IP3366 代理源
│ ├── ip89.py # IP89 代理源
│ ├── kuaidaili.py # 快代理源
│ ├── yundaili.py # 云代理源
│ ├── speedx.py # SpeedX 代理源
│ └── proxylist_download.py # ProxyList 代理源
├── frontend/ # Vue3 前端
│ ├── src/ │ ├── src/
│ │ ├── api/ # API 封装 │ │ ├── api/ # API 封装
│ │ ├── stores/ # Pinia 状态管理 │ │ ├── stores/ # Pinia 状态管理
@@ -113,6 +116,10 @@ ProxyPool/
│ ├── index.html │ ├── index.html
│ └── package.json │ └── package.json
├── script/ # 启动脚本
│ ├── start.bat # Windows 启动脚本
│ └── stop.bat # Windows 停止脚本
└── db/ # 数据存储目录 └── db/ # 数据存储目录
└── proxies.sqlite # SQLite 数据库 └── proxies.sqlite # SQLite 数据库
``` ```

View File

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

@@ -74,26 +74,42 @@
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="操作" width="200" fixed="right" align="center"> <el-table-column label="操作" width="220" fixed="right" align="center">
<template #default="{ row }"> <template #default="{ row }">
<el-button <div class="plugin-actions">
type="primary" <el-button
size="small" type="primary"
@click="handleOpenConfig(row)" size="small"
> @click="handleOpenConfig(row)"
<el-icon class="btn-icon"><Setting /></el-icon> >
配置 <el-icon class="btn-icon"><Setting /></el-icon>
</el-button> 配置
<el-button </el-button>
type="success" <el-button
size="small" type="success"
@click="handleCrawl(row.id)" size="small"
:loading="crawlingPlugin === row.id" @click="handleCrawl(row.id)"
:disabled="!row.enabled" :loading="crawlingPlugins.has(row.id)"
> :disabled="!row.enabled"
<el-icon class="btn-icon"><Promotion /></el-icon> >
爬取 <el-icon class="btn-icon"><Promotion /></el-icon>
</el-button> 爬取
</el-button>
</div>
<div v-if="crawlResults[row.id]" class="plugin-crawl-result">
<div class="result-mini" :class="crawlResults[row.id].type">
<el-icon v-if="crawlResults[row.id].type === 'success'" class="result-icon success"><CircleCheck /></el-icon>
<el-icon v-else class="result-icon failed"><CircleClose /></el-icon>
<span class="result-text">{{ crawlResults[row.id].message }}</span>
<span v-if="crawlResults[row.id].data?.valid_count !== undefined" class="result-count valid">
有效 {{ crawlResults[row.id].data.valid_count }}
</span>
<span v-if="crawlResults[row.id].data?.invalid_count !== undefined" class="result-count invalid">
无效 {{ crawlResults[row.id].data.invalid_count }}
</span>
<el-icon class="result-close" @click="clearCrawlResult(row.id)"><Close /></el-icon>
</div>
</div>
</template> </template>
</el-table-column> </el-table-column>
</el-table> </el-table>
@@ -104,28 +120,28 @@
:image-size="120" :image-size="120"
/> />
<!-- 爬取结果提示 --> <!-- 批量爬取结果提示 -->
<el-alert <el-alert
v-if="lastCrawlResult" v-if="allCrawlResult"
:title="lastCrawlResult.message" :title="allCrawlResult.message"
:type="lastCrawlResult.type" :type="allCrawlResult.type"
closable closable
class="crawl-result" class="crawl-result"
@close="lastCrawlResult = null" @close="allCrawlResult = null"
> >
<template v-if="lastCrawlResult.data"> <template v-if="allCrawlResult.data">
<div class="crawl-stats"> <div class="crawl-stats">
<span v-if="lastCrawlResult.data.total_crawled !== undefined"> <span v-if="allCrawlResult.data.total_crawled !== undefined">
爬取: {{ lastCrawlResult.data.total_crawled }} 爬取: {{ allCrawlResult.data.total_crawled }}
</span> </span>
<span v-if="lastCrawlResult.data.proxy_count !== undefined"> <span v-if="allCrawlResult.data.proxy_count !== undefined">
爬取: {{ lastCrawlResult.data.proxy_count }} 爬取: {{ allCrawlResult.data.proxy_count }}
</span> </span>
<span v-if="lastCrawlResult.data.valid_count !== undefined" class="valid-count"> <span v-if="allCrawlResult.data.valid_count !== undefined" class="valid-count">
有效: {{ lastCrawlResult.data.valid_count }} 有效: {{ allCrawlResult.data.valid_count }}
</span> </span>
<span v-if="lastCrawlResult.data.invalid_count !== undefined" class="invalid-count"> <span v-if="allCrawlResult.data.invalid_count !== undefined" class="invalid-count">
无效: {{ lastCrawlResult.data.invalid_count }} 无效: {{ allCrawlResult.data.invalid_count }}
</span> </span>
</div> </div>
</template> </template>
@@ -183,7 +199,8 @@ import {
CircleCheck, CircleCheck,
CircleClose, CircleClose,
Box, Box,
Setting Setting,
Close
} from '@element-plus/icons-vue' } from '@element-plus/icons-vue'
import { usePluginsStore } from '../stores/plugins' import { usePluginsStore } from '../stores/plugins'
import { pluginService } from '../services/pluginService' import { pluginService } from '../services/pluginService'
@@ -191,9 +208,10 @@ import { formatTime } from '../utils/format'
import PageHeader from '../components/PageHeader.vue' import PageHeader from '../components/PageHeader.vue'
const pluginsStore = usePluginsStore() const pluginsStore = usePluginsStore()
const crawlingPlugin = ref(null) const crawlingPlugins = ref(new Set())
const crawlingAll = ref(false) const crawlingAll = ref(false)
const lastCrawlResult = ref(null) const crawlResults = ref({})
const allCrawlResult = ref(null)
// //
const configDialogVisible = ref(false) const configDialogVisible = ref(false)
@@ -248,34 +266,36 @@ async function handleSaveConfig() {
async function handleCrawl(pluginId) { async function handleCrawl(pluginId) {
try { try {
crawlingPlugin.value = pluginId crawlingPlugins.value.add(pluginId)
lastCrawlResult.value = null
const response = await pluginService.crawlPlugin(pluginId) const response = await pluginService.crawlPlugin(pluginId)
if (response.code === 200) { if (response.code === 200) {
lastCrawlResult.value = { crawlResults.value[pluginId] = {
type: 'success', type: 'success',
message: response.message, message: response.message,
data: response.data data: response.data
} }
await pluginsStore.fetchPlugins()
} else { } else {
lastCrawlResult.value = { crawlResults.value[pluginId] = {
type: 'error', type: 'error',
message: response.message || '爬取失败' message: response.message || '爬取失败'
} }
} }
} catch (error) { } catch (error) {
lastCrawlResult.value = { crawlResults.value[pluginId] = {
type: 'error', type: 'error',
message: '爬取过程出错' message: '爬取过程出错'
} }
} finally { } finally {
crawlingPlugin.value = null crawlingPlugins.value.delete(pluginId)
} }
} }
function clearCrawlResult(pluginId) {
delete crawlResults.value[pluginId]
}
async function handleCrawlAll() { async function handleCrawlAll() {
try { try {
const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled) const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled)
@@ -295,20 +315,19 @@ async function handleCrawlAll() {
) )
crawlingAll.value = true crawlingAll.value = true
lastCrawlResult.value = null allCrawlResult.value = null
const response = await pluginService.crawlAll() const response = await pluginService.crawlAll()
if (response.code === 200) { if (response.code === 200) {
lastCrawlResult.value = { allCrawlResult.value = {
type: 'success', type: 'success',
message: response.message, message: response.message,
data: response.data data: response.data
} }
ElMessage.success('批量爬取完成') ElMessage.success('批量爬取完成')
await pluginsStore.fetchPlugins()
} else { } else {
lastCrawlResult.value = { allCrawlResult.value = {
type: 'error', type: 'error',
message: response.message || '批量爬取失败' message: response.message || '批量爬取失败'
} }
@@ -316,7 +335,7 @@ async function handleCrawlAll() {
} catch (error) { } catch (error) {
if (error !== 'cancel') { if (error !== 'cancel') {
console.error('批量爬取失败:', error) console.error('批量爬取失败:', error)
lastCrawlResult.value = { allCrawlResult.value = {
type: 'error', type: 'error',
message: '批量爬取过程出错' message: '批量爬取过程出错'
} }
@@ -460,4 +479,70 @@ onMounted(async () => {
margin-bottom: 16px; margin-bottom: 16px;
color: var(--text-primary); color: var(--text-primary);
} }
.plugin-actions {
display: flex;
justify-content: center;
gap: 8px;
}
.plugin-crawl-result {
margin-top: 8px;
}
.result-mini {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 4px 8px;
border-radius: 4px;
font-size: 12px;
line-height: 1.4;
}
.result-mini.success {
background: rgba(103, 194, 58, 0.15);
color: var(--success);
}
.result-mini.error {
background: rgba(245, 108, 108, 0.15);
color: var(--danger);
}
.result-icon {
font-size: 13px;
}
.result-text {
font-weight: 500;
}
.result-count {
font-weight: 600;
padding: 0 4px;
border-radius: 3px;
}
.result-count.valid {
background: rgba(103, 194, 58, 0.2);
color: var(--success);
}
.result-count.invalid {
background: rgba(245, 108, 108, 0.2);
color: var(--danger);
}
.result-close {
margin-left: 4px;
cursor: pointer;
font-size: 12px;
opacity: 0.7;
transition: opacity 0.2s;
}
.result-close:hover {
opacity: 1;
}
</style> </style>

View File

@@ -1,130 +0,0 @@
"""代理相关路由(含统计信息)"""
from typing import Optional
from fastapi import APIRouter, Depends, Query
from services.proxy_service import ProxyService
from services.scheduler_service import SchedulerService
from models.schemas import ProxyListRequest, BatchDeleteRequest
from api.deps import get_proxy_service, get_scheduler_service
from core.log import logger
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
def success_response(message: str, data=None):
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500):
return {"code": code, "message": message, "data": None}
@router.get("/stats")
async def get_stats(
proxy_service: ProxyService = Depends(get_proxy_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
try:
stats = await proxy_service.get_stats()
stats["scheduler_running"] = scheduler_service.running
return success_response("获取统计信息成功", stats)
except Exception as e:
logger.error(f"Get stats failed: {e}")
return error_response("获取统计信息失败")
@router.post("")
async def list_proxies(
request: ProxyListRequest,
service: ProxyService = Depends(get_proxy_service),
):
proxies, total = await service.list_proxies(
page=request.page,
page_size=request.page_size,
protocol=request.protocol,
min_score=request.min_score,
max_score=request.max_score,
sort_by=request.sort_by,
sort_order=request.sort_order,
)
return success_response(
"获取代理列表成功",
{
"list": [
{
"ip": p.ip,
"port": p.port,
"protocol": p.protocol,
"score": p.score,
"last_check": p.last_check.isoformat() if p.last_check else None,
}
for p in proxies
],
"total": total,
"page": request.page,
"page_size": request.page_size,
},
)
@router.get("/random")
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
proxy = await service.get_random_proxy()
if not proxy:
return error_response("没有找到可用的代理", 404)
return success_response(
"获取随机代理成功",
{
"ip": proxy.ip,
"port": proxy.port,
"protocol": proxy.protocol,
"score": proxy.score,
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
},
)
@router.get("/export/{fmt}")
async def export_proxies(
fmt: str,
protocol: Optional[str] = None,
limit: int = Query(default=10000, ge=1, le=100000),
service: ProxyService = Depends(get_proxy_service),
):
if fmt not in ("csv", "txt", "json"):
return error_response("不支持的导出格式", 400)
from fastapi.responses import StreamingResponse
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
async def generate():
async for chunk in service.export_proxies(fmt, protocol, limit):
yield chunk
return StreamingResponse(
generate(),
media_type=media_types[fmt],
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
)
@router.delete("/{ip}/{port}")
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
await service.delete_proxy(ip, port)
return success_response("删除代理成功")
@router.post("/batch-delete")
async def batch_delete(
request: BatchDeleteRequest,
service: ProxyService = Depends(get_proxy_service),
):
proxies = [(item.ip, item.port) for item in request.proxies]
deleted = await service.batch_delete(proxies)
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
@router.delete("/clean-invalid")
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
count = await service.clean_invalid()
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})

View File

@@ -1,80 +0,0 @@
"""调度器相关路由"""
from fastapi import APIRouter, Depends
from services.scheduler_service import SchedulerService
from repositories.settings_repo import SettingsRepository
from core.db import get_db
from api.deps import get_scheduler_service
from core.log import logger
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
settings_repo = SettingsRepository()
def success_response(message: str, data=None):
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500):
return {"code": code, "message": message, "data": None}
@router.post("/start")
async def start_scheduler(
scheduler: SchedulerService = Depends(get_scheduler_service),
):
try:
if scheduler.running:
return success_response("验证调度器已在运行", {"running": True})
await scheduler.start()
# 持久化设置
async with get_db() as db:
settings = await settings_repo.get_all(db)
settings["auto_validate"] = True
from models.schemas import SettingsSchema
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
return success_response("验证调度器已启动", {"running": True})
except Exception as e:
logger.error(f"Start scheduler failed: {e}")
return error_response(f"启动调度器失败: {str(e)}")
@router.post("/stop")
async def stop_scheduler(
scheduler: SchedulerService = Depends(get_scheduler_service),
):
try:
if not scheduler.running:
return success_response("验证调度器未运行", {"running": False})
await scheduler.stop()
# 持久化设置
async with get_db() as db:
settings = await settings_repo.get_all(db)
settings["auto_validate"] = False
from models.schemas import SettingsSchema
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
return success_response("验证调度器已停止", {"running": False})
except Exception as e:
logger.error(f"Stop scheduler failed: {e}")
return error_response(f"停止调度器失败: {str(e)}")
@router.post("/validate-now")
async def validate_now(
scheduler: SchedulerService = Depends(get_scheduler_service),
):
try:
scheduler.validate_all_now()
return success_response("已开始全量验证", {"started": True})
except Exception as e:
logger.error(f"Validate now failed: {e}")
return error_response(f"启动验证失败: {str(e)}")
@router.get("/status")
async def scheduler_status(
scheduler: SchedulerService = Depends(get_scheduler_service),
):
return success_response(
"获取状态成功",
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
)

View File

@@ -1,3 +1,4 @@
"""API 包"""
from .main import create_app from .main import create_app
__all__ = ["create_app"] __all__ = ["create_app"]

41
app/api/common.py Normal file
View File

@@ -0,0 +1,41 @@
"""API 通用工具函数"""
from typing import Any, Optional
from fastapi.responses import JSONResponse
def success_response(message: str, data: Any = None) -> dict:
"""成功响应"""
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500) -> JSONResponse:
"""错误响应"""
return JSONResponse(
status_code=code,
content={"code": code, "message": message, "data": None},
)
def format_proxy(proxy) -> dict:
"""格式化代理对象"""
return {
"ip": proxy.ip,
"port": proxy.port,
"protocol": proxy.protocol,
"score": proxy.score,
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
}
def format_plugin(plugin) -> dict:
"""格式化插件对象"""
return {
"id": plugin.id,
"name": plugin.display_name,
"display_name": plugin.display_name,
"description": plugin.description,
"enabled": plugin.enabled,
"last_run": plugin.last_run.isoformat() if plugin.last_run else None,
"success_count": plugin.success_count,
"failure_count": plugin.failure_count,
}

View File

@@ -1,12 +1,12 @@
"""依赖注入""" """依赖注入"""
from fastapi import Request from fastapi import Request
from services.proxy_service import ProxyService from app.services.proxy_service import ProxyService
from services.plugin_service import PluginService from app.services.plugin_service import PluginService
from services.scheduler_service import SchedulerService from app.services.scheduler_service import SchedulerService
from services.validator_service import ValidatorService from app.services.validator_service import ValidatorService
from repositories.proxy_repo import ProxyRepository from app.repositories.proxy_repo import ProxyRepository
from core.tasks.queue import ValidationQueue from app.core.tasks.queue import ValidationQueue
from core.config import settings as app_settings from app.core.config import settings as app_settings
def get_proxy_service() -> ProxyService: def get_proxy_service() -> ProxyService:

View File

@@ -2,8 +2,8 @@
from fastapi import Request from fastapi import Request
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from pydantic import ValidationError from pydantic import ValidationError
from core.exceptions import ProxyPoolException from app.core.exceptions import ProxyPoolException
from core.log import logger from app.core.log import logger
async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException): async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException):

View File

@@ -1,11 +1,11 @@
"""应用生命周期管理""" """应用生命周期管理"""
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI from fastapi import FastAPI
from core.db import init_db, get_db from app.core.db import init_db, get_db
from core.config import settings as app_settings from app.core.config import settings as app_settings
from core.log import logger from app.core.log import logger
from api.deps import create_scheduler_service from app.api.deps import create_scheduler_service
from repositories.settings_repo import SettingsRepository from app.repositories.settings_repo import SettingsRepository
settings_repo = SettingsRepository() settings_repo = SettingsRepository()

View File

@@ -1,15 +1,15 @@
"""FastAPI 应用工厂""" """FastAPI 应用工厂"""
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from api.lifespan import lifespan from app.api.lifespan import lifespan
from api.routes import api_router from app.api.routes import api_router
from api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler from app.api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
from core.exceptions import ProxyPoolException from app.core.exceptions import ProxyPoolException
from pydantic import ValidationError from pydantic import ValidationError
from core.config import settings as app_settings from app.core.config import settings as app_settings
# 导入并注册所有插件(显式注册模式) # 导入并注册所有插件(显式注册模式)
import plugins import app.plugins
def create_app() -> FastAPI: def create_app() -> FastAPI:

View File

@@ -1,5 +1,6 @@
"""路由包"""
from fastapi import APIRouter from fastapi import APIRouter
from . import proxies, plugins, scheduler, settings from app.api.routes import proxies, plugins, scheduler, settings
api_router = APIRouter() api_router = APIRouter()
api_router.include_router(proxies.router) api_router.include_router(proxies.router)

View File

@@ -1,42 +1,23 @@
"""插件相关路由""" """插件相关路由"""
import asyncio
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends
from services.plugin_service import PluginService from app.services.plugin_service import PluginService
from services.scheduler_service import SchedulerService from app.services.scheduler_service import SchedulerService
from api.deps import get_plugin_service, get_scheduler_service from app.api.deps import get_plugin_service, get_scheduler_service
from core.log import logger from app.api.common import success_response, error_response, format_plugin
from app.core.log import logger
router = APIRouter(prefix="/api/plugins", tags=["plugins"]) router = APIRouter(prefix="/api/plugins", tags=["plugins"])
def success_response(message: str, data=None):
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500):
return {"code": code, "message": message, "data": None}
@router.get("") @router.get("")
async def list_plugins(service: PluginService = Depends(get_plugin_service)): async def list_plugins(service: PluginService = Depends(get_plugin_service)):
plugins = await service.list_plugins() try:
return success_response( plugins = await service.list_plugins()
"获取插件列表成功", return success_response("获取插件列表成功", {"plugins": [format_plugin(p) for p in plugins]})
{ except Exception as e:
"plugins": [ logger.error(f"List plugins failed: {e}")
{ return error_response("获取插件列表失败", 500)
"id": p.id,
"name": p.display_name,
"display_name": p.display_name,
"description": p.description,
"enabled": p.enabled,
"last_run": p.last_run.isoformat() if p.last_run else None,
"success_count": p.success_count,
"failure_count": p.failure_count,
}
for p in plugins
]
},
)
@router.put("/{plugin_id}/toggle") @router.put("/{plugin_id}/toggle")
@@ -48,13 +29,18 @@ async def toggle_plugin(
enabled = request.get("enabled") enabled = request.get("enabled")
if enabled is None: if enabled is None:
return error_response("缺少 enabled 参数", 400) return error_response("缺少 enabled 参数", 400)
success = await service.toggle_plugin(plugin_id, enabled)
if not success: try:
return error_response("插件不存在", 404) success = await service.toggle_plugin(plugin_id, enabled)
return success_response( if not success:
f"插件 {plugin_id}{'启用' if enabled else '禁用'}", return error_response("插件不存在", 404)
{"plugin_id": plugin_id, "enabled": enabled}, return success_response(
) f"插件 {plugin_id}{'启用' if enabled else '禁用'}",
{"plugin_id": plugin_id, "enabled": enabled},
)
except Exception as e:
logger.error(f"Toggle plugin failed: {e}")
return error_response("切换插件状态失败", 500)
@router.get("/{plugin_id}/config") @router.get("/{plugin_id}/config")
@@ -62,10 +48,14 @@ async def get_plugin_config(
plugin_id: str, plugin_id: str,
service: PluginService = Depends(get_plugin_service), service: PluginService = Depends(get_plugin_service),
): ):
config = await service.get_plugin_config(plugin_id) try:
if config is None: config = await service.get_plugin_config(plugin_id)
return error_response("插件不存在", 404) if config is None:
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config}) return error_response("插件不存在", 404)
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
except Exception as e:
logger.error(f"Get plugin config failed: {e}")
return error_response("获取插件配置失败", 500)
@router.post("/{plugin_id}/config") @router.post("/{plugin_id}/config")
@@ -77,10 +67,15 @@ async def update_plugin_config(
config = request.get("config", {}) config = request.get("config", {})
if not isinstance(config, dict): if not isinstance(config, dict):
return error_response("config 必须是对象", 400) return error_response("config 必须是对象", 400)
success = await service.update_plugin_config(plugin_id, config)
if not success: try:
return error_response("插件不存在或配置无效", 404) success = await service.update_plugin_config(plugin_id, config)
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config}) if not success:
return error_response("插件不存在或配置无效", 404)
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
except Exception as e:
logger.error(f"Update plugin config failed: {e}")
return error_response("保存插件配置失败", 500)
@router.post("/{plugin_id}/crawl") @router.post("/{plugin_id}/crawl")
@@ -101,30 +96,27 @@ async def crawl_plugin(
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0}, {"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
) )
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies, sending to validation queue") logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies")
scheduler_service.validation_queue.reset_stats() scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results) await scheduler_service.validation_queue.submit(results)
# 等待队列排空(最多等 30 秒,避免前端超时)
try: try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0) await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
except asyncio.TimeoutError: except asyncio.TimeoutError:
pass pass
valid_count = scheduler_service.validation_queue.valid_count
invalid_count = scheduler_service.validation_queue.invalid_count
return success_response( return success_response(
f"插件 {plugin_id} 爬取并验证完成", f"插件 {plugin_id} 爬取并验证完成",
{ {
"plugin_id": plugin_id, "plugin_id": plugin_id,
"proxy_count": len(results), "proxy_count": len(results),
"valid_count": valid_count, "valid_count": scheduler_service.validation_queue.valid_count,
"invalid_count": invalid_count, "invalid_count": scheduler_service.validation_queue.invalid_count,
}, },
) )
except Exception as e: except Exception as e:
logger.error(f"Crawl plugin {plugin_id} failed: {e}") logger.error(f"Crawl plugin {plugin_id} failed: {e}")
return error_response(f"插件爬取失败: {str(e)}") return error_response(f"插件爬取失败: {str(e)}", 500)
@router.post("/crawl-all") @router.post("/crawl-all")
@@ -140,28 +132,23 @@ async def crawl_all(
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0}, {"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
) )
logger.info(f"All plugins crawled {len(results)} unique proxies, sending to validation queue") logger.info(f"All plugins crawled {len(results)} unique proxies")
scheduler_service.validation_queue.reset_stats() scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results) await scheduler_service.validation_queue.submit(results)
try: try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0) await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
except asyncio.TimeoutError: except asyncio.TimeoutError:
pass pass
valid_count = scheduler_service.validation_queue.valid_count
invalid_count = scheduler_service.validation_queue.invalid_count
return success_response( return success_response(
"所有插件爬取并验证完成", "所有插件爬取并验证完成",
{ {
"total_crawled": len(results), "total_crawled": len(results),
"valid_count": valid_count, "valid_count": scheduler_service.validation_queue.valid_count,
"invalid_count": invalid_count, "invalid_count": scheduler_service.validation_queue.invalid_count,
}, },
) )
except Exception as e: except Exception as e:
logger.error(f"Crawl all failed: {e}") logger.error(f"Crawl all failed: {e}")
return error_response(f"批量爬取失败: {str(e)}") return error_response(f"批量爬取失败: {str(e)}", 500)
import asyncio

125
app/api/routes/proxies.py Normal file
View File

@@ -0,0 +1,125 @@
"""代理相关路由(含统计信息)"""
from typing import Optional
from fastapi import APIRouter, Depends, Query
from app.services.proxy_service import ProxyService
from app.services.scheduler_service import SchedulerService
from app.models.schemas import ProxyListRequest, BatchDeleteRequest
from app.api.deps import get_proxy_service, get_scheduler_service
from app.api.common import success_response, error_response, format_proxy
from app.core.log import logger
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
@router.get("/stats")
async def get_stats(
proxy_service: ProxyService = Depends(get_proxy_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
try:
stats = await proxy_service.get_stats()
stats["scheduler_running"] = scheduler_service.running
return success_response("获取统计信息成功", stats)
except Exception as e:
logger.error(f"Get stats failed: {e}")
return error_response("获取统计信息失败", 500)
@router.post("")
async def list_proxies(
request: ProxyListRequest,
service: ProxyService = Depends(get_proxy_service),
):
try:
proxies, total = await service.list_proxies(
page=request.page,
page_size=request.page_size,
protocol=request.protocol,
min_score=request.min_score,
max_score=request.max_score,
sort_by=request.sort_by,
sort_order=request.sort_order,
)
return success_response(
"获取代理列表成功",
{
"list": [format_proxy(p) for p in proxies],
"total": total,
"page": request.page,
"page_size": request.page_size,
},
)
except Exception as e:
logger.error(f"List proxies failed: {e}")
return error_response("获取代理列表失败", 500)
@router.get("/random")
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
try:
proxy = await service.get_random_proxy()
if not proxy:
return error_response("没有找到可用的代理", 404)
return success_response("获取随机代理成功", format_proxy(proxy))
except Exception as e:
logger.error(f"Get random proxy failed: {e}")
return error_response("获取随机代理失败", 500)
@router.get("/export/{fmt}")
async def export_proxies(
fmt: str,
protocol: Optional[str] = None,
limit: int = Query(default=10000, ge=1, le=100000),
service: ProxyService = Depends(get_proxy_service),
):
if fmt not in ("csv", "txt", "json"):
return error_response("不支持的导出格式", 400)
from fastapi.responses import StreamingResponse
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
async def generate():
async for chunk in service.export_proxies(fmt, protocol, limit):
yield chunk
return StreamingResponse(
generate(),
media_type=media_types[fmt],
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
)
@router.delete("/{ip}/{port}")
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
try:
await service.delete_proxy(ip, port)
return success_response("删除代理成功")
except Exception as e:
logger.error(f"Delete proxy failed: {e}")
return error_response("删除代理失败", 500)
@router.post("/batch-delete")
async def batch_delete(
request: BatchDeleteRequest,
service: ProxyService = Depends(get_proxy_service),
):
try:
proxies = [(item.ip, item.port) for item in request.proxies]
deleted = await service.batch_delete(proxies)
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
except Exception as e:
logger.error(f"Batch delete failed: {e}")
return error_response("批量删除失败", 500)
@router.delete("/clean-invalid")
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
try:
count = await service.clean_invalid()
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
except Exception as e:
logger.error(f"Clean invalid failed: {e}")
return error_response("清理无效代理失败", 500)

View File

@@ -0,0 +1,64 @@
"""调度器相关路由"""
from fastapi import APIRouter, Depends
from app.services.scheduler_service import SchedulerService
from app.repositories.settings_repo import SettingsRepository
from app.core.db import get_db
from app.api.deps import get_scheduler_service
from app.api.common import success_response, error_response
from app.core.log import logger
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
settings_repo = SettingsRepository()
async def _save_auto_validate_setting(enabled: bool):
"""保存自动验证设置"""
async with get_db() as db:
settings = await settings_repo.get_all(db)
settings["auto_validate"] = enabled
from app.models.schemas import SettingsSchema
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
@router.post("/start")
async def start_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
if scheduler.running:
return success_response("验证调度器已在运行", {"running": True})
await scheduler.start()
await _save_auto_validate_setting(True)
return success_response("验证调度器已启动", {"running": True})
except Exception as e:
logger.error(f"Start scheduler failed: {e}")
return error_response(f"启动调度器失败: {str(e)}", 500)
@router.post("/stop")
async def stop_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
if not scheduler.running:
return success_response("验证调度器未运行", {"running": False})
await scheduler.stop()
await _save_auto_validate_setting(False)
return success_response("验证调度器已停止", {"running": False})
except Exception as e:
logger.error(f"Stop scheduler failed: {e}")
return error_response(f"停止调度器失败: {str(e)}", 500)
@router.post("/validate-now")
async def validate_now(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
scheduler.validate_all_now()
return success_response("已开始全量验证", {"started": True})
except Exception as e:
logger.error(f"Validate now failed: {e}")
return error_response(f"启动验证失败: {str(e)}", 500)
@router.get("/status")
async def scheduler_status(scheduler: SchedulerService = Depends(get_scheduler_service)):
return success_response(
"获取状态成功",
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
)

View File

@@ -1,22 +1,15 @@
"""设置相关路由""" """设置相关路由"""
from fastapi import APIRouter from fastapi import APIRouter
from core.db import get_db from app.core.db import get_db
from repositories.settings_repo import SettingsRepository from app.repositories.settings_repo import SettingsRepository
from models.schemas import SettingsSchema from app.models.schemas import SettingsSchema
from core.log import logger from app.api.common import success_response, error_response
from app.core.log import logger
router = APIRouter(prefix="/api/settings", tags=["settings"]) router = APIRouter(prefix="/api/settings", tags=["settings"])
settings_repo = SettingsRepository() settings_repo = SettingsRepository()
def success_response(message: str, data=None):
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500):
return {"code": code, "message": message, "data": None}
@router.get("") @router.get("")
async def get_settings(): async def get_settings():
try: try:
@@ -25,7 +18,7 @@ async def get_settings():
return success_response("获取设置成功", settings) return success_response("获取设置成功", settings)
except Exception as e: except Exception as e:
logger.error(f"Get settings failed: {e}") logger.error(f"Get settings failed: {e}")
return error_response("获取设置失败") return error_response("获取设置失败", 500)
@router.post("") @router.post("")
@@ -34,8 +27,8 @@ async def save_settings(request: SettingsSchema):
async with get_db() as db: async with get_db() as db:
success = await settings_repo.save(db, request.model_dump()) success = await settings_repo.save(db, request.model_dump())
if not success: if not success:
return error_response("保存设置失败") return error_response("保存设置失败", 500)
return success_response("保存设置成功", request.model_dump()) return success_response("保存设置成功", request.model_dump())
except Exception as e: except Exception as e:
logger.error(f"Save settings failed: {e}") logger.error(f"Save settings failed: {e}")
return error_response(f"保存设置失败: {str(e)}") return error_response(f"保存设置失败: {str(e)}", 500)

13
app/core/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
"""核心基础设施包"""
from .config import settings
from .log import logger
from .exceptions import ProxyPoolException, PluginNotFoundException, ProxyNotFoundException, ValidationException
__all__ = [
"settings",
"logger",
"ProxyPoolException",
"PluginNotFoundException",
"ProxyNotFoundException",
"ValidationException",
]

View File

@@ -52,7 +52,7 @@ class Settings(BaseSettings):
@property @property
def base_dir(self) -> str: def base_dir(self) -> str:
return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# 全局配置实例(启动时加载一次) # 全局配置实例(启动时加载一次)

View File

@@ -3,8 +3,8 @@ import os
import aiosqlite import aiosqlite
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from typing import AsyncIterator from typing import AsyncIterator
from core.config import settings from app.core.config import settings
from core.log import logger from app.core.log import logger
DB_PATH = os.path.join(settings.base_dir, settings.db_path) DB_PATH = os.path.join(settings.base_dir, settings.db_path)

View File

@@ -3,12 +3,13 @@ import os
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from datetime import datetime from datetime import datetime
class LogHandler(logging.Logger): class LogHandler(logging.Logger):
def __init__(self, name='ProxyPool', level=logging.INFO): def __init__(self, name='ProxyPool', level=logging.INFO):
super().__init__(name, level) super().__init__(name, level)
# 获取项目根目录并创建 logs 目录 # 获取项目根目录并创建 logs 目录
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
log_dir = os.path.join(base_dir, 'logs') log_dir = os.path.join(base_dir, 'logs')
if not os.path.exists(log_dir): if not os.path.exists(log_dir):
os.makedirs(log_dir) os.makedirs(log_dir)
@@ -38,6 +39,7 @@ class LogHandler(logging.Logger):
console_handler.setFormatter(formatter) console_handler.setFormatter(formatter)
self.addHandler(console_handler) self.addHandler(console_handler)
# 实例化一个默认 logger 供外部直接使用 # 实例化一个默认 logger 供外部直接使用
logger = LogHandler() logger = LogHandler()

View File

@@ -1,3 +1,4 @@
"""插件系统包"""
from .base import BaseCrawlerPlugin, ProxyRaw from .base import BaseCrawlerPlugin, ProxyRaw
from .registry import registry from .registry import registry

View File

@@ -3,8 +3,8 @@ import importlib
import inspect import inspect
import os import os
from typing import Dict, List, Type, Optional from typing import Dict, List, Type, Optional
from core.plugin_system.base import BaseCrawlerPlugin from app.core.plugin_system.base import BaseCrawlerPlugin
from core.log import logger from app.core.log import logger
class PluginRegistry: class PluginRegistry:

View File

@@ -1,3 +1,4 @@
"""任务队列包"""
from .queue import ValidationQueue from .queue import ValidationQueue
__all__ = ["ValidationQueue"] __all__ = ["ValidationQueue"]

View File

@@ -1,10 +1,10 @@
"""验证任务队列 - 解耦爬取与验证,支持背压控制和持久化""" """验证任务队列 - 解耦爬取与验证,支持背压控制和持久化"""
import asyncio import asyncio
from typing import Optional from typing import Optional
from models.domain import ProxyRaw from app.models.domain import ProxyRaw
from repositories.task_repo import ValidationTaskRepository from app.repositories.task_repo import ValidationTaskRepository
from core.db import get_db from app.core.db import get_db
from core.log import logger from app.core.log import logger
class ValidationQueue: class ValidationQueue:

30
app/models/__init__.py Normal file
View File

@@ -0,0 +1,30 @@
"""数据模型包"""
from .domain import ProxyRaw, Proxy, PluginInfo
from .schemas import (
ProxyCreate,
ProxyResponse,
PluginResponse,
SettingsSchema,
CrawlResult,
ProxyListRequest,
ProxyDeleteItem,
BatchDeleteRequest,
PluginToggleRequest,
ExportRequest,
)
__all__ = [
"ProxyRaw",
"Proxy",
"PluginInfo",
"ProxyCreate",
"ProxyResponse",
"PluginResponse",
"SettingsSchema",
"CrawlResult",
"ProxyListRequest",
"ProxyDeleteItem",
"BatchDeleteRequest",
"PluginToggleRequest",
"ExportRequest",
]

View File

@@ -1,5 +1,5 @@
"""插件包 - 在这里显式注册所有爬虫插件""" """插件包 - 在这里显式注册所有爬虫插件"""
from core.plugin_system import registry from app.core.plugin_system import registry
from .fate0 import Fate0Plugin from .fate0 import Fate0Plugin
from .proxylist_download import ProxyListDownloadPlugin from .proxylist_download import ProxyListDownloadPlugin

View File

@@ -3,7 +3,7 @@ import random
import asyncio import asyncio
import aiohttp import aiohttp
from typing import List from typing import List
from core.plugin_system import BaseCrawlerPlugin from app.core.plugin_system import BaseCrawlerPlugin
class BaseHTTPPlugin(BaseCrawlerPlugin): class BaseHTTPPlugin(BaseCrawlerPlugin):

View File

@@ -1,8 +1,8 @@
import json import json
from typing import List from typing import List
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
class Fate0Plugin(BaseHTTPPlugin): class Fate0Plugin(BaseHTTPPlugin):

View File

@@ -1,9 +1,9 @@
import re import re
from typing import List from typing import List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5") VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")

View File

@@ -1,9 +1,9 @@
import re import re
from typing import List from typing import List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
class Ip89Plugin(BaseHTTPPlugin): class Ip89Plugin(BaseHTTPPlugin):

View File

@@ -1,9 +1,9 @@
import re import re
from typing import List from typing import List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5") VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")

View File

@@ -1,7 +1,7 @@
from typing import List from typing import List
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
class ProxyListDownloadPlugin(BaseHTTPPlugin): class ProxyListDownloadPlugin(BaseHTTPPlugin):

View File

@@ -1,8 +1,8 @@
"""ProxyScrape 测试爬虫 - 用于验证架构,支持全协议类型""" """ProxyScrape 测试爬虫 - 用于验证架构,支持全协议类型"""
from typing import List from typing import List
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
class ProxyScrapePlugin(BaseHTTPPlugin): class ProxyScrapePlugin(BaseHTTPPlugin):

View File

@@ -1,8 +1,8 @@
import re import re
from typing import List from typing import List
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
class SpeedXPlugin(BaseHTTPPlugin): class SpeedXPlugin(BaseHTTPPlugin):

View File

@@ -1,9 +1,9 @@
import re import re
from typing import List from typing import List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from core.plugin_system import ProxyRaw from app.core.plugin_system import ProxyRaw
from plugins.base import BaseHTTPPlugin from app.plugins.base import BaseHTTPPlugin
from core.log import logger from app.core.log import logger
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5") VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")

View File

@@ -1,5 +1,11 @@
"""数据访问层包"""
from .proxy_repo import ProxyRepository from .proxy_repo import ProxyRepository
from .settings_repo import SettingsRepository, PluginSettingsRepository from .settings_repo import SettingsRepository, PluginSettingsRepository
from .task_repo import ValidationTaskRepository from .task_repo import ValidationTaskRepository
__all__ = ["ProxyRepository", "SettingsRepository", "PluginSettingsRepository", "ValidationTaskRepository"] __all__ = [
"ProxyRepository",
"SettingsRepository",
"PluginSettingsRepository",
"ValidationTaskRepository",
]

View File

@@ -2,8 +2,8 @@
import aiosqlite import aiosqlite
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import List, Optional, Tuple, Union from typing import List, Optional, Tuple, Union
from models.domain import Proxy from app.models.domain import Proxy
from core.log import logger from app.core.log import logger
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5") VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")

View File

@@ -2,7 +2,7 @@
import json import json
import aiosqlite import aiosqlite
from typing import Optional, Dict, Any from typing import Optional, Dict, Any
from core.log import logger from app.core.log import logger
DEFAULT_SETTINGS = { DEFAULT_SETTINGS = {

View File

@@ -1,8 +1,8 @@
"""验证任务队列持久化层""" """验证任务队列持久化层"""
import aiosqlite import aiosqlite
from typing import List, Optional from typing import List, Optional
from models.domain import ProxyRaw from app.models.domain import ProxyRaw
from core.log import logger from app.core.log import logger
class ValidationTaskRepository: class ValidationTaskRepository:

12
app/services/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""业务逻辑层包"""
from .proxy_service import ProxyService
from .plugin_service import PluginService
from .scheduler_service import SchedulerService
from .validator_service import ValidatorService
__all__ = [
"ProxyService",
"PluginService",
"SchedulerService",
"ValidatorService",
]

View File

@@ -1,12 +1,12 @@
"""插件业务服务""" """插件业务服务"""
from datetime import datetime from datetime import datetime
from typing import List, Optional from typing import List, Optional
from core.db import get_db from app.core.db import get_db
from core.plugin_system.registry import registry from app.core.plugin_system.registry import registry
from core.plugin_system.base import BaseCrawlerPlugin from app.core.plugin_system.base import BaseCrawlerPlugin
from repositories.settings_repo import PluginSettingsRepository from app.repositories.settings_repo import PluginSettingsRepository
from models.domain import PluginInfo, ProxyRaw from app.models.domain import PluginInfo, ProxyRaw
from core.log import logger from app.core.log import logger
class PluginService: class PluginService:

View File

@@ -4,10 +4,10 @@ import json
import io import io
from datetime import datetime from datetime import datetime
from typing import List, Optional, Tuple, AsyncIterator from typing import List, Optional, Tuple, AsyncIterator
from core.db import get_db from app.core.db import get_db
from repositories.proxy_repo import ProxyRepository from app.repositories.proxy_repo import ProxyRepository
from models.domain import Proxy from app.models.domain import Proxy
from core.log import logger from app.core.log import logger
class ProxyService: class ProxyService:

View File

@@ -1,11 +1,11 @@
"""调度器服务 - 定时验证存量代理""" """调度器服务 - 定时验证存量代理"""
import asyncio import asyncio
from datetime import datetime from datetime import datetime
from core.db import get_db from app.core.db import get_db
from repositories.proxy_repo import ProxyRepository from app.repositories.proxy_repo import ProxyRepository
from core.tasks.queue import ValidationQueue from app.core.tasks.queue import ValidationQueue
from core.config import settings as app_settings from app.core.config import settings as app_settings
from core.log import logger from app.core.log import logger
class SchedulerService: class SchedulerService:
@@ -70,7 +70,7 @@ class SchedulerService:
return return
logger.info(f"Validating {len(proxies)} proxies from database") logger.info(f"Validating {len(proxies)} proxies from database")
from models.domain import ProxyRaw from app.models.domain import ProxyRaw
# 批量提交到验证队列 # 批量提交到验证队列
batch_size = 100 batch_size = 100

View File

@@ -5,12 +5,18 @@ import time
import aiohttp import aiohttp
import aiohttp_socks import aiohttp_socks
from typing import Tuple from typing import Tuple
from core.log import logger from app.core.log import logger
class ValidatorService: class ValidatorService:
"""代理验证器""" """代理验证器"""
# 测试 URL
TEST_URLS = {
"http": ["http://httpbin.org/ip", "http://api.ipify.org"],
"https": ["https://httpbin.org/ip", "https://api.ipify.org"],
}
def __init__( def __init__(
self, self,
timeout: float = 5.0, timeout: float = 5.0,
@@ -20,33 +26,23 @@ class ValidatorService:
self.timeout = timeout self.timeout = timeout
self.connect_timeout = connect_timeout self.connect_timeout = connect_timeout
self.semaphore = asyncio.Semaphore(max_concurrency) self.semaphore = asyncio.Semaphore(max_concurrency)
self.http_sources = [
"http://httpbin.org/ip",
"http://api.ipify.org",
]
self.https_sources = [
"https://httpbin.org/ip",
"https://api.ipify.org",
]
def _get_test_url(self, protocol: str) -> str: def _get_test_url(self, protocol: str) -> str:
protocol = protocol.lower() """获取测试 URL"""
if protocol == "https": urls = self.TEST_URLS.get(protocol.lower(), self.TEST_URLS["http"])
return random.choice(self.https_sources) return random.choice(urls)
return random.choice(self.http_sources)
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]: async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
"""验证单个代理,返回 (是否有效, 延迟毫秒)""" """验证单个代理,返回 (是否有效, 延迟毫秒)"""
protocol = protocol.lower() protocol = protocol.lower()
test_url = self._get_test_url(protocol)
async with self.semaphore: async with self.semaphore:
start = time.time() start = time.time()
try: try:
if protocol in ("socks4", "socks5"): if protocol in ("socks4", "socks5"):
return await self._validate_socks(ip, port, protocol, test_url, start) return await self._validate_socks(ip, port, protocol, start)
else: else:
return await self._validate_http(ip, port, protocol, test_url, start) return await self._validate_http(ip, port, protocol, start)
except asyncio.TimeoutError: except asyncio.TimeoutError:
logger.debug(f"Validation timeout: {ip}:{port} ({protocol})") logger.debug(f"Validation timeout: {ip}:{port} ({protocol})")
return False, 0.0 return False, 0.0
@@ -54,18 +50,16 @@ class ValidatorService:
logger.debug(f"Validation error {ip}:{port} ({protocol}): {e}") logger.debug(f"Validation error {ip}:{port} ({protocol}): {e}")
return False, 0.0 return False, 0.0
async def _validate_http( async def _validate_http(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
self, ip: str, port: int, protocol: str, test_url: str, start: float """验证 HTTP/HTTPS 代理"""
) -> Tuple[bool, float]:
proxy_url = f"http://{ip}:{port}" proxy_url = f"http://{ip}:{port}"
connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True) connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout) timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
test_url = self._get_test_url(protocol)
try: try:
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
async with session.get( async with session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
test_url, proxy=proxy_url, allow_redirects=True
) as response:
if response.status in (200, 301, 302): if response.status in (200, 301, 302):
latency = round((time.time() - start) * 1000, 2) latency = round((time.time() - start) * 1000, 2)
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms") logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
@@ -74,9 +68,8 @@ class ValidatorService:
finally: finally:
await connector.close() await connector.close()
async def _validate_socks( async def _validate_socks(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
self, ip: str, port: int, protocol: str, test_url: str, start: float """验证 SOCKS4/SOCKS5 代理"""
) -> Tuple[bool, float]:
proxy_type = ( proxy_type = (
aiohttp_socks.ProxyType.SOCKS4 aiohttp_socks.ProxyType.SOCKS4
if protocol == "socks4" if protocol == "socks4"
@@ -90,6 +83,7 @@ class ValidatorService:
ssl=False, ssl=False,
) )
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout) timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
test_url = self._get_test_url("http")
try: try:
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:

View File

@@ -1,7 +1,7 @@
"""项目入口""" """项目入口"""
import uvicorn import uvicorn
from api import create_app from app.api import create_app
from core.config import settings from app.core.config import settings
app = create_app() app = create_app()

View File

@@ -1,13 +0,0 @@
from .domain import ProxyRaw, Proxy, PluginInfo
from .schemas import ProxyCreate, ProxyResponse, PluginResponse, SettingsSchema, CrawlResult
__all__ = [
"ProxyRaw",
"Proxy",
"PluginInfo",
"ProxyCreate",
"ProxyResponse",
"PluginResponse",
"SettingsSchema",
"CrawlResult",
]

17
pytest.ini Normal file
View File

@@ -0,0 +1,17 @@
[tool:pytest]
testpaths = tests
python_files = test_*.py
python_classes = Test*
python_functions = test_*
addopts =
-v
--tb=short
--strict-markers
-p no:warnings
markers =
unit: 单元测试
integration: 集成测试
e2e: 端到端测试
slow: 慢速测试
async_test: 异步测试
asyncio_mode = auto

View File

@@ -74,7 +74,7 @@ echo.
REM 4. Start Frontend REM 4. Start Frontend
echo [4/4] Starting frontend (Vite)... echo [4/4] Starting frontend (Vite)...
cd /d "%ROOT_PATH%\frontend" cd /d "%ROOT_PATH%\WebUI"
start /B "" cmd /c "npm run dev" start /B "" cmd /c "npm run dev"
cd /d "%ROOT_PATH%" cd /d "%ROOT_PATH%"
echo Frontend started echo Frontend started

159
tests/README.md Normal file
View File

@@ -0,0 +1,159 @@
# 测试说明
## 测试结构
```
tests/
├── conftest.py # pytest 配置和 fixtures
├── README.md # 本文件
├── unit/ # 单元测试
│ ├── test_models.py # 模型测试
│ └── test_repositories.py # 仓库层测试
├── integration/ # 集成测试
│ ├── test_proxies_api.py # 代理 API 测试
│ ├── test_plugins_api.py # 插件 API 测试
│ ├── test_scheduler_api.py # 调度器 API 测试
│ ├── test_settings_api.py # 设置 API 测试
│ └── test_health_api.py # 健康检查测试
└── e2e/ # 端到端测试
└── test_full_workflow.py # 完整工作流测试
```
## 运行测试
### 安装测试依赖
```bash
pip install pytest pytest-asyncio httpx
```
### 运行所有测试
```bash
pytest
```
### 运行特定类型的测试
```bash
# 仅运行单元测试
pytest tests/unit -v
# 仅运行集成测试
pytest tests/integration -v
# 仅运行 E2E 测试
pytest tests/e2e -v
```
### 运行特定测试文件
```bash
pytest tests/integration/test_proxies_api.py -v
```
### 运行特定测试函数
```bash
pytest tests/integration/test_proxies_api.py::TestProxiesAPI::test_get_stats -v
```
## 测试覆盖的 API
### 代理 API (`/api/proxies/*`)
-`GET /api/proxies/stats` - 获取统计信息
-`POST /api/proxies` - 列出代理
-`GET /api/proxies/random` - 获取随机代理
-`GET /api/proxies/export/{format}` - 导出代理 (csv, txt, json)
-`DELETE /api/proxies/{ip}/{port}` - 删除代理
-`POST /api/proxies/batch-delete` - 批量删除
-`DELETE /api/proxies/clean-invalid` - 清理无效代理
### 插件 API (`/api/plugins/*`)
-`GET /api/plugins` - 列出插件
-`PUT /api/plugins/{id}/toggle` - 切换插件状态
-`GET /api/plugins/{id}/config` - 获取插件配置
-`POST /api/plugins/{id}/config` - 更新插件配置
-`POST /api/plugins/{id}/crawl` - 触发单个插件爬取
-`POST /api/plugins/crawl-all` - 触发所有插件爬取
### 调度器 API (`/api/scheduler/*`)
-`GET /api/scheduler/status` - 获取调度器状态
-`POST /api/scheduler/start` - 启动调度器
-`POST /api/scheduler/stop` - 停止调度器
-`POST /api/scheduler/validate-now` - 立即验证
### 设置 API (`/api/settings`)
-`GET /api/settings` - 获取设置
-`POST /api/settings` - 保存设置
### 健康检查
-`GET /` - 根端点
-`GET /health` - 健康检查
## 测试 Fixtures
### `client`
异步 HTTP 客户端,用于发送请求到测试应用。
```python
async def test_example(client):
response = await client.get("/api/proxies/stats")
assert response.status_code == 200
```
### `db`
数据库连接 fixture。
```python
async def test_example(db, proxy_repo):
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
```
### `sample_proxy`
创建一个测试代理并自动清理。
```python
async def test_example(client, sample_proxy):
# sample_proxy = {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
response = await client.delete(f"/api/proxies/{sample_proxy['ip']}/{sample_proxy['port']}")
assert response.status_code == 200
```
## 编写新测试
### 单元测试示例
```python
# tests/unit/test_new_feature.py
import pytest
from app.models.domain import ProxyRaw
class TestProxyRaw:
def test_create(self):
proxy = ProxyRaw("192.168.1.1", 8080, "http")
assert proxy.ip == "192.168.1.1"
```
### 集成测试示例
```python
# tests/integration/test_new_api.py
import pytest
class TestNewAPI:
@pytest.mark.asyncio
async def test_new_endpoint(self, client):
response = await client.get("/api/new-endpoint")
assert response.status_code == 200
```

1
tests/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""测试包"""

5
tests/__main__.py Normal file
View File

@@ -0,0 +1,5 @@
"""直接运行 python -m tests 时执行的入口"""
from .run_tests import main
import sys
sys.exit(main())

Binary file not shown.

56
tests/conftest.py Normal file
View File

@@ -0,0 +1,56 @@
"""pytest 配置文件和 fixtures"""
import pytest
import asyncio
from typing import AsyncGenerator, Generator
from httpx import AsyncClient, ASGITransport
from app.api import create_app
from app.core.db import init_db, get_db
from app.repositories.proxy_repo import ProxyRepository
@pytest.fixture(scope="session")
def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]:
"""创建事件循环"""
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
loop.close()
@pytest.fixture(scope="session")
async def app():
"""创建应用实例"""
# 初始化测试数据库
await init_db()
app = create_app()
return app
@pytest.fixture
async def client(app) -> AsyncGenerator[AsyncClient, None]:
"""创建异步 HTTP 客户端"""
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
yield client
@pytest.fixture
async def db():
"""获取数据库连接"""
async with get_db() as db:
yield db
@pytest.fixture
async def proxy_repo():
"""获取代理仓库"""
return ProxyRepository()
@pytest.fixture
async def sample_proxy(db, proxy_repo):
"""创建一个测试代理"""
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
yield {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
# 清理
await proxy_repo.delete(db, "192.168.1.1", 8080)

1
tests/e2e/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""端到端测试"""

View File

@@ -0,0 +1,182 @@
"""完整工作流 E2E 测试
这些测试模拟真实用户场景,验证整个系统的集成功能。
"""
import pytest
class TestFullWorkflow:
"""测试完整工作流"""
@pytest.mark.asyncio
async def test_proxy_management_workflow(self, client):
"""测试代理管理完整工作流
场景:
1. 查看统计信息
2. 列出代理
3. 触发爬取
4. 查看更新后的统计
5. 导出代理
6. 清理无效代理
"""
# 1. 获取初始统计
response = await client.get("/api/proxies/stats")
assert response.status_code == 200
initial_stats = response.json()["data"]
# 2. 列出代理
response = await client.post("/api/proxies", json={
"page": 1,
"page_size": 20,
})
assert response.status_code == 200
# 3. 触发所有插件爬取
response = await client.post("/api/plugins/crawl-all")
assert response.status_code == 200
crawl_result = response.json()["data"]
# 4. 获取更新后的统计
response = await client.get("/api/proxies/stats")
updated_stats = response.json()["data"]
# 5. 导出代理(所有格式)
for fmt in ["csv", "txt", "json"]:
response = await client.get(f"/api/proxies/export/{fmt}")
assert response.status_code == 200
# 6. 清理无效代理
response = await client.delete("/api/proxies/clean-invalid")
assert response.status_code == 200
@pytest.mark.asyncio
async def test_plugin_management_workflow(self, client):
"""测试插件管理完整工作流
场景:
1. 列出所有插件
2. 禁用某个插件
3. 更新插件配置
4. 启用插件
5. 触发单个插件爬取
"""
# 1. 列出插件
response = await client.get("/api/plugins")
assert response.status_code == 200
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
# 2. 禁用插件
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
assert response.status_code == 200
# 3. 获取插件配置
response = await client.get(f"/api/plugins/{plugin_id}/config")
assert response.status_code == 200
# 4. 更新插件配置
response = await client.post(
f"/api/plugins/{plugin_id}/config",
json={"config": {"max_pages": 3}}
)
assert response.status_code == 200
# 5. 启用插件
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
assert response.status_code == 200
# 6. 触发爬取
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
assert response.status_code == 200
@pytest.mark.asyncio
async def test_scheduler_workflow(self, client):
"""测试调度器工作流
场景:
1. 启动调度器
2. 触发立即验证
3. 检查状态
4. 停止调度器
"""
# 1. 启动调度器
response = await client.post("/api/scheduler/start")
assert response.status_code == 200
# 2. 触发立即验证
response = await client.post("/api/scheduler/validate-now")
assert response.status_code == 200
# 3. 检查状态
response = await client.get("/api/scheduler/status")
assert response.status_code == 200
assert response.json()["data"]["running"] is True
# 4. 停止调度器
response = await client.post("/api/scheduler/stop")
assert response.status_code == 200
assert response.json()["data"]["running"] is False
@pytest.mark.asyncio
async def test_settings_workflow(self, client):
"""测试设置工作流
场景:
1. 获取当前设置
2. 修改设置
3. 验证设置已保存
4. 恢复默认设置
"""
# 1. 获取当前设置
response = await client.get("/api/settings")
assert response.status_code == 200
original_settings = response.json()["data"]
# 2. 修改设置
new_settings = original_settings.copy()
new_settings["crawl_timeout"] = 45
new_settings["auto_validate"] = not original_settings["auto_validate"]
response = await client.post("/api/settings", json=new_settings)
assert response.status_code == 200
# 3. 验证设置已保存
response = await client.get("/api/settings")
saved_settings = response.json()["data"]
assert saved_settings["crawl_timeout"] == 45
# 4. 恢复原始设置
response = await client.post("/api/settings", json=original_settings)
assert response.status_code == 200
@pytest.mark.asyncio
async def test_batch_operations_workflow(self, client):
"""测试批量操作工作流
场景:
1. 批量删除不存在的代理(幂等性测试)
2. 导出所有代理
3. 获取随机代理
"""
# 1. 批量删除(幂等性)
response = await client.post("/api/proxies/batch-delete", json={
"proxies": [
{"ip": "192.168.100.1", "port": 8080},
{"ip": "192.168.100.2", "port": 8081},
]
})
assert response.status_code == 200
# 2. 导出所有格式
for fmt in ["csv", "txt", "json"]:
response = await client.get(f"/api/proxies/export/{fmt}")
assert response.status_code == 200
# 3. 获取随机代理(可能返回 200 或 404
response = await client.get("/api/proxies/random")
assert response.status_code in [200, 404]

View File

@@ -0,0 +1 @@
"""集成测试"""

View File

@@ -0,0 +1,47 @@
"""健康检查 API 测试"""
import pytest
class TestHealthAPI:
"""测试健康检查端点"""
@pytest.mark.asyncio
async def test_root_endpoint(self, client):
"""测试根端点 /"""
response = await client.get("/")
assert response.status_code == 200
data = response.json()
assert "message" in data
assert "status" in data
assert data["status"] == "running"
@pytest.mark.asyncio
async def test_health_endpoint(self, client):
"""测试健康检查端点 /health"""
response = await client.get("/health")
assert response.status_code == 200
data = response.json()
assert data["status"] == "healthy"
assert "timestamp" in data
assert "database" in data
assert "scheduler" in data
assert "version" in data
@pytest.mark.asyncio
async def test_health_endpoint_structure(self, client):
"""测试健康检查端点返回结构"""
response = await client.get("/health")
data = response.json()
assert "status" in data
assert "timestamp" in data
assert "database" in data
assert "scheduler" in data
assert "version" in data
# 验证数据类型
assert isinstance(data["status"], str)
assert isinstance(data["timestamp"], str)
assert isinstance(data["database"], str)
assert isinstance(data["scheduler"], str)
assert isinstance(data["version"], str)

View File

@@ -0,0 +1,147 @@
"""插件 API 集成测试 - 测试 /api/plugins/* 所有接口"""
import pytest
class TestPluginsAPI:
"""测试插件相关 API"""
@pytest.mark.asyncio
async def test_list_plugins(self, client):
"""测试 GET /api/plugins"""
response = await client.get("/api/plugins")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "plugins" in data["data"]
assert isinstance(data["data"]["plugins"], list)
@pytest.mark.asyncio
async def test_list_plugins_structure(self, client):
"""测试 GET /api/plugins 返回结构"""
response = await client.get("/api/plugins")
data = response.json()
if data["data"]["plugins"]:
plugin = data["data"]["plugins"][0]
assert "id" in plugin
assert "name" in plugin
assert "display_name" in plugin
assert "description" in plugin
assert "enabled" in plugin
@pytest.mark.asyncio
async def test_toggle_plugin_enable(self, client):
"""测试 PUT /api/plugins/{id}/toggle - 启用"""
# 先获取一个插件 ID
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert data["data"]["enabled"] is True
@pytest.mark.asyncio
async def test_toggle_plugin_disable(self, client):
"""测试 PUT /api/plugins/{id}/toggle - 禁用"""
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert data["data"]["enabled"] is False
@pytest.mark.asyncio
async def test_toggle_plugin_missing_enabled(self, client):
"""测试 PUT /api/plugins/{id}/toggle - 缺少 enabled 参数"""
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={})
assert response.status_code == 400
@pytest.mark.asyncio
async def test_toggle_nonexistent_plugin(self, client):
"""测试 PUT /api/plugins/{id}/toggle - 不存在的插件"""
response = await client.put("/api/plugins/nonexistent_plugin/toggle", json={"enabled": True})
assert response.status_code == 404
@pytest.mark.asyncio
async def test_get_plugin_config(self, client):
"""测试 GET /api/plugins/{id}/config"""
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
response = await client.get(f"/api/plugins/{plugin_id}/config")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "config" in data["data"]
@pytest.mark.asyncio
async def test_get_nonexistent_plugin_config(self, client):
"""测试 GET /api/plugins/{id}/config - 不存在的插件"""
response = await client.get("/api/plugins/nonexistent_plugin/config")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_update_plugin_config(self, client):
"""测试 POST /api/plugins/{id}/config"""
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
response = await client.post(
f"/api/plugins/{plugin_id}/config",
json={"config": {"max_pages": 3}}
)
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
@pytest.mark.asyncio
async def test_crawl_plugin(self, client):
"""测试 POST /api/plugins/{id}/crawl"""
response = await client.get("/api/plugins")
plugins = response.json()["data"]["plugins"]
if not plugins:
pytest.skip("没有可用的插件")
plugin_id = plugins[0]["id"]
# 这个测试可能需要较长时间,设置较短的超时
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "proxy_count" in data["data"]
@pytest.mark.asyncio
async def test_crawl_nonexistent_plugin(self, client):
"""测试 POST /api/plugins/{id}/crawl - 不存在的插件"""
response = await client.post("/api/plugins/nonexistent_plugin/crawl")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_crawl_all_plugins(self, client):
"""测试 POST /api/plugins/crawl-all"""
response = await client.post("/api/plugins/crawl-all")
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "total_crawled" in data["data"]

Some files were not shown because too many files have changed in this diff Show More