重构: 迁移后端代码到 app 目录，前端移动到 WebUI，添加完整测试套件
主要变更: - 后端代码从根目录迁移到 app/ 目录 - 前端代码从 frontend/ 重命名为 WebUI/ - 更新所有导入路径以适配新结构 - 提取公共 API 响应函数到 app/api/common.py - 精简验证器服务代码 - 更新启动脚本和文档 测试: - 新增完整测试套件 (tests/) - 单元测试: 模型、仓库层 - 集成测试: 覆盖所有 22+ API 端点 - E2E 测试: 4个完整工作流场景 - 添加 pytest 配置和测试运行脚本
This commit is contained in:
祀梦
142
DESIGN.md
142
DESIGN.md
@@ -309,77 +309,87 @@ Store 只负责：
|
||||
|
||||
```
|
||||
ProxyPool/
|
||||
├── api/ # FastAPI 入口和路由
|
||||
│ ├── __init__.py
|
||||
│ ├── main.py # 应用工厂
|
||||
│ ├── lifespan.py # 生命周期管理
|
||||
│ ├── deps.py # 依赖注入
|
||||
│ ├── errors.py # 统一异常
|
||||
│ └── routes/
|
||||
│ ├── __init__.py
|
||||
│ ├── proxies.py
|
||||
│ ├── plugins.py
|
||||
│ ├── scheduler.py
|
||||
│ └── settings.py
|
||||
├── main.py # 项目入口
|
||||
├── requirements.txt # Python 依赖
|
||||
├── .env.example # 环境变量示例
|
||||
│
|
||||
├── core/ # 基础设施
|
||||
│ ├── __init__.py
|
||||
│ ├── config.py # Pydantic Settings
|
||||
│ ├── log.py # 日志
|
||||
│ ├── db.py # 数据库连接池/上下文
|
||||
│ └── exceptions.py # 业务异常
|
||||
│
|
||||
├── models/ # 数据模型
|
||||
│ ├── __init__.py
|
||||
│ ├── schemas.py # Pydantic 模型
|
||||
│ └── domain.py # 领域模型（ProxyRaw, PluginInfo 等）
|
||||
│
|
||||
├── repositories/ # 数据访问层
|
||||
│ ├── __init__.py
|
||||
│ └── proxy_repo.py # ProxyRepository
|
||||
│
|
||||
├── services/ # 业务逻辑层
|
||||
│ ├── __init__.py
|
||||
│ ├── proxy_service.py
|
||||
│ ├── plugin_service.py
|
||||
│ ├── scheduler_service.py
|
||||
│ └── validator_service.py
|
||||
│
|
||||
├── core/ # 任务与插件系统
|
||||
│ ├── plugin_system/
|
||||
├── app/ # 后端代码
|
||||
│ ├── api/ # FastAPI 入口和路由
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── base.py # BaseCrawlerPlugin
|
||||
│ │ └── registry.py # 插件注册中心
|
||||
│ └── tasks/
|
||||
│ │ ├── main.py # 应用工厂
|
||||
│ │ ├── lifespan.py # 生命周期管理
|
||||
│ │ ├── deps.py # 依赖注入
|
||||
│ │ ├── errors.py # 统一异常
|
||||
│ │ └── routes/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── proxies.py
|
||||
│ │ ├── plugins.py
|
||||
│ │ ├── scheduler.py
|
||||
│ │ └── settings.py
|
||||
│ │
|
||||
│ ├── core/ # 基础设施
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── config.py # Pydantic Settings
|
||||
│ │ ├── log.py # 日志
|
||||
│ │ ├── db.py # 数据库连接池/上下文
|
||||
│ │ ├── exceptions.py # 业务异常
|
||||
│ │ ├── plugin_system/ # 插件系统
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── base.py # BaseCrawlerPlugin
|
||||
│ │ │ └── registry.py # 插件注册中心
|
||||
│ │ └── tasks/ # 任务队列
|
||||
│ │ ├── __init__.py
|
||||
│ │ └── queue.py # ValidationQueue
|
||||
│ │
|
||||
│ ├── models/ # 数据模型
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── schemas.py # Pydantic 模型
|
||||
│ │ └── domain.py # 领域模型
|
||||
│ │
|
||||
│ ├── repositories/ # 数据访问层
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── proxy_repo.py
|
||||
│ │ ├── settings_repo.py
|
||||
│ │ └── task_repo.py
|
||||
│ │
|
||||
│ ├── services/ # 业务逻辑层
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── proxy_service.py
|
||||
│ │ ├── plugin_service.py
|
||||
│ │ ├── scheduler_service.py
|
||||
│ │ └── validator_service.py
|
||||
│ │
|
||||
│ └── plugins/ # 爬虫插件
|
||||
│ ├── __init__.py
|
||||
│ ├── queue.py # ValidationQueue
|
||||
│ └── workers.py # Worker Pool
|
||||
│ ├── base.py # 通用抓取基类
|
||||
│ ├── fate0.py
|
||||
│ ├── kuaidaili.py
|
||||
│ ├── ip3366.py
|
||||
│ ├── ip89.py
|
||||
│ ├── speedx.py
|
||||
│ ├── yundaili.py
|
||||
│ ├── proxylist_download.py
|
||||
│ └── proxyscrape.py
|
||||
│
|
||||
├── plugins/ # 爬虫插件
|
||||
│ ├── __init__.py
|
||||
│ ├── base.py # 通用抓取基类（HTTP 请求封装）
|
||||
│ ├── fate0.py
|
||||
│ ├── proxylist_download.py
|
||||
│ └── ...
|
||||
│
|
||||
├── frontend/ # Vue3 前端
|
||||
│ └── src/
|
||||
│ ├── services/ # 新增
|
||||
│ ├── stores/
|
||||
│ ├── api/
|
||||
│ └── ...
|
||||
├── WebUI/ # Vue3 前端
|
||||
│ ├── src/
|
||||
│ │ ├── api/ # API 封装
|
||||
│ │ ├── stores/ # Pinia 状态管理
|
||||
│ │ ├── views/ # 页面组件
|
||||
│ │ ├── router/ # 路由配置
|
||||
│ │ ├── components/ # 通用组件
|
||||
│ │ └── style.css # 全局样式
|
||||
│ ├── index.html
|
||||
│ └── package.json
|
||||
│
|
||||
├── tests/ # 测试目录
|
||||
│ ├── conftest.py
|
||||
│ ├── unit/
|
||||
│ └── integration/
|
||||
│
|
||||
├── script/
|
||||
├── data/
|
||||
├── db/
|
||||
├── logs/
|
||||
├── requirements.txt
|
||||
├── .env.example
|
||||
├── script/ # 启动脚本
|
||||
├── db/ # 数据存储
|
||||
├── logs/ # 日志文件
|
||||
└── DESIGN.md # 本文档
|
||||
```
|
||||
|
||||
@@ -426,11 +436,11 @@ ProxyPool/
|
||||
|
||||
假设要添加一个名为 `mynewsource` 的爬虫：
|
||||
|
||||
**Step 1**: 创建文件 `plugins/mynewsource.py`
|
||||
**Step 1**: 创建文件 `app/plugins/mynewsource.py`
|
||||
|
||||
```python
|
||||
from core.plugin_system import BaseCrawlerPlugin, ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin # 可选：如果基于 HTTP 爬取
|
||||
from app.core.plugin_system import BaseCrawlerPlugin, ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin # 可选：如果基于 HTTP 爬取
|
||||
|
||||
class MyNewSourcePlugin(BaseHTTPPlugin):
|
||||
name = "mynewsource"
|
||||
@@ -450,11 +460,11 @@ class MyNewSourcePlugin(BaseHTTPPlugin):
|
||||
return results
|
||||
```
|
||||
|
||||
**Step 2**: 在 `plugins/__init__.py` 中注册
|
||||
**Step 2**: 在 `app/plugins/__init__.py` 中注册
|
||||
|
||||
```python
|
||||
from .mynewsource import MyNewSourcePlugin
|
||||
from core.plugin_system import registry
|
||||
from app.core.plugin_system import registry
|
||||
|
||||
registry.register(MyNewSourcePlugin)
|
||||
```
|
||||
|
||||
55
README.md
55
README.md
@@ -36,7 +36,7 @@ pip install -r requirements.txt
|
||||
### 2. 安装前端依赖
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
cd WebUI
|
||||
npm install
|
||||
```
|
||||
|
||||
@@ -58,7 +58,7 @@ python api_server.py
|
||||
|
||||
**启动前端服务**（终端 2）
|
||||
```bash
|
||||
cd frontend
|
||||
cd WebUI
|
||||
npm run dev
|
||||
```
|
||||
|
||||
@@ -77,32 +77,35 @@ stop.bat
|
||||
|
||||
```
|
||||
ProxyPool/
|
||||
├── api_server.py # FastAPI 后端服务器
|
||||
├── config.py # 配置文件
|
||||
├── main.py # 项目入口
|
||||
├── requirements.txt # Python 依赖
|
||||
├── .env.example # 环境变量示例
|
||||
│
|
||||
├── script/ # 启动脚本
|
||||
│ ├── start.bat # Windows 启动脚本
|
||||
│ └── stop.bat # Windows 停止脚本
|
||||
├── app/ # 后端代码
|
||||
│ ├── api/ # FastAPI 路由
|
||||
│ │ ├── main.py # 应用工厂
|
||||
│ │ ├── routes/ # API 路由
|
||||
│ │ ├── deps.py # 依赖注入
|
||||
│ │ └── ...
|
||||
│ ├── core/ # 核心模块
|
||||
│ │ ├── config.py # 配置管理
|
||||
│ │ ├── db.py # 数据库连接
|
||||
│ │ ├── log.py # 日志配置
|
||||
│ │ ├── plugin_system/ # 插件系统
|
||||
│ │ └── tasks/ # 任务队列
|
||||
│ ├── models/ # 数据模型
|
||||
│ ├── repositories/ # 数据访问层
|
||||
│ ├── services/ # 业务逻辑层
|
||||
│ └── plugins/ # 代理源插件
|
||||
│ ├── fate0.py # Fate0 代理源
|
||||
│ ├── ip3366.py # IP3366 代理源
|
||||
│ ├── ip89.py # IP89 代理源
|
||||
│ ├── kuaidaili.py # 快代理源
|
||||
│ ├── yundaili.py # 云代理源
|
||||
│ ├── speedx.py # SpeedX 代理源
|
||||
│ └── proxylist_download.py # ProxyList 代理源
|
||||
│
|
||||
├── core/ # 核心模块
|
||||
│ ├── crawler.py # 爬虫基类
|
||||
│ ├── validator.py # 代理验证器
|
||||
│ ├── sqlite.py # 数据库管理
|
||||
│ ├── plugin_manager.py # 插件管理器
|
||||
│ └── log.py # 日志配置
|
||||
│
|
||||
├── plugins/ # 代理源插件
|
||||
│ ├── fate0.py # Fate0 代理源
|
||||
│ ├── ip3366.py # IP3366 代理源
|
||||
│ ├── ip89.py # IP89 代理源
|
||||
│ ├── kuaidaili.py # 快代理源
|
||||
│ ├── yundaili.py # 云代理源
|
||||
│ ├── speedx.py # SpeedX 代理源
|
||||
│ └── proxylist_download.py # ProxyList 代理源
|
||||
│
|
||||
├── frontend/ # Vue3 前端
|
||||
├── WebUI/ # Vue3 前端
|
||||
│ ├── src/
|
||||
│ │ ├── api/ # API 封装
|
||||
│ │ ├── stores/ # Pinia 状态管理
|
||||
@@ -113,6 +116,10 @@ ProxyPool/
|
||||
│ ├── index.html
|
||||
│ └── package.json
|
||||
│
|
||||
├── script/ # 启动脚本
|
||||
│ ├── start.bat # Windows 启动脚本
|
||||
│ └── stop.bat # Windows 停止脚本
|
||||
│
|
||||
└── db/ # 数据存储目录
|
||||
└── proxies.sqlite # SQLite 数据库
|
||||
```
|
||||
|
||||
0
frontend/.gitignore → WebUI/.gitignore
vendored
0
frontend/.gitignore → WebUI/.gitignore
vendored
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.5 KiB |
@@ -74,26 +74,42 @@
|
||||
</template>
|
||||
</el-table-column>
|
||||
|
||||
<el-table-column label="操作" width="200" fixed="right" align="center">
|
||||
<el-table-column label="操作" width="220" fixed="right" align="center">
|
||||
<template #default="{ row }">
|
||||
<el-button
|
||||
type="primary"
|
||||
size="small"
|
||||
@click="handleOpenConfig(row)"
|
||||
>
|
||||
<el-icon class="btn-icon"><Setting /></el-icon>
|
||||
配置
|
||||
</el-button>
|
||||
<el-button
|
||||
type="success"
|
||||
size="small"
|
||||
@click="handleCrawl(row.id)"
|
||||
:loading="crawlingPlugin === row.id"
|
||||
:disabled="!row.enabled"
|
||||
>
|
||||
<el-icon class="btn-icon"><Promotion /></el-icon>
|
||||
爬取
|
||||
</el-button>
|
||||
<div class="plugin-actions">
|
||||
<el-button
|
||||
type="primary"
|
||||
size="small"
|
||||
@click="handleOpenConfig(row)"
|
||||
>
|
||||
<el-icon class="btn-icon"><Setting /></el-icon>
|
||||
配置
|
||||
</el-button>
|
||||
<el-button
|
||||
type="success"
|
||||
size="small"
|
||||
@click="handleCrawl(row.id)"
|
||||
:loading="crawlingPlugins.has(row.id)"
|
||||
:disabled="!row.enabled"
|
||||
>
|
||||
<el-icon class="btn-icon"><Promotion /></el-icon>
|
||||
爬取
|
||||
</el-button>
|
||||
</div>
|
||||
<div v-if="crawlResults[row.id]" class="plugin-crawl-result">
|
||||
<div class="result-mini" :class="crawlResults[row.id].type">
|
||||
<el-icon v-if="crawlResults[row.id].type === 'success'" class="result-icon success"><CircleCheck /></el-icon>
|
||||
<el-icon v-else class="result-icon failed"><CircleClose /></el-icon>
|
||||
<span class="result-text">{{ crawlResults[row.id].message }}</span>
|
||||
<span v-if="crawlResults[row.id].data?.valid_count !== undefined" class="result-count valid">
|
||||
有效 {{ crawlResults[row.id].data.valid_count }}
|
||||
</span>
|
||||
<span v-if="crawlResults[row.id].data?.invalid_count !== undefined" class="result-count invalid">
|
||||
无效 {{ crawlResults[row.id].data.invalid_count }}
|
||||
</span>
|
||||
<el-icon class="result-close" @click="clearCrawlResult(row.id)"><Close /></el-icon>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
@@ -104,28 +120,28 @@
|
||||
:image-size="120"
|
||||
/>
|
||||
|
||||
<!-- 爬取结果提示 -->
|
||||
<!-- 批量爬取结果提示 -->
|
||||
<el-alert
|
||||
v-if="lastCrawlResult"
|
||||
:title="lastCrawlResult.message"
|
||||
:type="lastCrawlResult.type"
|
||||
v-if="allCrawlResult"
|
||||
:title="allCrawlResult.message"
|
||||
:type="allCrawlResult.type"
|
||||
closable
|
||||
class="crawl-result"
|
||||
@close="lastCrawlResult = null"
|
||||
@close="allCrawlResult = null"
|
||||
>
|
||||
<template v-if="lastCrawlResult.data">
|
||||
<template v-if="allCrawlResult.data">
|
||||
<div class="crawl-stats">
|
||||
<span v-if="lastCrawlResult.data.total_crawled !== undefined">
|
||||
爬取: {{ lastCrawlResult.data.total_crawled }}
|
||||
<span v-if="allCrawlResult.data.total_crawled !== undefined">
|
||||
爬取: {{ allCrawlResult.data.total_crawled }}
|
||||
</span>
|
||||
<span v-if="lastCrawlResult.data.proxy_count !== undefined">
|
||||
爬取: {{ lastCrawlResult.data.proxy_count }}
|
||||
<span v-if="allCrawlResult.data.proxy_count !== undefined">
|
||||
爬取: {{ allCrawlResult.data.proxy_count }}
|
||||
</span>
|
||||
<span v-if="lastCrawlResult.data.valid_count !== undefined" class="valid-count">
|
||||
有效: {{ lastCrawlResult.data.valid_count }}
|
||||
<span v-if="allCrawlResult.data.valid_count !== undefined" class="valid-count">
|
||||
有效: {{ allCrawlResult.data.valid_count }}
|
||||
</span>
|
||||
<span v-if="lastCrawlResult.data.invalid_count !== undefined" class="invalid-count">
|
||||
无效: {{ lastCrawlResult.data.invalid_count }}
|
||||
<span v-if="allCrawlResult.data.invalid_count !== undefined" class="invalid-count">
|
||||
无效: {{ allCrawlResult.data.invalid_count }}
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
@@ -183,7 +199,8 @@ import {
|
||||
CircleCheck,
|
||||
CircleClose,
|
||||
Box,
|
||||
Setting
|
||||
Setting,
|
||||
Close
|
||||
} from '@element-plus/icons-vue'
|
||||
import { usePluginsStore } from '../stores/plugins'
|
||||
import { pluginService } from '../services/pluginService'
|
||||
@@ -191,9 +208,10 @@ import { formatTime } from '../utils/format'
|
||||
import PageHeader from '../components/PageHeader.vue'
|
||||
|
||||
const pluginsStore = usePluginsStore()
|
||||
const crawlingPlugin = ref(null)
|
||||
const crawlingPlugins = ref(new Set())
|
||||
const crawlingAll = ref(false)
|
||||
const lastCrawlResult = ref(null)
|
||||
const crawlResults = ref({})
|
||||
const allCrawlResult = ref(null)
|
||||
|
||||
// 配置对话框
|
||||
const configDialogVisible = ref(false)
|
||||
@@ -248,34 +266,36 @@ async function handleSaveConfig() {
|
||||
|
||||
async function handleCrawl(pluginId) {
|
||||
try {
|
||||
crawlingPlugin.value = pluginId
|
||||
lastCrawlResult.value = null
|
||||
crawlingPlugins.value.add(pluginId)
|
||||
|
||||
const response = await pluginService.crawlPlugin(pluginId)
|
||||
|
||||
if (response.code === 200) {
|
||||
lastCrawlResult.value = {
|
||||
crawlResults.value[pluginId] = {
|
||||
type: 'success',
|
||||
message: response.message,
|
||||
data: response.data
|
||||
}
|
||||
await pluginsStore.fetchPlugins()
|
||||
} else {
|
||||
lastCrawlResult.value = {
|
||||
crawlResults.value[pluginId] = {
|
||||
type: 'error',
|
||||
message: response.message || '爬取失败'
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
lastCrawlResult.value = {
|
||||
crawlResults.value[pluginId] = {
|
||||
type: 'error',
|
||||
message: '爬取过程出错'
|
||||
}
|
||||
} finally {
|
||||
crawlingPlugin.value = null
|
||||
crawlingPlugins.value.delete(pluginId)
|
||||
}
|
||||
}
|
||||
|
||||
function clearCrawlResult(pluginId) {
|
||||
delete crawlResults.value[pluginId]
|
||||
}
|
||||
|
||||
async function handleCrawlAll() {
|
||||
try {
|
||||
const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled)
|
||||
@@ -295,20 +315,19 @@ async function handleCrawlAll() {
|
||||
)
|
||||
|
||||
crawlingAll.value = true
|
||||
lastCrawlResult.value = null
|
||||
allCrawlResult.value = null
|
||||
|
||||
const response = await pluginService.crawlAll()
|
||||
|
||||
if (response.code === 200) {
|
||||
lastCrawlResult.value = {
|
||||
allCrawlResult.value = {
|
||||
type: 'success',
|
||||
message: response.message,
|
||||
data: response.data
|
||||
}
|
||||
ElMessage.success('批量爬取完成')
|
||||
await pluginsStore.fetchPlugins()
|
||||
} else {
|
||||
lastCrawlResult.value = {
|
||||
allCrawlResult.value = {
|
||||
type: 'error',
|
||||
message: response.message || '批量爬取失败'
|
||||
}
|
||||
@@ -316,7 +335,7 @@ async function handleCrawlAll() {
|
||||
} catch (error) {
|
||||
if (error !== 'cancel') {
|
||||
console.error('批量爬取失败:', error)
|
||||
lastCrawlResult.value = {
|
||||
allCrawlResult.value = {
|
||||
type: 'error',
|
||||
message: '批量爬取过程出错'
|
||||
}
|
||||
@@ -460,4 +479,70 @@ onMounted(async () => {
|
||||
margin-bottom: 16px;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.plugin-actions {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.plugin-crawl-result {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.result-mini {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 4px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.result-mini.success {
|
||||
background: rgba(103, 194, 58, 0.15);
|
||||
color: var(--success);
|
||||
}
|
||||
|
||||
.result-mini.error {
|
||||
background: rgba(245, 108, 108, 0.15);
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.result-icon {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.result-text {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.result-count {
|
||||
font-weight: 600;
|
||||
padding: 0 4px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.result-count.valid {
|
||||
background: rgba(103, 194, 58, 0.2);
|
||||
color: var(--success);
|
||||
}
|
||||
|
||||
.result-count.invalid {
|
||||
background: rgba(245, 108, 108, 0.2);
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.result-close {
|
||||
margin-left: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
opacity: 0.7;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.result-close:hover {
|
||||
opacity: 1;
|
||||
}
|
||||
</style>
|
||||
@@ -1,130 +0,0 @@
|
||||
"""代理相关路由（含统计信息）"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from services.proxy_service import ProxyService
|
||||
from services.scheduler_service import SchedulerService
|
||||
from models.schemas import ProxyListRequest, BatchDeleteRequest
|
||||
from api.deps import get_proxy_service, get_scheduler_service
|
||||
from core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
|
||||
|
||||
|
||||
def success_response(message: str, data=None):
|
||||
return {"code": 200, "message": message, "data": data}
|
||||
|
||||
|
||||
def error_response(message: str, code: int = 500):
|
||||
return {"code": code, "message": message, "data": None}
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_stats(
|
||||
proxy_service: ProxyService = Depends(get_proxy_service),
|
||||
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
try:
|
||||
stats = await proxy_service.get_stats()
|
||||
stats["scheduler_running"] = scheduler_service.running
|
||||
return success_response("获取统计信息成功", stats)
|
||||
except Exception as e:
|
||||
logger.error(f"Get stats failed: {e}")
|
||||
return error_response("获取统计信息失败")
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def list_proxies(
|
||||
request: ProxyListRequest,
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
proxies, total = await service.list_proxies(
|
||||
page=request.page,
|
||||
page_size=request.page_size,
|
||||
protocol=request.protocol,
|
||||
min_score=request.min_score,
|
||||
max_score=request.max_score,
|
||||
sort_by=request.sort_by,
|
||||
sort_order=request.sort_order,
|
||||
)
|
||||
return success_response(
|
||||
"获取代理列表成功",
|
||||
{
|
||||
"list": [
|
||||
{
|
||||
"ip": p.ip,
|
||||
"port": p.port,
|
||||
"protocol": p.protocol,
|
||||
"score": p.score,
|
||||
"last_check": p.last_check.isoformat() if p.last_check else None,
|
||||
}
|
||||
for p in proxies
|
||||
],
|
||||
"total": total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/random")
|
||||
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
|
||||
proxy = await service.get_random_proxy()
|
||||
if not proxy:
|
||||
return error_response("没有找到可用的代理", 404)
|
||||
return success_response(
|
||||
"获取随机代理成功",
|
||||
{
|
||||
"ip": proxy.ip,
|
||||
"port": proxy.port,
|
||||
"protocol": proxy.protocol,
|
||||
"score": proxy.score,
|
||||
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/export/{fmt}")
|
||||
async def export_proxies(
|
||||
fmt: str,
|
||||
protocol: Optional[str] = None,
|
||||
limit: int = Query(default=10000, ge=1, le=100000),
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
if fmt not in ("csv", "txt", "json"):
|
||||
return error_response("不支持的导出格式", 400)
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
|
||||
|
||||
async def generate():
|
||||
async for chunk in service.export_proxies(fmt, protocol, limit):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(
|
||||
generate(),
|
||||
media_type=media_types[fmt],
|
||||
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{ip}/{port}")
|
||||
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
|
||||
await service.delete_proxy(ip, port)
|
||||
return success_response("删除代理成功")
|
||||
|
||||
|
||||
@router.post("/batch-delete")
|
||||
async def batch_delete(
|
||||
request: BatchDeleteRequest,
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
proxies = [(item.ip, item.port) for item in request.proxies]
|
||||
deleted = await service.batch_delete(proxies)
|
||||
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
|
||||
|
||||
|
||||
@router.delete("/clean-invalid")
|
||||
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
|
||||
count = await service.clean_invalid()
|
||||
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
|
||||
@@ -1,80 +0,0 @@
|
||||
"""调度器相关路由"""
|
||||
from fastapi import APIRouter, Depends
|
||||
from services.scheduler_service import SchedulerService
|
||||
from repositories.settings_repo import SettingsRepository
|
||||
from core.db import get_db
|
||||
from api.deps import get_scheduler_service
|
||||
from core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||
settings_repo = SettingsRepository()
|
||||
|
||||
|
||||
def success_response(message: str, data=None):
|
||||
return {"code": 200, "message": message, "data": data}
|
||||
|
||||
|
||||
def error_response(message: str, code: int = 500):
|
||||
return {"code": code, "message": message, "data": None}
|
||||
|
||||
|
||||
@router.post("/start")
|
||||
async def start_scheduler(
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
try:
|
||||
if scheduler.running:
|
||||
return success_response("验证调度器已在运行", {"running": True})
|
||||
await scheduler.start()
|
||||
# 持久化设置
|
||||
async with get_db() as db:
|
||||
settings = await settings_repo.get_all(db)
|
||||
settings["auto_validate"] = True
|
||||
from models.schemas import SettingsSchema
|
||||
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
||||
return success_response("验证调度器已启动", {"running": True})
|
||||
except Exception as e:
|
||||
logger.error(f"Start scheduler failed: {e}")
|
||||
return error_response(f"启动调度器失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/stop")
|
||||
async def stop_scheduler(
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
try:
|
||||
if not scheduler.running:
|
||||
return success_response("验证调度器未运行", {"running": False})
|
||||
await scheduler.stop()
|
||||
# 持久化设置
|
||||
async with get_db() as db:
|
||||
settings = await settings_repo.get_all(db)
|
||||
settings["auto_validate"] = False
|
||||
from models.schemas import SettingsSchema
|
||||
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
||||
return success_response("验证调度器已停止", {"running": False})
|
||||
except Exception as e:
|
||||
logger.error(f"Stop scheduler failed: {e}")
|
||||
return error_response(f"停止调度器失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/validate-now")
|
||||
async def validate_now(
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
try:
|
||||
scheduler.validate_all_now()
|
||||
return success_response("已开始全量验证", {"started": True})
|
||||
except Exception as e:
|
||||
logger.error(f"Validate now failed: {e}")
|
||||
return error_response(f"启动验证失败: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def scheduler_status(
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
return success_response(
|
||||
"获取状态成功",
|
||||
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
|
||||
)
|
||||
@@ -1,3 +1,4 @@
|
||||
"""API 包"""
|
||||
from .main import create_app
|
||||
|
||||
__all__ = ["create_app"]
|
||||
41
app/api/common.py
Normal file
41
app/api/common.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""API 通用工具函数"""
|
||||
from typing import Any, Optional
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
|
||||
def success_response(message: str, data: Any = None) -> dict:
|
||||
"""成功响应"""
|
||||
return {"code": 200, "message": message, "data": data}
|
||||
|
||||
|
||||
def error_response(message: str, code: int = 500) -> JSONResponse:
|
||||
"""错误响应"""
|
||||
return JSONResponse(
|
||||
status_code=code,
|
||||
content={"code": code, "message": message, "data": None},
|
||||
)
|
||||
|
||||
|
||||
def format_proxy(proxy) -> dict:
|
||||
"""格式化代理对象"""
|
||||
return {
|
||||
"ip": proxy.ip,
|
||||
"port": proxy.port,
|
||||
"protocol": proxy.protocol,
|
||||
"score": proxy.score,
|
||||
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
|
||||
}
|
||||
|
||||
|
||||
def format_plugin(plugin) -> dict:
|
||||
"""格式化插件对象"""
|
||||
return {
|
||||
"id": plugin.id,
|
||||
"name": plugin.display_name,
|
||||
"display_name": plugin.display_name,
|
||||
"description": plugin.description,
|
||||
"enabled": plugin.enabled,
|
||||
"last_run": plugin.last_run.isoformat() if plugin.last_run else None,
|
||||
"success_count": plugin.success_count,
|
||||
"failure_count": plugin.failure_count,
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
"""依赖注入"""
|
||||
from fastapi import Request
|
||||
from services.proxy_service import ProxyService
|
||||
from services.plugin_service import PluginService
|
||||
from services.scheduler_service import SchedulerService
|
||||
from services.validator_service import ValidatorService
|
||||
from repositories.proxy_repo import ProxyRepository
|
||||
from core.tasks.queue import ValidationQueue
|
||||
from core.config import settings as app_settings
|
||||
from app.services.proxy_service import ProxyService
|
||||
from app.services.plugin_service import PluginService
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
from app.services.validator_service import ValidatorService
|
||||
from app.repositories.proxy_repo import ProxyRepository
|
||||
from app.core.tasks.queue import ValidationQueue
|
||||
from app.core.config import settings as app_settings
|
||||
|
||||
|
||||
def get_proxy_service() -> ProxyService:
|
||||
@@ -2,8 +2,8 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import ValidationError
|
||||
from core.exceptions import ProxyPoolException
|
||||
from core.log import logger
|
||||
from app.core.exceptions import ProxyPoolException
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException):
|
||||
@@ -1,11 +1,11 @@
|
||||
"""应用生命周期管理"""
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from core.db import init_db, get_db
|
||||
from core.config import settings as app_settings
|
||||
from core.log import logger
|
||||
from api.deps import create_scheduler_service
|
||||
from repositories.settings_repo import SettingsRepository
|
||||
from app.core.db import init_db, get_db
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.log import logger
|
||||
from app.api.deps import create_scheduler_service
|
||||
from app.repositories.settings_repo import SettingsRepository
|
||||
|
||||
settings_repo = SettingsRepository()
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
"""FastAPI 应用工厂"""
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from api.lifespan import lifespan
|
||||
from api.routes import api_router
|
||||
from api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
|
||||
from core.exceptions import ProxyPoolException
|
||||
from app.api.lifespan import lifespan
|
||||
from app.api.routes import api_router
|
||||
from app.api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
|
||||
from app.core.exceptions import ProxyPoolException
|
||||
from pydantic import ValidationError
|
||||
from core.config import settings as app_settings
|
||||
from app.core.config import settings as app_settings
|
||||
|
||||
# 导入并注册所有插件（显式注册模式）
|
||||
import plugins
|
||||
import app.plugins
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@@ -1,5 +1,6 @@
|
||||
"""路由包"""
|
||||
from fastapi import APIRouter
|
||||
from . import proxies, plugins, scheduler, settings
|
||||
from app.api.routes import proxies, plugins, scheduler, settings
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(proxies.router)
|
||||
@@ -1,42 +1,23 @@
|
||||
"""插件相关路由"""
|
||||
import asyncio
|
||||
from fastapi import APIRouter, Depends
|
||||
from services.plugin_service import PluginService
|
||||
from services.scheduler_service import SchedulerService
|
||||
from api.deps import get_plugin_service, get_scheduler_service
|
||||
from core.log import logger
|
||||
from app.services.plugin_service import PluginService
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
from app.api.deps import get_plugin_service, get_scheduler_service
|
||||
from app.api.common import success_response, error_response, format_plugin
|
||||
from app.core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
|
||||
|
||||
|
||||
def success_response(message: str, data=None):
|
||||
return {"code": 200, "message": message, "data": data}
|
||||
|
||||
|
||||
def error_response(message: str, code: int = 500):
|
||||
return {"code": code, "message": message, "data": None}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_plugins(service: PluginService = Depends(get_plugin_service)):
|
||||
plugins = await service.list_plugins()
|
||||
return success_response(
|
||||
"获取插件列表成功",
|
||||
{
|
||||
"plugins": [
|
||||
{
|
||||
"id": p.id,
|
||||
"name": p.display_name,
|
||||
"display_name": p.display_name,
|
||||
"description": p.description,
|
||||
"enabled": p.enabled,
|
||||
"last_run": p.last_run.isoformat() if p.last_run else None,
|
||||
"success_count": p.success_count,
|
||||
"failure_count": p.failure_count,
|
||||
}
|
||||
for p in plugins
|
||||
]
|
||||
},
|
||||
)
|
||||
try:
|
||||
plugins = await service.list_plugins()
|
||||
return success_response("获取插件列表成功", {"plugins": [format_plugin(p) for p in plugins]})
|
||||
except Exception as e:
|
||||
logger.error(f"List plugins failed: {e}")
|
||||
return error_response("获取插件列表失败", 500)
|
||||
|
||||
|
||||
@router.put("/{plugin_id}/toggle")
|
||||
@@ -48,13 +29,18 @@ async def toggle_plugin(
|
||||
enabled = request.get("enabled")
|
||||
if enabled is None:
|
||||
return error_response("缺少 enabled 参数", 400)
|
||||
success = await service.toggle_plugin(plugin_id, enabled)
|
||||
if not success:
|
||||
return error_response("插件不存在", 404)
|
||||
return success_response(
|
||||
f"插件 {plugin_id} 已{'启用' if enabled else '禁用'}",
|
||||
{"plugin_id": plugin_id, "enabled": enabled},
|
||||
)
|
||||
|
||||
try:
|
||||
success = await service.toggle_plugin(plugin_id, enabled)
|
||||
if not success:
|
||||
return error_response("插件不存在", 404)
|
||||
return success_response(
|
||||
f"插件 {plugin_id} 已{'启用' if enabled else '禁用'}",
|
||||
{"plugin_id": plugin_id, "enabled": enabled},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Toggle plugin failed: {e}")
|
||||
return error_response("切换插件状态失败", 500)
|
||||
|
||||
|
||||
@router.get("/{plugin_id}/config")
|
||||
@@ -62,10 +48,14 @@ async def get_plugin_config(
|
||||
plugin_id: str,
|
||||
service: PluginService = Depends(get_plugin_service),
|
||||
):
|
||||
config = await service.get_plugin_config(plugin_id)
|
||||
if config is None:
|
||||
return error_response("插件不存在", 404)
|
||||
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||
try:
|
||||
config = await service.get_plugin_config(plugin_id)
|
||||
if config is None:
|
||||
return error_response("插件不存在", 404)
|
||||
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||
except Exception as e:
|
||||
logger.error(f"Get plugin config failed: {e}")
|
||||
return error_response("获取插件配置失败", 500)
|
||||
|
||||
|
||||
@router.post("/{plugin_id}/config")
|
||||
@@ -77,10 +67,15 @@ async def update_plugin_config(
|
||||
config = request.get("config", {})
|
||||
if not isinstance(config, dict):
|
||||
return error_response("config 必须是对象", 400)
|
||||
success = await service.update_plugin_config(plugin_id, config)
|
||||
if not success:
|
||||
return error_response("插件不存在或配置无效", 404)
|
||||
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||
|
||||
try:
|
||||
success = await service.update_plugin_config(plugin_id, config)
|
||||
if not success:
|
||||
return error_response("插件不存在或配置无效", 404)
|
||||
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||
except Exception as e:
|
||||
logger.error(f"Update plugin config failed: {e}")
|
||||
return error_response("保存插件配置失败", 500)
|
||||
|
||||
|
||||
@router.post("/{plugin_id}/crawl")
|
||||
@@ -101,30 +96,27 @@ async def crawl_plugin(
|
||||
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
|
||||
)
|
||||
|
||||
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies, sending to validation queue")
|
||||
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies")
|
||||
scheduler_service.validation_queue.reset_stats()
|
||||
await scheduler_service.validation_queue.submit(results)
|
||||
# 等待队列排空（最多等 30 秒，避免前端超时）
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
valid_count = scheduler_service.validation_queue.valid_count
|
||||
invalid_count = scheduler_service.validation_queue.invalid_count
|
||||
|
||||
return success_response(
|
||||
f"插件 {plugin_id} 爬取并验证完成",
|
||||
{
|
||||
"plugin_id": plugin_id,
|
||||
"proxy_count": len(results),
|
||||
"valid_count": valid_count,
|
||||
"invalid_count": invalid_count,
|
||||
"valid_count": scheduler_service.validation_queue.valid_count,
|
||||
"invalid_count": scheduler_service.validation_queue.invalid_count,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Crawl plugin {plugin_id} failed: {e}")
|
||||
return error_response(f"插件爬取失败: {str(e)}")
|
||||
return error_response(f"插件爬取失败: {str(e)}", 500)
|
||||
|
||||
|
||||
@router.post("/crawl-all")
|
||||
@@ -140,28 +132,23 @@ async def crawl_all(
|
||||
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
|
||||
)
|
||||
|
||||
logger.info(f"All plugins crawled {len(results)} unique proxies, sending to validation queue")
|
||||
logger.info(f"All plugins crawled {len(results)} unique proxies")
|
||||
scheduler_service.validation_queue.reset_stats()
|
||||
await scheduler_service.validation_queue.submit(results)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
valid_count = scheduler_service.validation_queue.valid_count
|
||||
invalid_count = scheduler_service.validation_queue.invalid_count
|
||||
|
||||
return success_response(
|
||||
"所有插件爬取并验证完成",
|
||||
{
|
||||
"total_crawled": len(results),
|
||||
"valid_count": valid_count,
|
||||
"invalid_count": invalid_count,
|
||||
"valid_count": scheduler_service.validation_queue.valid_count,
|
||||
"invalid_count": scheduler_service.validation_queue.invalid_count,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Crawl all failed: {e}")
|
||||
return error_response(f"批量爬取失败: {str(e)}")
|
||||
|
||||
|
||||
import asyncio
|
||||
return error_response(f"批量爬取失败: {str(e)}", 500)
|
||||
125
app/api/routes/proxies.py
Normal file
125
app/api/routes/proxies.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""代理相关路由（含统计信息）"""
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from app.services.proxy_service import ProxyService
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
from app.models.schemas import ProxyListRequest, BatchDeleteRequest
|
||||
from app.api.deps import get_proxy_service, get_scheduler_service
|
||||
from app.api.common import success_response, error_response, format_proxy
|
||||
from app.core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_stats(
|
||||
proxy_service: ProxyService = Depends(get_proxy_service),
|
||||
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
try:
|
||||
stats = await proxy_service.get_stats()
|
||||
stats["scheduler_running"] = scheduler_service.running
|
||||
return success_response("获取统计信息成功", stats)
|
||||
except Exception as e:
|
||||
logger.error(f"Get stats failed: {e}")
|
||||
return error_response("获取统计信息失败", 500)
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def list_proxies(
|
||||
request: ProxyListRequest,
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
try:
|
||||
proxies, total = await service.list_proxies(
|
||||
page=request.page,
|
||||
page_size=request.page_size,
|
||||
protocol=request.protocol,
|
||||
min_score=request.min_score,
|
||||
max_score=request.max_score,
|
||||
sort_by=request.sort_by,
|
||||
sort_order=request.sort_order,
|
||||
)
|
||||
return success_response(
|
||||
"获取代理列表成功",
|
||||
{
|
||||
"list": [format_proxy(p) for p in proxies],
|
||||
"total": total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"List proxies failed: {e}")
|
||||
return error_response("获取代理列表失败", 500)
|
||||
|
||||
|
||||
@router.get("/random")
|
||||
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
|
||||
try:
|
||||
proxy = await service.get_random_proxy()
|
||||
if not proxy:
|
||||
return error_response("没有找到可用的代理", 404)
|
||||
return success_response("获取随机代理成功", format_proxy(proxy))
|
||||
except Exception as e:
|
||||
logger.error(f"Get random proxy failed: {e}")
|
||||
return error_response("获取随机代理失败", 500)
|
||||
|
||||
|
||||
@router.get("/export/{fmt}")
|
||||
async def export_proxies(
|
||||
fmt: str,
|
||||
protocol: Optional[str] = None,
|
||||
limit: int = Query(default=10000, ge=1, le=100000),
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
if fmt not in ("csv", "txt", "json"):
|
||||
return error_response("不支持的导出格式", 400)
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
|
||||
|
||||
async def generate():
|
||||
async for chunk in service.export_proxies(fmt, protocol, limit):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(
|
||||
generate(),
|
||||
media_type=media_types[fmt],
|
||||
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{ip}/{port}")
|
||||
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
|
||||
try:
|
||||
await service.delete_proxy(ip, port)
|
||||
return success_response("删除代理成功")
|
||||
except Exception as e:
|
||||
logger.error(f"Delete proxy failed: {e}")
|
||||
return error_response("删除代理失败", 500)
|
||||
|
||||
|
||||
@router.post("/batch-delete")
|
||||
async def batch_delete(
|
||||
request: BatchDeleteRequest,
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
try:
|
||||
proxies = [(item.ip, item.port) for item in request.proxies]
|
||||
deleted = await service.batch_delete(proxies)
|
||||
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
|
||||
except Exception as e:
|
||||
logger.error(f"Batch delete failed: {e}")
|
||||
return error_response("批量删除失败", 500)
|
||||
|
||||
|
||||
@router.delete("/clean-invalid")
|
||||
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
|
||||
try:
|
||||
count = await service.clean_invalid()
|
||||
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
|
||||
except Exception as e:
|
||||
logger.error(f"Clean invalid failed: {e}")
|
||||
return error_response("清理无效代理失败", 500)
|
||||
64
app/api/routes/scheduler.py
Normal file
64
app/api/routes/scheduler.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""调度器相关路由"""
|
||||
from fastapi import APIRouter, Depends
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
from app.repositories.settings_repo import SettingsRepository
|
||||
from app.core.db import get_db
|
||||
from app.api.deps import get_scheduler_service
|
||||
from app.api.common import success_response, error_response
|
||||
from app.core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||
settings_repo = SettingsRepository()
|
||||
|
||||
|
||||
async def _save_auto_validate_setting(enabled: bool):
|
||||
"""保存自动验证设置"""
|
||||
async with get_db() as db:
|
||||
settings = await settings_repo.get_all(db)
|
||||
settings["auto_validate"] = enabled
|
||||
from app.models.schemas import SettingsSchema
|
||||
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
||||
|
||||
|
||||
@router.post("/start")
|
||||
async def start_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||
try:
|
||||
if scheduler.running:
|
||||
return success_response("验证调度器已在运行", {"running": True})
|
||||
await scheduler.start()
|
||||
await _save_auto_validate_setting(True)
|
||||
return success_response("验证调度器已启动", {"running": True})
|
||||
except Exception as e:
|
||||
logger.error(f"Start scheduler failed: {e}")
|
||||
return error_response(f"启动调度器失败: {str(e)}", 500)
|
||||
|
||||
|
||||
@router.post("/stop")
|
||||
async def stop_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||
try:
|
||||
if not scheduler.running:
|
||||
return success_response("验证调度器未运行", {"running": False})
|
||||
await scheduler.stop()
|
||||
await _save_auto_validate_setting(False)
|
||||
return success_response("验证调度器已停止", {"running": False})
|
||||
except Exception as e:
|
||||
logger.error(f"Stop scheduler failed: {e}")
|
||||
return error_response(f"停止调度器失败: {str(e)}", 500)
|
||||
|
||||
|
||||
@router.post("/validate-now")
|
||||
async def validate_now(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||
try:
|
||||
scheduler.validate_all_now()
|
||||
return success_response("已开始全量验证", {"started": True})
|
||||
except Exception as e:
|
||||
logger.error(f"Validate now failed: {e}")
|
||||
return error_response(f"启动验证失败: {str(e)}", 500)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def scheduler_status(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||
return success_response(
|
||||
"获取状态成功",
|
||||
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
|
||||
)
|
||||
@@ -1,22 +1,15 @@
|
||||
"""设置相关路由"""
|
||||
from fastapi import APIRouter
|
||||
from core.db import get_db
|
||||
from repositories.settings_repo import SettingsRepository
|
||||
from models.schemas import SettingsSchema
|
||||
from core.log import logger
|
||||
from app.core.db import get_db
|
||||
from app.repositories.settings_repo import SettingsRepository
|
||||
from app.models.schemas import SettingsSchema
|
||||
from app.api.common import success_response, error_response
|
||||
from app.core.log import logger
|
||||
|
||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
settings_repo = SettingsRepository()
|
||||
|
||||
|
||||
def success_response(message: str, data=None):
|
||||
return {"code": 200, "message": message, "data": data}
|
||||
|
||||
|
||||
def error_response(message: str, code: int = 500):
|
||||
return {"code": code, "message": message, "data": None}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_settings():
|
||||
try:
|
||||
@@ -25,7 +18,7 @@ async def get_settings():
|
||||
return success_response("获取设置成功", settings)
|
||||
except Exception as e:
|
||||
logger.error(f"Get settings failed: {e}")
|
||||
return error_response("获取设置失败")
|
||||
return error_response("获取设置失败", 500)
|
||||
|
||||
|
||||
@router.post("")
|
||||
@@ -34,8 +27,8 @@ async def save_settings(request: SettingsSchema):
|
||||
async with get_db() as db:
|
||||
success = await settings_repo.save(db, request.model_dump())
|
||||
if not success:
|
||||
return error_response("保存设置失败")
|
||||
return error_response("保存设置失败", 500)
|
||||
return success_response("保存设置成功", request.model_dump())
|
||||
except Exception as e:
|
||||
logger.error(f"Save settings failed: {e}")
|
||||
return error_response(f"保存设置失败: {str(e)}")
|
||||
return error_response(f"保存设置失败: {str(e)}", 500)
|
||||
13
app/core/__init__.py
Normal file
13
app/core/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""核心基础设施包"""
|
||||
from .config import settings
|
||||
from .log import logger
|
||||
from .exceptions import ProxyPoolException, PluginNotFoundException, ProxyNotFoundException, ValidationException
|
||||
|
||||
__all__ = [
|
||||
"settings",
|
||||
"logger",
|
||||
"ProxyPoolException",
|
||||
"PluginNotFoundException",
|
||||
"ProxyNotFoundException",
|
||||
"ValidationException",
|
||||
]
|
||||
@@ -52,7 +52,7 @@ class Settings(BaseSettings):
|
||||
|
||||
@property
|
||||
def base_dir(self) -> str:
|
||||
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
# 全局配置实例（启动时加载一次）
|
||||
@@ -3,8 +3,8 @@ import os
|
||||
import aiosqlite
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncIterator
|
||||
from core.config import settings
|
||||
from core.log import logger
|
||||
from app.core.config import settings
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
DB_PATH = os.path.join(settings.base_dir, settings.db_path)
|
||||
@@ -3,12 +3,13 @@ import os
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class LogHandler(logging.Logger):
|
||||
def __init__(self, name='ProxyPool', level=logging.INFO):
|
||||
super().__init__(name, level)
|
||||
|
||||
# 获取项目根目录并创建 logs 目录
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
log_dir = os.path.join(base_dir, 'logs')
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
@@ -38,6 +39,7 @@ class LogHandler(logging.Logger):
|
||||
console_handler.setFormatter(formatter)
|
||||
self.addHandler(console_handler)
|
||||
|
||||
|
||||
# 实例化一个默认 logger 供外部直接使用
|
||||
logger = LogHandler()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
"""插件系统包"""
|
||||
from .base import BaseCrawlerPlugin, ProxyRaw
|
||||
from .registry import registry
|
||||
|
||||
@@ -3,8 +3,8 @@ import importlib
|
||||
import inspect
|
||||
import os
|
||||
from typing import Dict, List, Type, Optional
|
||||
from core.plugin_system.base import BaseCrawlerPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system.base import BaseCrawlerPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class PluginRegistry:
|
||||
@@ -1,3 +1,4 @@
|
||||
"""任务队列包"""
|
||||
from .queue import ValidationQueue
|
||||
|
||||
__all__ = ["ValidationQueue"]
|
||||
@@ -1,10 +1,10 @@
|
||||
"""验证任务队列 - 解耦爬取与验证，支持背压控制和持久化"""
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
from models.domain import ProxyRaw
|
||||
from repositories.task_repo import ValidationTaskRepository
|
||||
from core.db import get_db
|
||||
from core.log import logger
|
||||
from app.models.domain import ProxyRaw
|
||||
from app.repositories.task_repo import ValidationTaskRepository
|
||||
from app.core.db import get_db
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ValidationQueue:
|
||||
30
app/models/__init__.py
Normal file
30
app/models/__init__.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""数据模型包"""
|
||||
from .domain import ProxyRaw, Proxy, PluginInfo
|
||||
from .schemas import (
|
||||
ProxyCreate,
|
||||
ProxyResponse,
|
||||
PluginResponse,
|
||||
SettingsSchema,
|
||||
CrawlResult,
|
||||
ProxyListRequest,
|
||||
ProxyDeleteItem,
|
||||
BatchDeleteRequest,
|
||||
PluginToggleRequest,
|
||||
ExportRequest,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ProxyRaw",
|
||||
"Proxy",
|
||||
"PluginInfo",
|
||||
"ProxyCreate",
|
||||
"ProxyResponse",
|
||||
"PluginResponse",
|
||||
"SettingsSchema",
|
||||
"CrawlResult",
|
||||
"ProxyListRequest",
|
||||
"ProxyDeleteItem",
|
||||
"BatchDeleteRequest",
|
||||
"PluginToggleRequest",
|
||||
"ExportRequest",
|
||||
]
|
||||
@@ -1,5 +1,5 @@
|
||||
"""插件包 - 在这里显式注册所有爬虫插件"""
|
||||
from core.plugin_system import registry
|
||||
from app.core.plugin_system import registry
|
||||
|
||||
from .fate0 import Fate0Plugin
|
||||
from .proxylist_download import ProxyListDownloadPlugin
|
||||
@@ -3,7 +3,7 @@ import random
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from typing import List
|
||||
from core.plugin_system import BaseCrawlerPlugin
|
||||
from app.core.plugin_system import BaseCrawlerPlugin
|
||||
|
||||
|
||||
class BaseHTTPPlugin(BaseCrawlerPlugin):
|
||||
@@ -1,8 +1,8 @@
|
||||
import json
|
||||
from typing import List
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class Fate0Plugin(BaseHTTPPlugin):
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
from typing import List
|
||||
from bs4 import BeautifulSoup
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
from typing import List
|
||||
from bs4 import BeautifulSoup
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class Ip89Plugin(BaseHTTPPlugin):
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
from typing import List
|
||||
from bs4 import BeautifulSoup
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import List
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ProxyListDownloadPlugin(BaseHTTPPlugin):
|
||||
@@ -1,8 +1,8 @@
|
||||
"""ProxyScrape 测试爬虫 - 用于验证架构，支持全协议类型"""
|
||||
from typing import List
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ProxyScrapePlugin(BaseHTTPPlugin):
|
||||
@@ -1,8 +1,8 @@
|
||||
import re
|
||||
from typing import List
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class SpeedXPlugin(BaseHTTPPlugin):
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
from typing import List
|
||||
from bs4 import BeautifulSoup
|
||||
from core.plugin_system import ProxyRaw
|
||||
from plugins.base import BaseHTTPPlugin
|
||||
from core.log import logger
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
from app.core.log import logger
|
||||
|
||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
"""数据访问层包"""
|
||||
from .proxy_repo import ProxyRepository
|
||||
from .settings_repo import SettingsRepository, PluginSettingsRepository
|
||||
from .task_repo import ValidationTaskRepository
|
||||
|
||||
__all__ = ["ProxyRepository", "SettingsRepository", "PluginSettingsRepository", "ValidationTaskRepository"]
|
||||
__all__ = [
|
||||
"ProxyRepository",
|
||||
"SettingsRepository",
|
||||
"PluginSettingsRepository",
|
||||
"ValidationTaskRepository",
|
||||
]
|
||||
@@ -2,8 +2,8 @@
|
||||
import aiosqlite
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Optional, Tuple, Union
|
||||
from models.domain import Proxy
|
||||
from core.log import logger
|
||||
from app.models.domain import Proxy
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||
@@ -2,7 +2,7 @@
|
||||
import json
|
||||
import aiosqlite
|
||||
from typing import Optional, Dict, Any
|
||||
from core.log import logger
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
@@ -1,8 +1,8 @@
|
||||
"""验证任务队列持久化层"""
|
||||
import aiosqlite
|
||||
from typing import List, Optional
|
||||
from models.domain import ProxyRaw
|
||||
from core.log import logger
|
||||
from app.models.domain import ProxyRaw
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ValidationTaskRepository:
|
||||
12
app/services/__init__.py
Normal file
12
app/services/__init__.py
Normal file
@@ -0,0 +1,12 @@
|
||||
"""业务逻辑层包"""
|
||||
from .proxy_service import ProxyService
|
||||
from .plugin_service import PluginService
|
||||
from .scheduler_service import SchedulerService
|
||||
from .validator_service import ValidatorService
|
||||
|
||||
__all__ = [
|
||||
"ProxyService",
|
||||
"PluginService",
|
||||
"SchedulerService",
|
||||
"ValidatorService",
|
||||
]
|
||||
@@ -1,12 +1,12 @@
|
||||
"""插件业务服务"""
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from core.db import get_db
|
||||
from core.plugin_system.registry import registry
|
||||
from core.plugin_system.base import BaseCrawlerPlugin
|
||||
from repositories.settings_repo import PluginSettingsRepository
|
||||
from models.domain import PluginInfo, ProxyRaw
|
||||
from core.log import logger
|
||||
from app.core.db import get_db
|
||||
from app.core.plugin_system.registry import registry
|
||||
from app.core.plugin_system.base import BaseCrawlerPlugin
|
||||
from app.repositories.settings_repo import PluginSettingsRepository
|
||||
from app.models.domain import PluginInfo, ProxyRaw
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class PluginService:
|
||||
@@ -4,10 +4,10 @@ import json
|
||||
import io
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Tuple, AsyncIterator
|
||||
from core.db import get_db
|
||||
from repositories.proxy_repo import ProxyRepository
|
||||
from models.domain import Proxy
|
||||
from core.log import logger
|
||||
from app.core.db import get_db
|
||||
from app.repositories.proxy_repo import ProxyRepository
|
||||
from app.models.domain import Proxy
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ProxyService:
|
||||
@@ -1,11 +1,11 @@
|
||||
"""调度器服务 - 定时验证存量代理"""
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from core.db import get_db
|
||||
from repositories.proxy_repo import ProxyRepository
|
||||
from core.tasks.queue import ValidationQueue
|
||||
from core.config import settings as app_settings
|
||||
from core.log import logger
|
||||
from app.core.db import get_db
|
||||
from app.repositories.proxy_repo import ProxyRepository
|
||||
from app.core.tasks.queue import ValidationQueue
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
@@ -70,7 +70,7 @@ class SchedulerService:
|
||||
return
|
||||
|
||||
logger.info(f"Validating {len(proxies)} proxies from database")
|
||||
from models.domain import ProxyRaw
|
||||
from app.models.domain import ProxyRaw
|
||||
|
||||
# 批量提交到验证队列
|
||||
batch_size = 100
|
||||
@@ -5,12 +5,18 @@ import time
|
||||
import aiohttp
|
||||
import aiohttp_socks
|
||||
from typing import Tuple
|
||||
from core.log import logger
|
||||
from app.core.log import logger
|
||||
|
||||
|
||||
class ValidatorService:
|
||||
"""代理验证器"""
|
||||
|
||||
# 测试 URL
|
||||
TEST_URLS = {
|
||||
"http": ["http://httpbin.org/ip", "http://api.ipify.org"],
|
||||
"https": ["https://httpbin.org/ip", "https://api.ipify.org"],
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timeout: float = 5.0,
|
||||
@@ -20,33 +26,23 @@ class ValidatorService:
|
||||
self.timeout = timeout
|
||||
self.connect_timeout = connect_timeout
|
||||
self.semaphore = asyncio.Semaphore(max_concurrency)
|
||||
self.http_sources = [
|
||||
"http://httpbin.org/ip",
|
||||
"http://api.ipify.org",
|
||||
]
|
||||
self.https_sources = [
|
||||
"https://httpbin.org/ip",
|
||||
"https://api.ipify.org",
|
||||
]
|
||||
|
||||
def _get_test_url(self, protocol: str) -> str:
|
||||
protocol = protocol.lower()
|
||||
if protocol == "https":
|
||||
return random.choice(self.https_sources)
|
||||
return random.choice(self.http_sources)
|
||||
"""获取测试 URL"""
|
||||
urls = self.TEST_URLS.get(protocol.lower(), self.TEST_URLS["http"])
|
||||
return random.choice(urls)
|
||||
|
||||
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
|
||||
"""验证单个代理，返回 (是否有效, 延迟毫秒)"""
|
||||
protocol = protocol.lower()
|
||||
test_url = self._get_test_url(protocol)
|
||||
|
||||
|
||||
async with self.semaphore:
|
||||
start = time.time()
|
||||
try:
|
||||
if protocol in ("socks4", "socks5"):
|
||||
return await self._validate_socks(ip, port, protocol, test_url, start)
|
||||
return await self._validate_socks(ip, port, protocol, start)
|
||||
else:
|
||||
return await self._validate_http(ip, port, protocol, test_url, start)
|
||||
return await self._validate_http(ip, port, protocol, start)
|
||||
except asyncio.TimeoutError:
|
||||
logger.debug(f"Validation timeout: {ip}:{port} ({protocol})")
|
||||
return False, 0.0
|
||||
@@ -54,18 +50,16 @@ class ValidatorService:
|
||||
logger.debug(f"Validation error {ip}:{port} ({protocol}): {e}")
|
||||
return False, 0.0
|
||||
|
||||
async def _validate_http(
|
||||
self, ip: str, port: int, protocol: str, test_url: str, start: float
|
||||
) -> Tuple[bool, float]:
|
||||
async def _validate_http(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||
"""验证 HTTP/HTTPS 代理"""
|
||||
proxy_url = f"http://{ip}:{port}"
|
||||
connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
||||
test_url = self._get_test_url(protocol)
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
async with session.get(
|
||||
test_url, proxy=proxy_url, allow_redirects=True
|
||||
) as response:
|
||||
async with session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
|
||||
if response.status in (200, 301, 302):
|
||||
latency = round((time.time() - start) * 1000, 2)
|
||||
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
|
||||
@@ -74,9 +68,8 @@ class ValidatorService:
|
||||
finally:
|
||||
await connector.close()
|
||||
|
||||
async def _validate_socks(
|
||||
self, ip: str, port: int, protocol: str, test_url: str, start: float
|
||||
) -> Tuple[bool, float]:
|
||||
async def _validate_socks(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||
"""验证 SOCKS4/SOCKS5 代理"""
|
||||
proxy_type = (
|
||||
aiohttp_socks.ProxyType.SOCKS4
|
||||
if protocol == "socks4"
|
||||
@@ -90,6 +83,7 @@ class ValidatorService:
|
||||
ssl=False,
|
||||
)
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
||||
test_url = self._get_test_url("http")
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
4
main.py
4
main.py
@@ -1,7 +1,7 @@
|
||||
"""项目入口"""
|
||||
import uvicorn
|
||||
from api import create_app
|
||||
from core.config import settings
|
||||
from app.api import create_app
|
||||
from app.core.config import settings
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
from .domain import ProxyRaw, Proxy, PluginInfo
|
||||
from .schemas import ProxyCreate, ProxyResponse, PluginResponse, SettingsSchema, CrawlResult
|
||||
|
||||
__all__ = [
|
||||
"ProxyRaw",
|
||||
"Proxy",
|
||||
"PluginInfo",
|
||||
"ProxyCreate",
|
||||
"ProxyResponse",
|
||||
"PluginResponse",
|
||||
"SettingsSchema",
|
||||
"CrawlResult",
|
||||
]
|
||||
17
pytest.ini
Normal file
17
pytest.ini
Normal file
@@ -0,0 +1,17 @@
|
||||
[tool:pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
addopts =
|
||||
-v
|
||||
--tb=short
|
||||
--strict-markers
|
||||
-p no:warnings
|
||||
markers =
|
||||
unit: 单元测试
|
||||
integration: 集成测试
|
||||
e2e: 端到端测试
|
||||
slow: 慢速测试
|
||||
async_test: 异步测试
|
||||
asyncio_mode = auto
|
||||
@@ -74,7 +74,7 @@ echo.
|
||||
|
||||
REM 4. Start Frontend
|
||||
echo [4/4] Starting frontend (Vite)...
|
||||
cd /d "%ROOT_PATH%\frontend"
|
||||
cd /d "%ROOT_PATH%\WebUI"
|
||||
start /B "" cmd /c "npm run dev"
|
||||
cd /d "%ROOT_PATH%"
|
||||
echo Frontend started
|
||||
|
||||
159
tests/README.md
Normal file
159
tests/README.md
Normal file
@@ -0,0 +1,159 @@
|
||||
# 测试说明
|
||||
|
||||
## 测试结构
|
||||
|
||||
```
|
||||
tests/
|
||||
├── conftest.py # pytest 配置和 fixtures
|
||||
├── README.md # 本文件
|
||||
├── unit/ # 单元测试
|
||||
│ ├── test_models.py # 模型测试
|
||||
│ └── test_repositories.py # 仓库层测试
|
||||
├── integration/ # 集成测试
|
||||
│ ├── test_proxies_api.py # 代理 API 测试
|
||||
│ ├── test_plugins_api.py # 插件 API 测试
|
||||
│ ├── test_scheduler_api.py # 调度器 API 测试
|
||||
│ ├── test_settings_api.py # 设置 API 测试
|
||||
│ └── test_health_api.py # 健康检查测试
|
||||
└── e2e/ # 端到端测试
|
||||
└── test_full_workflow.py # 完整工作流测试
|
||||
```
|
||||
|
||||
## 运行测试
|
||||
|
||||
### 安装测试依赖
|
||||
|
||||
```bash
|
||||
pip install pytest pytest-asyncio httpx
|
||||
```
|
||||
|
||||
### 运行所有测试
|
||||
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
### 运行特定类型的测试
|
||||
|
||||
```bash
|
||||
# 仅运行单元测试
|
||||
pytest tests/unit -v
|
||||
|
||||
# 仅运行集成测试
|
||||
pytest tests/integration -v
|
||||
|
||||
# 仅运行 E2E 测试
|
||||
pytest tests/e2e -v
|
||||
```
|
||||
|
||||
### 运行特定测试文件
|
||||
|
||||
```bash
|
||||
pytest tests/integration/test_proxies_api.py -v
|
||||
```
|
||||
|
||||
### 运行特定测试函数
|
||||
|
||||
```bash
|
||||
pytest tests/integration/test_proxies_api.py::TestProxiesAPI::test_get_stats -v
|
||||
```
|
||||
|
||||
## 测试覆盖的 API
|
||||
|
||||
### 代理 API (`/api/proxies/*`)
|
||||
|
||||
- ✅ `GET /api/proxies/stats` - 获取统计信息
|
||||
- ✅ `POST /api/proxies` - 列出代理
|
||||
- ✅ `GET /api/proxies/random` - 获取随机代理
|
||||
- ✅ `GET /api/proxies/export/{format}` - 导出代理 (csv, txt, json)
|
||||
- ✅ `DELETE /api/proxies/{ip}/{port}` - 删除代理
|
||||
- ✅ `POST /api/proxies/batch-delete` - 批量删除
|
||||
- ✅ `DELETE /api/proxies/clean-invalid` - 清理无效代理
|
||||
|
||||
### 插件 API (`/api/plugins/*`)
|
||||
|
||||
- ✅ `GET /api/plugins` - 列出插件
|
||||
- ✅ `PUT /api/plugins/{id}/toggle` - 切换插件状态
|
||||
- ✅ `GET /api/plugins/{id}/config` - 获取插件配置
|
||||
- ✅ `POST /api/plugins/{id}/config` - 更新插件配置
|
||||
- ✅ `POST /api/plugins/{id}/crawl` - 触发单个插件爬取
|
||||
- ✅ `POST /api/plugins/crawl-all` - 触发所有插件爬取
|
||||
|
||||
### 调度器 API (`/api/scheduler/*`)
|
||||
|
||||
- ✅ `GET /api/scheduler/status` - 获取调度器状态
|
||||
- ✅ `POST /api/scheduler/start` - 启动调度器
|
||||
- ✅ `POST /api/scheduler/stop` - 停止调度器
|
||||
- ✅ `POST /api/scheduler/validate-now` - 立即验证
|
||||
|
||||
### 设置 API (`/api/settings`)
|
||||
|
||||
- ✅ `GET /api/settings` - 获取设置
|
||||
- ✅ `POST /api/settings` - 保存设置
|
||||
|
||||
### 健康检查
|
||||
|
||||
- ✅ `GET /` - 根端点
|
||||
- ✅ `GET /health` - 健康检查
|
||||
|
||||
## 测试 Fixtures
|
||||
|
||||
### `client`
|
||||
|
||||
异步 HTTP 客户端，用于发送请求到测试应用。
|
||||
|
||||
```python
|
||||
async def test_example(client):
|
||||
response = await client.get("/api/proxies/stats")
|
||||
assert response.status_code == 200
|
||||
```
|
||||
|
||||
### `db`
|
||||
|
||||
数据库连接 fixture。
|
||||
|
||||
```python
|
||||
async def test_example(db, proxy_repo):
|
||||
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
|
||||
```
|
||||
|
||||
### `sample_proxy`
|
||||
|
||||
创建一个测试代理并自动清理。
|
||||
|
||||
```python
|
||||
async def test_example(client, sample_proxy):
|
||||
# sample_proxy = {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
|
||||
response = await client.delete(f"/api/proxies/{sample_proxy['ip']}/{sample_proxy['port']}")
|
||||
assert response.status_code == 200
|
||||
```
|
||||
|
||||
## 编写新测试
|
||||
|
||||
### 单元测试示例
|
||||
|
||||
```python
|
||||
# tests/unit/test_new_feature.py
|
||||
import pytest
|
||||
from app.models.domain import ProxyRaw
|
||||
|
||||
|
||||
class TestProxyRaw:
|
||||
def test_create(self):
|
||||
proxy = ProxyRaw("192.168.1.1", 8080, "http")
|
||||
assert proxy.ip == "192.168.1.1"
|
||||
```
|
||||
|
||||
### 集成测试示例
|
||||
|
||||
```python
|
||||
# tests/integration/test_new_api.py
|
||||
import pytest
|
||||
|
||||
|
||||
class TestNewAPI:
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_endpoint(self, client):
|
||||
response = await client.get("/api/new-endpoint")
|
||||
assert response.status_code == 200
|
||||
```
|
||||
1
tests/__init__.py
Normal file
1
tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""测试包"""
|
||||
5
tests/__main__.py
Normal file
5
tests/__main__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""直接运行 python -m tests 时执行的入口"""
|
||||
from .run_tests import main
|
||||
import sys
|
||||
|
||||
sys.exit(main())
|
||||
BIN
tests/__pycache__/conftest.cpython-311.pyc
Normal file
BIN
tests/__pycache__/conftest.cpython-311.pyc
Normal file
Binary file not shown.
56
tests/conftest.py
Normal file
56
tests/conftest.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""pytest 配置文件和 fixtures"""
|
||||
import pytest
|
||||
import asyncio
|
||||
from typing import AsyncGenerator, Generator
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from app.api import create_app
|
||||
from app.core.db import init_db, get_db
|
||||
from app.repositories.proxy_repo import ProxyRepository
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]:
|
||||
"""创建事件循环"""
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
async def app():
|
||||
"""创建应用实例"""
|
||||
# 初始化测试数据库
|
||||
await init_db()
|
||||
app = create_app()
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(app) -> AsyncGenerator[AsyncClient, None]:
|
||||
"""创建异步 HTTP 客户端"""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db():
|
||||
"""获取数据库连接"""
|
||||
async with get_db() as db:
|
||||
yield db
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def proxy_repo():
|
||||
"""获取代理仓库"""
|
||||
return ProxyRepository()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def sample_proxy(db, proxy_repo):
|
||||
"""创建一个测试代理"""
|
||||
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
|
||||
yield {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
|
||||
# 清理
|
||||
await proxy_repo.delete(db, "192.168.1.1", 8080)
|
||||
1
tests/e2e/__init__.py
Normal file
1
tests/e2e/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""端到端测试"""
|
||||
BIN
tests/e2e/__pycache__/test_full_workflow.cpython-311.pyc
Normal file
BIN
tests/e2e/__pycache__/test_full_workflow.cpython-311.pyc
Normal file
Binary file not shown.
182
tests/e2e/test_full_workflow.py
Normal file
182
tests/e2e/test_full_workflow.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""完整工作流 E2E 测试
|
||||
|
||||
这些测试模拟真实用户场景，验证整个系统的集成功能。
|
||||
"""
|
||||
import pytest
|
||||
|
||||
|
||||
class TestFullWorkflow:
|
||||
"""测试完整工作流"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_management_workflow(self, client):
|
||||
"""测试代理管理完整工作流
|
||||
|
||||
场景：
|
||||
1. 查看统计信息
|
||||
2. 列出代理
|
||||
3. 触发爬取
|
||||
4. 查看更新后的统计
|
||||
5. 导出代理
|
||||
6. 清理无效代理
|
||||
"""
|
||||
# 1. 获取初始统计
|
||||
response = await client.get("/api/proxies/stats")
|
||||
assert response.status_code == 200
|
||||
initial_stats = response.json()["data"]
|
||||
|
||||
# 2. 列出代理
|
||||
response = await client.post("/api/proxies", json={
|
||||
"page": 1,
|
||||
"page_size": 20,
|
||||
})
|
||||
assert response.status_code == 200
|
||||
|
||||
# 3. 触发所有插件爬取
|
||||
response = await client.post("/api/plugins/crawl-all")
|
||||
assert response.status_code == 200
|
||||
crawl_result = response.json()["data"]
|
||||
|
||||
# 4. 获取更新后的统计
|
||||
response = await client.get("/api/proxies/stats")
|
||||
updated_stats = response.json()["data"]
|
||||
|
||||
# 5. 导出代理（所有格式）
|
||||
for fmt in ["csv", "txt", "json"]:
|
||||
response = await client.get(f"/api/proxies/export/{fmt}")
|
||||
assert response.status_code == 200
|
||||
|
||||
# 6. 清理无效代理
|
||||
response = await client.delete("/api/proxies/clean-invalid")
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_management_workflow(self, client):
|
||||
"""测试插件管理完整工作流
|
||||
|
||||
场景：
|
||||
1. 列出所有插件
|
||||
2. 禁用某个插件
|
||||
3. 更新插件配置
|
||||
4. 启用插件
|
||||
5. 触发单个插件爬取
|
||||
"""
|
||||
# 1. 列出插件
|
||||
response = await client.get("/api/plugins")
|
||||
assert response.status_code == 200
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
|
||||
# 2. 禁用插件
|
||||
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
|
||||
assert response.status_code == 200
|
||||
|
||||
# 3. 获取插件配置
|
||||
response = await client.get(f"/api/plugins/{plugin_id}/config")
|
||||
assert response.status_code == 200
|
||||
|
||||
# 4. 更新插件配置
|
||||
response = await client.post(
|
||||
f"/api/plugins/{plugin_id}/config",
|
||||
json={"config": {"max_pages": 3}}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# 5. 启用插件
|
||||
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
|
||||
assert response.status_code == 200
|
||||
|
||||
# 6. 触发爬取
|
||||
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scheduler_workflow(self, client):
|
||||
"""测试调度器工作流
|
||||
|
||||
场景：
|
||||
1. 启动调度器
|
||||
2. 触发立即验证
|
||||
3. 检查状态
|
||||
4. 停止调度器
|
||||
"""
|
||||
# 1. 启动调度器
|
||||
response = await client.post("/api/scheduler/start")
|
||||
assert response.status_code == 200
|
||||
|
||||
# 2. 触发立即验证
|
||||
response = await client.post("/api/scheduler/validate-now")
|
||||
assert response.status_code == 200
|
||||
|
||||
# 3. 检查状态
|
||||
response = await client.get("/api/scheduler/status")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["running"] is True
|
||||
|
||||
# 4. 停止调度器
|
||||
response = await client.post("/api/scheduler/stop")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["running"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settings_workflow(self, client):
|
||||
"""测试设置工作流
|
||||
|
||||
场景：
|
||||
1. 获取当前设置
|
||||
2. 修改设置
|
||||
3. 验证设置已保存
|
||||
4. 恢复默认设置
|
||||
"""
|
||||
# 1. 获取当前设置
|
||||
response = await client.get("/api/settings")
|
||||
assert response.status_code == 200
|
||||
original_settings = response.json()["data"]
|
||||
|
||||
# 2. 修改设置
|
||||
new_settings = original_settings.copy()
|
||||
new_settings["crawl_timeout"] = 45
|
||||
new_settings["auto_validate"] = not original_settings["auto_validate"]
|
||||
|
||||
response = await client.post("/api/settings", json=new_settings)
|
||||
assert response.status_code == 200
|
||||
|
||||
# 3. 验证设置已保存
|
||||
response = await client.get("/api/settings")
|
||||
saved_settings = response.json()["data"]
|
||||
assert saved_settings["crawl_timeout"] == 45
|
||||
|
||||
# 4. 恢复原始设置
|
||||
response = await client.post("/api/settings", json=original_settings)
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_operations_workflow(self, client):
|
||||
"""测试批量操作工作流
|
||||
|
||||
场景：
|
||||
1. 批量删除不存在的代理（幂等性测试）
|
||||
2. 导出所有代理
|
||||
3. 获取随机代理
|
||||
"""
|
||||
# 1. 批量删除（幂等性）
|
||||
response = await client.post("/api/proxies/batch-delete", json={
|
||||
"proxies": [
|
||||
{"ip": "192.168.100.1", "port": 8080},
|
||||
{"ip": "192.168.100.2", "port": 8081},
|
||||
]
|
||||
})
|
||||
assert response.status_code == 200
|
||||
|
||||
# 2. 导出所有格式
|
||||
for fmt in ["csv", "txt", "json"]:
|
||||
response = await client.get(f"/api/proxies/export/{fmt}")
|
||||
assert response.status_code == 200
|
||||
|
||||
# 3. 获取随机代理（可能返回 200 或 404）
|
||||
response = await client.get("/api/proxies/random")
|
||||
assert response.status_code in [200, 404]
|
||||
1
tests/integration/__init__.py
Normal file
1
tests/integration/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""集成测试"""
|
||||
BIN
tests/integration/__pycache__/test_health_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_health_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_plugins_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_plugins_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_proxies_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_proxies_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_scheduler_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_scheduler_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_settings_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_settings_api.cpython-311.pyc
Normal file
Binary file not shown.
47
tests/integration/test_health_api.py
Normal file
47
tests/integration/test_health_api.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""健康检查 API 测试"""
|
||||
import pytest
|
||||
|
||||
|
||||
class TestHealthAPI:
|
||||
"""测试健康检查端点"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_endpoint(self, client):
|
||||
"""测试根端点 /"""
|
||||
response = await client.get("/")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "message" in data
|
||||
assert "status" in data
|
||||
assert data["status"] == "running"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint(self, client):
|
||||
"""测试健康检查端点 /health"""
|
||||
response = await client.get("/health")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
assert "timestamp" in data
|
||||
assert "database" in data
|
||||
assert "scheduler" in data
|
||||
assert "version" in data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_structure(self, client):
|
||||
"""测试健康检查端点返回结构"""
|
||||
response = await client.get("/health")
|
||||
data = response.json()
|
||||
|
||||
assert "status" in data
|
||||
assert "timestamp" in data
|
||||
assert "database" in data
|
||||
assert "scheduler" in data
|
||||
assert "version" in data
|
||||
|
||||
# 验证数据类型
|
||||
assert isinstance(data["status"], str)
|
||||
assert isinstance(data["timestamp"], str)
|
||||
assert isinstance(data["database"], str)
|
||||
assert isinstance(data["scheduler"], str)
|
||||
assert isinstance(data["version"], str)
|
||||
147
tests/integration/test_plugins_api.py
Normal file
147
tests/integration/test_plugins_api.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""插件 API 集成测试 - 测试 /api/plugins/* 所有接口"""
|
||||
import pytest
|
||||
|
||||
|
||||
class TestPluginsAPI:
|
||||
"""测试插件相关 API"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_plugins(self, client):
|
||||
"""测试 GET /api/plugins"""
|
||||
response = await client.get("/api/plugins")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert "plugins" in data["data"]
|
||||
assert isinstance(data["data"]["plugins"], list)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_plugins_structure(self, client):
|
||||
"""测试 GET /api/plugins 返回结构"""
|
||||
response = await client.get("/api/plugins")
|
||||
data = response.json()
|
||||
if data["data"]["plugins"]:
|
||||
plugin = data["data"]["plugins"][0]
|
||||
assert "id" in plugin
|
||||
assert "name" in plugin
|
||||
assert "display_name" in plugin
|
||||
assert "description" in plugin
|
||||
assert "enabled" in plugin
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_toggle_plugin_enable(self, client):
|
||||
"""测试 PUT /api/plugins/{id}/toggle - 启用"""
|
||||
# 先获取一个插件 ID
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert data["data"]["enabled"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_toggle_plugin_disable(self, client):
|
||||
"""测试 PUT /api/plugins/{id}/toggle - 禁用"""
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert data["data"]["enabled"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_toggle_plugin_missing_enabled(self, client):
|
||||
"""测试 PUT /api/plugins/{id}/toggle - 缺少 enabled 参数"""
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={})
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_toggle_nonexistent_plugin(self, client):
|
||||
"""测试 PUT /api/plugins/{id}/toggle - 不存在的插件"""
|
||||
response = await client.put("/api/plugins/nonexistent_plugin/toggle", json={"enabled": True})
|
||||
assert response.status_code == 404
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_plugin_config(self, client):
|
||||
"""测试 GET /api/plugins/{id}/config"""
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
response = await client.get(f"/api/plugins/{plugin_id}/config")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert "config" in data["data"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_nonexistent_plugin_config(self, client):
|
||||
"""测试 GET /api/plugins/{id}/config - 不存在的插件"""
|
||||
response = await client.get("/api/plugins/nonexistent_plugin/config")
|
||||
assert response.status_code == 404
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_plugin_config(self, client):
|
||||
"""测试 POST /api/plugins/{id}/config"""
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
response = await client.post(
|
||||
f"/api/plugins/{plugin_id}/config",
|
||||
json={"config": {"max_pages": 3}}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_plugin(self, client):
|
||||
"""测试 POST /api/plugins/{id}/crawl"""
|
||||
response = await client.get("/api/plugins")
|
||||
plugins = response.json()["data"]["plugins"]
|
||||
if not plugins:
|
||||
pytest.skip("没有可用的插件")
|
||||
|
||||
plugin_id = plugins[0]["id"]
|
||||
# 这个测试可能需要较长时间，设置较短的超时
|
||||
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert "proxy_count" in data["data"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_nonexistent_plugin(self, client):
|
||||
"""测试 POST /api/plugins/{id}/crawl - 不存在的插件"""
|
||||
response = await client.post("/api/plugins/nonexistent_plugin/crawl")
|
||||
assert response.status_code == 404
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_all_plugins(self, client):
|
||||
"""测试 POST /api/plugins/crawl-all"""
|
||||
response = await client.post("/api/plugins/crawl-all")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["code"] == 200
|
||||
assert "total_crawled" in data["data"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user