重构: 迁移后端代码到 app 目录,前端移动到 WebUI,添加完整测试套件
主要变更: - 后端代码从根目录迁移到 app/ 目录 - 前端代码从 frontend/ 重命名为 WebUI/ - 更新所有导入路径以适配新结构 - 提取公共 API 响应函数到 app/api/common.py - 精简验证器服务代码 - 更新启动脚本和文档 测试: - 新增完整测试套件 (tests/) - 单元测试: 模型、仓库层 - 集成测试: 覆盖所有 22+ API 端点 - E2E 测试: 4个完整工作流场景 - 添加 pytest 配置和测试运行脚本
This commit is contained in:
138
DESIGN.md
138
DESIGN.md
@@ -309,77 +309,87 @@ Store 只负责:
|
|||||||
|
|
||||||
```
|
```
|
||||||
ProxyPool/
|
ProxyPool/
|
||||||
├── api/ # FastAPI 入口和路由
|
├── main.py # 项目入口
|
||||||
│ ├── __init__.py
|
├── requirements.txt # Python 依赖
|
||||||
│ ├── main.py # 应用工厂
|
├── .env.example # 环境变量示例
|
||||||
│ ├── lifespan.py # 生命周期管理
|
|
||||||
│ ├── deps.py # 依赖注入
|
|
||||||
│ ├── errors.py # 统一异常
|
|
||||||
│ └── routes/
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ ├── proxies.py
|
|
||||||
│ ├── plugins.py
|
|
||||||
│ ├── scheduler.py
|
|
||||||
│ └── settings.py
|
|
||||||
│
|
│
|
||||||
├── core/ # 基础设施
|
├── app/ # 后端代码
|
||||||
│ ├── __init__.py
|
│ ├── api/ # FastAPI 入口和路由
|
||||||
│ ├── config.py # Pydantic Settings
|
|
||||||
│ ├── log.py # 日志
|
|
||||||
│ ├── db.py # 数据库连接池/上下文
|
|
||||||
│ └── exceptions.py # 业务异常
|
|
||||||
│
|
|
||||||
├── models/ # 数据模型
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ ├── schemas.py # Pydantic 模型
|
|
||||||
│ └── domain.py # 领域模型(ProxyRaw, PluginInfo 等)
|
|
||||||
│
|
|
||||||
├── repositories/ # 数据访问层
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ └── proxy_repo.py # ProxyRepository
|
|
||||||
│
|
|
||||||
├── services/ # 业务逻辑层
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ ├── proxy_service.py
|
|
||||||
│ ├── plugin_service.py
|
|
||||||
│ ├── scheduler_service.py
|
|
||||||
│ └── validator_service.py
|
|
||||||
│
|
|
||||||
├── core/ # 任务与插件系统
|
|
||||||
│ ├── plugin_system/
|
|
||||||
│ │ ├── __init__.py
|
│ │ ├── __init__.py
|
||||||
│ │ ├── base.py # BaseCrawlerPlugin
|
│ │ ├── main.py # 应用工厂
|
||||||
│ │ └── registry.py # 插件注册中心
|
│ │ ├── lifespan.py # 生命周期管理
|
||||||
│ └── tasks/
|
│ │ ├── deps.py # 依赖注入
|
||||||
|
│ │ ├── errors.py # 统一异常
|
||||||
|
│ │ └── routes/
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ ├── proxies.py
|
||||||
|
│ │ ├── plugins.py
|
||||||
|
│ │ ├── scheduler.py
|
||||||
|
│ │ └── settings.py
|
||||||
|
│ │
|
||||||
|
│ ├── core/ # 基础设施
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ ├── config.py # Pydantic Settings
|
||||||
|
│ │ ├── log.py # 日志
|
||||||
|
│ │ ├── db.py # 数据库连接池/上下文
|
||||||
|
│ │ ├── exceptions.py # 业务异常
|
||||||
|
│ │ ├── plugin_system/ # 插件系统
|
||||||
|
│ │ │ ├── __init__.py
|
||||||
|
│ │ │ ├── base.py # BaseCrawlerPlugin
|
||||||
|
│ │ │ └── registry.py # 插件注册中心
|
||||||
|
│ │ └── tasks/ # 任务队列
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ └── queue.py # ValidationQueue
|
||||||
|
│ │
|
||||||
|
│ ├── models/ # 数据模型
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ ├── schemas.py # Pydantic 模型
|
||||||
|
│ │ └── domain.py # 领域模型
|
||||||
|
│ │
|
||||||
|
│ ├── repositories/ # 数据访问层
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ ├── proxy_repo.py
|
||||||
|
│ │ ├── settings_repo.py
|
||||||
|
│ │ └── task_repo.py
|
||||||
|
│ │
|
||||||
|
│ ├── services/ # 业务逻辑层
|
||||||
|
│ │ ├── __init__.py
|
||||||
|
│ │ ├── proxy_service.py
|
||||||
|
│ │ ├── plugin_service.py
|
||||||
|
│ │ ├── scheduler_service.py
|
||||||
|
│ │ └── validator_service.py
|
||||||
|
│ │
|
||||||
|
│ └── plugins/ # 爬虫插件
|
||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ ├── queue.py # ValidationQueue
|
│ ├── base.py # 通用抓取基类
|
||||||
│ └── workers.py # Worker Pool
|
|
||||||
│
|
|
||||||
├── plugins/ # 爬虫插件
|
|
||||||
│ ├── __init__.py
|
|
||||||
│ ├── base.py # 通用抓取基类(HTTP 请求封装)
|
|
||||||
│ ├── fate0.py
|
│ ├── fate0.py
|
||||||
|
│ ├── kuaidaili.py
|
||||||
|
│ ├── ip3366.py
|
||||||
|
│ ├── ip89.py
|
||||||
|
│ ├── speedx.py
|
||||||
|
│ ├── yundaili.py
|
||||||
│ ├── proxylist_download.py
|
│ ├── proxylist_download.py
|
||||||
│ └── ...
|
│ └── proxyscrape.py
|
||||||
│
|
│
|
||||||
├── frontend/ # Vue3 前端
|
├── WebUI/ # Vue3 前端
|
||||||
│ └── src/
|
│ ├── src/
|
||||||
│ ├── services/ # 新增
|
│ │ ├── api/ # API 封装
|
||||||
│ ├── stores/
|
│ │ ├── stores/ # Pinia 状态管理
|
||||||
│ ├── api/
|
│ │ ├── views/ # 页面组件
|
||||||
│ └── ...
|
│ │ ├── router/ # 路由配置
|
||||||
|
│ │ ├── components/ # 通用组件
|
||||||
|
│ │ └── style.css # 全局样式
|
||||||
|
│ ├── index.html
|
||||||
|
│ └── package.json
|
||||||
│
|
│
|
||||||
├── tests/ # 测试目录
|
├── tests/ # 测试目录
|
||||||
│ ├── conftest.py
|
│ ├── conftest.py
|
||||||
│ ├── unit/
|
│ ├── unit/
|
||||||
│ └── integration/
|
│ └── integration/
|
||||||
│
|
│
|
||||||
├── script/
|
├── script/ # 启动脚本
|
||||||
├── data/
|
├── db/ # 数据存储
|
||||||
├── db/
|
├── logs/ # 日志文件
|
||||||
├── logs/
|
|
||||||
├── requirements.txt
|
|
||||||
├── .env.example
|
|
||||||
└── DESIGN.md # 本文档
|
└── DESIGN.md # 本文档
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -426,11 +436,11 @@ ProxyPool/
|
|||||||
|
|
||||||
假设要添加一个名为 `mynewsource` 的爬虫:
|
假设要添加一个名为 `mynewsource` 的爬虫:
|
||||||
|
|
||||||
**Step 1**: 创建文件 `plugins/mynewsource.py`
|
**Step 1**: 创建文件 `app/plugins/mynewsource.py`
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from core.plugin_system import BaseCrawlerPlugin, ProxyRaw
|
from app.core.plugin_system import BaseCrawlerPlugin, ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin # 可选:如果基于 HTTP 爬取
|
from app.plugins.base import BaseHTTPPlugin # 可选:如果基于 HTTP 爬取
|
||||||
|
|
||||||
class MyNewSourcePlugin(BaseHTTPPlugin):
|
class MyNewSourcePlugin(BaseHTTPPlugin):
|
||||||
name = "mynewsource"
|
name = "mynewsource"
|
||||||
@@ -450,11 +460,11 @@ class MyNewSourcePlugin(BaseHTTPPlugin):
|
|||||||
return results
|
return results
|
||||||
```
|
```
|
||||||
|
|
||||||
**Step 2**: 在 `plugins/__init__.py` 中注册
|
**Step 2**: 在 `app/plugins/__init__.py` 中注册
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from .mynewsource import MyNewSourcePlugin
|
from .mynewsource import MyNewSourcePlugin
|
||||||
from core.plugin_system import registry
|
from app.core.plugin_system import registry
|
||||||
|
|
||||||
registry.register(MyNewSourcePlugin)
|
registry.register(MyNewSourcePlugin)
|
||||||
```
|
```
|
||||||
|
|||||||
41
README.md
41
README.md
@@ -36,7 +36,7 @@ pip install -r requirements.txt
|
|||||||
### 2. 安装前端依赖
|
### 2. 安装前端依赖
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd frontend
|
cd WebUI
|
||||||
npm install
|
npm install
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -58,7 +58,7 @@ python api_server.py
|
|||||||
|
|
||||||
**启动前端服务**(终端 2)
|
**启动前端服务**(终端 2)
|
||||||
```bash
|
```bash
|
||||||
cd frontend
|
cd WebUI
|
||||||
npm run dev
|
npm run dev
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -77,23 +77,26 @@ stop.bat
|
|||||||
|
|
||||||
```
|
```
|
||||||
ProxyPool/
|
ProxyPool/
|
||||||
├── api_server.py # FastAPI 后端服务器
|
├── main.py # 项目入口
|
||||||
├── config.py # 配置文件
|
|
||||||
├── requirements.txt # Python 依赖
|
├── requirements.txt # Python 依赖
|
||||||
├── .env.example # 环境变量示例
|
├── .env.example # 环境变量示例
|
||||||
│
|
│
|
||||||
├── script/ # 启动脚本
|
├── app/ # 后端代码
|
||||||
│ ├── start.bat # Windows 启动脚本
|
│ ├── api/ # FastAPI 路由
|
||||||
│ └── stop.bat # Windows 停止脚本
|
│ │ ├── main.py # 应用工厂
|
||||||
│
|
│ │ ├── routes/ # API 路由
|
||||||
├── core/ # 核心模块
|
│ │ ├── deps.py # 依赖注入
|
||||||
│ ├── crawler.py # 爬虫基类
|
│ │ └── ...
|
||||||
│ ├── validator.py # 代理验证器
|
│ ├── core/ # 核心模块
|
||||||
│ ├── sqlite.py # 数据库管理
|
│ │ ├── config.py # 配置管理
|
||||||
│ ├── plugin_manager.py # 插件管理器
|
│ │ ├── db.py # 数据库连接
|
||||||
│ └── log.py # 日志配置
|
│ │ ├── log.py # 日志配置
|
||||||
│
|
│ │ ├── plugin_system/ # 插件系统
|
||||||
├── plugins/ # 代理源插件
|
│ │ └── tasks/ # 任务队列
|
||||||
|
│ ├── models/ # 数据模型
|
||||||
|
│ ├── repositories/ # 数据访问层
|
||||||
|
│ ├── services/ # 业务逻辑层
|
||||||
|
│ └── plugins/ # 代理源插件
|
||||||
│ ├── fate0.py # Fate0 代理源
|
│ ├── fate0.py # Fate0 代理源
|
||||||
│ ├── ip3366.py # IP3366 代理源
|
│ ├── ip3366.py # IP3366 代理源
|
||||||
│ ├── ip89.py # IP89 代理源
|
│ ├── ip89.py # IP89 代理源
|
||||||
@@ -102,7 +105,7 @@ ProxyPool/
|
|||||||
│ ├── speedx.py # SpeedX 代理源
|
│ ├── speedx.py # SpeedX 代理源
|
||||||
│ └── proxylist_download.py # ProxyList 代理源
|
│ └── proxylist_download.py # ProxyList 代理源
|
||||||
│
|
│
|
||||||
├── frontend/ # Vue3 前端
|
├── WebUI/ # Vue3 前端
|
||||||
│ ├── src/
|
│ ├── src/
|
||||||
│ │ ├── api/ # API 封装
|
│ │ ├── api/ # API 封装
|
||||||
│ │ ├── stores/ # Pinia 状态管理
|
│ │ ├── stores/ # Pinia 状态管理
|
||||||
@@ -113,6 +116,10 @@ ProxyPool/
|
|||||||
│ ├── index.html
|
│ ├── index.html
|
||||||
│ └── package.json
|
│ └── package.json
|
||||||
│
|
│
|
||||||
|
├── script/ # 启动脚本
|
||||||
|
│ ├── start.bat # Windows 启动脚本
|
||||||
|
│ └── stop.bat # Windows 停止脚本
|
||||||
|
│
|
||||||
└── db/ # 数据存储目录
|
└── db/ # 数据存储目录
|
||||||
└── proxies.sqlite # SQLite 数据库
|
└── proxies.sqlite # SQLite 数据库
|
||||||
```
|
```
|
||||||
|
|||||||
0
frontend/.gitignore → WebUI/.gitignore
vendored
0
frontend/.gitignore → WebUI/.gitignore
vendored
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.5 KiB |
@@ -74,8 +74,9 @@
|
|||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
|
|
||||||
<el-table-column label="操作" width="200" fixed="right" align="center">
|
<el-table-column label="操作" width="220" fixed="right" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
|
<div class="plugin-actions">
|
||||||
<el-button
|
<el-button
|
||||||
type="primary"
|
type="primary"
|
||||||
size="small"
|
size="small"
|
||||||
@@ -88,12 +89,27 @@
|
|||||||
type="success"
|
type="success"
|
||||||
size="small"
|
size="small"
|
||||||
@click="handleCrawl(row.id)"
|
@click="handleCrawl(row.id)"
|
||||||
:loading="crawlingPlugin === row.id"
|
:loading="crawlingPlugins.has(row.id)"
|
||||||
:disabled="!row.enabled"
|
:disabled="!row.enabled"
|
||||||
>
|
>
|
||||||
<el-icon class="btn-icon"><Promotion /></el-icon>
|
<el-icon class="btn-icon"><Promotion /></el-icon>
|
||||||
爬取
|
爬取
|
||||||
</el-button>
|
</el-button>
|
||||||
|
</div>
|
||||||
|
<div v-if="crawlResults[row.id]" class="plugin-crawl-result">
|
||||||
|
<div class="result-mini" :class="crawlResults[row.id].type">
|
||||||
|
<el-icon v-if="crawlResults[row.id].type === 'success'" class="result-icon success"><CircleCheck /></el-icon>
|
||||||
|
<el-icon v-else class="result-icon failed"><CircleClose /></el-icon>
|
||||||
|
<span class="result-text">{{ crawlResults[row.id].message }}</span>
|
||||||
|
<span v-if="crawlResults[row.id].data?.valid_count !== undefined" class="result-count valid">
|
||||||
|
有效 {{ crawlResults[row.id].data.valid_count }}
|
||||||
|
</span>
|
||||||
|
<span v-if="crawlResults[row.id].data?.invalid_count !== undefined" class="result-count invalid">
|
||||||
|
无效 {{ crawlResults[row.id].data.invalid_count }}
|
||||||
|
</span>
|
||||||
|
<el-icon class="result-close" @click="clearCrawlResult(row.id)"><Close /></el-icon>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
@@ -104,28 +120,28 @@
|
|||||||
:image-size="120"
|
:image-size="120"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<!-- 爬取结果提示 -->
|
<!-- 批量爬取结果提示 -->
|
||||||
<el-alert
|
<el-alert
|
||||||
v-if="lastCrawlResult"
|
v-if="allCrawlResult"
|
||||||
:title="lastCrawlResult.message"
|
:title="allCrawlResult.message"
|
||||||
:type="lastCrawlResult.type"
|
:type="allCrawlResult.type"
|
||||||
closable
|
closable
|
||||||
class="crawl-result"
|
class="crawl-result"
|
||||||
@close="lastCrawlResult = null"
|
@close="allCrawlResult = null"
|
||||||
>
|
>
|
||||||
<template v-if="lastCrawlResult.data">
|
<template v-if="allCrawlResult.data">
|
||||||
<div class="crawl-stats">
|
<div class="crawl-stats">
|
||||||
<span v-if="lastCrawlResult.data.total_crawled !== undefined">
|
<span v-if="allCrawlResult.data.total_crawled !== undefined">
|
||||||
爬取: {{ lastCrawlResult.data.total_crawled }}
|
爬取: {{ allCrawlResult.data.total_crawled }}
|
||||||
</span>
|
</span>
|
||||||
<span v-if="lastCrawlResult.data.proxy_count !== undefined">
|
<span v-if="allCrawlResult.data.proxy_count !== undefined">
|
||||||
爬取: {{ lastCrawlResult.data.proxy_count }}
|
爬取: {{ allCrawlResult.data.proxy_count }}
|
||||||
</span>
|
</span>
|
||||||
<span v-if="lastCrawlResult.data.valid_count !== undefined" class="valid-count">
|
<span v-if="allCrawlResult.data.valid_count !== undefined" class="valid-count">
|
||||||
有效: {{ lastCrawlResult.data.valid_count }}
|
有效: {{ allCrawlResult.data.valid_count }}
|
||||||
</span>
|
</span>
|
||||||
<span v-if="lastCrawlResult.data.invalid_count !== undefined" class="invalid-count">
|
<span v-if="allCrawlResult.data.invalid_count !== undefined" class="invalid-count">
|
||||||
无效: {{ lastCrawlResult.data.invalid_count }}
|
无效: {{ allCrawlResult.data.invalid_count }}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -183,7 +199,8 @@ import {
|
|||||||
CircleCheck,
|
CircleCheck,
|
||||||
CircleClose,
|
CircleClose,
|
||||||
Box,
|
Box,
|
||||||
Setting
|
Setting,
|
||||||
|
Close
|
||||||
} from '@element-plus/icons-vue'
|
} from '@element-plus/icons-vue'
|
||||||
import { usePluginsStore } from '../stores/plugins'
|
import { usePluginsStore } from '../stores/plugins'
|
||||||
import { pluginService } from '../services/pluginService'
|
import { pluginService } from '../services/pluginService'
|
||||||
@@ -191,9 +208,10 @@ import { formatTime } from '../utils/format'
|
|||||||
import PageHeader from '../components/PageHeader.vue'
|
import PageHeader from '../components/PageHeader.vue'
|
||||||
|
|
||||||
const pluginsStore = usePluginsStore()
|
const pluginsStore = usePluginsStore()
|
||||||
const crawlingPlugin = ref(null)
|
const crawlingPlugins = ref(new Set())
|
||||||
const crawlingAll = ref(false)
|
const crawlingAll = ref(false)
|
||||||
const lastCrawlResult = ref(null)
|
const crawlResults = ref({})
|
||||||
|
const allCrawlResult = ref(null)
|
||||||
|
|
||||||
// 配置对话框
|
// 配置对话框
|
||||||
const configDialogVisible = ref(false)
|
const configDialogVisible = ref(false)
|
||||||
@@ -248,34 +266,36 @@ async function handleSaveConfig() {
|
|||||||
|
|
||||||
async function handleCrawl(pluginId) {
|
async function handleCrawl(pluginId) {
|
||||||
try {
|
try {
|
||||||
crawlingPlugin.value = pluginId
|
crawlingPlugins.value.add(pluginId)
|
||||||
lastCrawlResult.value = null
|
|
||||||
|
|
||||||
const response = await pluginService.crawlPlugin(pluginId)
|
const response = await pluginService.crawlPlugin(pluginId)
|
||||||
|
|
||||||
if (response.code === 200) {
|
if (response.code === 200) {
|
||||||
lastCrawlResult.value = {
|
crawlResults.value[pluginId] = {
|
||||||
type: 'success',
|
type: 'success',
|
||||||
message: response.message,
|
message: response.message,
|
||||||
data: response.data
|
data: response.data
|
||||||
}
|
}
|
||||||
await pluginsStore.fetchPlugins()
|
|
||||||
} else {
|
} else {
|
||||||
lastCrawlResult.value = {
|
crawlResults.value[pluginId] = {
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: response.message || '爬取失败'
|
message: response.message || '爬取失败'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
lastCrawlResult.value = {
|
crawlResults.value[pluginId] = {
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: '爬取过程出错'
|
message: '爬取过程出错'
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
crawlingPlugin.value = null
|
crawlingPlugins.value.delete(pluginId)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function clearCrawlResult(pluginId) {
|
||||||
|
delete crawlResults.value[pluginId]
|
||||||
|
}
|
||||||
|
|
||||||
async function handleCrawlAll() {
|
async function handleCrawlAll() {
|
||||||
try {
|
try {
|
||||||
const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled)
|
const enabledPlugins = pluginsStore.plugins.filter(p => p.enabled)
|
||||||
@@ -295,20 +315,19 @@ async function handleCrawlAll() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
crawlingAll.value = true
|
crawlingAll.value = true
|
||||||
lastCrawlResult.value = null
|
allCrawlResult.value = null
|
||||||
|
|
||||||
const response = await pluginService.crawlAll()
|
const response = await pluginService.crawlAll()
|
||||||
|
|
||||||
if (response.code === 200) {
|
if (response.code === 200) {
|
||||||
lastCrawlResult.value = {
|
allCrawlResult.value = {
|
||||||
type: 'success',
|
type: 'success',
|
||||||
message: response.message,
|
message: response.message,
|
||||||
data: response.data
|
data: response.data
|
||||||
}
|
}
|
||||||
ElMessage.success('批量爬取完成')
|
ElMessage.success('批量爬取完成')
|
||||||
await pluginsStore.fetchPlugins()
|
|
||||||
} else {
|
} else {
|
||||||
lastCrawlResult.value = {
|
allCrawlResult.value = {
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: response.message || '批量爬取失败'
|
message: response.message || '批量爬取失败'
|
||||||
}
|
}
|
||||||
@@ -316,7 +335,7 @@ async function handleCrawlAll() {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error !== 'cancel') {
|
if (error !== 'cancel') {
|
||||||
console.error('批量爬取失败:', error)
|
console.error('批量爬取失败:', error)
|
||||||
lastCrawlResult.value = {
|
allCrawlResult.value = {
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: '批量爬取过程出错'
|
message: '批量爬取过程出错'
|
||||||
}
|
}
|
||||||
@@ -460,4 +479,70 @@ onMounted(async () => {
|
|||||||
margin-bottom: 16px;
|
margin-bottom: 16px;
|
||||||
color: var(--text-primary);
|
color: var(--text-primary);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.plugin-actions {
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.plugin-crawl-result {
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-mini {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
padding: 4px 8px;
|
||||||
|
border-radius: 4px;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-mini.success {
|
||||||
|
background: rgba(103, 194, 58, 0.15);
|
||||||
|
color: var(--success);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-mini.error {
|
||||||
|
background: rgba(245, 108, 108, 0.15);
|
||||||
|
color: var(--danger);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-icon {
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-text {
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-count {
|
||||||
|
font-weight: 600;
|
||||||
|
padding: 0 4px;
|
||||||
|
border-radius: 3px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-count.valid {
|
||||||
|
background: rgba(103, 194, 58, 0.2);
|
||||||
|
color: var(--success);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-count.invalid {
|
||||||
|
background: rgba(245, 108, 108, 0.2);
|
||||||
|
color: var(--danger);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-close {
|
||||||
|
margin-left: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 12px;
|
||||||
|
opacity: 0.7;
|
||||||
|
transition: opacity 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-close:hover {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
"""代理相关路由(含统计信息)"""
|
|
||||||
from typing import Optional
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
|
||||||
from services.proxy_service import ProxyService
|
|
||||||
from services.scheduler_service import SchedulerService
|
|
||||||
from models.schemas import ProxyListRequest, BatchDeleteRequest
|
|
||||||
from api.deps import get_proxy_service, get_scheduler_service
|
|
||||||
from core.log import logger
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
|
|
||||||
|
|
||||||
|
|
||||||
def success_response(message: str, data=None):
|
|
||||||
return {"code": 200, "message": message, "data": data}
|
|
||||||
|
|
||||||
|
|
||||||
def error_response(message: str, code: int = 500):
|
|
||||||
return {"code": code, "message": message, "data": None}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/stats")
|
|
||||||
async def get_stats(
|
|
||||||
proxy_service: ProxyService = Depends(get_proxy_service),
|
|
||||||
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
stats = await proxy_service.get_stats()
|
|
||||||
stats["scheduler_running"] = scheduler_service.running
|
|
||||||
return success_response("获取统计信息成功", stats)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Get stats failed: {e}")
|
|
||||||
return error_response("获取统计信息失败")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
|
||||||
async def list_proxies(
|
|
||||||
request: ProxyListRequest,
|
|
||||||
service: ProxyService = Depends(get_proxy_service),
|
|
||||||
):
|
|
||||||
proxies, total = await service.list_proxies(
|
|
||||||
page=request.page,
|
|
||||||
page_size=request.page_size,
|
|
||||||
protocol=request.protocol,
|
|
||||||
min_score=request.min_score,
|
|
||||||
max_score=request.max_score,
|
|
||||||
sort_by=request.sort_by,
|
|
||||||
sort_order=request.sort_order,
|
|
||||||
)
|
|
||||||
return success_response(
|
|
||||||
"获取代理列表成功",
|
|
||||||
{
|
|
||||||
"list": [
|
|
||||||
{
|
|
||||||
"ip": p.ip,
|
|
||||||
"port": p.port,
|
|
||||||
"protocol": p.protocol,
|
|
||||||
"score": p.score,
|
|
||||||
"last_check": p.last_check.isoformat() if p.last_check else None,
|
|
||||||
}
|
|
||||||
for p in proxies
|
|
||||||
],
|
|
||||||
"total": total,
|
|
||||||
"page": request.page,
|
|
||||||
"page_size": request.page_size,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/random")
|
|
||||||
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
|
|
||||||
proxy = await service.get_random_proxy()
|
|
||||||
if not proxy:
|
|
||||||
return error_response("没有找到可用的代理", 404)
|
|
||||||
return success_response(
|
|
||||||
"获取随机代理成功",
|
|
||||||
{
|
|
||||||
"ip": proxy.ip,
|
|
||||||
"port": proxy.port,
|
|
||||||
"protocol": proxy.protocol,
|
|
||||||
"score": proxy.score,
|
|
||||||
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/export/{fmt}")
|
|
||||||
async def export_proxies(
|
|
||||||
fmt: str,
|
|
||||||
protocol: Optional[str] = None,
|
|
||||||
limit: int = Query(default=10000, ge=1, le=100000),
|
|
||||||
service: ProxyService = Depends(get_proxy_service),
|
|
||||||
):
|
|
||||||
if fmt not in ("csv", "txt", "json"):
|
|
||||||
return error_response("不支持的导出格式", 400)
|
|
||||||
|
|
||||||
from fastapi.responses import StreamingResponse
|
|
||||||
|
|
||||||
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
|
|
||||||
|
|
||||||
async def generate():
|
|
||||||
async for chunk in service.export_proxies(fmt, protocol, limit):
|
|
||||||
yield chunk
|
|
||||||
|
|
||||||
return StreamingResponse(
|
|
||||||
generate(),
|
|
||||||
media_type=media_types[fmt],
|
|
||||||
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{ip}/{port}")
|
|
||||||
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
|
|
||||||
await service.delete_proxy(ip, port)
|
|
||||||
return success_response("删除代理成功")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/batch-delete")
|
|
||||||
async def batch_delete(
|
|
||||||
request: BatchDeleteRequest,
|
|
||||||
service: ProxyService = Depends(get_proxy_service),
|
|
||||||
):
|
|
||||||
proxies = [(item.ip, item.port) for item in request.proxies]
|
|
||||||
deleted = await service.batch_delete(proxies)
|
|
||||||
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/clean-invalid")
|
|
||||||
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
|
|
||||||
count = await service.clean_invalid()
|
|
||||||
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
"""调度器相关路由"""
|
|
||||||
from fastapi import APIRouter, Depends
|
|
||||||
from services.scheduler_service import SchedulerService
|
|
||||||
from repositories.settings_repo import SettingsRepository
|
|
||||||
from core.db import get_db
|
|
||||||
from api.deps import get_scheduler_service
|
|
||||||
from core.log import logger
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
|
||||||
settings_repo = SettingsRepository()
|
|
||||||
|
|
||||||
|
|
||||||
def success_response(message: str, data=None):
|
|
||||||
return {"code": 200, "message": message, "data": data}
|
|
||||||
|
|
||||||
|
|
||||||
def error_response(message: str, code: int = 500):
|
|
||||||
return {"code": code, "message": message, "data": None}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/start")
|
|
||||||
async def start_scheduler(
|
|
||||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
if scheduler.running:
|
|
||||||
return success_response("验证调度器已在运行", {"running": True})
|
|
||||||
await scheduler.start()
|
|
||||||
# 持久化设置
|
|
||||||
async with get_db() as db:
|
|
||||||
settings = await settings_repo.get_all(db)
|
|
||||||
settings["auto_validate"] = True
|
|
||||||
from models.schemas import SettingsSchema
|
|
||||||
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
|
||||||
return success_response("验证调度器已启动", {"running": True})
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Start scheduler failed: {e}")
|
|
||||||
return error_response(f"启动调度器失败: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/stop")
|
|
||||||
async def stop_scheduler(
|
|
||||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
if not scheduler.running:
|
|
||||||
return success_response("验证调度器未运行", {"running": False})
|
|
||||||
await scheduler.stop()
|
|
||||||
# 持久化设置
|
|
||||||
async with get_db() as db:
|
|
||||||
settings = await settings_repo.get_all(db)
|
|
||||||
settings["auto_validate"] = False
|
|
||||||
from models.schemas import SettingsSchema
|
|
||||||
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
|
||||||
return success_response("验证调度器已停止", {"running": False})
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Stop scheduler failed: {e}")
|
|
||||||
return error_response(f"停止调度器失败: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/validate-now")
|
|
||||||
async def validate_now(
|
|
||||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
scheduler.validate_all_now()
|
|
||||||
return success_response("已开始全量验证", {"started": True})
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Validate now failed: {e}")
|
|
||||||
return error_response(f"启动验证失败: {str(e)}")
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
|
||||||
async def scheduler_status(
|
|
||||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
|
||||||
):
|
|
||||||
return success_response(
|
|
||||||
"获取状态成功",
|
|
||||||
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
|
|
||||||
)
|
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
"""API 包"""
|
||||||
from .main import create_app
|
from .main import create_app
|
||||||
|
|
||||||
__all__ = ["create_app"]
|
__all__ = ["create_app"]
|
||||||
41
app/api/common.py
Normal file
41
app/api/common.py
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
"""API 通用工具函数"""
|
||||||
|
from typing import Any, Optional
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
|
||||||
|
def success_response(message: str, data: Any = None) -> dict:
|
||||||
|
"""成功响应"""
|
||||||
|
return {"code": 200, "message": message, "data": data}
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(message: str, code: int = 500) -> JSONResponse:
|
||||||
|
"""错误响应"""
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=code,
|
||||||
|
content={"code": code, "message": message, "data": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def format_proxy(proxy) -> dict:
|
||||||
|
"""格式化代理对象"""
|
||||||
|
return {
|
||||||
|
"ip": proxy.ip,
|
||||||
|
"port": proxy.port,
|
||||||
|
"protocol": proxy.protocol,
|
||||||
|
"score": proxy.score,
|
||||||
|
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def format_plugin(plugin) -> dict:
|
||||||
|
"""格式化插件对象"""
|
||||||
|
return {
|
||||||
|
"id": plugin.id,
|
||||||
|
"name": plugin.display_name,
|
||||||
|
"display_name": plugin.display_name,
|
||||||
|
"description": plugin.description,
|
||||||
|
"enabled": plugin.enabled,
|
||||||
|
"last_run": plugin.last_run.isoformat() if plugin.last_run else None,
|
||||||
|
"success_count": plugin.success_count,
|
||||||
|
"failure_count": plugin.failure_count,
|
||||||
|
}
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
"""依赖注入"""
|
"""依赖注入"""
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from services.proxy_service import ProxyService
|
from app.services.proxy_service import ProxyService
|
||||||
from services.plugin_service import PluginService
|
from app.services.plugin_service import PluginService
|
||||||
from services.scheduler_service import SchedulerService
|
from app.services.scheduler_service import SchedulerService
|
||||||
from services.validator_service import ValidatorService
|
from app.services.validator_service import ValidatorService
|
||||||
from repositories.proxy_repo import ProxyRepository
|
from app.repositories.proxy_repo import ProxyRepository
|
||||||
from core.tasks.queue import ValidationQueue
|
from app.core.tasks.queue import ValidationQueue
|
||||||
from core.config import settings as app_settings
|
from app.core.config import settings as app_settings
|
||||||
|
|
||||||
|
|
||||||
def get_proxy_service() -> ProxyService:
|
def get_proxy_service() -> ProxyService:
|
||||||
@@ -2,8 +2,8 @@
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from core.exceptions import ProxyPoolException
|
from app.core.exceptions import ProxyPoolException
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException):
|
async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException):
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
"""应用生命周期管理"""
|
"""应用生命周期管理"""
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from core.db import init_db, get_db
|
from app.core.db import init_db, get_db
|
||||||
from core.config import settings as app_settings
|
from app.core.config import settings as app_settings
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
from api.deps import create_scheduler_service
|
from app.api.deps import create_scheduler_service
|
||||||
from repositories.settings_repo import SettingsRepository
|
from app.repositories.settings_repo import SettingsRepository
|
||||||
|
|
||||||
settings_repo = SettingsRepository()
|
settings_repo = SettingsRepository()
|
||||||
|
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
"""FastAPI 应用工厂"""
|
"""FastAPI 应用工厂"""
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from api.lifespan import lifespan
|
from app.api.lifespan import lifespan
|
||||||
from api.routes import api_router
|
from app.api.routes import api_router
|
||||||
from api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
|
from app.api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
|
||||||
from core.exceptions import ProxyPoolException
|
from app.core.exceptions import ProxyPoolException
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from core.config import settings as app_settings
|
from app.core.config import settings as app_settings
|
||||||
|
|
||||||
# 导入并注册所有插件(显式注册模式)
|
# 导入并注册所有插件(显式注册模式)
|
||||||
import plugins
|
import app.plugins
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
|
"""路由包"""
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from . import proxies, plugins, scheduler, settings
|
from app.api.routes import proxies, plugins, scheduler, settings
|
||||||
|
|
||||||
api_router = APIRouter()
|
api_router = APIRouter()
|
||||||
api_router.include_router(proxies.router)
|
api_router.include_router(proxies.router)
|
||||||
@@ -1,42 +1,23 @@
|
|||||||
"""插件相关路由"""
|
"""插件相关路由"""
|
||||||
|
import asyncio
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from services.plugin_service import PluginService
|
from app.services.plugin_service import PluginService
|
||||||
from services.scheduler_service import SchedulerService
|
from app.services.scheduler_service import SchedulerService
|
||||||
from api.deps import get_plugin_service, get_scheduler_service
|
from app.api.deps import get_plugin_service, get_scheduler_service
|
||||||
from core.log import logger
|
from app.api.common import success_response, error_response, format_plugin
|
||||||
|
from app.core.log import logger
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
|
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
|
||||||
|
|
||||||
|
|
||||||
def success_response(message: str, data=None):
|
|
||||||
return {"code": 200, "message": message, "data": data}
|
|
||||||
|
|
||||||
|
|
||||||
def error_response(message: str, code: int = 500):
|
|
||||||
return {"code": code, "message": message, "data": None}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
async def list_plugins(service: PluginService = Depends(get_plugin_service)):
|
async def list_plugins(service: PluginService = Depends(get_plugin_service)):
|
||||||
|
try:
|
||||||
plugins = await service.list_plugins()
|
plugins = await service.list_plugins()
|
||||||
return success_response(
|
return success_response("获取插件列表成功", {"plugins": [format_plugin(p) for p in plugins]})
|
||||||
"获取插件列表成功",
|
except Exception as e:
|
||||||
{
|
logger.error(f"List plugins failed: {e}")
|
||||||
"plugins": [
|
return error_response("获取插件列表失败", 500)
|
||||||
{
|
|
||||||
"id": p.id,
|
|
||||||
"name": p.display_name,
|
|
||||||
"display_name": p.display_name,
|
|
||||||
"description": p.description,
|
|
||||||
"enabled": p.enabled,
|
|
||||||
"last_run": p.last_run.isoformat() if p.last_run else None,
|
|
||||||
"success_count": p.success_count,
|
|
||||||
"failure_count": p.failure_count,
|
|
||||||
}
|
|
||||||
for p in plugins
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/{plugin_id}/toggle")
|
@router.put("/{plugin_id}/toggle")
|
||||||
@@ -48,6 +29,8 @@ async def toggle_plugin(
|
|||||||
enabled = request.get("enabled")
|
enabled = request.get("enabled")
|
||||||
if enabled is None:
|
if enabled is None:
|
||||||
return error_response("缺少 enabled 参数", 400)
|
return error_response("缺少 enabled 参数", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
success = await service.toggle_plugin(plugin_id, enabled)
|
success = await service.toggle_plugin(plugin_id, enabled)
|
||||||
if not success:
|
if not success:
|
||||||
return error_response("插件不存在", 404)
|
return error_response("插件不存在", 404)
|
||||||
@@ -55,6 +38,9 @@ async def toggle_plugin(
|
|||||||
f"插件 {plugin_id} 已{'启用' if enabled else '禁用'}",
|
f"插件 {plugin_id} 已{'启用' if enabled else '禁用'}",
|
||||||
{"plugin_id": plugin_id, "enabled": enabled},
|
{"plugin_id": plugin_id, "enabled": enabled},
|
||||||
)
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Toggle plugin failed: {e}")
|
||||||
|
return error_response("切换插件状态失败", 500)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{plugin_id}/config")
|
@router.get("/{plugin_id}/config")
|
||||||
@@ -62,10 +48,14 @@ async def get_plugin_config(
|
|||||||
plugin_id: str,
|
plugin_id: str,
|
||||||
service: PluginService = Depends(get_plugin_service),
|
service: PluginService = Depends(get_plugin_service),
|
||||||
):
|
):
|
||||||
|
try:
|
||||||
config = await service.get_plugin_config(plugin_id)
|
config = await service.get_plugin_config(plugin_id)
|
||||||
if config is None:
|
if config is None:
|
||||||
return error_response("插件不存在", 404)
|
return error_response("插件不存在", 404)
|
||||||
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
|
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Get plugin config failed: {e}")
|
||||||
|
return error_response("获取插件配置失败", 500)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{plugin_id}/config")
|
@router.post("/{plugin_id}/config")
|
||||||
@@ -77,10 +67,15 @@ async def update_plugin_config(
|
|||||||
config = request.get("config", {})
|
config = request.get("config", {})
|
||||||
if not isinstance(config, dict):
|
if not isinstance(config, dict):
|
||||||
return error_response("config 必须是对象", 400)
|
return error_response("config 必须是对象", 400)
|
||||||
|
|
||||||
|
try:
|
||||||
success = await service.update_plugin_config(plugin_id, config)
|
success = await service.update_plugin_config(plugin_id, config)
|
||||||
if not success:
|
if not success:
|
||||||
return error_response("插件不存在或配置无效", 404)
|
return error_response("插件不存在或配置无效", 404)
|
||||||
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
|
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Update plugin config failed: {e}")
|
||||||
|
return error_response("保存插件配置失败", 500)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{plugin_id}/crawl")
|
@router.post("/{plugin_id}/crawl")
|
||||||
@@ -101,30 +96,27 @@ async def crawl_plugin(
|
|||||||
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
|
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies, sending to validation queue")
|
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies")
|
||||||
scheduler_service.validation_queue.reset_stats()
|
scheduler_service.validation_queue.reset_stats()
|
||||||
await scheduler_service.validation_queue.submit(results)
|
await scheduler_service.validation_queue.submit(results)
|
||||||
# 等待队列排空(最多等 30 秒,避免前端超时)
|
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
|
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
valid_count = scheduler_service.validation_queue.valid_count
|
|
||||||
invalid_count = scheduler_service.validation_queue.invalid_count
|
|
||||||
|
|
||||||
return success_response(
|
return success_response(
|
||||||
f"插件 {plugin_id} 爬取并验证完成",
|
f"插件 {plugin_id} 爬取并验证完成",
|
||||||
{
|
{
|
||||||
"plugin_id": plugin_id,
|
"plugin_id": plugin_id,
|
||||||
"proxy_count": len(results),
|
"proxy_count": len(results),
|
||||||
"valid_count": valid_count,
|
"valid_count": scheduler_service.validation_queue.valid_count,
|
||||||
"invalid_count": invalid_count,
|
"invalid_count": scheduler_service.validation_queue.invalid_count,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Crawl plugin {plugin_id} failed: {e}")
|
logger.error(f"Crawl plugin {plugin_id} failed: {e}")
|
||||||
return error_response(f"插件爬取失败: {str(e)}")
|
return error_response(f"插件爬取失败: {str(e)}", 500)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/crawl-all")
|
@router.post("/crawl-all")
|
||||||
@@ -140,28 +132,23 @@ async def crawl_all(
|
|||||||
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
|
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f"All plugins crawled {len(results)} unique proxies, sending to validation queue")
|
logger.info(f"All plugins crawled {len(results)} unique proxies")
|
||||||
scheduler_service.validation_queue.reset_stats()
|
scheduler_service.validation_queue.reset_stats()
|
||||||
await scheduler_service.validation_queue.submit(results)
|
await scheduler_service.validation_queue.submit(results)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
|
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
valid_count = scheduler_service.validation_queue.valid_count
|
|
||||||
invalid_count = scheduler_service.validation_queue.invalid_count
|
|
||||||
|
|
||||||
return success_response(
|
return success_response(
|
||||||
"所有插件爬取并验证完成",
|
"所有插件爬取并验证完成",
|
||||||
{
|
{
|
||||||
"total_crawled": len(results),
|
"total_crawled": len(results),
|
||||||
"valid_count": valid_count,
|
"valid_count": scheduler_service.validation_queue.valid_count,
|
||||||
"invalid_count": invalid_count,
|
"invalid_count": scheduler_service.validation_queue.invalid_count,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Crawl all failed: {e}")
|
logger.error(f"Crawl all failed: {e}")
|
||||||
return error_response(f"批量爬取失败: {str(e)}")
|
return error_response(f"批量爬取失败: {str(e)}", 500)
|
||||||
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
125
app/api/routes/proxies.py
Normal file
125
app/api/routes/proxies.py
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
"""代理相关路由(含统计信息)"""
|
||||||
|
from typing import Optional
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from app.services.proxy_service import ProxyService
|
||||||
|
from app.services.scheduler_service import SchedulerService
|
||||||
|
from app.models.schemas import ProxyListRequest, BatchDeleteRequest
|
||||||
|
from app.api.deps import get_proxy_service, get_scheduler_service
|
||||||
|
from app.api.common import success_response, error_response, format_proxy
|
||||||
|
from app.core.log import logger
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/stats")
|
||||||
|
async def get_stats(
|
||||||
|
proxy_service: ProxyService = Depends(get_proxy_service),
|
||||||
|
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
stats = await proxy_service.get_stats()
|
||||||
|
stats["scheduler_running"] = scheduler_service.running
|
||||||
|
return success_response("获取统计信息成功", stats)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Get stats failed: {e}")
|
||||||
|
return error_response("获取统计信息失败", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("")
|
||||||
|
async def list_proxies(
|
||||||
|
request: ProxyListRequest,
|
||||||
|
service: ProxyService = Depends(get_proxy_service),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
proxies, total = await service.list_proxies(
|
||||||
|
page=request.page,
|
||||||
|
page_size=request.page_size,
|
||||||
|
protocol=request.protocol,
|
||||||
|
min_score=request.min_score,
|
||||||
|
max_score=request.max_score,
|
||||||
|
sort_by=request.sort_by,
|
||||||
|
sort_order=request.sort_order,
|
||||||
|
)
|
||||||
|
return success_response(
|
||||||
|
"获取代理列表成功",
|
||||||
|
{
|
||||||
|
"list": [format_proxy(p) for p in proxies],
|
||||||
|
"total": total,
|
||||||
|
"page": request.page,
|
||||||
|
"page_size": request.page_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"List proxies failed: {e}")
|
||||||
|
return error_response("获取代理列表失败", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/random")
|
||||||
|
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
|
||||||
|
try:
|
||||||
|
proxy = await service.get_random_proxy()
|
||||||
|
if not proxy:
|
||||||
|
return error_response("没有找到可用的代理", 404)
|
||||||
|
return success_response("获取随机代理成功", format_proxy(proxy))
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Get random proxy failed: {e}")
|
||||||
|
return error_response("获取随机代理失败", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/export/{fmt}")
|
||||||
|
async def export_proxies(
|
||||||
|
fmt: str,
|
||||||
|
protocol: Optional[str] = None,
|
||||||
|
limit: int = Query(default=10000, ge=1, le=100000),
|
||||||
|
service: ProxyService = Depends(get_proxy_service),
|
||||||
|
):
|
||||||
|
if fmt not in ("csv", "txt", "json"):
|
||||||
|
return error_response("不支持的导出格式", 400)
|
||||||
|
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
|
||||||
|
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
|
||||||
|
|
||||||
|
async def generate():
|
||||||
|
async for chunk in service.export_proxies(fmt, protocol, limit):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
generate(),
|
||||||
|
media_type=media_types[fmt],
|
||||||
|
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{ip}/{port}")
|
||||||
|
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
|
||||||
|
try:
|
||||||
|
await service.delete_proxy(ip, port)
|
||||||
|
return success_response("删除代理成功")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Delete proxy failed: {e}")
|
||||||
|
return error_response("删除代理失败", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/batch-delete")
|
||||||
|
async def batch_delete(
|
||||||
|
request: BatchDeleteRequest,
|
||||||
|
service: ProxyService = Depends(get_proxy_service),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
proxies = [(item.ip, item.port) for item in request.proxies]
|
||||||
|
deleted = await service.batch_delete(proxies)
|
||||||
|
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Batch delete failed: {e}")
|
||||||
|
return error_response("批量删除失败", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/clean-invalid")
|
||||||
|
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
|
||||||
|
try:
|
||||||
|
count = await service.clean_invalid()
|
||||||
|
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Clean invalid failed: {e}")
|
||||||
|
return error_response("清理无效代理失败", 500)
|
||||||
64
app/api/routes/scheduler.py
Normal file
64
app/api/routes/scheduler.py
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
"""调度器相关路由"""
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from app.services.scheduler_service import SchedulerService
|
||||||
|
from app.repositories.settings_repo import SettingsRepository
|
||||||
|
from app.core.db import get_db
|
||||||
|
from app.api.deps import get_scheduler_service
|
||||||
|
from app.api.common import success_response, error_response
|
||||||
|
from app.core.log import logger
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||||
|
settings_repo = SettingsRepository()
|
||||||
|
|
||||||
|
|
||||||
|
async def _save_auto_validate_setting(enabled: bool):
|
||||||
|
"""保存自动验证设置"""
|
||||||
|
async with get_db() as db:
|
||||||
|
settings = await settings_repo.get_all(db)
|
||||||
|
settings["auto_validate"] = enabled
|
||||||
|
from app.models.schemas import SettingsSchema
|
||||||
|
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/start")
|
||||||
|
async def start_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||||
|
try:
|
||||||
|
if scheduler.running:
|
||||||
|
return success_response("验证调度器已在运行", {"running": True})
|
||||||
|
await scheduler.start()
|
||||||
|
await _save_auto_validate_setting(True)
|
||||||
|
return success_response("验证调度器已启动", {"running": True})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Start scheduler failed: {e}")
|
||||||
|
return error_response(f"启动调度器失败: {str(e)}", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/stop")
|
||||||
|
async def stop_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||||
|
try:
|
||||||
|
if not scheduler.running:
|
||||||
|
return success_response("验证调度器未运行", {"running": False})
|
||||||
|
await scheduler.stop()
|
||||||
|
await _save_auto_validate_setting(False)
|
||||||
|
return success_response("验证调度器已停止", {"running": False})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Stop scheduler failed: {e}")
|
||||||
|
return error_response(f"停止调度器失败: {str(e)}", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/validate-now")
|
||||||
|
async def validate_now(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||||
|
try:
|
||||||
|
scheduler.validate_all_now()
|
||||||
|
return success_response("已开始全量验证", {"started": True})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Validate now failed: {e}")
|
||||||
|
return error_response(f"启动验证失败: {str(e)}", 500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status")
|
||||||
|
async def scheduler_status(scheduler: SchedulerService = Depends(get_scheduler_service)):
|
||||||
|
return success_response(
|
||||||
|
"获取状态成功",
|
||||||
|
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
|
||||||
|
)
|
||||||
@@ -1,22 +1,15 @@
|
|||||||
"""设置相关路由"""
|
"""设置相关路由"""
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from core.db import get_db
|
from app.core.db import get_db
|
||||||
from repositories.settings_repo import SettingsRepository
|
from app.repositories.settings_repo import SettingsRepository
|
||||||
from models.schemas import SettingsSchema
|
from app.models.schemas import SettingsSchema
|
||||||
from core.log import logger
|
from app.api.common import success_response, error_response
|
||||||
|
from app.core.log import logger
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||||
settings_repo = SettingsRepository()
|
settings_repo = SettingsRepository()
|
||||||
|
|
||||||
|
|
||||||
def success_response(message: str, data=None):
|
|
||||||
return {"code": 200, "message": message, "data": data}
|
|
||||||
|
|
||||||
|
|
||||||
def error_response(message: str, code: int = 500):
|
|
||||||
return {"code": code, "message": message, "data": None}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
async def get_settings():
|
async def get_settings():
|
||||||
try:
|
try:
|
||||||
@@ -25,7 +18,7 @@ async def get_settings():
|
|||||||
return success_response("获取设置成功", settings)
|
return success_response("获取设置成功", settings)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Get settings failed: {e}")
|
logger.error(f"Get settings failed: {e}")
|
||||||
return error_response("获取设置失败")
|
return error_response("获取设置失败", 500)
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
@@ -34,8 +27,8 @@ async def save_settings(request: SettingsSchema):
|
|||||||
async with get_db() as db:
|
async with get_db() as db:
|
||||||
success = await settings_repo.save(db, request.model_dump())
|
success = await settings_repo.save(db, request.model_dump())
|
||||||
if not success:
|
if not success:
|
||||||
return error_response("保存设置失败")
|
return error_response("保存设置失败", 500)
|
||||||
return success_response("保存设置成功", request.model_dump())
|
return success_response("保存设置成功", request.model_dump())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Save settings failed: {e}")
|
logger.error(f"Save settings failed: {e}")
|
||||||
return error_response(f"保存设置失败: {str(e)}")
|
return error_response(f"保存设置失败: {str(e)}", 500)
|
||||||
13
app/core/__init__.py
Normal file
13
app/core/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
"""核心基础设施包"""
|
||||||
|
from .config import settings
|
||||||
|
from .log import logger
|
||||||
|
from .exceptions import ProxyPoolException, PluginNotFoundException, ProxyNotFoundException, ValidationException
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"settings",
|
||||||
|
"logger",
|
||||||
|
"ProxyPoolException",
|
||||||
|
"PluginNotFoundException",
|
||||||
|
"ProxyNotFoundException",
|
||||||
|
"ValidationException",
|
||||||
|
]
|
||||||
@@ -52,7 +52,7 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def base_dir(self) -> str:
|
def base_dir(self) -> str:
|
||||||
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
# 全局配置实例(启动时加载一次)
|
# 全局配置实例(启动时加载一次)
|
||||||
@@ -3,8 +3,8 @@ import os
|
|||||||
import aiosqlite
|
import aiosqlite
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from typing import AsyncIterator
|
from typing import AsyncIterator
|
||||||
from core.config import settings
|
from app.core.config import settings
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
DB_PATH = os.path.join(settings.base_dir, settings.db_path)
|
DB_PATH = os.path.join(settings.base_dir, settings.db_path)
|
||||||
@@ -3,12 +3,13 @@ import os
|
|||||||
from logging.handlers import RotatingFileHandler
|
from logging.handlers import RotatingFileHandler
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
class LogHandler(logging.Logger):
|
class LogHandler(logging.Logger):
|
||||||
def __init__(self, name='ProxyPool', level=logging.INFO):
|
def __init__(self, name='ProxyPool', level=logging.INFO):
|
||||||
super().__init__(name, level)
|
super().__init__(name, level)
|
||||||
|
|
||||||
# 获取项目根目录并创建 logs 目录
|
# 获取项目根目录并创建 logs 目录
|
||||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
log_dir = os.path.join(base_dir, 'logs')
|
log_dir = os.path.join(base_dir, 'logs')
|
||||||
if not os.path.exists(log_dir):
|
if not os.path.exists(log_dir):
|
||||||
os.makedirs(log_dir)
|
os.makedirs(log_dir)
|
||||||
@@ -38,6 +39,7 @@ class LogHandler(logging.Logger):
|
|||||||
console_handler.setFormatter(formatter)
|
console_handler.setFormatter(formatter)
|
||||||
self.addHandler(console_handler)
|
self.addHandler(console_handler)
|
||||||
|
|
||||||
|
|
||||||
# 实例化一个默认 logger 供外部直接使用
|
# 实例化一个默认 logger 供外部直接使用
|
||||||
logger = LogHandler()
|
logger = LogHandler()
|
||||||
|
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
"""插件系统包"""
|
||||||
from .base import BaseCrawlerPlugin, ProxyRaw
|
from .base import BaseCrawlerPlugin, ProxyRaw
|
||||||
from .registry import registry
|
from .registry import registry
|
||||||
|
|
||||||
@@ -3,8 +3,8 @@ import importlib
|
|||||||
import inspect
|
import inspect
|
||||||
import os
|
import os
|
||||||
from typing import Dict, List, Type, Optional
|
from typing import Dict, List, Type, Optional
|
||||||
from core.plugin_system.base import BaseCrawlerPlugin
|
from app.core.plugin_system.base import BaseCrawlerPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class PluginRegistry:
|
class PluginRegistry:
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
"""任务队列包"""
|
||||||
from .queue import ValidationQueue
|
from .queue import ValidationQueue
|
||||||
|
|
||||||
__all__ = ["ValidationQueue"]
|
__all__ = ["ValidationQueue"]
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
"""验证任务队列 - 解耦爬取与验证,支持背压控制和持久化"""
|
"""验证任务队列 - 解耦爬取与验证,支持背压控制和持久化"""
|
||||||
import asyncio
|
import asyncio
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from models.domain import ProxyRaw
|
from app.models.domain import ProxyRaw
|
||||||
from repositories.task_repo import ValidationTaskRepository
|
from app.repositories.task_repo import ValidationTaskRepository
|
||||||
from core.db import get_db
|
from app.core.db import get_db
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ValidationQueue:
|
class ValidationQueue:
|
||||||
30
app/models/__init__.py
Normal file
30
app/models/__init__.py
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
"""数据模型包"""
|
||||||
|
from .domain import ProxyRaw, Proxy, PluginInfo
|
||||||
|
from .schemas import (
|
||||||
|
ProxyCreate,
|
||||||
|
ProxyResponse,
|
||||||
|
PluginResponse,
|
||||||
|
SettingsSchema,
|
||||||
|
CrawlResult,
|
||||||
|
ProxyListRequest,
|
||||||
|
ProxyDeleteItem,
|
||||||
|
BatchDeleteRequest,
|
||||||
|
PluginToggleRequest,
|
||||||
|
ExportRequest,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ProxyRaw",
|
||||||
|
"Proxy",
|
||||||
|
"PluginInfo",
|
||||||
|
"ProxyCreate",
|
||||||
|
"ProxyResponse",
|
||||||
|
"PluginResponse",
|
||||||
|
"SettingsSchema",
|
||||||
|
"CrawlResult",
|
||||||
|
"ProxyListRequest",
|
||||||
|
"ProxyDeleteItem",
|
||||||
|
"BatchDeleteRequest",
|
||||||
|
"PluginToggleRequest",
|
||||||
|
"ExportRequest",
|
||||||
|
]
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
"""插件包 - 在这里显式注册所有爬虫插件"""
|
"""插件包 - 在这里显式注册所有爬虫插件"""
|
||||||
from core.plugin_system import registry
|
from app.core.plugin_system import registry
|
||||||
|
|
||||||
from .fate0 import Fate0Plugin
|
from .fate0 import Fate0Plugin
|
||||||
from .proxylist_download import ProxyListDownloadPlugin
|
from .proxylist_download import ProxyListDownloadPlugin
|
||||||
@@ -3,7 +3,7 @@ import random
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from typing import List
|
from typing import List
|
||||||
from core.plugin_system import BaseCrawlerPlugin
|
from app.core.plugin_system import BaseCrawlerPlugin
|
||||||
|
|
||||||
|
|
||||||
class BaseHTTPPlugin(BaseCrawlerPlugin):
|
class BaseHTTPPlugin(BaseCrawlerPlugin):
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
from typing import List
|
from typing import List
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class Fate0Plugin(BaseHTTPPlugin):
|
class Fate0Plugin(BaseHTTPPlugin):
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import re
|
import re
|
||||||
from typing import List
|
from typing import List
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||||
|
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import re
|
import re
|
||||||
from typing import List
|
from typing import List
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class Ip89Plugin(BaseHTTPPlugin):
|
class Ip89Plugin(BaseHTTPPlugin):
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import re
|
import re
|
||||||
from typing import List
|
from typing import List
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||||
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from typing import List
|
from typing import List
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ProxyListDownloadPlugin(BaseHTTPPlugin):
|
class ProxyListDownloadPlugin(BaseHTTPPlugin):
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
"""ProxyScrape 测试爬虫 - 用于验证架构,支持全协议类型"""
|
"""ProxyScrape 测试爬虫 - 用于验证架构,支持全协议类型"""
|
||||||
from typing import List
|
from typing import List
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ProxyScrapePlugin(BaseHTTPPlugin):
|
class ProxyScrapePlugin(BaseHTTPPlugin):
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
import re
|
import re
|
||||||
from typing import List
|
from typing import List
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class SpeedXPlugin(BaseHTTPPlugin):
|
class SpeedXPlugin(BaseHTTPPlugin):
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import re
|
import re
|
||||||
from typing import List
|
from typing import List
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from core.plugin_system import ProxyRaw
|
from app.core.plugin_system import ProxyRaw
|
||||||
from plugins.base import BaseHTTPPlugin
|
from app.plugins.base import BaseHTTPPlugin
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||||
|
|
||||||
@@ -1,5 +1,11 @@
|
|||||||
|
"""数据访问层包"""
|
||||||
from .proxy_repo import ProxyRepository
|
from .proxy_repo import ProxyRepository
|
||||||
from .settings_repo import SettingsRepository, PluginSettingsRepository
|
from .settings_repo import SettingsRepository, PluginSettingsRepository
|
||||||
from .task_repo import ValidationTaskRepository
|
from .task_repo import ValidationTaskRepository
|
||||||
|
|
||||||
__all__ = ["ProxyRepository", "SettingsRepository", "PluginSettingsRepository", "ValidationTaskRepository"]
|
__all__ = [
|
||||||
|
"ProxyRepository",
|
||||||
|
"SettingsRepository",
|
||||||
|
"PluginSettingsRepository",
|
||||||
|
"ValidationTaskRepository",
|
||||||
|
]
|
||||||
@@ -2,8 +2,8 @@
|
|||||||
import aiosqlite
|
import aiosqlite
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import List, Optional, Tuple, Union
|
from typing import List, Optional, Tuple, Union
|
||||||
from models.domain import Proxy
|
from app.models.domain import Proxy
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
import json
|
import json
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
from typing import Optional, Dict, Any
|
from typing import Optional, Dict, Any
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_SETTINGS = {
|
DEFAULT_SETTINGS = {
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
"""验证任务队列持久化层"""
|
"""验证任务队列持久化层"""
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from models.domain import ProxyRaw
|
from app.models.domain import ProxyRaw
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ValidationTaskRepository:
|
class ValidationTaskRepository:
|
||||||
12
app/services/__init__.py
Normal file
12
app/services/__init__.py
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
"""业务逻辑层包"""
|
||||||
|
from .proxy_service import ProxyService
|
||||||
|
from .plugin_service import PluginService
|
||||||
|
from .scheduler_service import SchedulerService
|
||||||
|
from .validator_service import ValidatorService
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ProxyService",
|
||||||
|
"PluginService",
|
||||||
|
"SchedulerService",
|
||||||
|
"ValidatorService",
|
||||||
|
]
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
"""插件业务服务"""
|
"""插件业务服务"""
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from core.db import get_db
|
from app.core.db import get_db
|
||||||
from core.plugin_system.registry import registry
|
from app.core.plugin_system.registry import registry
|
||||||
from core.plugin_system.base import BaseCrawlerPlugin
|
from app.core.plugin_system.base import BaseCrawlerPlugin
|
||||||
from repositories.settings_repo import PluginSettingsRepository
|
from app.repositories.settings_repo import PluginSettingsRepository
|
||||||
from models.domain import PluginInfo, ProxyRaw
|
from app.models.domain import PluginInfo, ProxyRaw
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class PluginService:
|
class PluginService:
|
||||||
@@ -4,10 +4,10 @@ import json
|
|||||||
import io
|
import io
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Optional, Tuple, AsyncIterator
|
from typing import List, Optional, Tuple, AsyncIterator
|
||||||
from core.db import get_db
|
from app.core.db import get_db
|
||||||
from repositories.proxy_repo import ProxyRepository
|
from app.repositories.proxy_repo import ProxyRepository
|
||||||
from models.domain import Proxy
|
from app.models.domain import Proxy
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ProxyService:
|
class ProxyService:
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
"""调度器服务 - 定时验证存量代理"""
|
"""调度器服务 - 定时验证存量代理"""
|
||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from core.db import get_db
|
from app.core.db import get_db
|
||||||
from repositories.proxy_repo import ProxyRepository
|
from app.repositories.proxy_repo import ProxyRepository
|
||||||
from core.tasks.queue import ValidationQueue
|
from app.core.tasks.queue import ValidationQueue
|
||||||
from core.config import settings as app_settings
|
from app.core.config import settings as app_settings
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class SchedulerService:
|
class SchedulerService:
|
||||||
@@ -70,7 +70,7 @@ class SchedulerService:
|
|||||||
return
|
return
|
||||||
|
|
||||||
logger.info(f"Validating {len(proxies)} proxies from database")
|
logger.info(f"Validating {len(proxies)} proxies from database")
|
||||||
from models.domain import ProxyRaw
|
from app.models.domain import ProxyRaw
|
||||||
|
|
||||||
# 批量提交到验证队列
|
# 批量提交到验证队列
|
||||||
batch_size = 100
|
batch_size = 100
|
||||||
@@ -5,12 +5,18 @@ import time
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import aiohttp_socks
|
import aiohttp_socks
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
from core.log import logger
|
from app.core.log import logger
|
||||||
|
|
||||||
|
|
||||||
class ValidatorService:
|
class ValidatorService:
|
||||||
"""代理验证器"""
|
"""代理验证器"""
|
||||||
|
|
||||||
|
# 测试 URL
|
||||||
|
TEST_URLS = {
|
||||||
|
"http": ["http://httpbin.org/ip", "http://api.ipify.org"],
|
||||||
|
"https": ["https://httpbin.org/ip", "https://api.ipify.org"],
|
||||||
|
}
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
timeout: float = 5.0,
|
timeout: float = 5.0,
|
||||||
@@ -20,33 +26,23 @@ class ValidatorService:
|
|||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.connect_timeout = connect_timeout
|
self.connect_timeout = connect_timeout
|
||||||
self.semaphore = asyncio.Semaphore(max_concurrency)
|
self.semaphore = asyncio.Semaphore(max_concurrency)
|
||||||
self.http_sources = [
|
|
||||||
"http://httpbin.org/ip",
|
|
||||||
"http://api.ipify.org",
|
|
||||||
]
|
|
||||||
self.https_sources = [
|
|
||||||
"https://httpbin.org/ip",
|
|
||||||
"https://api.ipify.org",
|
|
||||||
]
|
|
||||||
|
|
||||||
def _get_test_url(self, protocol: str) -> str:
|
def _get_test_url(self, protocol: str) -> str:
|
||||||
protocol = protocol.lower()
|
"""获取测试 URL"""
|
||||||
if protocol == "https":
|
urls = self.TEST_URLS.get(protocol.lower(), self.TEST_URLS["http"])
|
||||||
return random.choice(self.https_sources)
|
return random.choice(urls)
|
||||||
return random.choice(self.http_sources)
|
|
||||||
|
|
||||||
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
|
async def validate(self, ip: str, port: int, protocol: str = "http") -> Tuple[bool, float]:
|
||||||
"""验证单个代理,返回 (是否有效, 延迟毫秒)"""
|
"""验证单个代理,返回 (是否有效, 延迟毫秒)"""
|
||||||
protocol = protocol.lower()
|
protocol = protocol.lower()
|
||||||
test_url = self._get_test_url(protocol)
|
|
||||||
|
|
||||||
async with self.semaphore:
|
async with self.semaphore:
|
||||||
start = time.time()
|
start = time.time()
|
||||||
try:
|
try:
|
||||||
if protocol in ("socks4", "socks5"):
|
if protocol in ("socks4", "socks5"):
|
||||||
return await self._validate_socks(ip, port, protocol, test_url, start)
|
return await self._validate_socks(ip, port, protocol, start)
|
||||||
else:
|
else:
|
||||||
return await self._validate_http(ip, port, protocol, test_url, start)
|
return await self._validate_http(ip, port, protocol, start)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
logger.debug(f"Validation timeout: {ip}:{port} ({protocol})")
|
logger.debug(f"Validation timeout: {ip}:{port} ({protocol})")
|
||||||
return False, 0.0
|
return False, 0.0
|
||||||
@@ -54,18 +50,16 @@ class ValidatorService:
|
|||||||
logger.debug(f"Validation error {ip}:{port} ({protocol}): {e}")
|
logger.debug(f"Validation error {ip}:{port} ({protocol}): {e}")
|
||||||
return False, 0.0
|
return False, 0.0
|
||||||
|
|
||||||
async def _validate_http(
|
async def _validate_http(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||||
self, ip: str, port: int, protocol: str, test_url: str, start: float
|
"""验证 HTTP/HTTPS 代理"""
|
||||||
) -> Tuple[bool, float]:
|
|
||||||
proxy_url = f"http://{ip}:{port}"
|
proxy_url = f"http://{ip}:{port}"
|
||||||
connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
connector = aiohttp.TCPConnector(ssl=False, limit=0, force_close=True)
|
||||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
||||||
|
test_url = self._get_test_url(protocol)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||||
async with session.get(
|
async with session.get(test_url, proxy=proxy_url, allow_redirects=True) as response:
|
||||||
test_url, proxy=proxy_url, allow_redirects=True
|
|
||||||
) as response:
|
|
||||||
if response.status in (200, 301, 302):
|
if response.status in (200, 301, 302):
|
||||||
latency = round((time.time() - start) * 1000, 2)
|
latency = round((time.time() - start) * 1000, 2)
|
||||||
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
|
logger.info(f"HTTP valid: {ip}:{port} ({protocol}) {latency}ms")
|
||||||
@@ -74,9 +68,8 @@ class ValidatorService:
|
|||||||
finally:
|
finally:
|
||||||
await connector.close()
|
await connector.close()
|
||||||
|
|
||||||
async def _validate_socks(
|
async def _validate_socks(self, ip: str, port: int, protocol: str, start: float) -> Tuple[bool, float]:
|
||||||
self, ip: str, port: int, protocol: str, test_url: str, start: float
|
"""验证 SOCKS4/SOCKS5 代理"""
|
||||||
) -> Tuple[bool, float]:
|
|
||||||
proxy_type = (
|
proxy_type = (
|
||||||
aiohttp_socks.ProxyType.SOCKS4
|
aiohttp_socks.ProxyType.SOCKS4
|
||||||
if protocol == "socks4"
|
if protocol == "socks4"
|
||||||
@@ -90,6 +83,7 @@ class ValidatorService:
|
|||||||
ssl=False,
|
ssl=False,
|
||||||
)
|
)
|
||||||
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
timeout = aiohttp.ClientTimeout(total=self.timeout, connect=self.connect_timeout)
|
||||||
|
test_url = self._get_test_url("http")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||||
4
main.py
4
main.py
@@ -1,7 +1,7 @@
|
|||||||
"""项目入口"""
|
"""项目入口"""
|
||||||
import uvicorn
|
import uvicorn
|
||||||
from api import create_app
|
from app.api import create_app
|
||||||
from core.config import settings
|
from app.core.config import settings
|
||||||
|
|
||||||
app = create_app()
|
app = create_app()
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
from .domain import ProxyRaw, Proxy, PluginInfo
|
|
||||||
from .schemas import ProxyCreate, ProxyResponse, PluginResponse, SettingsSchema, CrawlResult
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"ProxyRaw",
|
|
||||||
"Proxy",
|
|
||||||
"PluginInfo",
|
|
||||||
"ProxyCreate",
|
|
||||||
"ProxyResponse",
|
|
||||||
"PluginResponse",
|
|
||||||
"SettingsSchema",
|
|
||||||
"CrawlResult",
|
|
||||||
]
|
|
||||||
17
pytest.ini
Normal file
17
pytest.ini
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
[tool:pytest]
|
||||||
|
testpaths = tests
|
||||||
|
python_files = test_*.py
|
||||||
|
python_classes = Test*
|
||||||
|
python_functions = test_*
|
||||||
|
addopts =
|
||||||
|
-v
|
||||||
|
--tb=short
|
||||||
|
--strict-markers
|
||||||
|
-p no:warnings
|
||||||
|
markers =
|
||||||
|
unit: 单元测试
|
||||||
|
integration: 集成测试
|
||||||
|
e2e: 端到端测试
|
||||||
|
slow: 慢速测试
|
||||||
|
async_test: 异步测试
|
||||||
|
asyncio_mode = auto
|
||||||
@@ -74,7 +74,7 @@ echo.
|
|||||||
|
|
||||||
REM 4. Start Frontend
|
REM 4. Start Frontend
|
||||||
echo [4/4] Starting frontend (Vite)...
|
echo [4/4] Starting frontend (Vite)...
|
||||||
cd /d "%ROOT_PATH%\frontend"
|
cd /d "%ROOT_PATH%\WebUI"
|
||||||
start /B "" cmd /c "npm run dev"
|
start /B "" cmd /c "npm run dev"
|
||||||
cd /d "%ROOT_PATH%"
|
cd /d "%ROOT_PATH%"
|
||||||
echo Frontend started
|
echo Frontend started
|
||||||
|
|||||||
159
tests/README.md
Normal file
159
tests/README.md
Normal file
@@ -0,0 +1,159 @@
|
|||||||
|
# 测试说明
|
||||||
|
|
||||||
|
## 测试结构
|
||||||
|
|
||||||
|
```
|
||||||
|
tests/
|
||||||
|
├── conftest.py # pytest 配置和 fixtures
|
||||||
|
├── README.md # 本文件
|
||||||
|
├── unit/ # 单元测试
|
||||||
|
│ ├── test_models.py # 模型测试
|
||||||
|
│ └── test_repositories.py # 仓库层测试
|
||||||
|
├── integration/ # 集成测试
|
||||||
|
│ ├── test_proxies_api.py # 代理 API 测试
|
||||||
|
│ ├── test_plugins_api.py # 插件 API 测试
|
||||||
|
│ ├── test_scheduler_api.py # 调度器 API 测试
|
||||||
|
│ ├── test_settings_api.py # 设置 API 测试
|
||||||
|
│ └── test_health_api.py # 健康检查测试
|
||||||
|
└── e2e/ # 端到端测试
|
||||||
|
└── test_full_workflow.py # 完整工作流测试
|
||||||
|
```
|
||||||
|
|
||||||
|
## 运行测试
|
||||||
|
|
||||||
|
### 安装测试依赖
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install pytest pytest-asyncio httpx
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行所有测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pytest
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行特定类型的测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 仅运行单元测试
|
||||||
|
pytest tests/unit -v
|
||||||
|
|
||||||
|
# 仅运行集成测试
|
||||||
|
pytest tests/integration -v
|
||||||
|
|
||||||
|
# 仅运行 E2E 测试
|
||||||
|
pytest tests/e2e -v
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行特定测试文件
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pytest tests/integration/test_proxies_api.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行特定测试函数
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pytest tests/integration/test_proxies_api.py::TestProxiesAPI::test_get_stats -v
|
||||||
|
```
|
||||||
|
|
||||||
|
## 测试覆盖的 API
|
||||||
|
|
||||||
|
### 代理 API (`/api/proxies/*`)
|
||||||
|
|
||||||
|
- ✅ `GET /api/proxies/stats` - 获取统计信息
|
||||||
|
- ✅ `POST /api/proxies` - 列出代理
|
||||||
|
- ✅ `GET /api/proxies/random` - 获取随机代理
|
||||||
|
- ✅ `GET /api/proxies/export/{format}` - 导出代理 (csv, txt, json)
|
||||||
|
- ✅ `DELETE /api/proxies/{ip}/{port}` - 删除代理
|
||||||
|
- ✅ `POST /api/proxies/batch-delete` - 批量删除
|
||||||
|
- ✅ `DELETE /api/proxies/clean-invalid` - 清理无效代理
|
||||||
|
|
||||||
|
### 插件 API (`/api/plugins/*`)
|
||||||
|
|
||||||
|
- ✅ `GET /api/plugins` - 列出插件
|
||||||
|
- ✅ `PUT /api/plugins/{id}/toggle` - 切换插件状态
|
||||||
|
- ✅ `GET /api/plugins/{id}/config` - 获取插件配置
|
||||||
|
- ✅ `POST /api/plugins/{id}/config` - 更新插件配置
|
||||||
|
- ✅ `POST /api/plugins/{id}/crawl` - 触发单个插件爬取
|
||||||
|
- ✅ `POST /api/plugins/crawl-all` - 触发所有插件爬取
|
||||||
|
|
||||||
|
### 调度器 API (`/api/scheduler/*`)
|
||||||
|
|
||||||
|
- ✅ `GET /api/scheduler/status` - 获取调度器状态
|
||||||
|
- ✅ `POST /api/scheduler/start` - 启动调度器
|
||||||
|
- ✅ `POST /api/scheduler/stop` - 停止调度器
|
||||||
|
- ✅ `POST /api/scheduler/validate-now` - 立即验证
|
||||||
|
|
||||||
|
### 设置 API (`/api/settings`)
|
||||||
|
|
||||||
|
- ✅ `GET /api/settings` - 获取设置
|
||||||
|
- ✅ `POST /api/settings` - 保存设置
|
||||||
|
|
||||||
|
### 健康检查
|
||||||
|
|
||||||
|
- ✅ `GET /` - 根端点
|
||||||
|
- ✅ `GET /health` - 健康检查
|
||||||
|
|
||||||
|
## 测试 Fixtures
|
||||||
|
|
||||||
|
### `client`
|
||||||
|
|
||||||
|
异步 HTTP 客户端,用于发送请求到测试应用。
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def test_example(client):
|
||||||
|
response = await client.get("/api/proxies/stats")
|
||||||
|
assert response.status_code == 200
|
||||||
|
```
|
||||||
|
|
||||||
|
### `db`
|
||||||
|
|
||||||
|
数据库连接 fixture。
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def test_example(db, proxy_repo):
|
||||||
|
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
|
||||||
|
```
|
||||||
|
|
||||||
|
### `sample_proxy`
|
||||||
|
|
||||||
|
创建一个测试代理并自动清理。
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def test_example(client, sample_proxy):
|
||||||
|
# sample_proxy = {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
|
||||||
|
response = await client.delete(f"/api/proxies/{sample_proxy['ip']}/{sample_proxy['port']}")
|
||||||
|
assert response.status_code == 200
|
||||||
|
```
|
||||||
|
|
||||||
|
## 编写新测试
|
||||||
|
|
||||||
|
### 单元测试示例
|
||||||
|
|
||||||
|
```python
|
||||||
|
# tests/unit/test_new_feature.py
|
||||||
|
import pytest
|
||||||
|
from app.models.domain import ProxyRaw
|
||||||
|
|
||||||
|
|
||||||
|
class TestProxyRaw:
|
||||||
|
def test_create(self):
|
||||||
|
proxy = ProxyRaw("192.168.1.1", 8080, "http")
|
||||||
|
assert proxy.ip == "192.168.1.1"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 集成测试示例
|
||||||
|
|
||||||
|
```python
|
||||||
|
# tests/integration/test_new_api.py
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class TestNewAPI:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_new_endpoint(self, client):
|
||||||
|
response = await client.get("/api/new-endpoint")
|
||||||
|
assert response.status_code == 200
|
||||||
|
```
|
||||||
1
tests/__init__.py
Normal file
1
tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""测试包"""
|
||||||
5
tests/__main__.py
Normal file
5
tests/__main__.py
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
"""直接运行 python -m tests 时执行的入口"""
|
||||||
|
from .run_tests import main
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.exit(main())
|
||||||
BIN
tests/__pycache__/conftest.cpython-311.pyc
Normal file
BIN
tests/__pycache__/conftest.cpython-311.pyc
Normal file
Binary file not shown.
56
tests/conftest.py
Normal file
56
tests/conftest.py
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
"""pytest 配置文件和 fixtures"""
|
||||||
|
import pytest
|
||||||
|
import asyncio
|
||||||
|
from typing import AsyncGenerator, Generator
|
||||||
|
from httpx import AsyncClient, ASGITransport
|
||||||
|
|
||||||
|
from app.api import create_app
|
||||||
|
from app.core.db import init_db, get_db
|
||||||
|
from app.repositories.proxy_repo import ProxyRepository
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]:
|
||||||
|
"""创建事件循环"""
|
||||||
|
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||||
|
yield loop
|
||||||
|
loop.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
async def app():
|
||||||
|
"""创建应用实例"""
|
||||||
|
# 初始化测试数据库
|
||||||
|
await init_db()
|
||||||
|
app = create_app()
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def client(app) -> AsyncGenerator[AsyncClient, None]:
|
||||||
|
"""创建异步 HTTP 客户端"""
|
||||||
|
transport = ASGITransport(app=app)
|
||||||
|
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
yield client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def db():
|
||||||
|
"""获取数据库连接"""
|
||||||
|
async with get_db() as db:
|
||||||
|
yield db
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def proxy_repo():
|
||||||
|
"""获取代理仓库"""
|
||||||
|
return ProxyRepository()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def sample_proxy(db, proxy_repo):
|
||||||
|
"""创建一个测试代理"""
|
||||||
|
await proxy_repo.insert_or_update(db, "192.168.1.1", 8080, "http", 50)
|
||||||
|
yield {"ip": "192.168.1.1", "port": 8080, "protocol": "http", "score": 50}
|
||||||
|
# 清理
|
||||||
|
await proxy_repo.delete(db, "192.168.1.1", 8080)
|
||||||
1
tests/e2e/__init__.py
Normal file
1
tests/e2e/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""端到端测试"""
|
||||||
BIN
tests/e2e/__pycache__/test_full_workflow.cpython-311.pyc
Normal file
BIN
tests/e2e/__pycache__/test_full_workflow.cpython-311.pyc
Normal file
Binary file not shown.
182
tests/e2e/test_full_workflow.py
Normal file
182
tests/e2e/test_full_workflow.py
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
"""完整工作流 E2E 测试
|
||||||
|
|
||||||
|
这些测试模拟真实用户场景,验证整个系统的集成功能。
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class TestFullWorkflow:
|
||||||
|
"""测试完整工作流"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_proxy_management_workflow(self, client):
|
||||||
|
"""测试代理管理完整工作流
|
||||||
|
|
||||||
|
场景:
|
||||||
|
1. 查看统计信息
|
||||||
|
2. 列出代理
|
||||||
|
3. 触发爬取
|
||||||
|
4. 查看更新后的统计
|
||||||
|
5. 导出代理
|
||||||
|
6. 清理无效代理
|
||||||
|
"""
|
||||||
|
# 1. 获取初始统计
|
||||||
|
response = await client.get("/api/proxies/stats")
|
||||||
|
assert response.status_code == 200
|
||||||
|
initial_stats = response.json()["data"]
|
||||||
|
|
||||||
|
# 2. 列出代理
|
||||||
|
response = await client.post("/api/proxies", json={
|
||||||
|
"page": 1,
|
||||||
|
"page_size": 20,
|
||||||
|
})
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 3. 触发所有插件爬取
|
||||||
|
response = await client.post("/api/plugins/crawl-all")
|
||||||
|
assert response.status_code == 200
|
||||||
|
crawl_result = response.json()["data"]
|
||||||
|
|
||||||
|
# 4. 获取更新后的统计
|
||||||
|
response = await client.get("/api/proxies/stats")
|
||||||
|
updated_stats = response.json()["data"]
|
||||||
|
|
||||||
|
# 5. 导出代理(所有格式)
|
||||||
|
for fmt in ["csv", "txt", "json"]:
|
||||||
|
response = await client.get(f"/api/proxies/export/{fmt}")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 6. 清理无效代理
|
||||||
|
response = await client.delete("/api/proxies/clean-invalid")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_plugin_management_workflow(self, client):
|
||||||
|
"""测试插件管理完整工作流
|
||||||
|
|
||||||
|
场景:
|
||||||
|
1. 列出所有插件
|
||||||
|
2. 禁用某个插件
|
||||||
|
3. 更新插件配置
|
||||||
|
4. 启用插件
|
||||||
|
5. 触发单个插件爬取
|
||||||
|
"""
|
||||||
|
# 1. 列出插件
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
assert response.status_code == 200
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
|
||||||
|
# 2. 禁用插件
|
||||||
|
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 3. 获取插件配置
|
||||||
|
response = await client.get(f"/api/plugins/{plugin_id}/config")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 4. 更新插件配置
|
||||||
|
response = await client.post(
|
||||||
|
f"/api/plugins/{plugin_id}/config",
|
||||||
|
json={"config": {"max_pages": 3}}
|
||||||
|
)
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 5. 启用插件
|
||||||
|
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 6. 触发爬取
|
||||||
|
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_scheduler_workflow(self, client):
|
||||||
|
"""测试调度器工作流
|
||||||
|
|
||||||
|
场景:
|
||||||
|
1. 启动调度器
|
||||||
|
2. 触发立即验证
|
||||||
|
3. 检查状态
|
||||||
|
4. 停止调度器
|
||||||
|
"""
|
||||||
|
# 1. 启动调度器
|
||||||
|
response = await client.post("/api/scheduler/start")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 2. 触发立即验证
|
||||||
|
response = await client.post("/api/scheduler/validate-now")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 3. 检查状态
|
||||||
|
response = await client.get("/api/scheduler/status")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["data"]["running"] is True
|
||||||
|
|
||||||
|
# 4. 停止调度器
|
||||||
|
response = await client.post("/api/scheduler/stop")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["data"]["running"] is False
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_settings_workflow(self, client):
|
||||||
|
"""测试设置工作流
|
||||||
|
|
||||||
|
场景:
|
||||||
|
1. 获取当前设置
|
||||||
|
2. 修改设置
|
||||||
|
3. 验证设置已保存
|
||||||
|
4. 恢复默认设置
|
||||||
|
"""
|
||||||
|
# 1. 获取当前设置
|
||||||
|
response = await client.get("/api/settings")
|
||||||
|
assert response.status_code == 200
|
||||||
|
original_settings = response.json()["data"]
|
||||||
|
|
||||||
|
# 2. 修改设置
|
||||||
|
new_settings = original_settings.copy()
|
||||||
|
new_settings["crawl_timeout"] = 45
|
||||||
|
new_settings["auto_validate"] = not original_settings["auto_validate"]
|
||||||
|
|
||||||
|
response = await client.post("/api/settings", json=new_settings)
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 3. 验证设置已保存
|
||||||
|
response = await client.get("/api/settings")
|
||||||
|
saved_settings = response.json()["data"]
|
||||||
|
assert saved_settings["crawl_timeout"] == 45
|
||||||
|
|
||||||
|
# 4. 恢复原始设置
|
||||||
|
response = await client.post("/api/settings", json=original_settings)
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_batch_operations_workflow(self, client):
|
||||||
|
"""测试批量操作工作流
|
||||||
|
|
||||||
|
场景:
|
||||||
|
1. 批量删除不存在的代理(幂等性测试)
|
||||||
|
2. 导出所有代理
|
||||||
|
3. 获取随机代理
|
||||||
|
"""
|
||||||
|
# 1. 批量删除(幂等性)
|
||||||
|
response = await client.post("/api/proxies/batch-delete", json={
|
||||||
|
"proxies": [
|
||||||
|
{"ip": "192.168.100.1", "port": 8080},
|
||||||
|
{"ip": "192.168.100.2", "port": 8081},
|
||||||
|
]
|
||||||
|
})
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 2. 导出所有格式
|
||||||
|
for fmt in ["csv", "txt", "json"]:
|
||||||
|
response = await client.get(f"/api/proxies/export/{fmt}")
|
||||||
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
# 3. 获取随机代理(可能返回 200 或 404)
|
||||||
|
response = await client.get("/api/proxies/random")
|
||||||
|
assert response.status_code in [200, 404]
|
||||||
1
tests/integration/__init__.py
Normal file
1
tests/integration/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""集成测试"""
|
||||||
BIN
tests/integration/__pycache__/test_health_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_health_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_plugins_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_plugins_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_proxies_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_proxies_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_scheduler_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_scheduler_api.cpython-311.pyc
Normal file
Binary file not shown.
BIN
tests/integration/__pycache__/test_settings_api.cpython-311.pyc
Normal file
BIN
tests/integration/__pycache__/test_settings_api.cpython-311.pyc
Normal file
Binary file not shown.
47
tests/integration/test_health_api.py
Normal file
47
tests/integration/test_health_api.py
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
"""健康检查 API 测试"""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class TestHealthAPI:
|
||||||
|
"""测试健康检查端点"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_root_endpoint(self, client):
|
||||||
|
"""测试根端点 /"""
|
||||||
|
response = await client.get("/")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert "message" in data
|
||||||
|
assert "status" in data
|
||||||
|
assert data["status"] == "running"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_health_endpoint(self, client):
|
||||||
|
"""测试健康检查端点 /health"""
|
||||||
|
response = await client.get("/health")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["status"] == "healthy"
|
||||||
|
assert "timestamp" in data
|
||||||
|
assert "database" in data
|
||||||
|
assert "scheduler" in data
|
||||||
|
assert "version" in data
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_health_endpoint_structure(self, client):
|
||||||
|
"""测试健康检查端点返回结构"""
|
||||||
|
response = await client.get("/health")
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
assert "status" in data
|
||||||
|
assert "timestamp" in data
|
||||||
|
assert "database" in data
|
||||||
|
assert "scheduler" in data
|
||||||
|
assert "version" in data
|
||||||
|
|
||||||
|
# 验证数据类型
|
||||||
|
assert isinstance(data["status"], str)
|
||||||
|
assert isinstance(data["timestamp"], str)
|
||||||
|
assert isinstance(data["database"], str)
|
||||||
|
assert isinstance(data["scheduler"], str)
|
||||||
|
assert isinstance(data["version"], str)
|
||||||
147
tests/integration/test_plugins_api.py
Normal file
147
tests/integration/test_plugins_api.py
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
"""插件 API 集成测试 - 测试 /api/plugins/* 所有接口"""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
class TestPluginsAPI:
|
||||||
|
"""测试插件相关 API"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_list_plugins(self, client):
|
||||||
|
"""测试 GET /api/plugins"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert "plugins" in data["data"]
|
||||||
|
assert isinstance(data["data"]["plugins"], list)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_list_plugins_structure(self, client):
|
||||||
|
"""测试 GET /api/plugins 返回结构"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
data = response.json()
|
||||||
|
if data["data"]["plugins"]:
|
||||||
|
plugin = data["data"]["plugins"][0]
|
||||||
|
assert "id" in plugin
|
||||||
|
assert "name" in plugin
|
||||||
|
assert "display_name" in plugin
|
||||||
|
assert "description" in plugin
|
||||||
|
assert "enabled" in plugin
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_toggle_plugin_enable(self, client):
|
||||||
|
"""测试 PUT /api/plugins/{id}/toggle - 启用"""
|
||||||
|
# 先获取一个插件 ID
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": True})
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert data["data"]["enabled"] is True
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_toggle_plugin_disable(self, client):
|
||||||
|
"""测试 PUT /api/plugins/{id}/toggle - 禁用"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={"enabled": False})
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert data["data"]["enabled"] is False
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_toggle_plugin_missing_enabled(self, client):
|
||||||
|
"""测试 PUT /api/plugins/{id}/toggle - 缺少 enabled 参数"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
response = await client.put(f"/api/plugins/{plugin_id}/toggle", json={})
|
||||||
|
assert response.status_code == 400
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_toggle_nonexistent_plugin(self, client):
|
||||||
|
"""测试 PUT /api/plugins/{id}/toggle - 不存在的插件"""
|
||||||
|
response = await client.put("/api/plugins/nonexistent_plugin/toggle", json={"enabled": True})
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_plugin_config(self, client):
|
||||||
|
"""测试 GET /api/plugins/{id}/config"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
response = await client.get(f"/api/plugins/{plugin_id}/config")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert "config" in data["data"]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_nonexistent_plugin_config(self, client):
|
||||||
|
"""测试 GET /api/plugins/{id}/config - 不存在的插件"""
|
||||||
|
response = await client.get("/api/plugins/nonexistent_plugin/config")
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_update_plugin_config(self, client):
|
||||||
|
"""测试 POST /api/plugins/{id}/config"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
response = await client.post(
|
||||||
|
f"/api/plugins/{plugin_id}/config",
|
||||||
|
json={"config": {"max_pages": 3}}
|
||||||
|
)
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_crawl_plugin(self, client):
|
||||||
|
"""测试 POST /api/plugins/{id}/crawl"""
|
||||||
|
response = await client.get("/api/plugins")
|
||||||
|
plugins = response.json()["data"]["plugins"]
|
||||||
|
if not plugins:
|
||||||
|
pytest.skip("没有可用的插件")
|
||||||
|
|
||||||
|
plugin_id = plugins[0]["id"]
|
||||||
|
# 这个测试可能需要较长时间,设置较短的超时
|
||||||
|
response = await client.post(f"/api/plugins/{plugin_id}/crawl")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert "proxy_count" in data["data"]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_crawl_nonexistent_plugin(self, client):
|
||||||
|
"""测试 POST /api/plugins/{id}/crawl - 不存在的插件"""
|
||||||
|
response = await client.post("/api/plugins/nonexistent_plugin/crawl")
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_crawl_all_plugins(self, client):
|
||||||
|
"""测试 POST /api/plugins/crawl-all"""
|
||||||
|
response = await client.post("/api/plugins/crawl-all")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["code"] == 200
|
||||||
|
assert "total_crawled" in data["data"]
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user