重构: 迁移后端代码到 app 目录,前端移动到 WebUI,添加完整测试套件

主要变更:
- 后端代码从根目录迁移到 app/ 目录
- 前端代码从 frontend/ 重命名为 WebUI/
- 更新所有导入路径以适配新结构
- 提取公共 API 响应函数到 app/api/common.py
- 精简验证器服务代码
- 更新启动脚本和文档

测试:
- 新增完整测试套件 (tests/)
- 单元测试: 模型、仓库层
- 集成测试: 覆盖所有 22+ API 端点
- E2E 测试: 4个完整工作流场景
- 添加 pytest 配置和测试运行脚本
This commit is contained in:
祀梦
2026-04-04 13:32:36 +08:00
parent df3cc87f88
commit 38bd66128b
109 changed files with 2017 additions and 548 deletions

4
app/api/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
"""API 包"""
from .main import create_app
__all__ = ["create_app"]

41
app/api/common.py Normal file
View File

@@ -0,0 +1,41 @@
"""API 通用工具函数"""
from typing import Any, Optional
from fastapi.responses import JSONResponse
def success_response(message: str, data: Any = None) -> dict:
"""成功响应"""
return {"code": 200, "message": message, "data": data}
def error_response(message: str, code: int = 500) -> JSONResponse:
"""错误响应"""
return JSONResponse(
status_code=code,
content={"code": code, "message": message, "data": None},
)
def format_proxy(proxy) -> dict:
"""格式化代理对象"""
return {
"ip": proxy.ip,
"port": proxy.port,
"protocol": proxy.protocol,
"score": proxy.score,
"last_check": proxy.last_check.isoformat() if proxy.last_check else None,
}
def format_plugin(plugin) -> dict:
"""格式化插件对象"""
return {
"id": plugin.id,
"name": plugin.display_name,
"display_name": plugin.display_name,
"description": plugin.description,
"enabled": plugin.enabled,
"last_run": plugin.last_run.isoformat() if plugin.last_run else None,
"success_count": plugin.success_count,
"failure_count": plugin.failure_count,
}

45
app/api/deps.py Normal file
View File

@@ -0,0 +1,45 @@
"""依赖注入"""
from fastapi import Request
from app.services.proxy_service import ProxyService
from app.services.plugin_service import PluginService
from app.services.scheduler_service import SchedulerService
from app.services.validator_service import ValidatorService
from app.repositories.proxy_repo import ProxyRepository
from app.core.tasks.queue import ValidationQueue
from app.core.config import settings as app_settings
def get_proxy_service() -> ProxyService:
return ProxyService()
def get_plugin_service() -> PluginService:
return PluginService()
def get_scheduler_service(request: Request) -> SchedulerService:
return request.app.state.scheduler_service
def get_validation_queue(request: Request) -> ValidationQueue:
return request.app.state.validation_queue
def create_scheduler_service() -> SchedulerService:
"""在应用启动时创建 SchedulerService非请求上下文"""
validator = ValidatorService(
timeout=app_settings.validator_timeout,
connect_timeout=app_settings.validator_connect_timeout,
max_concurrency=app_settings.validator_max_concurrency,
)
proxy_repo = ProxyRepository()
queue = ValidationQueue(
validator=validator,
proxy_repo=proxy_repo,
worker_count=app_settings.validator_max_concurrency,
score_valid=app_settings.score_valid,
score_invalid=app_settings.score_invalid,
score_min=app_settings.score_min,
score_max=app_settings.score_max,
)
return SchedulerService(validation_queue=queue, proxy_repo=proxy_repo)

33
app/api/errors.py Normal file
View File

@@ -0,0 +1,33 @@
"""统一异常处理"""
from fastapi import Request
from fastapi.responses import JSONResponse
from pydantic import ValidationError
from app.core.exceptions import ProxyPoolException
from app.core.log import logger
async def proxy_pool_exception_handler(request: Request, exc: ProxyPoolException):
return JSONResponse(
status_code=exc.code,
content={"code": exc.code, "message": exc.message, "data": None},
)
async def pydantic_validation_handler(request: Request, exc: ValidationError):
logger.error(f"Validation error: {exc}")
return JSONResponse(
status_code=422,
content={
"code": 422,
"message": "参数验证失败",
"data": exc.errors(),
},
)
async def general_exception_handler(request: Request, exc: Exception):
logger.error(f"Unhandled exception: {exc}", exc_info=True)
return JSONResponse(
status_code=500,
content={"code": 500, "message": "服务器内部错误", "data": None},
)

41
app/api/lifespan.py Normal file
View File

@@ -0,0 +1,41 @@
"""应用生命周期管理"""
from contextlib import asynccontextmanager
from fastapi import FastAPI
from app.core.db import init_db, get_db
from app.core.config import settings as app_settings
from app.core.log import logger
from app.api.deps import create_scheduler_service
from app.repositories.settings_repo import SettingsRepository
settings_repo = SettingsRepository()
@asynccontextmanager
async def lifespan(app: FastAPI):
"""应用启动和关闭时的生命周期管理"""
# 初始化数据库
await init_db()
# 创建调度器并挂载到 app.state
scheduler_service = create_scheduler_service()
app.state.scheduler_service = scheduler_service
app.state.validation_queue = scheduler_service.validation_queue
# 加载设置并决定是否启动调度器
try:
async with get_db() as db:
settings = await settings_repo.get_all(db)
scheduler_service.interval_minutes = settings.get(
"validate_interval_minutes", app_settings.validator_timeout
)
if settings.get("auto_validate", True):
await scheduler_service.start()
except Exception as e:
logger.error(f"Failed to load settings on startup: {e}")
logger.info("API server started")
yield
# 关闭调度器
await scheduler_service.stop()
logger.info("API server shutdown")

55
app/api/main.py Normal file
View File

@@ -0,0 +1,55 @@
"""FastAPI 应用工厂"""
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.api.lifespan import lifespan
from app.api.routes import api_router
from app.api.errors import proxy_pool_exception_handler, pydantic_validation_handler, general_exception_handler
from app.core.exceptions import ProxyPoolException
from pydantic import ValidationError
from app.core.config import settings as app_settings
# 导入并注册所有插件(显式注册模式)
import app.plugins
def create_app() -> FastAPI:
app = FastAPI(
title="代理池API",
version="2.0.0",
lifespan=lifespan,
)
# CORS
app.add_middleware(
CORSMiddleware,
allow_origins=app_settings.cors_origins_list,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 异常处理
app.add_exception_handler(ProxyPoolException, proxy_pool_exception_handler)
app.add_exception_handler(ValidationError, pydantic_validation_handler)
app.add_exception_handler(Exception, general_exception_handler)
# 路由
app.include_router(api_router)
@app.get("/")
async def root():
return {"message": "欢迎使用代理池API", "status": "running", "data": None}
@app.get("/health")
async def health_check():
from datetime import datetime
scheduler = app.state.scheduler_service
return {
"status": "healthy",
"timestamp": datetime.now().isoformat(),
"database": "connected",
"scheduler": "running" if scheduler.running else "stopped",
"version": "2.0.0",
}
return app

View File

@@ -0,0 +1,9 @@
"""路由包"""
from fastapi import APIRouter
from app.api.routes import proxies, plugins, scheduler, settings
api_router = APIRouter()
api_router.include_router(proxies.router)
api_router.include_router(plugins.router)
api_router.include_router(scheduler.router)
api_router.include_router(settings.router)

154
app/api/routes/plugins.py Normal file
View File

@@ -0,0 +1,154 @@
"""插件相关路由"""
import asyncio
from fastapi import APIRouter, Depends
from app.services.plugin_service import PluginService
from app.services.scheduler_service import SchedulerService
from app.api.deps import get_plugin_service, get_scheduler_service
from app.api.common import success_response, error_response, format_plugin
from app.core.log import logger
router = APIRouter(prefix="/api/plugins", tags=["plugins"])
@router.get("")
async def list_plugins(service: PluginService = Depends(get_plugin_service)):
try:
plugins = await service.list_plugins()
return success_response("获取插件列表成功", {"plugins": [format_plugin(p) for p in plugins]})
except Exception as e:
logger.error(f"List plugins failed: {e}")
return error_response("获取插件列表失败", 500)
@router.put("/{plugin_id}/toggle")
async def toggle_plugin(
plugin_id: str,
request: dict,
service: PluginService = Depends(get_plugin_service),
):
enabled = request.get("enabled")
if enabled is None:
return error_response("缺少 enabled 参数", 400)
try:
success = await service.toggle_plugin(plugin_id, enabled)
if not success:
return error_response("插件不存在", 404)
return success_response(
f"插件 {plugin_id}{'启用' if enabled else '禁用'}",
{"plugin_id": plugin_id, "enabled": enabled},
)
except Exception as e:
logger.error(f"Toggle plugin failed: {e}")
return error_response("切换插件状态失败", 500)
@router.get("/{plugin_id}/config")
async def get_plugin_config(
plugin_id: str,
service: PluginService = Depends(get_plugin_service),
):
try:
config = await service.get_plugin_config(plugin_id)
if config is None:
return error_response("插件不存在", 404)
return success_response("获取插件配置成功", {"plugin_id": plugin_id, "config": config})
except Exception as e:
logger.error(f"Get plugin config failed: {e}")
return error_response("获取插件配置失败", 500)
@router.post("/{plugin_id}/config")
async def update_plugin_config(
plugin_id: str,
request: dict,
service: PluginService = Depends(get_plugin_service),
):
config = request.get("config", {})
if not isinstance(config, dict):
return error_response("config 必须是对象", 400)
try:
success = await service.update_plugin_config(plugin_id, config)
if not success:
return error_response("插件不存在或配置无效", 404)
return success_response("保存插件配置成功", {"plugin_id": plugin_id, "config": config})
except Exception as e:
logger.error(f"Update plugin config failed: {e}")
return error_response("保存插件配置失败", 500)
@router.post("/{plugin_id}/crawl")
async def crawl_plugin(
plugin_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
plugin = plugin_service.get_plugin(plugin_id)
if not plugin:
return error_response("插件不存在", 404)
try:
results = await plugin_service.run_plugin(plugin_id)
if not results:
return success_response(
f"插件 {plugin_id} 爬取完成,未获取到代理",
{"plugin_id": plugin_id, "proxy_count": 0, "valid_count": 0},
)
logger.info(f"Plugin {plugin_id} crawled {len(results)} proxies")
scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results)
try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=30.0)
except asyncio.TimeoutError:
pass
return success_response(
f"插件 {plugin_id} 爬取并验证完成",
{
"plugin_id": plugin_id,
"proxy_count": len(results),
"valid_count": scheduler_service.validation_queue.valid_count,
"invalid_count": scheduler_service.validation_queue.invalid_count,
},
)
except Exception as e:
logger.error(f"Crawl plugin {plugin_id} failed: {e}")
return error_response(f"插件爬取失败: {str(e)}", 500)
@router.post("/crawl-all")
async def crawl_all(
plugin_service: PluginService = Depends(get_plugin_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
try:
results = await plugin_service.run_all_plugins()
if not results:
return success_response(
"所有插件爬取完成,未获取到代理",
{"total_crawled": 0, "valid_count": 0, "invalid_count": 0},
)
logger.info(f"All plugins crawled {len(results)} unique proxies")
scheduler_service.validation_queue.reset_stats()
await scheduler_service.validation_queue.submit(results)
try:
await asyncio.wait_for(scheduler_service.validation_queue.drain(), timeout=60.0)
except asyncio.TimeoutError:
pass
return success_response(
"所有插件爬取并验证完成",
{
"total_crawled": len(results),
"valid_count": scheduler_service.validation_queue.valid_count,
"invalid_count": scheduler_service.validation_queue.invalid_count,
},
)
except Exception as e:
logger.error(f"Crawl all failed: {e}")
return error_response(f"批量爬取失败: {str(e)}", 500)

125
app/api/routes/proxies.py Normal file
View File

@@ -0,0 +1,125 @@
"""代理相关路由(含统计信息)"""
from typing import Optional
from fastapi import APIRouter, Depends, Query
from app.services.proxy_service import ProxyService
from app.services.scheduler_service import SchedulerService
from app.models.schemas import ProxyListRequest, BatchDeleteRequest
from app.api.deps import get_proxy_service, get_scheduler_service
from app.api.common import success_response, error_response, format_proxy
from app.core.log import logger
router = APIRouter(prefix="/api/proxies", tags=["proxies"])
@router.get("/stats")
async def get_stats(
proxy_service: ProxyService = Depends(get_proxy_service),
scheduler_service: SchedulerService = Depends(get_scheduler_service),
):
try:
stats = await proxy_service.get_stats()
stats["scheduler_running"] = scheduler_service.running
return success_response("获取统计信息成功", stats)
except Exception as e:
logger.error(f"Get stats failed: {e}")
return error_response("获取统计信息失败", 500)
@router.post("")
async def list_proxies(
request: ProxyListRequest,
service: ProxyService = Depends(get_proxy_service),
):
try:
proxies, total = await service.list_proxies(
page=request.page,
page_size=request.page_size,
protocol=request.protocol,
min_score=request.min_score,
max_score=request.max_score,
sort_by=request.sort_by,
sort_order=request.sort_order,
)
return success_response(
"获取代理列表成功",
{
"list": [format_proxy(p) for p in proxies],
"total": total,
"page": request.page,
"page_size": request.page_size,
},
)
except Exception as e:
logger.error(f"List proxies failed: {e}")
return error_response("获取代理列表失败", 500)
@router.get("/random")
async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
try:
proxy = await service.get_random_proxy()
if not proxy:
return error_response("没有找到可用的代理", 404)
return success_response("获取随机代理成功", format_proxy(proxy))
except Exception as e:
logger.error(f"Get random proxy failed: {e}")
return error_response("获取随机代理失败", 500)
@router.get("/export/{fmt}")
async def export_proxies(
fmt: str,
protocol: Optional[str] = None,
limit: int = Query(default=10000, ge=1, le=100000),
service: ProxyService = Depends(get_proxy_service),
):
if fmt not in ("csv", "txt", "json"):
return error_response("不支持的导出格式", 400)
from fastapi.responses import StreamingResponse
media_types = {"csv": "text/csv", "txt": "text/plain", "json": "application/json"}
async def generate():
async for chunk in service.export_proxies(fmt, protocol, limit):
yield chunk
return StreamingResponse(
generate(),
media_type=media_types[fmt],
headers={"Content-Disposition": f"attachment; filename=proxies.{fmt}"},
)
@router.delete("/{ip}/{port}")
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
try:
await service.delete_proxy(ip, port)
return success_response("删除代理成功")
except Exception as e:
logger.error(f"Delete proxy failed: {e}")
return error_response("删除代理失败", 500)
@router.post("/batch-delete")
async def batch_delete(
request: BatchDeleteRequest,
service: ProxyService = Depends(get_proxy_service),
):
try:
proxies = [(item.ip, item.port) for item in request.proxies]
deleted = await service.batch_delete(proxies)
return success_response(f"批量删除 {deleted} 个代理成功", {"deleted_count": deleted})
except Exception as e:
logger.error(f"Batch delete failed: {e}")
return error_response("批量删除失败", 500)
@router.delete("/clean-invalid")
async def clean_invalid(service: ProxyService = Depends(get_proxy_service)):
try:
count = await service.clean_invalid()
return success_response(f"清理了 {count} 个无效代理", {"deleted_count": count})
except Exception as e:
logger.error(f"Clean invalid failed: {e}")
return error_response("清理无效代理失败", 500)

View File

@@ -0,0 +1,64 @@
"""调度器相关路由"""
from fastapi import APIRouter, Depends
from app.services.scheduler_service import SchedulerService
from app.repositories.settings_repo import SettingsRepository
from app.core.db import get_db
from app.api.deps import get_scheduler_service
from app.api.common import success_response, error_response
from app.core.log import logger
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
settings_repo = SettingsRepository()
async def _save_auto_validate_setting(enabled: bool):
"""保存自动验证设置"""
async with get_db() as db:
settings = await settings_repo.get_all(db)
settings["auto_validate"] = enabled
from app.models.schemas import SettingsSchema
await settings_repo.save(db, SettingsSchema(**settings).model_dump())
@router.post("/start")
async def start_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
if scheduler.running:
return success_response("验证调度器已在运行", {"running": True})
await scheduler.start()
await _save_auto_validate_setting(True)
return success_response("验证调度器已启动", {"running": True})
except Exception as e:
logger.error(f"Start scheduler failed: {e}")
return error_response(f"启动调度器失败: {str(e)}", 500)
@router.post("/stop")
async def stop_scheduler(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
if not scheduler.running:
return success_response("验证调度器未运行", {"running": False})
await scheduler.stop()
await _save_auto_validate_setting(False)
return success_response("验证调度器已停止", {"running": False})
except Exception as e:
logger.error(f"Stop scheduler failed: {e}")
return error_response(f"停止调度器失败: {str(e)}", 500)
@router.post("/validate-now")
async def validate_now(scheduler: SchedulerService = Depends(get_scheduler_service)):
try:
scheduler.validate_all_now()
return success_response("已开始全量验证", {"started": True})
except Exception as e:
logger.error(f"Validate now failed: {e}")
return error_response(f"启动验证失败: {str(e)}", 500)
@router.get("/status")
async def scheduler_status(scheduler: SchedulerService = Depends(get_scheduler_service)):
return success_response(
"获取状态成功",
{"running": scheduler.running, "interval_minutes": scheduler.interval_minutes},
)

View File

@@ -0,0 +1,34 @@
"""设置相关路由"""
from fastapi import APIRouter
from app.core.db import get_db
from app.repositories.settings_repo import SettingsRepository
from app.models.schemas import SettingsSchema
from app.api.common import success_response, error_response
from app.core.log import logger
router = APIRouter(prefix="/api/settings", tags=["settings"])
settings_repo = SettingsRepository()
@router.get("")
async def get_settings():
try:
async with get_db() as db:
settings = await settings_repo.get_all(db)
return success_response("获取设置成功", settings)
except Exception as e:
logger.error(f"Get settings failed: {e}")
return error_response("获取设置失败", 500)
@router.post("")
async def save_settings(request: SettingsSchema):
try:
async with get_db() as db:
success = await settings_repo.save(db, request.model_dump())
if not success:
return error_response("保存设置失败", 500)
return success_response("保存设置成功", request.model_dump())
except Exception as e:
logger.error(f"Save settings failed: {e}")
return error_response(f"保存设置失败: {str(e)}", 500)