feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation
- Add Free_Proxy_Website-style fpw_* plugins and register them - Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting - Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore - Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait - Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys - WebSocket dashboard stats; ProxyList pool_filter and API alignment - POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404 - pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB - .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import Made-with: Cursor
This commit is contained in:
祀梦
@@ -1,9 +1,10 @@
|
||||
"""路由包"""
|
||||
from fastapi import APIRouter
|
||||
from app.api.routes import proxies, plugins, scheduler, settings, tasks
|
||||
from app.api.routes import proxies, plugins, scheduler, settings, tasks, ws
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(proxies.router)
|
||||
api_router.include_router(ws.router)
|
||||
api_router.include_router(plugins.router)
|
||||
api_router.include_router(scheduler.router)
|
||||
api_router.include_router(settings.router)
|
||||
|
||||
@@ -113,8 +113,8 @@ def _create_crawl_all_aggregator(job_ids, executor):
|
||||
class CrawlAllAggregator(Job):
|
||||
async def run(self):
|
||||
self._set_running()
|
||||
# 等待所有子 job 完成（最多等 30 秒）
|
||||
for _ in range(300):
|
||||
# 等待所有子 job 完成（最多约 5 分钟，与前端轮询一致）
|
||||
for _ in range(3000):
|
||||
if self.is_cancelled:
|
||||
break
|
||||
all_done = all(
|
||||
@@ -125,15 +125,56 @@ def _create_crawl_all_aggregator(job_ids, executor):
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
total = 0
|
||||
valid = 0
|
||||
invalid = 0
|
||||
plugins_failed = 0
|
||||
per_plugin = []
|
||||
for jid in job_ids:
|
||||
job = executor.get_job(jid)
|
||||
if job and job.result:
|
||||
total += job.result.get("proxy_count", 0)
|
||||
valid += job.result.get("success_count", 0)
|
||||
invalid += job.result.get("failure_count", 0)
|
||||
result = {"total_crawled": total, "valid_count": valid, "invalid_count": invalid}
|
||||
plugin_id = getattr(job, "plugin_id", "") if job else ""
|
||||
proxy_count = 0
|
||||
crawl_failed = False
|
||||
err_msg = None
|
||||
job_status = job.status.value if job else "missing"
|
||||
|
||||
if not job:
|
||||
per_plugin.append({
|
||||
"plugin_id": plugin_id,
|
||||
"proxy_count": 0,
|
||||
"crawl_failed": True,
|
||||
"error": "任务不存在",
|
||||
"job_status": job_status,
|
||||
})
|
||||
plugins_failed += 1
|
||||
continue
|
||||
|
||||
if job.status.value == "failed":
|
||||
crawl_failed = True
|
||||
plugins_failed += 1
|
||||
err_msg = job.error or "任务失败"
|
||||
elif job.result:
|
||||
r = job.result
|
||||
plugin_id = r.get("plugin_id") or plugin_id
|
||||
proxy_count = r.get("proxy_count", 0)
|
||||
total += proxy_count
|
||||
if r.get("crawl_failed") or r.get("failure_count", 0) > 0:
|
||||
crawl_failed = True
|
||||
plugins_failed += 1
|
||||
err_msg = r.get("error")
|
||||
else:
|
||||
total += 0
|
||||
|
||||
per_plugin.append({
|
||||
"plugin_id": plugin_id,
|
||||
"proxy_count": proxy_count,
|
||||
"crawl_failed": crawl_failed,
|
||||
"error": err_msg,
|
||||
"job_status": job_status,
|
||||
})
|
||||
|
||||
result = {
|
||||
"total_crawled": total,
|
||||
"plugins_failed": plugins_failed,
|
||||
"per_plugin": per_plugin,
|
||||
}
|
||||
if self.is_cancelled:
|
||||
result["cancelled"] = True
|
||||
return result
|
||||
|
||||
@@ -5,7 +5,8 @@ from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.services.proxy_service import ProxyService
|
||||
from app.services.scheduler_service import SchedulerService
|
||||
from app.models.schemas import ProxyListRequest, BatchDeleteRequest
|
||||
from app.services.dashboard_stats import get_dashboard_stats
|
||||
from app.models.schemas import ProxyListRequest, BatchDeleteRequest, ProxyDeleteItem
|
||||
from app.api.deps import get_proxy_service, get_scheduler_service
|
||||
from app.api.common import success_response, format_proxy
|
||||
from app.core.exceptions import ProxyPoolException, ProxyNotFoundException
|
||||
@@ -15,11 +16,9 @@ router = APIRouter(prefix="/api/proxies", tags=["proxies"])
|
||||
|
||||
@router.get("/stats")
|
||||
async def get_stats(
|
||||
proxy_service: ProxyService = Depends(get_proxy_service),
|
||||
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
stats = await proxy_service.get_stats()
|
||||
stats["scheduler_running"] = scheduler_service.running
|
||||
stats = await get_dashboard_stats(scheduler_service.running)
|
||||
return success_response("获取统计信息成功", stats)
|
||||
|
||||
|
||||
@@ -36,6 +35,7 @@ async def list_proxies(
|
||||
max_score=request.max_score,
|
||||
sort_by=request.sort_by,
|
||||
sort_order=request.sort_order,
|
||||
pool_filter=request.pool_filter,
|
||||
)
|
||||
return success_response(
|
||||
"获取代理列表成功",
|
||||
@@ -75,6 +75,16 @@ async def export_proxies(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/delete-one")
|
||||
async def delete_proxy_one(
|
||||
item: ProxyDeleteItem,
|
||||
service: ProxyService = Depends(get_proxy_service),
|
||||
):
|
||||
"""JSON 删除（推荐）：IPv6 等含冒号 IP 不受路径分段影响。"""
|
||||
await service.delete_proxy(item.ip, item.port)
|
||||
return success_response("删除代理成功")
|
||||
|
||||
|
||||
@router.delete("/{ip}/{port}")
|
||||
async def delete_proxy(ip: str, port: int, service: ProxyService = Depends(get_proxy_service)):
|
||||
await service.delete_proxy(ip, port)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""设置相关路由"""
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, Request, Depends
|
||||
from app.core.db import get_db
|
||||
from app.repositories.settings_repo import SettingsRepository
|
||||
from app.models.schemas import SettingsSchema
|
||||
from app.api.common import success_response
|
||||
from app.api.deps import get_settings_repo
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.exceptions import ProxyPoolException
|
||||
from app.core.log import logger
|
||||
|
||||
@@ -47,17 +50,21 @@ async def save_settings(
|
||||
|
||||
# 热更新验证器超时和并发（下次验证时生效）
|
||||
if validator:
|
||||
validator._init_timeout = request.validation_timeout
|
||||
validator._init_connect_timeout = request.validation_timeout
|
||||
vt = float(request.validation_timeout)
|
||||
validator._init_timeout = vt
|
||||
# 连接阶段单独收紧：勿与 total 等同，否则死代理会在 connect 上耗满整段超时
|
||||
validator._init_connect_timeout = min(
|
||||
float(app_settings.validator_connect_timeout), vt
|
||||
)
|
||||
validator._init_max_concurrency = request.default_concurrency
|
||||
if request.validation_targets is not None:
|
||||
validator.update_test_urls(request.validation_targets)
|
||||
# 延迟关闭旧 session：让正在验证的代理继续使用旧 session，
|
||||
# 新请求会通过 _ensure_session() 自动创建使用新配置的 session
|
||||
await validator.close_socks_sessions()
|
||||
old_session = validator._http_session
|
||||
validator._http_session = None
|
||||
validator._http_connector = None
|
||||
validator._semaphore = None
|
||||
if old_session and not old_session.closed:
|
||||
asyncio.create_task(old_session.close())
|
||||
logger.info(f"Validator config updated: timeout={request.validation_timeout}, concurrency={request.default_concurrency}, targets={request.validation_targets}")
|
||||
|
||||
32
app/api/routes/ws.py
Normal file
32
app/api/routes/ws.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""WebSocket 实时推送"""
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, WebSocket
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from app.services.dashboard_stats import get_dashboard_stats
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["websocket"])
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def websocket_dashboard(websocket: WebSocket):
|
||||
app = websocket.app
|
||||
await websocket.accept()
|
||||
manager = app.state.ws_manager
|
||||
await manager.connect(websocket)
|
||||
try:
|
||||
stats = await get_dashboard_stats(app.state.scheduler.running)
|
||||
await websocket.send_json({"type": "stats", "data": stats})
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if msg.get("type") == "ping":
|
||||
await websocket.send_json({"type": "pong"})
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
await manager.disconnect(websocket)
|
||||
Reference in New Issue
Block a user