commit b06044c91ccf3938786ff2ef474d88e181744c17 Author: 祀梦 <3501646051@qq.com> Date: Tue Jan 27 21:17:36 2026 +0800 first commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ed49a16 --- /dev/null +++ b/.env.example @@ -0,0 +1,62 @@ +# 代理池系统配置文件示例 +# 复制此文件为 .env 并根据实际情况修改配置 + +# ==================== 数据库配置 ==================== +DB_PATH=db/proxies.sqlite + +# ==================== API服务配置 ==================== +HOST=0.0.0.0 +PORT=3000 + +# ==================== 验证器配置 ==================== +VALIDATOR_TIMEOUT=5 +VALIDATOR_MAX_CONCURRENCY=200 +VALIDATOR_CONNECT_TIMEOUT=3 + +# ==================== 爬虫配置 ==================== +CRAWLER_NUM_VALIDATORS=50 +CRAWLER_MAX_QUEUE_SIZE=500 + +# ==================== 定时任务配置 ==================== +SCHEDULER_INTERVAL_MINUTES=60 +SCHEDULER_ENABLED=false + +# ==================== 日志配置 ==================== +LOG_LEVEL=INFO +LOG_DIR=logs + +# ==================== 导出配置 ==================== +EXPORT_MAX_RECORDS=10000 + +# ==================== 代理评分配置 ==================== +SCORE_VALID=10 +SCORE_INVALID=-5 +SCORE_MIN=0 +SCORE_MAX=100 + +# ==================== WebSocket配置 ==================== +WS_PING_INTERVAL=20 +WS_PING_TIMEOUT=20 + +# ==================== 插件配置 ==================== +PLUGINS_DIR=plugins + +# ==================== CORS配置 ==================== +# 允许的来源域名，用逗号分隔 +# 开发环境示例: http://localhost:8080,http://localhost:5173 +# 生产环境示例: https://yourdomain.com,https://api.yourdomain.com +CORS_ORIGINS=http://localhost:8080,http://localhost:5173 + +# ==================== API Key配置 ==================== +# 普通用户API Key（只读权限） +# 请修改为强随机字符串，例如: openssl rand -hex 32 +API_KEY=your-api-key-here + +# 管理员API Key（读写权限） +# 请修改为强随机字符串 +ADMIN_API_KEY=your-admin-api-key-here + +# ==================== 认证开关 ==================== +# 是否启用API认证 +# 开发环境可设为 false，生产环境务必设为 true +REQUIRE_AUTH=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e49221 --- /dev/null +++ b/.gitignore @@ -0,0 +1,87 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual Environment +venv/ +ENV/ +env/ +.venv + +# Database +*.sqlite +*.sqlite3 +*.db +*.db-shm +*.db-wal + +# Logs +logs/ +*.log + +# Environment Variables +.env +.env.local +.env.*.local + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +.trae/ + +# Test +test/ +tests/ + +# Share Directory +share/ + +# Node.js +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +package-lock.json + +# Frontend Build +frontend/dist/ + +# Cache +.cache/ +*.cache + +# OS +Thumbs.db +.DS_Store + +# Temporary Files +*.tmp +*.bak +*.old +*~ + +# ProxyPool Specific +db/ +proxies.sqlite* diff --git a/README.md b/README.md new file mode 100644 index 0000000..ac49300 --- /dev/null +++ b/README.md @@ -0,0 +1,225 @@ +# 代理池管理系统 + +现代化、科技风的代理池 WebUI 管理系统，基于 Python + Vue3 开发。 + +## 🌟 特性 + +- 🔮 **科技风设计** - 现代化的深色科技主题 +- 📊 **实时监控** - WebSocket 实时推送任务进度 +- 🎯 **智能管理** - 代理查询、筛选、排序、批量操作 +- 📥 **多格式导出** - 支持 CSV、TXT、JSON 格式 +- ⏰ **定时任务** - 自动定期更新代理池 +- 🚀 **高性能** - 异步爬取和验证，支持高并发 + +## 📦 技术栈 + +### 后端 +- **框架**: FastAPI (端口 3000) +- **数据库**: SQLite + aiosqlite +- **异步**: asyncio +- **实时通信**: WebSocket + +### 前端 +- **框架**: Vue 3 + Vite (端口 8080) +- **UI库**: Element Plus +- **状态管理**: Pinia +- **图表**: ECharts +- **样式**: CSS Variables + 深色科技风 + +## 🚀 快速开始 + +### 1. 安装后端依赖 + +```bash +pip install -r requirements.txt +``` + +### 2. 安装前端依赖 + +```bash +cd frontend +npm install +``` + +### 3. 启动服务 + +#### 方式一：使用启动脚本（推荐） +双击运行 `start.bat` 选择启动方式 + +#### 方式二：手动启动 + +**启动后端服务**（终端 1） +```bash +python api_server.py +``` + +**启动前端服务**（终端 2） +```bash +cd frontend +npm run dev +``` + +### 4. 访问 WebUI + +打开浏览器访问：**http://localhost:8080** + +## 📁 项目结构 + +``` +ProxyPool/ +├── api_server.py # FastAPI 后端服务器 +├── tasks_manager.py # 任务管理器 +├── main.py # 原始命令行入口 +├── requirements.txt # Python 依赖 +├── start.bat # 启动脚本 +│ +├── core/ # 核心模块 +│ ├── crawler.py # 爬虫基类 +│ ├── validator.py # 代理验证器 +│ ├── sqlite.py # 数据库管理 +│ ├── plugin_manager.py# 插件管理器 +│ └── log.py # 日志配置 +│ +├── plugins/ # 代理源插件 +│ ├── __init__.py +│ └── speedx.py # SpeedX 代理源 +│ +├── frontend/ # Vue3 前端 +│ ├── src/ +│ │ ├── api/ # API 封装 +│ │ ├── stores/ # Pinia 状态管理 +│ │ ├── views/ # 页面组件 +│ │ ├── router/ # 路由配置 +│ │ ├── App.vue +│ │ ├── main.js +│ │ └── style.css # 全局样式 +│ ├── index.html +│ ├── package.json +│ └── vite.config.js +│ +└── data/ # 数据存储目录 + └── proxy_pool.db # SQLite 数据库 +``` + +## 🎨 主题切换 + +在设置页面可以切换三种科技风主题： + +- **🔮 科技蓝** - 默认主题，蓝色霓虹风格 +- **💜 星空紫** - 紫色星空风格 +- **💚 矩阵绿** - 绿色黑客风格 + +## 📡 API 接口 + +### 统计信息 +``` +GET /api/stats +``` + +### 代理列表 +``` +POST /api/proxies +``` + +### 获取随机代理 +``` +GET /api/proxies/random +``` + +### 启动爬虫 +``` +POST /api/crawler/start +``` + +### 停止爬虫 +``` +POST /api/crawler/stop +``` + +### 定时任务 +``` +POST /api/scheduler +GET /api/scheduler +``` + +### WebSocket 连接 +``` +ws://localhost:3000/ws +``` + +## 🐛 调试指南 + +### 任务进度不显示？ + +1. **检查 WebSocket 连接** + - 打开浏览器控制台（F12） + - 查看 Console 标签 + - 应该看到 "WebSocket连接成功啦~" + - 应该看到 "收到WebSocket消息:" 日志 + +2. **检查后端任务** + - 查看后端终端输出 + - 确认任务正在运行 + - 查看是否有错误日志 + +3. **检查插件可用性** + - 确保 `plugins/` 目录下有插件文件 + - 插件能正常抓取代理 + +### 数据不更新？ + +1. **检查数据库** + - 确认 `data/proxy_pool.db` 文件存在 + - 使用 SQLite 客户端打开查看数据 + +2. **手动测试 API** + ```bash + # 获取统计信息 + curl http://localhost:3000/api/stats + + # 获取代理列表 + curl -X POST http://localhost:3000/api/proxies \ + -H "Content-Type: application/json" \ + -d '{"page": 1, "page_size": 20}' + ``` + +3. **查看浏览器网络请求** + - 打开开发者工具 Network 标签 + - 刷新页面查看 API 请求 + - 检查响应状态码和数据 + +## 📝 配置说明 + +### 爬虫配置 +- **最大并发数**: 10-500，默认 200 +- **验证超时**: 3-30秒，默认 5秒 +- **验证线程数**: 10-200，默认 50 + +### 定时任务 +- **执行间隔**: 10-1440分钟，默认 60分钟 +- **自动清理**: 可选，清理无效代理 + +## 🔧 常见问题 + +### Q: 启动后端口被占用？ +A: 修改 `api_server.py` 最后一行的端口号（默认3000）或 `frontend/vite.config.js` 中的端口号（默认8080） + +### Q: 爬虫无法抓取代理？ +A: 检查网络连接，确保能访问目标网站，或尝试更换代理源插件 + +### Q: 代理验证失败率高？ +A: 增加验证超时时间，或减少并发验证数量 + +### Q: 数据库文件在哪里？ +A: 默认在 `data/proxy_pool.db`，可在 `core/sqlite.py` 中修改 `db_path` + +## 📄 License + +MIT License + +## 🙏 致谢 + +- FastAPI - 高性能 Python Web 框架 +- Vue 3 - 渐进式 JavaScript 框架 +- Element Plus - 优秀的 Vue 3 UI 库 +- ECharts - 强大的数据可视化库 diff --git a/api_server.py b/api_server.py new file mode 100644 index 0000000..9883a6b --- /dev/null +++ b/api_server.py @@ -0,0 +1,553 @@ +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Header, Request, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse, JSONResponse +from pydantic import BaseModel, Field, field_validator, ValidationError +from typing import Optional, List +import asyncio +import io +import csv +import json +from datetime import datetime +import re +from contextlib import asynccontextmanager + +from core.sqlite import SQLiteManager +from core.validator import ProxyValidator +from core.plugin_manager import PluginManager +from tasks_manager import TasksManager, ScheduledTasks +from core.log import logger +from config import Config +from core.auth import verify_api_key, require_admin, PermissionLevel + +@asynccontextmanager +async def lifespan(app: FastAPI): + """应用生命周期管理""" + db = SQLiteManager() + await db.init_db() + logger.info("API服务器启动啦~") + yield + logger.info("API服务器关闭啦~") + +app = FastAPI(title="代理池API", version="1.1.0", lifespan=lifespan) + +def format_datetime(datetime_str: str) -> str: + """将数据库时间格式统一转换为ISO 8601格式""" + if not datetime_str: + return None + + if isinstance(datetime_str, str): + if 'T' in datetime_str: + return datetime_str + + if re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', datetime_str): + return datetime_str.replace(' ', 'T') + '.000Z' + + return datetime_str + +@app.exception_handler(ValidationError) +async def validation_exception_handler(request: Request, exc: ValidationError): + logger.error(f"参数验证失败: {exc}") + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content={"code": 422, "message": "参数验证失败呢~", "data": exc.errors()} + ) + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + logger.error(f"HTTP异常: {exc.status_code} - {exc.detail}") + return JSONResponse( + status_code=exc.status_code, + content={"code": exc.status_code, "message": exc.detail, "data": None} + ) + +@app.exception_handler(Exception) +async def general_exception_handler(request: Request, exc: Exception): + logger.error(f"未处理的异常: {exc}", exc_info=True) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"code": 500, "message": "服务器内部错误呢~", "data": None} + ) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +tasks_manager = TasksManager() +scheduled_tasks = ScheduledTasks(tasks_manager) +plugin_manager = PluginManager() +active_websockets = set() +websockets_lock = asyncio.Lock() + +def optional_auth(): + if Config.REQUIRE_AUTH: + return Depends(verify_api_key) + return None + +async def broadcast_message(message: dict): + """向所有WebSocket客户端广播消息""" + async with websockets_lock: + websockets_to_remove = [] + tasks = [] + + for ws in active_websockets: + try: + tasks.append(ws.send_json(message)) + except Exception as e: + logger.error(f"发送WebSocket消息失败: {e}") + websockets_to_remove.append(ws) + + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + for ws in websockets_to_remove: + active_websockets.discard(ws) + +class ProxyRequest(BaseModel): + page: int = Field(default=1, ge=1, description="页码，必须大于等于1") + page_size: int = Field(default=20, ge=1, le=100, description="每页数量，必须在1-100之间") + protocol: Optional[str] = None + min_score: int = Field(default=0, ge=0, description="最低分数") + max_score: Optional[int] = Field(default=None, ge=0, description="最高分数") + sort_by: str = 'last_check' + sort_order: str = 'DESC' + + @field_validator('protocol') + @classmethod + def validate_protocol(cls, v): + if v is not None and v.lower() not in ['http', 'https', 'socks4', 'socks5']: + raise ValueError('协议类型必须是 http, https, socks4 或 socks5') + return v.lower() if v else v + + @field_validator('sort_by') + @classmethod + def validate_sort_by(cls, v): + if v not in ['ip', 'port', 'protocol', 'score', 'last_check']: + raise ValueError('排序字段必须是 ip, port, protocol, score 或 last_check') + return v + + @field_validator('sort_order') + @classmethod + def validate_sort_order(cls, v): + if v.upper() not in ['ASC', 'DESC']: + raise ValueError('排序方式必须是 ASC 或 DESC') + return v.upper() + +class ProxyDeleteItem(BaseModel): + ip: str + port: int + + @field_validator('port') + @classmethod + def validate_port(cls, v): + if not 1 <= v <= 65535: + raise ValueError('端口号必须在1-65535范围内') + return v + +class DeleteProxiesRequest(BaseModel): + proxies: List[ProxyDeleteItem] + + @field_validator('proxies') + @classmethod + def validate_proxies_count(cls, v): + if len(v) > 1000: + raise ValueError('单次最多删除1000个代理') + return v + +class CrawlerRequest(BaseModel): + num_validators: int = 50 + +class ScheduleRequest(BaseModel): + enabled: bool + interval_minutes: int = 60 + +@app.get("/") +async def root(): + return {"message": "欢迎使用代理池API~", "status": "running", "data": None} + +@app.get("/health") +async def health_check(): + try: + db = SQLiteManager() + await db.count_proxies() + return { + "status": "healthy", + "timestamp": datetime.now().isoformat(), + "database": "connected", + "version": "1.0.0" + } + except Exception as e: + logger.error(f"健康检查失败: {e}") + return { + "status": "unhealthy", + "timestamp": datetime.now().isoformat(), + "database": "disconnected", + "error": str(e) + } + +@app.get("/api/stats") +async def get_stats(_permission: str = optional_auth()): + try: + db = SQLiteManager() + stats = await db.get_stats() + today_new = await db.get_today_new_count() + stats['today_new'] = today_new + return {"code": 200, "message": "获取统计信息成功啦~", "data": stats} + except Exception as e: + logger.error(f"获取统计信息失败: {e}") + return {"code": 500, "message": "获取统计信息失败呢~", "data": None} + +@app.post("/api/proxies") +async def get_proxies(request: ProxyRequest, _permission: str = optional_auth()): + try: + db = SQLiteManager() + proxies = await db.get_proxies_paginated( + page=request.page, + page_size=request.page_size, + protocol=request.protocol, + min_score=request.min_score, + max_score=request.max_score, + sort_by=request.sort_by, + sort_order=request.sort_order + ) + total = await db.get_proxies_total( + protocol=request.protocol, + min_score=request.min_score, + max_score=request.max_score + ) + + proxy_list = [] + for proxy in proxies: + proxy_list.append({ + "ip": proxy[0], + "port": proxy[1], + "protocol": proxy[2], + "score": proxy[3], + "last_check": format_datetime(proxy[4]) + }) + + return { + "code": 200, + "message": "获取代理列表成功啦~", + "data": { + "list": proxy_list, + "total": total, + "page": request.page, + "page_size": request.page_size + } + } + except Exception as e: + logger.error(f"获取代理列表失败: {e}") + return {"code": 500, "message": "获取代理列表失败呢~", "data": None} + +@app.get("/api/proxies/random") +async def get_random_proxy(_permission: str = optional_auth()): + db = SQLiteManager() + proxy = await db.get_random_proxy() + if proxy: + return { + "code": 200, + "message": "获取随机代理成功啦~", + "data": { + "ip": proxy[0], + "port": proxy[1], + "protocol": proxy[2], + "score": proxy[3], + "last_check": format_datetime(proxy[4]) + } + } + return {"code": 404, "message": "没有找到可用的代理呢~", "data": None} + +@app.get("/api/proxies/{ip}/{port}") +async def get_proxy_detail(ip: str, port: int, _permission: str = optional_auth()): + db = SQLiteManager() + proxy = await db.get_proxy_detail(ip, port) + if proxy: + return { + "code": 200, + "message": "获取代理详情成功啦~", + "data": { + "ip": proxy[0], + "port": proxy[1], + "protocol": proxy[2], + "score": proxy[3], + "last_check": format_datetime(proxy[4]) + } + } + return {"code": 404, "message": "代理不存在呢~", "data": None} + +@app.delete("/api/proxies/{ip}/{port}") +async def delete_proxy(ip: str, port: int, _permission: str = Depends(require_admin)): + db = SQLiteManager() + await db.delete_proxy(ip, port) + return {"code": 200, "message": "删除代理成功啦~", "data": None} + +@app.post("/api/proxies/batch-delete") +async def batch_delete_proxies(request: DeleteProxiesRequest, _permission: str = Depends(require_admin)): + db = SQLiteManager() + proxy_tuples = [(item.ip, item.port) for item in request.proxies] + deleted_count = await db.batch_delete_proxies(proxy_tuples) + return {"code": 200, "message": f"批量删除 {deleted_count} 个代理成功啦~", "data": {"deleted_count": deleted_count}} + +@app.delete("/api/proxies/clean-invalid") +async def clean_invalid_proxies(_permission: str = Depends(require_admin)): + db = SQLiteManager() + deleted_count = await db.clean_invalid_proxies() + return {"code": 200, "message": f"清理了 {deleted_count} 个无效代理啦~", "data": {"deleted_count": deleted_count}} + +@app.get("/api/proxies/export/{format}") +async def export_proxies(format: str, protocol: Optional[str] = None, _permission: str = optional_auth(), limit: int = 10000): + try: + db = SQLiteManager() + + if format not in ['csv', 'txt', 'json']: + raise HTTPException(status_code=400, detail="不支持的导出格式呢~") + + if limit > 100000: + raise HTTPException(status_code=400, detail="导出数量不能超过100000条呢~") + + async def generate_csv(): + proxies = await db.get_all_proxies() + if protocol: + proxies = [p for p in proxies if p[2].lower() == protocol.lower()] + + proxies = proxies[:limit] + + output = [] + output.append('IP,Port,Protocol,Score,Last Check') + for proxy in proxies: + output.append(f"{proxy[0]},{proxy[1]},{proxy[2]},{proxy[3]},{format_datetime(proxy[4])}") + + for line in output: + yield line + '\n' + + async def generate_txt(): + proxies = await db.get_all_proxies() + if protocol: + proxies = [p for p in proxies if p[2].lower() == protocol.lower()] + + proxies = proxies[:limit] + + for proxy in proxies: + yield f"{proxy[0]}:{proxy[1]}\n" + + async def generate_json(): + proxies = await db.get_all_proxies() + if protocol: + proxies = [p for p in proxies if p[2].lower() == protocol.lower()] + + proxies = proxies[:limit] + + proxy_list = [] + for proxy in proxies: + proxy_list.append({'ip': proxy[0], 'port': proxy[1], 'protocol': proxy[2], 'score': proxy[3], 'last_check': format_datetime(proxy[4])}) + + yield '[\n' + for i, item in enumerate(proxy_list): + if i > 0: + yield ',\n' + yield json.dumps(item, ensure_ascii=False, indent=2) + yield '\n]' + + if format == 'csv': + return StreamingResponse( + generate_csv(), + media_type='text/csv', + headers={'Content-Disposition': 'attachment; filename=proxies.csv'} + ) + + elif format == 'txt': + return StreamingResponse( + generate_txt(), + media_type='text/plain', + headers={'Content-Disposition': 'attachment; filename=proxies.txt'} + ) + + elif format == 'json': + return StreamingResponse( + generate_json(), + media_type='application/json', + headers={'Content-Disposition': 'attachment; filename=proxies.json'} + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"导出代理失败: {e}") + raise HTTPException(status_code=500, detail="导出代理失败呢~") + +@app.post("/api/crawler/start") +async def start_crawler(request: CrawlerRequest, _permission: str = Depends(require_admin)): + try: + if tasks_manager.is_task_running(): + return {"code": 400, "message": "任务正在运行中呢~"} + + async def progress_callback(data): + await broadcast_message({"type": "progress", "data": data}) + + async def status_callback(data): + await broadcast_message({"type": "status", "data": data}) + + tasks_manager.set_callbacks(progress_callback, status_callback) + + db = SQLiteManager() + async with ProxyValidator(max_concurrency=200) as validator: + asyncio.create_task(tasks_manager.start_task(db, validator, request.num_validators)) + + return {"code": 200, "message": "爬虫任务开始啦~", "data": None} + except Exception as e: + logger.error(f"启动爬虫失败: {e}") + return {"code": 500, "message": "启动爬虫失败呢~", "data": None} + +@app.post("/api/crawler/stop") +async def stop_crawler(_permission: str = Depends(require_admin)): + if not tasks_manager.is_task_running(): + return {"code": 400, "message": "没有运行中的任务呢~", "data": None} + + await tasks_manager.stop_task() + return {"code": 200, "message": "爬虫任务停止啦~", "data": None} + +@app.get("/api/crawler/status") +async def get_crawler_status(_permission: str = optional_auth()): + return { + "code": 200, + "message": "获取爬虫状态成功啦~", + "data": { + "running": tasks_manager.is_task_running(), + "stats": tasks_manager.get_stats() + } + } + +@app.post("/api/scheduler") +async def set_scheduler(request: ScheduleRequest, _permission: str = Depends(require_admin)): + if request.enabled: + scheduled_tasks.start_scheduled(request.interval_minutes) + return {"code": 200, "message": f"定时任务已启动，间隔 {request.interval_minutes} 分钟~", "data": None} + else: + scheduled_tasks.stop_scheduled() + return {"code": 200, "message": "定时任务已停止~", "data": None} + +@app.get("/api/scheduler") +async def get_scheduler_status(_permission: str = optional_auth()): + return { + "code": 200, + "message": "获取定时任务状态成功啦~", + "data": { + "enabled": scheduled_tasks.is_scheduled, + "interval_minutes": scheduled_tasks.interval_minutes + } + } + +@app.websocket("/ws") +async def websocket_endpoint(websocket: WebSocket, token: Optional[str] = None): + if Config.REQUIRE_AUTH: + if not token: + await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason="缺少认证token") + logger.warning("WebSocket连接被拒绝：缺少token") + return + + if token != Config.API_KEY and token != Config.ADMIN_API_KEY: + await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason="无效的token") + logger.warning(f"WebSocket连接被拒绝：无效的token {token[:8]}...") + return + + permission_level = PermissionLevel.ADMIN if token == Config.ADMIN_API_KEY else PermissionLevel.READ_ONLY + logger.info(f"WebSocket连接成功，权限级别: {permission_level}") + + await websocket.accept() + + async with websockets_lock: + active_websockets.add(websocket) + + try: + await websocket.send_json({ + "type": "status", + "data": { + "status": "connected", + "message": "WebSocket连接成功啦~", + "timestamp": datetime.now().isoformat() + } + }) + + while True: + await websocket.receive_text() + except WebSocketDisconnect: + async with websockets_lock: + active_websockets.discard(websocket) + logger.info("WebSocket断开连接") + except Exception as e: + logger.error(f"WebSocket错误: {e}") + async with websockets_lock: + active_websockets.discard(websocket) + +@app.get("/api/plugins") +async def get_plugins(_permission: str = optional_auth()): + try: + plugins_info = plugin_manager.get_all_plugin_info() + return { + "code": 200, + "message": "获取插件列表成功啦~", + "data": { + "plugins": plugins_info + } + } + except Exception as e: + logger.error(f"获取插件列表失败: {e}") + return {"code": 500, "message": "获取插件列表失败呢~", "data": None} + +class PluginToggleRequest(BaseModel): + enabled: bool + +@app.put("/api/plugins/{plugin_id}/toggle") +async def toggle_plugin(plugin_id: str, request: PluginToggleRequest, _permission: str = Depends(require_admin)): + try: + success = plugin_manager.toggle_plugin(plugin_id, request.enabled) + if success: + return { + "code": 200, + "message": f"插件 {plugin_id} 已{'启用' if request.enabled else '禁用'}啦~", + "data": { + "plugin_id": plugin_id, + "enabled": request.enabled + } + } + else: + return {"code": 404, "message": "插件不存在呢~", "data": None} + except Exception as e: + logger.error(f"切换插件状态失败: {e}") + return {"code": 500, "message": "切换插件状态失败呢~", "data": None} + +@app.post("/api/plugins/{plugin_id}/crawl") +async def crawl_plugin(plugin_id: str, _permission: str = Depends(require_admin)): + try: + async def progress_callback(data): + await broadcast_message({"type": "progress", "data": data}) + + async def status_callback(data): + await broadcast_message({"type": "status", "data": data}) + + tasks_manager.set_callbacks(progress_callback, status_callback) + + db = SQLiteManager() + results = await plugin_manager.run_plugin(plugin_id) + + for ip, port, protocol in results: + await db.insert_proxy(ip, port, protocol) + + return { + "code": 200, + "message": f"插件 {plugin_id} 开始爬取啦~", + "data": { + "plugin_id": plugin_id, + "proxy_count": len(results) + } + } + except Exception as e: + logger.error(f"插件爬取失败: {e}") + return {"code": 500, "message": "插件爬取失败呢~", "data": None} + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8923) diff --git a/clean_protocol_data.py b/clean_protocol_data.py new file mode 100644 index 0000000..d47fcfc --- /dev/null +++ b/clean_protocol_data.py @@ -0,0 +1,49 @@ +import asyncio +import aiosqlite +import os + +async def clean_protocol_data(): + """清理数据库中协议字段异常的数据""" + + db_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'db') + db_path = os.path.join(db_dir, 'proxies.sqlite') + + VALID_PROTOCOLS = ['http', 'https', 'socks4', 'socks5'] + + async with aiosqlite.connect(db_path) as db: + # 查询异常的协议数据 + async with db.execute('SELECT ip, port, protocol FROM proxies WHERE protocol NOT IN (?, ?, ?, ?)', VALID_PROTOCOLS) as cursor: + invalid_proxies = await cursor.fetchall() + + if invalid_proxies: + print(f"发现 {len(invalid_proxies)} 条异常协议数据：") + for ip, port, protocol in invalid_proxies: + print(f" - {ip}:{port} (protocol={protocol})") + + # 更新所有不是有效协议类型的记录为 'http' + cursor = await db.execute(''' + UPDATE proxies + SET protocol = 'http' + WHERE protocol NOT IN (?, ?, ?, ?) + ''', VALID_PROTOCOLS) + + updated_count = cursor.rowcount + await db.commit() + + print(f"\n已将 {updated_count} 条记录的协议更新为 'http'") + + # 统计修复后的协议分布 + print("\n修复后的协议分布：") + for protocol in VALID_PROTOCOLS: + async with db.execute('SELECT COUNT(*) FROM proxies WHERE protocol = ?', (protocol,)) as cursor: + count = (await cursor.fetchone())[0] + print(f" - {protocol}: {count} 条") + +if __name__ == "__main__": + print("=" * 60) + print("开始清理数据库中的异常协议数据...") + print("=" * 60) + asyncio.run(clean_protocol_data()) + print("=" * 60) + print("清理完成！") + print("=" * 60) diff --git a/clear_database.py b/clear_database.py new file mode 100644 index 0000000..028e0be --- /dev/null +++ b/clear_database.py @@ -0,0 +1,33 @@ +import asyncio +import sys +import os +import aiosqlite + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from core.sqlite import SQLiteManager +from core.log import logger + +async def clear_proxies(): + db = SQLiteManager() + + try: + count_before = await db.count_proxies() + logger.info(f"清空前共有 {count_before} 个代理") + + async with aiosqlite.connect(db.db_path) as conn: + await conn.execute('DELETE FROM proxies') + await conn.commit() + + count_after = await db.count_proxies() + logger.info(f"清空后共有 {count_after} 个代理") + + print(f"✨ 成功清空数据库！删除了 {count_before} 个代理~") + return True + except Exception as e: + logger.error(f"清空数据库失败: {e}") + print(f"❌ 清空数据库失败: {e}") + return False + +if __name__ == "__main__": + asyncio.run(clear_proxies()) diff --git a/config.py b/config.py new file mode 100644 index 0000000..324874d --- /dev/null +++ b/config.py @@ -0,0 +1,75 @@ +""" +代理池系统配置管理 +统一管理所有配置项，支持环境变量覆盖 +""" +import os +from typing import Optional + +class Config: + # 数据库配置 + DB_PATH: str = os.getenv("DB_PATH", "db/proxies.db") + + # API服务配置 + HOST: str = os.getenv("HOST", "0.0.0.0") + PORT: int = int(os.getenv("PORT", "3000")) + + # 验证器配置 + VALIDATOR_TIMEOUT: int = int(os.getenv("VALIDATOR_TIMEOUT", "5")) + VALIDATOR_MAX_CONCURRENCY: int = int(os.getenv("VALIDATOR_MAX_CONCURRENCY", "200")) + VALIDATOR_CONNECT_TIMEOUT: int = int(os.getenv("VALIDATOR_CONNECT_TIMEOUT", "3")) + + # 爬虫配置 + CRAWLER_NUM_VALIDATORS: int = int(os.getenv("CRAWLER_NUM_VALIDATORS", "50")) + CRAWLER_MAX_QUEUE_SIZE: int = int(os.getenv("CRAWLER_MAX_QUEUE_SIZE", "500")) + + # 定时任务配置 + SCHEDULER_INTERVAL_MINUTES: int = int(os.getenv("SCHEDULER_INTERVAL_MINUTES", "60")) + SCHEDULER_ENABLED: bool = os.getenv("SCHEDULER_ENABLED", "true").lower() == "true" + + # 日志配置 + LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO") + LOG_DIR: str = os.getenv("LOG_DIR", "logs") + + # 导出配置 + EXPORT_MAX_RECORDS: int = int(os.getenv("EXPORT_MAX_RECORDS", "10000")) + + # 代理评分配置 + SCORE_VALID: int = int(os.getenv("SCORE_VALID", "10")) + SCORE_INVALID: int = int(os.getenv("SCORE_INVALID", "-5")) + SCORE_MIN: int = int(os.getenv("SCORE_MIN", "0")) + SCORE_MAX: int = int(os.getenv("SCORE_MAX", "100")) + + # WebSocket配置 + WS_PING_INTERVAL: int = int(os.getenv("WS_PING_INTERVAL", "20")) + WS_PING_TIMEOUT: int = int(os.getenv("WS_PING_TIMEOUT", "20")) + + # 插件配置 + PLUGINS_DIR: str = os.getenv("PLUGINS_DIR", "plugins") + + # CORS配置 + CORS_ORIGINS: str = os.getenv("CORS_ORIGINS", "http://localhost:8080,http://localhost:5173") + + # API Key配置 + API_KEY: str = os.getenv("API_KEY", "your-api-key-here") + ADMIN_API_KEY: str = os.getenv("ADMIN_API_KEY", "your-admin-api-key-here") + REQUIRE_AUTH: bool = os.getenv("REQUIRE_AUTH", "false").lower() == "true" + + @classmethod + def get(cls, key: str, default=None): + """获取配置项""" + return getattr(cls, key, default) + + @classmethod + def set(cls, key: str, value): + """设置配置项（仅限运行时）""" + setattr(cls, key, value) + + @classmethod + def update(cls, updates: dict): + """批量更新配置""" + for key, value in updates.items(): + if hasattr(cls, key): + setattr(cls, key, value) + +# 全局配置实例 +config = Config() diff --git a/core/auth.py b/core/auth.py new file mode 100644 index 0000000..a981f2d --- /dev/null +++ b/core/auth.py @@ -0,0 +1,89 @@ +from fastapi import HTTPException, Depends, Header, status +from typing import Optional +from config import Config +from core.log import logger + +class PermissionLevel: + READ_ONLY = "read_only" + ADMIN = "admin" + +def verify_api_key( + x_api_key: Optional[str] = Header(None, alias="X-API-Key"), + authorization: Optional[str] = Header(None) +) -> str: + """ + 验证API Key并返回权限级别 + + Args: + x_api_key: X-API-Key header中的API Key + authorization: Authorization header中的Bearer token + + Returns: + str: 权限级别 + + Raises: + HTTPException: 认证失败时抛出401错误 + """ + api_key = x_api_key + + if authorization and authorization.startswith("Bearer "): + api_key = authorization.replace("Bearer ", "") + + if not api_key: + logger.warning("API请求缺少API Key") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="缺少API Key，请在请求头中添加 X-API-Key 或 Authorization: Bearer ", + headers={"WWW-Authenticate": "Bearer"}, + ) + + if api_key == Config.ADMIN_API_KEY: + logger.info(f"管理员API认证成功: {api_key[:8]}...") + return PermissionLevel.ADMIN + elif api_key == Config.API_KEY: + logger.info(f"普通用户API认证成功: {api_key[:8]}...") + return PermissionLevel.READ_ONLY + else: + logger.warning(f"无效的API Key尝试: {api_key[:8]}...") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="无效的API Key", + headers={"WWW-Authenticate": "Bearer"}, + ) + +def require_admin(permission_level: str = Depends(verify_api_key)) -> str: + """ + 要求管理员权限的依赖函数 + + Args: + permission_level: 从verify_api_key获得的权限级别 + + Returns: + str: 权限级别 + + Raises: + HTTPException: 权限不足时抛出403错误 + """ + if permission_level != PermissionLevel.ADMIN: + logger.warning(f"非管理员用户尝试访问管理接口") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="需要管理员权限才能执行此操作" + ) + return permission_level + +def skip_auth_for_dev() -> Optional[str]: + """ + 开发环境跳过认证（仅在开发模式下使用） + + Returns: + Optional[str]: 返回管理员权限级别 + + Warning: + 仅用于开发环境，生产环境务必使用真实认证 + """ + import os + if os.getenv("SKIP_AUTH", "false").lower() == "true": + logger.warning("开发模式：跳过API Key认证") + return PermissionLevel.ADMIN + return None diff --git a/core/crawler.py b/core/crawler.py new file mode 100644 index 0000000..5521a51 --- /dev/null +++ b/core/crawler.py @@ -0,0 +1,86 @@ +import aiohttp +import asyncio +import random +from core.log import logger + +class BaseCrawler: + def __init__(self): + self.user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1.2 Mobile/15E148 Safari/604.1" + ] + + def get_headers(self): + return { + 'User-Agent': random.choice(self.user_agents), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', + 'Connection': 'keep-alive', + } + + async def fetch(self, url, method='GET', params=None, data=None, proxies=None, timeout=10, retry_count=3): + """异步抓取方法""" + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + } + async with aiohttp.ClientSession(headers=headers) as session: + for i in range(retry_count): + try: + # 注意：aiohttp 的代理格式与 requests 不同，通常为 http://user:pass@host:port + async with session.request( + method=method, + url=url, + params=params, + data=data, + proxy=proxies, + timeout=aiohttp.ClientTimeout(total=timeout) + ) as response: + if response.status == 200: + # 先读取内容，再处理编码 + content = await response.read() + + # 尝试获取编码 + encoding = response.get_encoding() + if encoding == 'utf-8' or not encoding: + try: + return content.decode('utf-8') + except UnicodeDecodeError: + # 尝试从内容中检测编码或手动设置为 gbk (国内网站常见) + return content.decode('gbk', errors='ignore') + + return content.decode(encoding, errors='ignore') + else: + logger.warning(f"请求失败 [{response.status}]: {url}, 正在进行第 {i+1} 次重试...") + except Exception as e: + logger.error(f"请求异常: {url}, 错误: {e}, 正在进行第 {i+1} 次重试...") + + await asyncio.sleep(random.uniform(1, 3)) + + return None + +class BasePlugin(BaseCrawler): + def __init__(self): + super().__init__() + self.name = "BasePlugin" + self.urls = [] + self.enabled = True + + async def parse(self, html): + """异步解析网页内容，需在子类中实现""" + raise NotImplementedError("Please implement parse method") + + async def run(self): + """异步运行插件""" + logger.info(f"正在运行插件: {self.name}") + results = [] + for url in self.urls: + self.current_url = url # 记录当前正在抓取的 URL，供 parse 使用 + html = await self.fetch(url) + if html: + async for proxy in self.parse(html): + results.append(proxy) + await asyncio.sleep(random.uniform(1, 2)) + return results diff --git a/core/log.py b/core/log.py new file mode 100644 index 0000000..53d016f --- /dev/null +++ b/core/log.py @@ -0,0 +1,38 @@ +import logging +import os +from datetime import datetime + +class LogHandler(logging.Logger): + def __init__(self, name='ProxyPool', level=logging.INFO): + super().__init__(name, level) + + # 获取项目根目录并创建 logs 目录 + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + log_dir = os.path.join(base_dir, 'logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + # 仅使用日期作为文件名 + log_filename = f"{datetime.now().strftime('%Y-%m-%d')}.log" + log_file = os.path.join(log_dir, log_filename) + + # 设置格式 + formatter = logging.Formatter( + '[%(asctime)s] %(name)s [%(levelname)s] %(filename)s[line:%(lineno)d]: %(message)s' + ) + + # 文件处理器 + file_handler = logging.FileHandler(log_file, encoding='utf-8') + file_handler.setFormatter(formatter) + self.addHandler(file_handler) + + # 控制台处理器 + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + self.addHandler(console_handler) + +# 实例化一个默认 logger 供外部直接使用 +logger = LogHandler() + +if __name__ == '__main__': + logger.info('这是一条按日期存储的日志测试') diff --git a/core/plugin_manager.py b/core/plugin_manager.py new file mode 100644 index 0000000..d580943 --- /dev/null +++ b/core/plugin_manager.py @@ -0,0 +1,125 @@ +import os +import importlib +import inspect +import asyncio +from typing import List, Dict, Optional +from core.crawler import BasePlugin +from core.log import logger + +class PluginManager: + def __init__(self, plugin_dir='plugins'): + self.plugin_dir = plugin_dir + self.plugins = [] + self.plugin_stats = {} + self._load_plugins() + self._init_stats() + + def _init_stats(self): + for plugin in self.plugins: + self.plugin_stats[plugin.name] = { + 'success_count': 0, + 'failure_count': 0, + 'last_run': None + } + + def _load_plugins(self): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + full_plugin_path = os.path.join(base_dir, self.plugin_dir) + + if not os.path.exists(full_plugin_path): + logger.error(f"插件目录不存在: {full_plugin_path}") + return + + for filename in os.listdir(full_plugin_path): + if filename.endswith('.py') and not filename.startswith('__'): + module_name = f"{self.plugin_dir}.{filename[:-3]}" + try: + module = importlib.import_module(module_name) + for name, obj in inspect.getmembers(module): + if inspect.isclass(obj) and issubclass(obj, BasePlugin) and obj is not BasePlugin: + plugin_instance = obj() + if plugin_instance.enabled: + logger.info(f"成功加载插件: {name} 来自 {module_name}") + self.plugins.append(plugin_instance) + else: + logger.info(f"插件已禁用，跳过加载: {name} 来自 {module_name}") + except Exception as e: + logger.error(f"加载插件失败 {module_name}: {e}") + + def get_plugin_by_name(self, plugin_name: str) -> Optional[BasePlugin]: + for plugin in self.plugins: + if plugin.name == plugin_name: + return plugin + return None + + def get_all_plugin_info(self) -> List[Dict]: + plugins_info = [] + for plugin in self.plugins: + stats = self.plugin_stats.get(plugin.name, { + 'success_count': 0, + 'failure_count': 0, + 'last_run': None + }) + plugins_info.append({ + 'id': plugin.name, + 'name': plugin.name, + 'enabled': plugin.enabled, + 'description': getattr(plugin, 'description', f'从{plugin.name}网站爬取代理'), + 'last_run': stats['last_run'], + 'success_count': stats['success_count'], + 'failure_count': stats['failure_count'] + }) + return plugins_info + + def toggle_plugin(self, plugin_name: str, enabled: bool) -> bool: + plugin = self.get_plugin_by_name(plugin_name) + if plugin: + plugin.enabled = enabled + logger.info(f"插件 {plugin_name} 已{'启用' if enabled else '禁用'}") + return True + return False + + async def run_plugin(self, plugin_name: str): + plugin = self.get_plugin_by_name(plugin_name) + if not plugin: + logger.error(f"插件不存在: {plugin_name}") + return [] + + if not plugin.enabled: + logger.warning(f"插件已禁用: {plugin_name}") + return [] + + try: + results = await plugin.run() + success_count = len(results) + failure_count = 0 + + from datetime import datetime + self.plugin_stats[plugin.name] = { + 'success_count': self.plugin_stats[plugin.name]['success_count'] + success_count, + 'failure_count': self.plugin_stats[plugin.name]['failure_count'] + failure_count, + 'last_run': datetime.now().isoformat() + } + + logger.info(f"插件 {plugin_name} 执行完成，成功: {success_count}") + return results + except Exception as e: + logger.error(f"插件 {plugin_name} 执行失败: {e}") + from datetime import datetime + self.plugin_stats[plugin.name] = { + 'success_count': self.plugin_stats[plugin.name]['success_count'], + 'failure_count': self.plugin_stats[plugin.name]['failure_count'] + 1, + 'last_run': datetime.now().isoformat() + } + return [] + + async def run_all(self): + """并发运行所有插件""" + tasks = [plugin.run() for plugin in self.plugins] + # 并发执行并收集结果 + results_list = await asyncio.gather(*tasks) + + # 将嵌套列表扁平化并产出结果 + for results in results_list: + for proxy in results: + yield proxy diff --git a/core/sqlite.py b/core/sqlite.py new file mode 100644 index 0000000..a003dd6 --- /dev/null +++ b/core/sqlite.py @@ -0,0 +1,334 @@ +import aiosqlite +import os +import asyncio +from core.log import logger + +VALID_PROTOCOLS = ['http', 'https', 'socks4', 'socks5'] + +class SQLiteManager: + _instance = None + _connection = None + _lock = asyncio.Lock() + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(SQLiteManager, cls).__new__(cls) + return cls._instance + + def __init__(self, db_path=None): + if hasattr(self, 'initialized') and self.initialized: + return + + if db_path is None: + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + db_dir = os.path.join(base_dir, 'db') + if not os.path.exists(db_dir): + os.makedirs(db_dir) + self.db_path = os.path.join(db_dir, 'proxies.sqlite') + else: + self.db_path = db_path + + self.initialized = True + + async def get_connection(self): + async with self._lock: + if self._connection is None: + self._connection = await aiosqlite.connect(self.db_path) + await self._connection.execute("PRAGMA journal_mode=WAL") + await self._connection.execute("PRAGMA synchronous=NORMAL") + await self._connection.execute("PRAGMA cache_size=-64000") + await self._connection.execute("PRAGMA temp_store=MEMORY") + return self._connection + + async def close_connection(self): + async with self._lock: + if self._connection is not None: + await self._connection.close() + self._connection = None + + async def init_db(self): + """初始化数据库和表结构""" + db = await self.get_connection() + await db.execute(''' + CREATE TABLE IF NOT EXISTS proxies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, + port INTEGER NOT NULL, + protocol TEXT DEFAULT 'http', + score INTEGER DEFAULT 10, + last_check TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(ip, port) + ) + ''') + + await db.execute('CREATE INDEX IF NOT EXISTS idx_score ON proxies(score)') + await db.execute('CREATE INDEX IF NOT EXISTS idx_protocol ON proxies(protocol)') + await db.execute('CREATE INDEX IF NOT EXISTS idx_last_check ON proxies(last_check)') + await db.execute('CREATE INDEX IF NOT EXISTS idx_ip_port ON proxies(ip, port)') + + await db.commit() + + async def insert_proxy(self, ip, port, protocol='http', score=10): + """异步插入或更新代理""" + try: + # 验证协议类型 + if protocol not in VALID_PROTOCOLS: + protocol = 'http' + logger.warning(f"无效的协议类型 {protocol}，默认使用 http") + + db = await self.get_connection() + # 先检查是否存在 + async with db.execute('SELECT score FROM proxies WHERE ip = ? AND port = ?', (ip, port)) as cursor: + row = await cursor.fetchone() + if row: + # 如果存在，则更新最后检查时间和分数 + await db.execute(''' + UPDATE proxies SET last_check = CURRENT_TIMESTAMP, score = ?, protocol = ? WHERE ip = ? AND port = ? + ''', (score, protocol, ip, port)) + else: + # 如果不存在，则插入新记录 + await db.execute(''' + INSERT INTO proxies (ip, port, protocol, score, last_check) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ''', (ip, port, protocol, score)) + await db.commit() + return True + except aiosqlite.IntegrityError as e: + # 处理唯一性约束冲突 + if "UNIQUE" in str(e): + # 代理已存在，更新它 + if protocol not in VALID_PROTOCOLS: + protocol = 'http' + db = await self.get_connection() + await db.execute(''' + UPDATE proxies SET last_check = CURRENT_TIMESTAMP, score = ?, protocol = ? WHERE ip = ? AND port = ? + ''', (score, protocol, ip, port)) + await db.commit() + return True + else: + logger.error(f"数据库完整性错误: {e}") + return False + except Exception as e: + logger.error(f"插入代理失败 {ip}:{port} - {e}") + return False + + async def get_all_proxies(self): + """异步获取所有代理""" + db = await self.get_connection() + async with db.execute('SELECT ip, port, protocol, score, last_check FROM proxies') as cursor: + return await cursor.fetchall() + + async def get_random_proxy(self): + """异步随机获取一个高分代理""" + db = await self.get_connection() + async with db.execute('SELECT ip, port, protocol, score, last_check FROM proxies WHERE score > 0 ORDER BY RANDOM() LIMIT 1') as cursor: + return await cursor.fetchone() + + async def update_score(self, ip, port, delta, min_score=0, max_score=100): + """异步更新代理分数（增量更新，带分数限制）""" + try: + db = await self.get_connection() + # 获取当前分数 + async with db.execute('SELECT score FROM proxies WHERE ip = ? AND port = ?', (ip, port)) as cursor: + row = await cursor.fetchone() + if row: + current_score = row[0] + new_score = max(min_score, min(max_score, current_score + delta)) + await db.execute(''' + UPDATE proxies SET score = ?, last_check = CURRENT_TIMESTAMP WHERE ip = ? AND port = ? + ''', (new_score, ip, port)) + if new_score <= 0: + await db.execute('DELETE FROM proxies WHERE score <= 0') + await db.commit() + return True + return False + except Exception as e: + logger.error(f"更新代理分数失败 {ip}:{port} - {e}") + return False + + async def delete_proxy(self, ip, port): + """异步删除指定代理""" + db = await self.get_connection() + await db.execute('DELETE FROM proxies WHERE ip = ? AND port = ?', (ip, port)) + await db.commit() + + async def count_proxies(self): + """异步统计代理数量""" + db = await self.get_connection() + async with db.execute('SELECT COUNT(*) FROM proxies') as cursor: + row = await cursor.fetchone() + return row[0] if row else 0 + + async def get_proxies_paginated_with_total(self, page: int = 1, page_size: int = 20, + protocol: str = None, min_score: int = 0, + max_score: int = None, + sort_by: str = 'last_check', + sort_order: str = 'DESC'): + """分页获取代理列表（一次查询返回数据和总数）""" + db = await self.get_connection() + conditions = ['score >= ?'] + params = [min_score] + + if protocol: + conditions.append('protocol = ?') + params.append(protocol) + + if max_score is not None: + conditions.append('score <= ?') + params.append(max_score) + + where_clause = ' AND '.join(conditions) + + order_by_clause = f'{sort_by} {sort_order}' + + offset = (page - 1) * page_size + query = f''' + SELECT ip, port, protocol, score, last_check, + COUNT(*) OVER() as total_count + FROM proxies + WHERE {where_clause} + ORDER BY {order_by_clause} + LIMIT ? OFFSET ? + ''' + params.extend([page_size, offset]) + + async with db.execute(query, params) as cursor: + rows = await cursor.fetchall() + total = rows[0][5] if rows else 0 + proxies = [(row[0], row[1], row[2], row[3], row[4]) for row in rows] + return proxies, total + + async def get_proxies_paginated(self, page: int = 1, page_size: int = 20, + protocol: str = None, min_score: int = 0, + max_score: int = None, + sort_by: str = 'last_check', + sort_order: str = 'DESC'): + """分页获取代理列表""" + db = await self.get_connection() + conditions = ['score >= ?'] + params = [min_score] + + if protocol: + conditions.append('protocol = ?') + params.append(protocol) + + if max_score is not None: + conditions.append('score <= ?') + params.append(max_score) + + where_clause = ' AND '.join(conditions) + + order_by_clause = f'{sort_by} {sort_order}' + + offset = (page - 1) * page_size + query = f''' + SELECT ip, port, protocol, score, last_check + FROM proxies + WHERE {where_clause} + ORDER BY {order_by_clause} + LIMIT ? OFFSET ? + ''' + params.extend([page_size, offset]) + + async with db.execute(query, params) as cursor: + return await cursor.fetchall() + + async def get_proxies_total(self, protocol: str = None, min_score: int = 0, max_score: int = None): + """获取符合条件的代理总数""" + db = await self.get_connection() + conditions = ['score >= ?'] + params = [min_score] + + if protocol: + conditions.append('protocol = ?') + params.append(protocol) + + if max_score is not None: + conditions.append('score <= ?') + params.append(max_score) + + where_clause = ' AND '.join(conditions) + + query = f'SELECT COUNT(*) FROM proxies WHERE {where_clause}' + + async with db.execute(query, params) as cursor: + row = await cursor.fetchone() + return row[0] if row else 0 + + async def get_proxy_detail(self, ip: str, port: int): + """获取单个代理的详细信息""" + db = await self.get_connection() + async with db.execute( + 'SELECT ip, port, protocol, score, last_check FROM proxies WHERE ip = ? AND port = ?', + (ip, port) + ) as cursor: + row = await cursor.fetchone() + return row + + async def batch_delete_proxies(self, proxy_list: list): + """批量删除代理，返回实际删除的数量""" + deleted_count = 0 + db = await self.get_connection() + for ip, port in proxy_list: + cursor = await db.execute('DELETE FROM proxies WHERE ip = ? AND port = ?', (ip, port)) + deleted_count += cursor.rowcount + await db.commit() + return deleted_count + + async def get_stats(self): + """获取统计信息""" + db = await self.get_connection() + stats = {} + + async with db.execute('SELECT COUNT(*) FROM proxies') as cursor: + row = await cursor.fetchone() + stats['total'] = row[0] if row else 0 + + async with db.execute('SELECT COUNT(*) FROM proxies WHERE score > 0') as cursor: + row = await cursor.fetchone() + stats['available'] = row[0] if row else 0 + + async with db.execute('SELECT COUNT(*) FROM proxies WHERE protocol = "http"') as cursor: + row = await cursor.fetchone() + stats['http_count'] = row[0] if row else 0 + + async with db.execute('SELECT COUNT(*) FROM proxies WHERE protocol = "https"') as cursor: + row = await cursor.fetchone() + stats['https_count'] = row[0] if row else 0 + + async with db.execute('SELECT COUNT(*) FROM proxies WHERE protocol = "socks4"') as cursor: + row = await cursor.fetchone() + stats['socks4_count'] = row[0] if row else 0 + + async with db.execute('SELECT COUNT(*) FROM proxies WHERE protocol = "socks5"') as cursor: + row = await cursor.fetchone() + stats['socks5_count'] = row[0] if row else 0 + + async with db.execute('SELECT AVG(score) FROM proxies') as cursor: + row = await cursor.fetchone() + stats['avg_score'] = row[0] if row and row[0] else 0 + + return stats + + async def get_today_new_count(self): + """获取今日新增代理数量""" + try: + db = await self.get_connection() + query = ''' + SELECT COUNT(*) FROM proxies + WHERE DATE(last_check) = DATE('now', 'localtime') + ''' + async with db.execute(query) as cursor: + row = await cursor.fetchone() + return row[0] if row else 0 + except Exception as e: + logger.error(f"获取今日新增数量失败: {e}") + return 0 + + async def clean_invalid_proxies(self): + """清理无效代理（分数<=0）""" + db = await self.get_connection() + async with db.execute('DELETE FROM proxies WHERE score <= 0') as cursor: + deleted_count = cursor.rowcount + await db.commit() + return deleted_count diff --git a/core/validator.py b/core/validator.py new file mode 100644 index 0000000..a826c21 --- /dev/null +++ b/core/validator.py @@ -0,0 +1,76 @@ +import asyncio +import aiohttp +import random +import time +from core.log import logger + +class ProxyValidator: + def __init__(self, max_concurrency=50, timeout=5): + # 验证目标源（使用更适合代理验证的源） + self.http_sources = [ + "http://httpbin.org/ip", + "http://api.ipify.org" + ] + self.https_sources = [ + "https://httpbin.org/ip", + "https://api.ipify.org" + ] + self.semaphore = asyncio.Semaphore(max_concurrency) + self.timeout = timeout + self.session = None + + async def __aenter__(self): + # 允许通过 async with 管理 session + if not self.session: + self.session = aiohttp.ClientSession( + connector=aiohttp.TCPConnector(ssl=False, limit=0, force_close=True), + timeout=aiohttp.ClientTimeout(total=self.timeout, connect=3) + ) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if self.session: + await self.session.close() + + async def validate(self, ip, port, protocol='http'): + """ + 验证单个代理是否可用 + """ + protocol = protocol.lower() + sources = self.https_sources if protocol == 'https' else self.http_sources + test_url = random.choice(sources) + + # aiohttp 代理 URL 格式 + proxy_url = f"http://{ip}:{port}" + + async with self.semaphore: + start_time = time.time() + try: + # 复用 session + async with self.session.get( + test_url, + proxy=proxy_url, + allow_redirects=True, + timeout=aiohttp.ClientTimeout(total=self.timeout, connect=3) + ) as response: + # 检查状态码和响应内容 + if response.status in [200, 301, 302]: + try: + content = await response.text() + # 确保返回了有效的JSON响应 + if 'ip' in content.lower() or 'origin' in content.lower(): + latency = round((time.time() - start_time) * 1000, 2) + logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms") + return True, latency + except: + # 即使无法解析内容，如果状态码正常也认为可用 + latency = round((time.time() - start_time) * 1000, 2) + logger.info(f"验证成功: {ip}:{port} ({protocol}) - 延迟: {latency}ms") + return True, latency + return False, 0 + except asyncio.TimeoutError: + logger.warning(f"验证超时: {ip}:{port} ({protocol})") + return False, 0 + except Exception as e: + logger.warning(f"验证失败: {ip}:{port} ({protocol}) - {e}") + return False, 0 diff --git a/frontend/.eslintrc.json b/frontend/.eslintrc.json new file mode 100644 index 0000000..fd59231 --- /dev/null +++ b/frontend/.eslintrc.json @@ -0,0 +1,22 @@ +{ + "env": { + "browser": true, + "es2021": true, + "node": true + }, + "extends": [ + "eslint:recommended" + ], + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "rules": { + "no-console": "warn", + "no-unused-vars": "warn", + "semi": ["error", "never"], + "quotes": ["error", "single"], + "indent": ["error", 2], + "comma-dangle": ["error", "never"] + } +} diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/.prettierrc.json b/frontend/.prettierrc.json new file mode 100644 index 0000000..f4b0dfa --- /dev/null +++ b/frontend/.prettierrc.json @@ -0,0 +1,9 @@ +{ + "semi": false, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "none", + "printWidth": 100, + "arrowParens": "always", + "endOfLine": "lf" +} diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..1511959 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,5 @@ +# Vue 3 + Vite + +This template should help get you started developing with Vue 3 in Vite. The template uses Vue 3 ` + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..1ba150b --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,28 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs --fix --ignore-path .gitignore", + "format": "prettier --write src/" + }, + "dependencies": { + "@element-plus/icons-vue": "^2.3.2", + "axios": "^1.13.3", + "echarts": "^6.0.0", + "element-plus": "^2.13.1", + "pinia": "^3.0.4", + "vue": "^3.5.24", + "vue-router": "^4.6.4" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^6.0.1", + "vite": "^7.2.4", + "eslint": "^9.0.0", + "prettier": "^3.0.0" + } +} diff --git a/frontend/public/vite.svg b/frontend/public/vite.svg new file mode 100644 index 0000000..e7b8dfb --- /dev/null +++ b/frontend/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..0f4f41e --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,391 @@ + + + + + + + + + + + diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js new file mode 100644 index 0000000..4cc81b9 --- /dev/null +++ b/frontend/src/api/index.js @@ -0,0 +1,52 @@ +import axios from 'axios' +import { showError } from '../utils/message' + +const api = axios.create({ + baseURL: import.meta.env.VITE_API_BASE_URL || 'http://localhost:8923', + timeout: 30000 +}) + +api.interceptors.response.use( + response => response.data, + error => { + console.error('API请求错误:', error) + showError(error) + return Promise.reject(error) + } +) + +export const statsAPI = { + getStats: () => api.get('/api/stats') +} + +export const proxiesAPI = { + getProxies: (params) => api.post('/api/proxies', params), + getRandomProxy: () => api.get('/api/proxies/random'), + getProxyDetail: (ip, port) => api.get(`/api/proxies/${ip}/${port}`), + deleteProxy: (ip, port) => api.delete(`/api/proxies/${ip}/${port}`), + batchDeleteProxies: (proxies) => api.post('/api/proxies/batch-delete', { proxies }), + cleanInvalidProxies: () => api.delete('/api/proxies/clean-invalid'), + exportProxies: (format, protocol) => api.get(`/api/proxies/export/${format}`, { + params: { protocol }, + responseType: 'blob' + }) +} + +export const crawlerAPI = { + start: (numValidators = 50) => api.post('/api/crawler/start', { num_validators: numValidators }), + stop: () => api.post('/api/crawler/stop'), + getStatus: () => api.get('/api/crawler/status') +} + +export const schedulerAPI = { + setScheduler: (enabled, intervalMinutes = 60) => api.post('/api/scheduler', { enabled, interval_minutes: intervalMinutes }), + getStatus: () => api.get('/api/scheduler') +} + +export const pluginsAPI = { + getPlugins: () => api.get('/api/plugins'), + togglePlugin: (pluginId, enabled) => api.put(`/api/plugins/${pluginId}/toggle`, { enabled }), + crawlPlugin: (pluginId) => api.post(`/api/plugins/${pluginId}/crawl`) +} + +export default api diff --git a/frontend/src/assets/vue.svg b/frontend/src/assets/vue.svg new file mode 100644 index 0000000..770e9d3 --- /dev/null +++ b/frontend/src/assets/vue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/components/HelloWorld.vue b/frontend/src/components/HelloWorld.vue new file mode 100644 index 0000000..546ebbc --- /dev/null +++ b/frontend/src/components/HelloWorld.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/frontend/src/components/PageHeader.vue b/frontend/src/components/PageHeader.vue new file mode 100644 index 0000000..9bbcaf5 --- /dev/null +++ b/frontend/src/components/PageHeader.vue @@ -0,0 +1,36 @@ + + + + + diff --git a/frontend/src/components/ProtocolChart.vue b/frontend/src/components/ProtocolChart.vue new file mode 100644 index 0000000..bbadea8 --- /dev/null +++ b/frontend/src/components/ProtocolChart.vue @@ -0,0 +1,155 @@ + + + + + diff --git a/frontend/src/components/QuickActions.vue b/frontend/src/components/QuickActions.vue new file mode 100644 index 0000000..6a35e6b --- /dev/null +++ b/frontend/src/components/QuickActions.vue @@ -0,0 +1,114 @@ + + + + + diff --git a/frontend/src/components/StatCard.vue b/frontend/src/components/StatCard.vue new file mode 100644 index 0000000..ddb339d --- /dev/null +++ b/frontend/src/components/StatCard.vue @@ -0,0 +1,98 @@ + + + + + diff --git a/frontend/src/composables/useWebSocket.js b/frontend/src/composables/useWebSocket.js new file mode 100644 index 0000000..ff61770 --- /dev/null +++ b/frontend/src/composables/useWebSocket.js @@ -0,0 +1,76 @@ +import { ref } from 'vue' + +export function useWebSocket() { + const ws = ref(null) + const isExplicitDisconnect = ref(false) + let reconnectTimer = null + + function connect(url, onMessage, onError, onClose, onOpen, token) { + isExplicitDisconnect.value = false + + if (ws.value && ws.value.readyState === WebSocket.OPEN) { + console.log('WebSocket已经连接啦~') + return + } + + const wsUrl = token ? `${url}?token=${token}` : url + console.log('尝试连接WebSocket:', wsUrl) + ws.value = new WebSocket(wsUrl) + + ws.value.onopen = () => { + console.log('WebSocket连接成功啦~', ws.value.readyState) + if (reconnectTimer) { + clearTimeout(reconnectTimer) + reconnectTimer = null + } + onOpen?.() + } + + ws.value.onmessage = (event) => { + try { + const data = JSON.parse(event.data) + onMessage?.(data) + } catch (error) { + console.error('解析WebSocket消息失败:', error, event.data) + } + } + + ws.value.onerror = (error) => { + console.error('WebSocket错误:', error) + onError?.(error) + } + + ws.value.onclose = (event) => { + console.log('WebSocket连接关闭:', event.code, event.reason) + ws.value = null + + onClose?.(event) + + if (!isExplicitDisconnect.value) { + console.log('检测到异常断开，3秒后尝试重连...') + if (reconnectTimer) clearTimeout(reconnectTimer) + reconnectTimer = setTimeout(() => { + connect(url, onMessage, onError, onClose, onOpen) + }, 3000) + } + } + } + + function disconnect() { + isExplicitDisconnect.value = true + if (ws.value) { + ws.value.close() + ws.value = null + } + if (reconnectTimer) { + clearTimeout(reconnectTimer) + reconnectTimer = null + } + } + + return { + ws, + connect, + disconnect + } +} diff --git a/frontend/src/main.js b/frontend/src/main.js new file mode 100644 index 0000000..3a4b343 --- /dev/null +++ b/frontend/src/main.js @@ -0,0 +1,17 @@ +import { createApp } from 'vue' +import { createPinia } from 'pinia' +import ElementPlus from 'element-plus' +import 'element-plus/dist/index.css' +import router from './router' +import './style.css' +import './styles/element-plus.css' +import App from './App.vue' + +const app = createApp(App) +const pinia = createPinia() + +app.use(pinia) +app.use(router) +app.use(ElementPlus) + +app.mount('#app') diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js new file mode 100644 index 0000000..4055074 --- /dev/null +++ b/frontend/src/router/index.js @@ -0,0 +1,40 @@ +import { createRouter, createWebHistory } from 'vue-router' + +const routes = [ + { + path: '/', + redirect: '/dashboard' + }, + { + path: '/dashboard', + name: 'Dashboard', + component: () => import('../views/Dashboard.vue') + }, + { + path: '/proxies', + name: 'ProxyList', + component: () => import('../views/ProxyList.vue') + }, + { + path: '/crawler', + name: 'CrawlerTasks', + component: () => import('../views/CrawlerTasks.vue') + }, + { + path: '/plugins', + name: 'Plugins', + component: () => import('../views/Plugins.vue') + }, + { + path: '/settings', + name: 'Settings', + component: () => import('../views/Settings.vue') + } +] + +const router = createRouter({ + history: createWebHistory(), + routes +}) + +export default router diff --git a/frontend/src/stores/crawler.js b/frontend/src/stores/crawler.js new file mode 100644 index 0000000..bc5a4af --- /dev/null +++ b/frontend/src/stores/crawler.js @@ -0,0 +1,144 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { crawlerAPI, schedulerAPI } from '../api' +import { useWebSocket } from '../composables/useWebSocket' + +export const useCrawlerStore = defineStore('crawler', () => { + const running = ref(false) + const stats = ref({}) + const scheduled = ref(false) + const intervalMinutes = ref(60) + const progress = ref({ + total: 0, + current: 0, + success: 0, + failed: 0 + }) + const statusMessage = ref('') + + const { connect, disconnect } = useWebSocket() + + async function fetchStatus() { + try { + const response = await crawlerAPI.getStatus() + if (response.code === 200) { + running.value = response.data.running + stats.value = response.data.stats || {} + } + } catch (error) { + console.error('获取爬虫状态失败:', error) + } + } + + async function startCrawler(numValidators = 50) { + try { + const response = await crawlerAPI.start(numValidators) + if (response.code === 200) { + running.value = true + return true + } + } catch (error) { + console.error('启动爬虫失败:', error) + } + return false + } + + async function stopCrawler() { + try { + const response = await crawlerAPI.stop() + if (response.code === 200) { + running.value = false + return true + } + } catch (error) { + console.error('停止爬虫失败:', error) + } + return false + } + + async function fetchSchedulerStatus() { + try { + const response = await schedulerAPI.getStatus() + if (response.code === 200) { + scheduled.value = response.data.enabled + intervalMinutes.value = response.data.interval_minutes + } + } catch (error) { + console.error('获取定时任务状态失败:', error) + } + } + + async function setScheduler(enabled, interval = 60) { + try { + const response = await schedulerAPI.setScheduler(enabled, interval) + if (response.code === 200) { + scheduled.value = enabled + intervalMinutes.value = interval + return true + } + } catch (error) { + console.error('设置定时任务失败:', error) + } + return false + } + + function connectWebSocket() { + const wsUrl = import.meta.env.VITE_WS_BASE_URL || 'ws://localhost:8923' + const token = import.meta.env.VITE_API_KEY + + connect( + `${wsUrl}/ws`, + (data) => { + console.log('收到WebSocket消息:', data) + if (data.type === 'progress') { + console.log('更新进度:', data.data) + progress.value = { + found: data.data.found || 0, + verified: data.data.verified || 0, + success_rate: data.data.success_rate || 0 + } + console.log('进度更新后:', progress.value) + } else if (data.type === 'status') { + statusMessage.value = data.data.message + if (data.data.status === 'completed') { + running.value = false + } else if (data.data.status === 'stopped') { + running.value = false + } else if (data.data.status === 'running') { + running.value = true + } + } + }, + (error) => { + console.error('WebSocket错误:', error) + }, + (event) => { + console.log('WebSocket连接关闭:', event.code, event.reason) + }, + () => { + console.log('WebSocket连接成功啦~') + }, + token + ) + } + + function disconnectWebSocket() { + disconnect() + } + + return { + running, + stats, + scheduled, + intervalMinutes, + progress, + statusMessage, + fetchStatus, + startCrawler, + stopCrawler, + fetchSchedulerStatus, + setScheduler, + connectWebSocket, + disconnectWebSocket + } +}) diff --git a/frontend/src/stores/plugins.js b/frontend/src/stores/plugins.js new file mode 100644 index 0000000..2969939 --- /dev/null +++ b/frontend/src/stores/plugins.js @@ -0,0 +1,58 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { pluginsAPI } from '../api' + +export const usePluginsStore = defineStore('plugins', () => { + const plugins = ref([]) + const loading = ref(false) + + async function fetchPlugins() { + loading.value = true + try { + const response = await pluginsAPI.getPlugins() + if (response.code === 200) { + plugins.value = response.data.plugins || [] + } + } catch (error) { + console.error('获取插件列表失败:', error) + } finally { + loading.value = false + } + } + + async function togglePlugin(pluginId, enabled) { + try { + const response = await pluginsAPI.togglePlugin(pluginId, enabled) + if (response.code === 200) { + const plugin = plugins.value.find(p => p.id === pluginId) + if (plugin) { + plugin.enabled = enabled + } + return true + } + } catch (error) { + console.error('切换插件状态失败:', error) + } + return false + } + + async function crawlPlugin(pluginId) { + try { + const response = await pluginsAPI.crawlPlugin(pluginId) + if (response.code === 200) { + return true + } + } catch (error) { + console.error('触发插件爬取失败:', error) + } + return false + } + + return { + plugins, + loading, + fetchPlugins, + togglePlugin, + crawlPlugin + } +}) diff --git a/frontend/src/stores/proxy.js b/frontend/src/stores/proxy.js new file mode 100644 index 0000000..16e64e7 --- /dev/null +++ b/frontend/src/stores/proxy.js @@ -0,0 +1,108 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { proxiesAPI, statsAPI } from '../api' + +export const useProxyStore = defineStore('proxy', () => { + const proxies = ref([]) + const total = ref(0) + const loading = ref(false) + const stats = ref({}) + + const availableCount = computed(() => stats.value.available || 0) + const totalCount = computed(() => stats.value.total || 0) + + async function fetchStats() { + try { + const response = await statsAPI.getStats() + if (response.code === 200) { + stats.value = response.data + } + } catch (error) { + console.error('获取统计信息失败:', error) + } + } + + async function fetchProxies(params) { + loading.value = true + try { + const response = await proxiesAPI.getProxies(params) + if (response.code === 200) { + proxies.value = response.data.list + total.value = response.data.total + } + } catch (error) { + console.error('获取代理列表失败:', error) + } finally { + loading.value = false + } + } + + async function deleteProxy(ip, port) { + try { + const response = await proxiesAPI.deleteProxy(ip, port) + if (response.code === 200) { + return true + } + } catch (error) { + console.error('删除代理失败:', error) + } + return false + } + + async function batchDeleteProxies(proxyList) { + try { + const response = await proxiesAPI.batchDeleteProxies(proxyList) + if (response.code === 200) { + return response.data.deleted_count + } + } catch (error) { + console.error('批量删除代理失败:', error) + } + return 0 + } + + async function cleanInvalidProxies() { + try { + const response = await proxiesAPI.cleanInvalidProxies() + if (response.code === 200) { + return response.data.deleted_count + } + } catch (error) { + console.error('清理无效代理失败:', error) + } + return 0 + } + + async function exportProxies(format, protocol) { + try { + const response = await proxiesAPI.exportProxies(format, protocol) + const url = window.URL.createObjectURL(new Blob([response])) + const link = document.createElement('a') + link.href = url + link.setAttribute('download', `proxies.${format}`) + document.body.appendChild(link) + link.click() + link.remove() + window.URL.revokeObjectURL(url) + return true + } catch (error) { + console.error('导出代理失败:', error) + } + return false + } + + return { + proxies, + total, + loading, + stats, + availableCount, + totalCount, + fetchStats, + fetchProxies, + deleteProxy, + batchDeleteProxies, + cleanInvalidProxies, + exportProxies + } +}) diff --git a/frontend/src/style.css b/frontend/src/style.css new file mode 100644 index 0000000..63c8e3f --- /dev/null +++ b/frontend/src/style.css @@ -0,0 +1,569 @@ +:root { + --theme-primary: #00D4FF; + --theme-primary-light: #00B8E0; + --theme-primary-dark: #0090B0; + --theme-bg: linear-gradient(135deg, #0A0E27 0%, #1A1F3A 50%, #162032 100%); + --theme-bg-solid: #0A0E27; + --theme-bg-light: #1A1F3A; + --theme-bg-card: rgba(26, 31, 58, 0.95); + --theme-text: #E0E6FF; + --theme-text-secondary: #9CA3AF; + --theme-border: #2D3748; + --theme-border-light: #3A4558; + --theme-gradient-1: linear-gradient(135deg, #00D4FF 0%, #00B8E0 100%); + --theme-gradient-2: linear-gradient(135deg, #FF6B9D 0%, #FF8E53 100%); + --theme-gradient-3: linear-gradient(135deg, #00FF88 0%, #00CC6A 100%); + + --el-color-primary: #00D4FF; + --el-color-primary-light-3: #00B8E0; + --el-color-primary-light-5: #4A90E2; + --el-color-primary-light-7: #00B8FF; + --el-color-primary-light-8: #00D4FF; + --el-color-primary-light-9: #00E5FF; + --el-color-primary-dark-2: #0090B0; + --el-color-success: #00FF88; + --el-color-warning: #FFB800; + --el-color-danger: #FF3366; + --el-color-info: #A855F7; + --el-bg-color: #0A0E27; + --el-bg-color-page: #0A0E27; + --el-text-color-primary: #E0E6FF; + --el-text-color-regular: #9CA3AF; + --el-border-color: #2D3748; + --el-border-color-light: #2D3748; + --el-fill-color-blank: #1A1F3A; + --el-fill-color-light: #1A1F3A; +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; + line-height: 1.6; + color: var(--theme-text); + background: var(--theme-bg); + background-attachment: fixed; + background-size: 400% 400%; + animation: gradientShift 15s ease infinite; + overflow-x: hidden; +} + +@keyframes gradientShift { + 0%, 100% { + background-position: 0% 50%; + } + 50% { + background-position: 100% 50%; + } +} + +a { + text-decoration: none; + color: var(--theme-primary); + transition: color 0.3s ease; +} + +a:hover { + color: var(--theme-primary-light); +} + +.el-card { + border-radius: 16px; + border: 1px solid rgba(0, 212, 255, 0.15); + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); + background-color: var(--theme-bg-card); + backdrop-filter: blur(10px); +} + +.el-card:hover { + box-shadow: 0 8px 32px rgba(0, 212, 255, 0.2); + transform: translateY(-4px); + border-color: rgba(0, 212, 255, 0.4); +} + +.el-button--primary { + background: var(--theme-gradient-1); + border: none; + border-radius: 8px; + padding: 10px 24px; + font-weight: 600; + transition: all 0.3s ease; + color: #0A0E27; + position: relative; + overflow: hidden; +} + +.el-button--primary::before { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.3), transparent); + transition: left 0.5s ease; +} + +.el-button--primary:hover::before { + left: 100%; +} + +.el-button--primary:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(0, 212, 255, 0.4); +} + +.el-button--success { + background: var(--theme-gradient-3); + border: none; + border-radius: 8px; + padding: 10px 24px; + font-weight: 600; + transition: all 0.3s ease; + color: #0A0E27; + position: relative; + overflow: hidden; +} + +.el-button--success::before { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.3), transparent); + transition: left 0.5s ease; +} + +.el-button--success:hover::before { + left: 100%; +} + +.el-button--success:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(0, 255, 136, 0.4); +} + +.el-button--danger { + background: var(--theme-gradient-2); + border: none; + border-radius: 8px; + padding: 10px 24px; + font-weight: 600; + transition: all 0.3s ease; + color: white; + position: relative; + overflow: hidden; +} + +.el-button--danger::before { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.3), transparent); + transition: left 0.5s ease; +} + +.el-button--danger:hover::before { + left: 100%; +} + +.el-button--danger:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(255, 107, 157, 0.4); +} + +.el-button--warning { + background-color: #FFB800; + border-color: #FFB800; + border-radius: 8px; + padding: 10px 24px; + font-weight: 600; + transition: all 0.3s ease; + color: #0A0E27; +} + +.el-button--warning:hover { + background-color: #E5A600; + border-color: #E5A600; + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(255, 184, 0, 0.3); +} + +.el-button--default { + border: 1px solid var(--theme-border); + border-radius: 8px; + padding: 10px 24px; + font-weight: 600; + background-color: var(--theme-bg-light); + color: var(--theme-text); + transition: all 0.3s ease; +} + +.el-button--default:hover { + border-color: var(--theme-primary); + color: var(--theme-primary); + transform: translateY(-2px); + box-shadow: 0 2px 8px rgba(0, 212, 255, 0.2); +} + +.el-input__wrapper { + border-radius: 8px; + box-shadow: 0 0 0 1px var(--theme-border) inset; + transition: all 0.3s ease; + background-color: var(--theme-bg-light); +} + +.el-input__wrapper:hover { + box-shadow: 0 0 0 1px var(--theme-primary-light) inset; +} + +.el-input__wrapper.is-focus { + box-shadow: 0 0 0 2px var(--theme-primary) inset; +} + +.el-select .el-input__wrapper { + border-radius: 8px; +} + +.el-table { + border-radius: 12px; + overflow: hidden; + border: 1px solid var(--theme-border); + background-color: var(--theme-bg-card); +} + +.el-table th { + background-color: var(--theme-bg-light); + color: var(--theme-primary); + font-weight: 600; + border-bottom: 2px solid var(--theme-primary); + text-transform: uppercase; + letter-spacing: 1px; +} + +.el-table td { + border-bottom: 1px solid var(--theme-border); + background-color: var(--theme-bg-card); +} + +.el-table tr:hover > td { + background-color: var(--theme-bg-light); +} + +.el-tag { + border-radius: 6px; + border: 1px solid var(--theme-border); + padding: 4px 12px; + font-weight: 600; + font-size: 13px; +} + +.el-tag--primary { + background-color: rgba(0, 212, 255, 0.15); + color: var(--theme-primary); + border-color: var(--theme-primary); +} + +.el-tag--success { + background-color: rgba(0, 255, 136, 0.15); + color: #00FF88; + border-color: #00FF88; +} + +.el-tag--warning { + background-color: rgba(255, 184, 0, 0.15); + color: #FFB800; + border-color: #FFB800; +} + +.el-tag--danger { + background-color: rgba(255, 51, 102, 0.15); + color: #FF3366; + border-color: #FF3366; +} + +.el-tag--info { + background-color: rgba(168, 85, 247, 0.15); + color: #A855F7; + border-color: #A855F7; +} + +.el-rate__icon { + color: var(--theme-primary); +} + +.el-pagination.is-background .el-pager li:not(.is-disabled).is-active { + background-color: var(--theme-primary); + color: #0A0E27; + font-weight: 600; +} + +.el-pagination.is-background .btn-next, +.el-pagination.is-background .btn-prev { + background-color: var(--theme-bg-light); + color: var(--theme-primary); + border-radius: 6px; + border: 1px solid var(--theme-border); + transition: all 0.3s ease; +} + +.el-pagination.is-background .btn-next:hover, +.el-pagination.is-background .btn-prev:hover { + background-color: var(--theme-primary); + color: #0A0E27; + border-color: var(--theme-primary); +} + +.el-pagination.is-background .el-pager li { + background-color: var(--theme-bg-light); + color: var(--theme-primary); + border-radius: 6px; + margin: 0 4px; + border: 1px solid var(--theme-border); + transition: all 0.3s ease; +} + +.el-pagination.is-background .el-pager li:hover { + background-color: var(--theme-primary-light); + color: #0A0E27; +} + +.el-message { + border-radius: 12px; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); + background-color: var(--theme-bg-card); + border: 1px solid var(--theme-border); +} + +.el-message--success .el-message__content { + color: #00FF88; + font-weight: 600; +} + +.el-message--error .el-message__content { + color: #FF3366; + font-weight: 600; +} + +.el-message--warning .el-message__content { + color: #FFB800; + font-weight: 600; +} + +.el-message--info .el-message__content { + color: #A855F7; + font-weight: 600; +} + +.el-progress-bar__inner { + background: var(--theme-gradient-1); + border-radius: 8px; + position: relative; + overflow: hidden; +} + +.el-progress-bar__inner::after { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.2) 50%, + transparent 100% + ); + animation: progressShine 2s infinite; +} + +@keyframes progressShine { + 0% { + transform: translateX(-100%); + } + 100% { + transform: translateX(100%); + } +} + +.el-switch.is-checked .el-switch__core { + background-color: var(--theme-primary); +} + +.el-switch.is-checked .el-switch__action { + background-color: white; +} + +.el-alert { + border-radius: 12px; + border: 1px solid var(--theme-border); + background-color: var(--theme-bg-card); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.el-alert--success { + background-color: rgba(0, 255, 136, 0.1); + border-color: #00FF88; +} + +.el-alert--info { + background-color: rgba(168, 85, 247, 0.1); + border-color: #A855F7; +} + +.el-alert--warning { + background-color: rgba(255, 184, 0, 0.1); + border-color: #FFB800; +} + +.el-alert--error { + background-color: rgba(255, 51, 102, 0.1); + border-color: #FF3366; +} + +.el-dialog { + border-radius: 16px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4); + background-color: var(--theme-bg-card); + border: 1px solid var(--theme-border); +} + +.el-dialog__header { + background-color: var(--theme-bg-light); + border-radius: 16px 16px 0 0; + padding: 20px; + border-bottom: 1px solid var(--theme-border); +} + +.el-dialog__title { + color: var(--theme-primary); + font-weight: 600; + font-size: 18px; +} + +.el-dialog__body { + color: var(--theme-text); +} + +.el-dropdown-menu { + border-radius: 12px; + border: 1px solid var(--theme-border); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3); + background-color: var(--theme-bg-card); +} + +.el-dropdown-menu__item:hover { + background-color: var(--theme-bg-light); + color: var(--theme-primary); + border-radius: 6px; + margin: 2px 6px; +} + +.el-notification { + border-radius: 16px; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + background-color: var(--theme-bg-card); + border: 1px solid var(--theme-border); +} + +.el-notification__title { + color: var(--theme-primary); + font-weight: 600; +} + +.el-card__header { + border-bottom: 1px solid var(--theme-border); + padding: 16px 20px; + background-color: var(--theme-bg-light); +} + +.el-card__body { + padding: 20px; + color: var(--theme-text); +} + +.el-form-item__label { + color: var(--theme-text); + font-weight: 600; +} + +.el-input-number { + border-radius: 8px; +} + +.el-input-number .el-input__wrapper { + border-radius: 8px; + background-color: var(--theme-bg-light); +} + +.el-radio-button__inner { + border-radius: 8px !important; + border: 1px solid var(--theme-border); + background-color: var(--theme-bg-light); + color: var(--theme-text); + transition: all 0.3s ease; + font-weight: 600; +} + +.el-radio-button__original-radio:checked + .el-radio-button__inner { + background-color: var(--theme-primary); + border-color: var(--theme-primary); + color: #0A0E27; + font-weight: 700; +} + +.el-menu { + border-right: 1px solid var(--theme-border); + background-color: var(--theme-bg); +} + +.el-menu-item { + border-radius: 8px; + margin: 4px 8px; + transition: all 0.3s ease; + font-weight: 600; + color: var(--theme-text-secondary); +} + +.el-menu-item:hover { + background-color: var(--theme-bg-light); + color: var(--theme-primary); + transform: translateX(4px); +} + +.el-menu-item.is-active { + background-color: var(--theme-primary); + color: #0A0E27; + font-weight: 700; +} + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background-color: var(--theme-bg); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb { + background-color: var(--theme-border-light); + border-radius: 4px; + transition: all 0.3s ease; +} + +::-webkit-scrollbar-thumb:hover { + background-color: var(--theme-primary-light); +} diff --git a/frontend/src/styles/element-plus.css b/frontend/src/styles/element-plus.css new file mode 100644 index 0000000..e1bdf82 --- /dev/null +++ b/frontend/src/styles/element-plus.css @@ -0,0 +1,377 @@ +/* Element Plus 全局样式覆盖 - 强制去除所有黑色边框 */ + +/* 输入框 */ +.el-input__wrapper { + box-shadow: 0 0 0 1px #FFE4EC inset !important; +} + +.el-input__wrapper:hover { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +.el-input__wrapper.is-focus { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +/* 下拉选择框 */ +.el-select__wrapper { + box-shadow: 0 0 0 1px #FFE4EC inset !important; +} + +.el-select__wrapper:hover { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +.el-select__wrapper.is-focused { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +.el-select__placeholder { + color: #999999 !important; +} + +.el-select__caret { + color: #FF6B9D !important; +} + +.el-select-dropdown { + border: 1px solid #FFE4EC !important; + box-shadow: 0 2px 12px rgba(255, 107, 157, 0.1) !important; + background: white !important; +} + +.el-select-dropdown__item { + color: #333333 !important; +} + +.el-select-dropdown__item:hover { + background: rgba(255, 107, 157, 0.1) !important; + color: #FF6B9D !important; +} + +.el-select-dropdown__item.is-selected { + color: #FF6B9D !important; + font-weight: 600; +} + +/* 数字输入框 */ +.el-input-number__decrease, +.el-input-number__increase { + background: #FFF9FB !important; + color: #999999 !important; + border: 1px solid #FFE4EC !important; +} + +.el-input-number__decrease:hover, +.el-input-number__increase:hover { + background: rgba(255, 107, 157, 0.1) !important; + color: #FF6B9D !important; + border-color: #FF6B9D !important; +} + +.el-input-number__decrease.is-disabled, +.el-input-number__increase.is-disabled { + color: #cccccc !important; + border-color: #FFE4EC !important; +} + +.el-input-number__wrapper { + box-shadow: 0 0 0 1px #FFE4EC inset !important; +} + +.el-input-number__wrapper:hover { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +.el-input-number__wrapper.is-focus { + box-shadow: 0 0 0 1px #FF6B9D inset !important; +} + +/* 按钮 */ +.el-button { + border: 1px solid #FFE4EC !important; +} + +.el-button--primary { + background: linear-gradient(135deg, #FF6B9D 0%, #FF8FB3 100%) !important; + border-color: #FF6B9D !important; + color: white !important; +} + +.el-button--primary:hover { + background: linear-gradient(135deg, #FF5A8F 0%, #FF7FA7 100%) !important; + box-shadow: 0 4px 12px rgba(255, 107, 157, 0.3) !important; +} + +.el-button--success { + background: linear-gradient(135deg, #00D4FF 0%, #00E5FF 100%) !important; + border-color: #00D4FF !important; + color: white !important; +} + +.el-button--success:hover { + background: linear-gradient(135deg, #00C4F0 0%, #00D4E8 100%) !important; + box-shadow: 0 4px 12px rgba(0, 212, 255, 0.3) !important; +} + +.el-button--warning { + background: linear-gradient(135deg, #FFB800 0%, #FFD000 100%) !important; + border-color: #FFB800 !important; + color: white !important; +} + +.el-button--warning:hover { + background: linear-gradient(135deg, #FFA700 0%, #FFC000 100%) !important; + box-shadow: 0 4px 12px rgba(255, 184, 0, 0.3) !important; +} + +.el-button--danger { + background: linear-gradient(135deg, #FF6B6B 0%, #FF8B8B 100%) !important; + border-color: #FF6B6B !important; + color: white !important; +} + +.el-button--danger:hover { + background: linear-gradient(135deg, #FF5A5A 0%, #FF7A7A 100%) !important; + box-shadow: 0 4px 12px rgba(255, 107, 107, 0.3) !important; +} + +/* 卡片 */ +.el-card { + border: 1px solid #FFE4EC !important; + box-shadow: 0 2px 12px rgba(255, 107, 157, 0.08) !important; +} + +.el-card__header { + border-bottom: 1px solid #FFE4EC !important; +} + +.el-card__body { + background: rgba(255, 255, 255, 0.95) !important; +} + +/* 表格 */ +.el-table { + border: 1px solid #FFE4EC !important; + background: white !important; +} + +.el-table th.el-table__cell { + background: #FFF9FB !important; + color: #333333 !important; + border-bottom: 1px solid #FFE4EC !important; +} + +.el-table td.el-table__cell { + border-bottom: 1px solid #FFE4EC !important; +} + +.el-table__border-left { + border-left: 1px solid #FFE4EC !important; +} + +.el-table__border-right { + border-right: 1px solid #FFE4EC !important; +} + +.el-table tr:hover > td { + background: #FFF0F5 !important; +} + +.el-table__body tr.current-row > td.el-table__cell { + background: #FFE4EC !important; +} + +/* Checkbox */ +.el-checkbox__inner { + border: 1px solid #FFE4EC !important; + background: white !important; +} + +.el-checkbox__inner:hover { + border-color: #FF6B9D !important; +} + +.el-checkbox__input.is-checked .el-checkbox__inner { + background: #FF6B9D !important; + border-color: #FF6B9D !important; +} + +.el-checkbox__input.is-disabled .el-checkbox__inner { + background: #f5f5f5 !important; + border-color: #e4e7ed !important; +} + +/* 分页器 */ +.el-pagination button { + border: 1px solid #FFE4EC !important; + background: #FFF9FB !important; + color: #999999 !important; +} + +.el-pagination button:hover { + background: rgba(255, 107, 157, 0.1) !important; + color: #FF6B9D !important; +} + +.el-pagination li.is-active { + background: #FF6B9D !important; + color: white !important; + border-color: #FF6B9D !important; +} + +.el-pager li { + background: #FFF9FB !important; + color: #999999 !important; + border: 1px solid #FFE4EC !important; +} + +.el-pager li:hover { + color: #FF6B9D !important; +} + +/* Tag */ +.el-tag { + border: 1px solid #FFE4EC !important; +} + +.el-tag--primary { + background: rgba(255, 107, 157, 0.1) !important; + color: #FF6B9D !important; + border-color: rgba(255, 107, 157, 0.3) !important; +} + +.el-tag--success { + background: rgba(0, 212, 255, 0.1) !important; + color: #00D4FF !important; + border-color: rgba(0, 212, 255, 0.3) !important; +} + +.el-tag--warning { + background: rgba(255, 184, 0, 0.1) !important; + color: #FFB800 !important; + border-color: rgba(255, 184, 0, 0.3) !important; +} + +.el-tag--danger { + background: rgba(255, 107, 107, 0.1) !important; + color: #FF6B6B !important; + border-color: rgba(255, 107, 107, 0.3) !important; +} + +/* Rate 评分 */ +.el-rate__icon { + color: #FFE4EC !important; +} + +.el-rate__icon.hover { + color: #FF6B9D !important; +} + +/* Dialog 对话框 */ +.el-dialog { + border: 1px solid #FFE4EC !important; +} + +.el-dialog__header { + border-bottom: 1px solid #FFE4EC !important; +} + +.el-dialog__body { + background: white !important; +} + +.el-dialog__footer { + border-top: 1px solid #FFE4EC !important; +} + +/* Dropdown 下拉菜单 */ +.el-dropdown-menu { + border: 1px solid #FFE4EC !important; + box-shadow: 0 2px 12px rgba(255, 107, 157, 0.1) !important; +} + +.el-dropdown-menu__item { + color: #333333 !important; +} + +.el-dropdown-menu__item:hover { + background: rgba(255, 107, 157, 0.1) !important; + color: #FF6B9D !important; +} + +/* Scrollbar 滚动条 */ +.el-scrollbar__wrap::-webkit-scrollbar { + width: 6px; + height: 6px; +} + +.el-scrollbar__wrap::-webkit-scrollbar-thumb { + background: #FFE4EC; + border-radius: 3px; +} + +.el-scrollbar__wrap::-webkit-scrollbar-thumb:hover { + background: #FF6B9D; +} + +/* Form 表单 */ +.el-form-item__label { + color: #666666 !important; +} + +.el-form-item__error { + color: #FF6B6B !important; +} + +/* Message 消息提示 */ +.el-message { + border: 1px solid #FFE4EC !important; + box-shadow: 0 4px 16px rgba(255, 107, 157, 0.15) !important; +} + +.el-message--success { + background: rgba(0, 212, 255, 0.1) !important; + border-color: rgba(0, 212, 255, 0.3) !important; + color: #00D4FF !important; +} + +.el-message--error { + background: rgba(255, 107, 107, 0.1) !important; + border-color: rgba(255, 107, 107, 0.3) !important; + color: #FF6B6B !important; +} + +.el-message--warning { + background: rgba(255, 184, 0, 0.1) !important; + border-color: rgba(255, 184, 0, 0.3) !important; + color: #FFB800 !important; +} + +.el-message--info { + background: rgba(255, 107, 157, 0.1) !important; + border-color: rgba(255, 107, 157, 0.3) !important; + color: #FF6B9D !important; +} + +/* MessageBox 弹窗 */ +.el-message-box { + border: 1px solid #FFE4EC !important; + box-shadow: 0 4px 16px rgba(255, 107, 157, 0.15) !important; +} + +.el-message-box__header { + border-bottom: 1px solid #FFE4EC !important; +} + +.el-message-box__title { + color: #FF6B9D !important; +} + +.el-message-box__content { + color: #333333 !important; +} + +.el-message-box__btns { + border-top: 1px solid #FFE4EC !important; +} diff --git a/frontend/src/utils/message.js b/frontend/src/utils/message.js new file mode 100644 index 0000000..1a795cd --- /dev/null +++ b/frontend/src/utils/message.js @@ -0,0 +1,36 @@ +import { ElMessage } from 'element-plus' + +export const showSuccess = (message) => { + ElMessage.success(message) +} + +export const showError = (error) => { + let message = '操作失败啦~' + + if (error) { + if (typeof error === 'string') { + message = error + } else if (error.response) { + const { data, status } = error.response + if (data && data.message) { + message = data.message + } else if (data && data.error) { + message = data.error + } else { + message = `请求失败 (${status})` + } + } else if (error.message) { + message = error.message + } + } + + ElMessage.error(message) +} + +export const showWarning = (message) => { + ElMessage.warning(message) +} + +export const showInfo = (message) => { + ElMessage.info(message) +} diff --git a/frontend/src/views/CrawlerTasks.vue b/frontend/src/views/CrawlerTasks.vue new file mode 100644 index 0000000..4f06031 --- /dev/null +++ b/frontend/src/views/CrawlerTasks.vue @@ -0,0 +1,421 @@ + + + + + diff --git a/frontend/src/views/Dashboard.vue b/frontend/src/views/Dashboard.vue new file mode 100644 index 0000000..4c33d4b --- /dev/null +++ b/frontend/src/views/Dashboard.vue @@ -0,0 +1,209 @@ + + + + + diff --git a/frontend/src/views/Plugins.vue b/frontend/src/views/Plugins.vue new file mode 100644 index 0000000..c06c84f --- /dev/null +++ b/frontend/src/views/Plugins.vue @@ -0,0 +1,209 @@ + + + + + diff --git a/frontend/src/views/ProxyList.vue b/frontend/src/views/ProxyList.vue new file mode 100644 index 0000000..f77faa5 --- /dev/null +++ b/frontend/src/views/ProxyList.vue @@ -0,0 +1,332 @@ + + + + + diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue new file mode 100644 index 0000000..998196d --- /dev/null +++ b/frontend/src/views/Settings.vue @@ -0,0 +1,200 @@ + + + + + diff --git a/frontend/vite.config.js b/frontend/vite.config.js new file mode 100644 index 0000000..f9684cb --- /dev/null +++ b/frontend/vite.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [vue()], + server: { + port: 6173 + } +}) diff --git a/main.py b/main.py new file mode 100644 index 0000000..f34e30e --- /dev/null +++ b/main.py @@ -0,0 +1,80 @@ +import asyncio +from core.plugin_manager import PluginManager +from core.sqlite import SQLiteManager +from core.validator import ProxyValidator +from core.log import logger + +# 异步队列，增大缓冲区以适应更高并发 +proxy_queue = asyncio.Queue(maxsize=500) + +async def run_crawler(): + """生产者：抓取代理并放入队列""" + logger.info("后台爬虫任务启动...") + manager = PluginManager() + + count = 0 + async for ip, port, protocol in manager.run_all(): + await proxy_queue.put((ip, port, protocol)) + count += 1 + + logger.info(f"爬虫抓取阶段完成，共发现 {count} 个潜在代理。") + +async def run_validator(db, validator): + """消费者：从队列获取代理并验证入库""" + verified_count = 0 + + while True: + proxy = await proxy_queue.get() + if proxy is None: + proxy_queue.task_done() + break + + ip, port, protocol = proxy + try: + is_valid, latency = await validator.validate(ip, port, protocol) + if is_valid: + logger.info(f"验证通过: {ip}:{port} ({protocol}) - 延迟: {latency}ms") + await db.insert_proxy(ip, port, protocol) + verified_count += 1 + except Exception as e: + logger.error(f"验证器异常: {e}") + finally: + proxy_queue.task_done() + + if verified_count > 0: + logger.info(f"验证协程完成，入库 {verified_count} 个代理。") + +async def main(): + logger.info("=== ProxyPool 加速启动 ===") + + db = SQLiteManager() + await db.init_db() + + # 大幅提升并发参数 + # max_concurrency 限制底层请求并发，num_validators 决定上层消费速度 + async with ProxyValidator(max_concurrency=200) as validator: + num_validators = 100 + + # 启动生产者 + crawler_task = asyncio.create_task(run_crawler()) + + # 启动验证协程 + validator_tasks = [asyncio.create_task(run_validator(db, validator)) for _ in range(num_validators)] + + await crawler_task + + # 发送退出信号 + for _ in range(num_validators): + await proxy_queue.put(None) + + await proxy_queue.join() + await asyncio.gather(*validator_tasks) + + total = await db.count_proxies() + logger.info(f"=== 运行结束，当前池内总数: {total} ===") + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("程序手动停止") diff --git a/plugins/fate0.py b/plugins/fate0.py new file mode 100644 index 0000000..a34d182 --- /dev/null +++ b/plugins/fate0.py @@ -0,0 +1,61 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +import json +import asyncio + +class Fate0Plugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "Fate0聚合源" + # 这是一个持续更新的高质量代理聚合列表 + self.urls = ["https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list"] + + async def parse(self, html): + if not html: + return + + count = 0 + # fate0 的数据格式是每行一个 JSON 对象 + for line in html.split('\n'): + if not line.strip(): + continue + try: + data = json.loads(line) + ip = data.get('host') + port = data.get('port') + protocol = data.get('type', 'http') + + if ip and port: + yield ip, int(port), protocol + count += 1 + except Exception: + continue + + if count > 0: + logger.info(f"{self.name} 解析完成，获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = Fate0Plugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/ip3366.py b/plugins/ip3366.py new file mode 100644 index 0000000..70c277e --- /dev/null +++ b/plugins/ip3366.py @@ -0,0 +1,74 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +from bs4 import BeautifulSoup +import re +import asyncio + +VALID_PROTOCOLS = ['http', 'https', 'socks4', 'socks5'] + +class Ip3366Plugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "IP3366" + # 抓取高匿和普通代理的前 5 页 + self.urls = [ + f"http://www.ip3366.net/free/?stype=1&page={i}" for i in range(1, 6) + ] + [ + f"http://www.ip3366.net/free/?stype=2&page={i}" for i in range(1, 6) + ] + + async def parse(self, html): + if not html: + return + + soup = BeautifulSoup(html, 'lxml') + list_div = soup.find('div', id='list') + if not list_div: return + + table = list_div.find('table') + if not table: return + + rows = table.find_all('tr') + count = 0 + for row in rows: + tds = row.find_all('td') + if len(tds) >= 5: + ip = tds[0].get_text(strip=True) + port = tds[1].get_text(strip=True) + protocol = tds[4].get_text(strip=True).lower() if len(tds) > 4 else 'http' + + if protocol not in VALID_PROTOCOLS: + protocol = 'http' + + if re.match(r'^\d+\.\d+\.\d+\.\d+$', ip) and port.isdigit(): + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = Ip3366Plugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/ip89.py b/plugins/ip89.py new file mode 100644 index 0000000..7038bd1 --- /dev/null +++ b/plugins/ip89.py @@ -0,0 +1,69 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +from bs4 import BeautifulSoup +import re +import asyncio + +class Ip89Plugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "89免费代理" + # 抓取前 5 页 + self.urls = [ + f"https://www.89ip.cn/index_{i}.html" for i in range(1, 6) + ] + + async def parse(self, html): + """ + 解析 89ip 页面 + """ + if not html: + return + + soup = BeautifulSoup(html, 'lxml') + table = soup.find('table', class_='layui-table') + if not table: + return + + rows = table.find_all('tr') + count = 0 + for row in rows: + tds = row.find_all('td') + if len(tds) >= 2: + ip = tds[0].get_text(strip=True) + port = tds[1].get_text(strip=True) + # 89ip 通常不直接写协议，默认尝试 http + protocol = 'http' + + if re.match(r'^\d+\.\d+\.\d+\.\d+$', ip) and port.isdigit(): + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = Ip89Plugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/kuaidaili.py b/plugins/kuaidaili.py new file mode 100644 index 0000000..fc88292 --- /dev/null +++ b/plugins/kuaidaili.py @@ -0,0 +1,79 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +from bs4 import BeautifulSoup +import re +import asyncio + +VALID_PROTOCOLS = ['http', 'https', 'socks4', 'socks5'] + +class KuaiDaiLiPlugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "快代理" + # 抓取国内高匿和国内普通代理的前 10 页 + self.urls = [ + f"https://www.kuaidaili.com/free/inha/{i}/" for i in range(1, 11) + ] + [ + f"https://www.kuaidaili.com/free/intr/{i}/" for i in range(1, 11) + ] + + async def parse(self, html): + """ + 解析快代理页面 + """ + if not html: + return + + soup = BeautifulSoup(html, 'lxml') + # 快代理的表格在 tbody 中 + table = soup.find('table') + if not table: + # 尝试通过正则表达式匹配可能被加密或特殊处理的数据 + logger.warning(f"{self.name} 未能找到表格，可能是触发了反爬或结构变化") + return + + rows = table.find_all('tr') + count = 0 + for row in rows: + tds = row.find_all('td') + if len(tds) >= 5: + ip = tds[0].get_text(strip=True) + port = tds[1].get_text(strip=True) + protocol = tds[4].get_text(strip=True).lower() if len(tds) > 4 else 'http' + + if protocol not in VALID_PROTOCOLS: + protocol = 'http' + + # 简单校验格式 + if re.match(r'^\d+\.\d+\.\d+\.\d+$', ip) and port.isdigit(): + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = KuaiDaiLiPlugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/proxylist_download.py b/plugins/proxylist_download.py new file mode 100644 index 0000000..e963e79 --- /dev/null +++ b/plugins/proxylist_download.py @@ -0,0 +1,64 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +import asyncio + +class ProxyListDownloadPlugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "ProxyListDownload" + self.urls = [ + "https://www.proxy-list.download/api/v1/get?type=http", + "https://www.proxy-list.download/api/v1/get?type=https" + ] + + async def parse(self, html): + if not html: + return + + lines = html.split('\r\n') + if len(lines) <= 1: + lines = html.split('\n') + + count = 0 + for line in lines: + line = line.strip() + if not line: + continue + + if ':' in line: + parts = line.split(':') + if len(parts) >= 2: + ip = parts[0] + port = parts[1] + protocol = 'http' if 'type=http' in self.current_url else 'https' + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，从 {self.current_url} 获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = ProxyListDownloadPlugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/speedx.py b/plugins/speedx.py new file mode 100644 index 0000000..5088c84 --- /dev/null +++ b/plugins/speedx.py @@ -0,0 +1,78 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +import re +import asyncio + +class SpeedXPlugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "SpeedX代理源" + self.urls = [ + "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt", + "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt", + "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt" + ] + + async def parse(self, html): + if not html: + return + + lines = html.split('\n') + count = 0 + for line in lines: + line = line.strip() + if not line: + continue + + if ':' in line: + parts = line.split(':') + if len(parts) >= 2: + ip = parts[0].strip() + port = parts[1].strip() + + # 验证IP地址格式 + if not re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', ip): + continue + + # 验证端口是数字 + if not port.isdigit() or not (1 <= int(port) <= 65535): + continue + + # 根据 URL 判断协议 + protocol = 'http' + if 'socks5' in self.current_url: + protocol = 'socks5' + elif 'socks4' in self.current_url: + protocol = 'socks4' + + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，从 {self.current_url} 获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = SpeedXPlugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/plugins/yundaili.py b/plugins/yundaili.py new file mode 100644 index 0000000..f472c98 --- /dev/null +++ b/plugins/yundaili.py @@ -0,0 +1,79 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from core.crawler import BasePlugin +from core.log import logger +from bs4 import BeautifulSoup +import re +import asyncio + +VALID_PROTOCOLS = ['http', 'https', 'socks4', 'socks5'] + +class YunDaiLiPlugin(BasePlugin): + def __init__(self): + super().__init__() + self.name = "云代理" + # 抓取高匿和普通代理的前 5 页 + self.urls = [ + f"http://www.ip3366.net/free/?stype=1&page={i}" for i in range(1, 6) + ] + [ + f"http://www.ip3366.net/free/?stype=2&page={i}" for i in range(1, 6) + ] + + async def parse(self, html): + """ + 解析云代理/IP3366 页面 (两者结构相似) + """ + if not html: + return + + soup = BeautifulSoup(html, 'lxml') + list_table = soup.find('div', id='list') + if not list_table: + return + + table = list_table.find('table') + if not table: + return + + rows = table.find_all('tr') + count = 0 + for row in rows: + tds = row.find_all('td') + if len(tds) >= 5: + ip = tds[0].get_text(strip=True) + port = tds[1].get_text(strip=True) + protocol = tds[4].get_text(strip=True).lower() if len(tds) > 4 else 'http' + + if protocol not in VALID_PROTOCOLS: + protocol = 'http' + + if re.match(r'^\d+\.\d+\.\d+\.\d+$', ip) and port.isdigit(): + yield ip, int(port), protocol + count += 1 + + if count > 0: + logger.info(f"{self.name} 解析完成，获得 {count} 个潜在代理") + + +if __name__ == "__main__": + async def test_plugin(): + plugin = YunDaiLiPlugin() + print(f"========== 测试 {plugin.name} ==========") + print(f"目标URL数量: {len(plugin.urls)}") + print(f"开始抓取...\n") + + proxies = await plugin.run() + + print(f"\n========== 抓取结果 ==========") + print(f"总计获取 {len(proxies)} 个代理:") + print("-" * 60) + + for idx, (ip, port, protocol) in enumerate(proxies, 1): + print(f"{idx:3d}. {ip:15s} : {str(port):5s} | {protocol}") + + print("-" * 60) + print(f"完成！共 {len(proxies)} 个代理~") + + asyncio.run(test_plugin()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7b2a148 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +websockets==12.0 +aiosqlite==0.19.0 +aiohttp==3.9.1 +beautifulsoup4==4.12.3 +lxml==5.1.0 diff --git a/script/README.md b/script/README.md new file mode 100644 index 0000000..562d086 --- /dev/null +++ b/script/README.md @@ -0,0 +1,142 @@ +# Proxy Pool Startup Scripts + +## File List + +- **start_backend.bat** - Start backend service +- **start_frontend.bat** - Start frontend service +- **stop_all.bat** - Stop all services + +## Quick Start + +### Start Services Separately +- Backend: Double-click `start_backend.bat` +- Frontend: Double-click `start_frontend.bat` + +### Stop Services +Double-click `stop_all.bat` to stop all services + +## Script Features + +### Smart Process Management +- Automatically detect and stop running processes +- Prevent duplicate startup of multiple instances +- Automatically clean up port conflicts + +### Log Management +- All output written to log files +- Backend log: `backend.log` +- Frontend log: `frontend.log` +- Logs include timestamps for troubleshooting + +### PID Management +- Automatically record process ID to PID files +- Facilitates subsequent service stopping +- Automatically clean up PID files after process stops + +### Port Cleanup +- Automatically detect and clean up port conflicts +- Backend port: 3000 +- Frontend port: 8080 + +## Access Addresses + +After successful startup: +- Backend API: http://localhost:3000 +- Frontend UI: http://localhost:8080 + +## Manual Operations + +### View Logs +```bash +# View backend log +type backend.log + +# View frontend log +type frontend.log +``` + +### Manual Stop Process +```bash +# View PID file content +type backend.pid +type frontend.pid + +# Stop process using PID +taskkill /F /PID +``` + +### Check Port Usage +```bash +# Check backend port +netstat -ano | findstr :3000 + +# Check frontend port +netstat -ano | findstr :8080 +``` + +## Log Examples + +### backend.log +``` +[13:30:15.00] ======================================== +[13:30:15.00] Starting backend service... +INFO: Started server process [12345] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:3000 +``` + +### frontend.log +``` +[13:30:20.00] ======================================== +[13:30:20.00] Starting frontend service... +VITE v5.0.0 ready in 1234 ms + +➜ Local: http://localhost:8080/ +➜ Network: use --host to expose +``` + +## Notes + +1. **First-time frontend startup**: If dependencies are not installed, the script will automatically run `npm install` +2. **Virtual environment**: Ensure backend uses Python virtual environment (venv) +3. **Firewall**: Ensure firewall allows ports 3000 and 8080 +4. **Antivirus**: Some antivirus software may block scripts, need to add to whitelist + +## Troubleshooting + +### Backend Won't Start +1. Check if Python virtual environment is correctly installed +2. View `backend.log` log file +3. Confirm port 3000 is not in use +4. Check if dependency packages are complete: `venv\Scripts\pip list` + +### Frontend Won't Start +1. Check if Node.js is installed: `node --version` +2. View `frontend.log` log file +3. Confirm port 8080 is not in use +4. Manually install dependencies: Enter frontend directory and run `npm install` + +### Process Won't Stop +1. Manually find process: `tasklist | findstr python` +2. Force stop: `taskkill /F /IM python.exe` +3. Check port: `netstat -ano | findstr :3000` + +## Advanced Usage + +### Modify Ports +- Backend: Modify port number in `api_server.py` +- Frontend: Modify port number in `vite.config.js` +- After modification, need to sync update port checking logic in scripts + +### Custom Log Location +- Modify `LOG_FILE` variable in scripts +- Ensure directory exists and has write permissions + +## Technical Support + +If you encounter issues, please check: +1. Log files (backend.log, frontend.log) +2. PID files (backend.pid, frontend.pid) +3. Port usage (netstat -ano) +4. Process list (tasklist) diff --git a/script/start.bat b/script/start.bat new file mode 100644 index 0000000..22a5053 --- /dev/null +++ b/script/start.bat @@ -0,0 +1,9 @@ +@echo off +chcp 65001 >nul +setlocal +cd /d %~dp0 + +REM Launch via PowerShell to avoid encoding issues with Chinese characters +powershell -ExecutionPolicy Bypass -File start.ps1 + +timeout /t 3 diff --git a/script/start.ps1 b/script/start.ps1 new file mode 100644 index 0000000..4a5489e --- /dev/null +++ b/script/start.ps1 @@ -0,0 +1,109 @@ +# ProxyPool Startup Script +$rootPath = Split-Path $PSScriptRoot -Parent + +Write-Host "=== ProxyPool Startup ===" -ForegroundColor Cyan +Write-Host "" + +# 1. Clean processes on ports 8923 and 6173 +Write-Host "[1/4] Cleaning old processes..." -ForegroundColor Cyan +$ports = @(8923, 6173) +foreach ($port in $ports) { + try { + $conn = Get-NetTCPConnection -LocalPort $port -ErrorAction SilentlyContinue + if ($conn) { + $processId = $conn.OwningProcess + Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue + Write-Host " Stopped port $port (PID: $processId)" -ForegroundColor Gray + } + } catch {} +} +Write-Host " Cleanup complete!" -ForegroundColor Green +Write-Host "" + +# 2. Start Backend (FastAPI) +Write-Host "[2/4] Starting backend (FastAPI)..." -ForegroundColor Cyan + +$venvPython = "$rootPath\venv\Scripts\python.exe" +if (Test-Path $venvPython) { + $pythonPath = $venvPython + Write-Host " Using venv: $venvPython" -ForegroundColor Green +} else { + $pythonPath = (Get-Command python).Source + Write-Host " Using system Python: $pythonPath" -ForegroundColor Yellow +} + +$env:PYTHONIOENCODING = "utf-8" + +$backendLog = "$rootPath\logs\backend_startup.log" +$backendErr = "$rootPath\logs\backend_error.log" + +# Clear old logs +if (Test-Path $backendLog) { Remove-Item $backendLog -Force } +if (Test-Path $backendErr) { Remove-Item $backendErr -Force } + +# Start backend with -u flag for unbuffered output and redirect logs +$backendProcess = Start-Process -FilePath $pythonPath -ArgumentList "-u", "api_server.py" -WorkingDirectory "$rootPath" -RedirectStandardOutput $backendLog -RedirectStandardError $backendErr -WindowStyle Hidden -PassThru + +Write-Host " Backend started (PID: $($backendProcess.Id))" -ForegroundColor Green +Write-Host "" + +# 3. Wait for backend to be ready (max 10 seconds) +Write-Host "[3/4] Waiting for backend..." -ForegroundColor Cyan +$maxRetries = 5 +$retryCount = 0 +$backendReady = $false + +while (-not $backendReady -and $retryCount -lt $maxRetries) { + Start-Sleep -Seconds 2 + $retryCount++ + + try { + $response = Invoke-RestMethod -Uri "http://127.0.0.1:8923/" -Method Get -TimeoutSec 2 -ErrorAction Stop + if ($response) { + $backendReady = $true + Write-Host " Backend is ready!" -ForegroundColor Green + } + } catch { + $errMessage = $_.Exception.Message + Write-Host " Waiting... ($retryCount/$maxRetries)" -ForegroundColor Yellow + + if (Test-Path $backendLog) { + $lastLog = Get-Content $backendLog -Tail 1 -ErrorAction SilentlyContinue + if ($lastLog) { Write-Host " Log: $lastLog" -ForegroundColor DarkGray } + } + + if ($backendProcess.HasExited) { + Write-Host " Backend process exited!" -ForegroundColor Red + Write-Host " Exit code: $($backendProcess.ExitCode)" -ForegroundColor Red + if (Test-Path $backendErr) { + Write-Host "" -ForegroundColor Red + Write-Host "Error log:" -ForegroundColor Red + Get-Content $backendErr -Tail 20 | ForEach-Object { Write-Host " $_" -ForegroundColor Red } + } + $backendReady = $false + break + } + } +} + +if (-not $backendReady) { + Write-Host "" -ForegroundColor Red + Write-Host "Backend failed to start!" -ForegroundColor Red + Write-Host "Check error log: $backendErr" -ForegroundColor Red + pause + exit +} + +Write-Host "" + +# 4. Start Frontend (Vite) +Write-Host "[4/4] Starting frontend (Vite)..." -ForegroundColor Cyan +Start-Process -FilePath "cmd" -ArgumentList "/c npm run dev" -WorkingDirectory "$rootPath\frontend" -WindowStyle Hidden +Write-Host " Frontend started" -ForegroundColor Green +Write-Host "" + +Write-Host "=== All services started ===" -ForegroundColor Cyan +Write-Host "Backend: http://127.0.0.1:8923" -ForegroundColor Green +Write-Host "Frontend: http://localhost:6173" -ForegroundColor Green +Write-Host "" +Write-Host "Please open frontend in browser" -ForegroundColor Magenta diff --git a/script/stop.bat b/script/stop.bat new file mode 100644 index 0000000..114920f --- /dev/null +++ b/script/stop.bat @@ -0,0 +1,9 @@ +@echo off +chcp 65001 >nul +setlocal +cd /d %~dp0 + +REM Stop processes using PowerShell +powershell -ExecutionPolicy Bypass -Command "& { Write-Host 'Stopping processes on 8923 and 6173...' -ForegroundColor Cyan; $ports = @(8923, 6173); foreach ($port in $ports) { $p = Get-NetTCPConnection -LocalPort $port -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p.OwningProcess -Force; Write-Host \"Stopped port $port\" } }; Write-Host 'Done.' -ForegroundColor Green }" + +timeout /t 2 diff --git a/tasks_manager.py b/tasks_manager.py new file mode 100644 index 0000000..0929dcd --- /dev/null +++ b/tasks_manager.py @@ -0,0 +1,224 @@ +import asyncio +from datetime import datetime +from core.plugin_manager import PluginManager +from core.sqlite import SQLiteManager +from core.validator import ProxyValidator +from core.log import logger +from typing import Optional, Callable + +class TasksManager: + def __init__(self): + self.is_running = False + self.stop_requested = False + self.current_task = None + self.validator_tasks = [] + self.progress_callback = None + self.status_callback = None + self.proxy_queue = asyncio.Queue(maxsize=500) + self.stats = { + 'total_found': 0, + 'total_verified': 0, + 'start_time': None, + 'current_url': None, + 'plugins': [] + } + + def set_callbacks(self, progress_callback: Optional[Callable] = None, status_callback: Optional[Callable] = None): + self.progress_callback = progress_callback + self.status_callback = status_callback + + async def _notify_progress(self, data: dict): + if self.progress_callback: + data['timestamp'] = datetime.now().isoformat() + + if 'found' in data and 'verified' in data: + data['success_rate'] = round((data['verified'] / data['found'] * 100), 2) if data['found'] > 0 else 0 + + await self.progress_callback(data) + + async def _notify_status(self, status: str, message: str): + if self.status_callback: + await self.status_callback({ + 'status': status, + 'message': message, + 'timestamp': datetime.now().isoformat() + }) + + async def run_crawler(self): + await self._notify_status('crawling', '开始爬取代理啦~') + manager = PluginManager() + + count = 0 + self.stats['plugins'] = [plugin.name for plugin in manager.plugins] + + async for ip, port, protocol in manager.run_all(): + if self.stop_requested: + logger.info("爬虫收到停止信号") + break + await self.proxy_queue.put((ip, port, protocol)) + count += 1 + self.stats['total_found'] = count + + if count % 10 == 0: + await self._notify_progress({ + 'type': 'crawling', + 'found': count, + 'verified': self.stats['total_verified'] + }) + + if self.stop_requested: + await self._notify_status('stopped', '爬虫已停止啦~') + else: + await self._notify_status('crawling_done', f'爬虫抓取完成啦，共发现 {count} 个潜在代理~') + logger.info(f"爬虫抓取阶段完成，共发现 {count} 个潜在代理。") + + async def run_validator(self, db: SQLiteManager, validator: ProxyValidator): + await self._notify_status('validating', '开始验证代理啦~') + verified_count = 0 + + while True: + proxy = await self.proxy_queue.get() + if proxy is None or self.stop_requested: + self.proxy_queue.task_done() + break + + ip, port, protocol = proxy + try: + is_valid, latency = await validator.validate(ip, port, protocol) + if is_valid: + logger.info(f"验证通过: {ip}:{port} ({protocol}) - 延迟: {latency}ms") + await db.insert_proxy(ip, port, protocol) + verified_count += 1 + self.stats['total_verified'] = verified_count + + if verified_count % 5 == 0: + await self._notify_progress({ + 'type': 'validating', + 'found': self.stats['total_found'], + 'verified': verified_count, + 'current_proxy': f"{ip}:{port}" + }) + else: + logger.info(f"验证失败: {ip}:{port} ({protocol})") + except Exception as e: + logger.error(f"验证器异常: {e}") + finally: + self.proxy_queue.task_done() + + if self.stop_requested: + await self._notify_status('stopped', '验证器已停止啦~') + elif verified_count > 0: + await self._notify_status('validating_done', f'验证完成啦，入库 {verified_count} 个代理~') + logger.info(f"验证协程完成，入库 {verified_count} 个代理。") + + async def start_task(self, db: SQLiteManager, validator: ProxyValidator, num_validators: int = 50): + if self.is_running: + await self._notify_status('error', '任务正在运行中呢~') + return False + + self.is_running = True + self.stop_requested = False + self.stats = { + 'total_found': 0, + 'total_verified': 0, + 'start_time': datetime.now().isoformat(), + 'current_url': None, + 'plugins': [] + } + + await self._notify_status('running', '任务开始啦~') + + crawler_task = asyncio.create_task(self.run_crawler()) + self.validator_tasks = [asyncio.create_task(self.run_validator(db, validator)) for _ in range(num_validators)] + + await crawler_task + + for _ in range(num_validators): + await self.proxy_queue.put(None) + + await self.proxy_queue.join() + await asyncio.gather(*self.validator_tasks, return_exceptions=True) + + total = await db.count_proxies() + self.is_running = False + self.stop_requested = False + + if not self.stop_requested: + await self._notify_status('completed', f'任务完成啦，当前池内总数: {total}~') + await self._notify_progress({ + 'type': 'completed', + 'found': self.stats['total_found'], + 'verified': self.stats['total_verified'], + 'total': total + }) + + logger.info(f"=== 运行结束，当前池内总数: {total} ===") + return True + + async def stop_task(self): + if not self.is_running: + return False + + self.stop_requested = True + + # 取消所有验证器任务 + for task in self.validator_tasks: + if not task.done(): + task.cancel() + + # 清空队列并添加停止信号 + while not self.proxy_queue.empty(): + try: + self.proxy_queue.get_nowait() + except asyncio.QueueEmpty: + break + + # 添加停止信号到队列 + for _ in range(len(self.validator_tasks)): + await self.proxy_queue.put(None) + + await self._notify_status('stopped', '任务已停止~') + logger.info("任务被手动停止") + return True + + def get_stats(self) -> dict: + return self.stats.copy() + + def is_task_running(self) -> bool: + return self.is_running + +class ScheduledTasks: + def __init__(self, tasks_manager: TasksManager): + self.tasks_manager = tasks_manager + self.scheduler_task = None + self.is_scheduled = False + self.interval_minutes = 60 + + async def scheduler(self): + from core.validator import ProxyValidator + from core.sqlite import SQLiteManager + + while self.is_scheduled: + try: + db = SQLiteManager() + await db.init_db() + + async with ProxyValidator(max_concurrency=200) as validator: + await self.tasks_manager.start_task(db, validator, num_validators=50) + + await asyncio.sleep(self.interval_minutes * 60) + except Exception as e: + logger.error(f"定时任务异常: {e}") + await asyncio.sleep(60) + + def start_scheduled(self, interval_minutes: int = 60): + self.interval_minutes = interval_minutes + self.is_scheduled = True + self.scheduler_task = asyncio.create_task(self.scheduler()) + logger.info(f"定时任务已启动，间隔: {interval_minutes} 分钟") + + def stop_scheduled(self): + self.is_scheduled = False + if self.scheduler_task: + self.scheduler_task.cancel() + logger.info("定时任务已停止")