新增 ProxyScrape 测试爬虫插件
- 支持 HTTP/HTTPS/SOCKS4/SOCKS5 全协议类型 - 优先从公开 GitHub 代理列表获取真实数据 - 提供 fallback 测试数据生成模式,确保在无网络环境也能验证插件系统 - 在 plugins/__init__.py 显式注册
This commit is contained in:
@@ -8,6 +8,7 @@ from .ip89 import Ip89Plugin
|
|||||||
from .kuaidaili import KuaiDaiLiPlugin
|
from .kuaidaili import KuaiDaiLiPlugin
|
||||||
from .speedx import SpeedXPlugin
|
from .speedx import SpeedXPlugin
|
||||||
from .yundaili import YunDaiLiPlugin
|
from .yundaili import YunDaiLiPlugin
|
||||||
|
from .proxyscrape import ProxyScrapePlugin
|
||||||
|
|
||||||
# 显式注册所有插件
|
# 显式注册所有插件
|
||||||
registry.register(Fate0Plugin)
|
registry.register(Fate0Plugin)
|
||||||
@@ -17,3 +18,4 @@ registry.register(Ip89Plugin)
|
|||||||
registry.register(KuaiDaiLiPlugin)
|
registry.register(KuaiDaiLiPlugin)
|
||||||
registry.register(SpeedXPlugin)
|
registry.register(SpeedXPlugin)
|
||||||
registry.register(YunDaiLiPlugin)
|
registry.register(YunDaiLiPlugin)
|
||||||
|
registry.register(ProxyScrapePlugin)
|
||||||
|
|||||||
75
plugins/proxyscrape.py
Normal file
75
plugins/proxyscrape.py
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
"""ProxyScrape 测试爬虫 - 用于验证架构,支持全协议类型"""
|
||||||
|
from typing import List
|
||||||
|
from core.plugin_system import ProxyRaw
|
||||||
|
from plugins.base import BaseHTTPPlugin
|
||||||
|
from core.log import logger
|
||||||
|
|
||||||
|
|
||||||
|
class ProxyScrapePlugin(BaseHTTPPlugin):
|
||||||
|
"""
|
||||||
|
从 ProxyScrape 公开 API 获取代理。
|
||||||
|
覆盖 http/https/socks4/socks5 全协议,专门用于测试插件系统的可扩展性。
|
||||||
|
"""
|
||||||
|
|
||||||
|
name = "proxyscrape"
|
||||||
|
display_name = "ProxyScrape测试源"
|
||||||
|
description = "从 ProxyScrape API 获取各类型代理(HTTP/HTTPS/SOCKS4/SOCKS5),用于测试架构扩展"
|
||||||
|
enabled = True
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
# 使用多个公开 GitHub 代理列表作为源,稳定性较高
|
||||||
|
self.urls = [
|
||||||
|
("http", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt"),
|
||||||
|
("https", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/https.txt"),
|
||||||
|
("socks4", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt"),
|
||||||
|
("socks5", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt"),
|
||||||
|
]
|
||||||
|
|
||||||
|
async def crawl(self) -> List[ProxyRaw]:
|
||||||
|
results: List[ProxyRaw] = []
|
||||||
|
for protocol, url in self.urls:
|
||||||
|
try:
|
||||||
|
html = await self.fetch(url, timeout=30)
|
||||||
|
if not html:
|
||||||
|
logger.warning(f"ProxyScrape {protocol.upper()} 返回空内容")
|
||||||
|
continue
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for line in html.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or ":" not in line:
|
||||||
|
continue
|
||||||
|
parts = line.split(":")
|
||||||
|
if len(parts) >= 2:
|
||||||
|
ip = parts[0].strip()
|
||||||
|
port_str = parts[1].strip()
|
||||||
|
if port_str.isdigit():
|
||||||
|
results.append(ProxyRaw(ip, int(port_str), protocol))
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
logger.info(f"ProxyScrape {protocol.upper()} 获取 {count} 个代理")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"ProxyScrape {protocol.upper()} 爬取失败: {e}")
|
||||||
|
|
||||||
|
if results:
|
||||||
|
logger.info(f"ProxyScrape 总计获取 {len(results)} 个代理")
|
||||||
|
else:
|
||||||
|
# Fallback:生成测试代理,确保在测试环境也能验证完整流程
|
||||||
|
logger.warning("ProxyScrape 所有真实源均不可用,生成测试代理用于架构验证")
|
||||||
|
results = self._generate_test_proxies()
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _generate_test_proxies(self) -> List[ProxyRaw]:
|
||||||
|
"""生成测试代理数据,覆盖全协议类型,用于验证插件系统"""
|
||||||
|
import random
|
||||||
|
test_proxies = []
|
||||||
|
protocols = ["http", "https", "socks4", "socks5"]
|
||||||
|
for protocol in protocols:
|
||||||
|
for _ in range(3):
|
||||||
|
# 生成随机公网格式 IP(仅用于测试流程)
|
||||||
|
ip = f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}"
|
||||||
|
port = random.randint(1024, 65535)
|
||||||
|
test_proxies.append(ProxyRaw(ip, port, protocol))
|
||||||
|
logger.info(f"生成 {len(test_proxies)} 个测试代理: HTTP/HTTPS/SOCKS4/SOCKS5 各 3 个")
|
||||||
|
return test_proxies
|
||||||
Reference in New Issue
Block a user