si-meng-spec/ProxyPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(backend): optimize database safety, validator performance, and scheduler concurrency

Browse Source 
- Fix SQL injection risks in proxy_repo and task_repo
- Atomic acquire_pending with UPDATE ... RETURNING
- Reuse aiohttp ClientSession in ValidatorService
- Replace polling with asyncio.Event in SchedulerService
- Optimize ValidationQueue.drain with asyncio.Condition
- Concurrent plugin crawling with asyncio.gather
- Unify ProxyRaw model import path
- Fix test baseline and remove tracked __pycache__ files
This commit is contained in:
祀梦
2026-04-04 14:43:31 +08:00
parent abb8b32ed3
commit 635c524a7e
27 changed files with 103 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/plugins/fate0.py
View File

@@ -6,9 +6,10 @@ from app.core.log import logger
class Fate0Plugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "fate0"
display_name = "Fate0聚合源"
description = "从 GitHub 持续更新的高质量代理聚合列表"
display_name = "Fate0聚合站"
description = "来自 GitHub 持续更新的高质量代理聚合列表"
def __init__(self):
super().__init__()
@@ -34,5 +35,5 @@ class Fate0Plugin(BaseHTTPPlugin):
except Exception:
continue
if results:
logger.info(f"{self.display_name} 解析完成，获得 {len(results)} 个潜在代理")
logger.info(f"{self.display_name} 解析完成，获取 {len(results)} 个潜在代理")
return results

3
app/plugins/ip89.py
View File

@@ -7,6 +7,7 @@ from app.core.log import logger
class Ip89Plugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "ip89"
display_name = "89免费代理"
description = "从 89ip.cn 爬取免费代理"
@@ -35,5 +36,5 @@ class Ip89Plugin(BaseHTTPPlugin):
results.append(ProxyRaw(ip, int(port), "http"))
if results:
logger.info(f"{self.display_name} 解析完成，获得 {len(results)} 个潜在代理")
logger.info(f"{self.display_name} 解析完成，获取 {len(results)} 个潜在代理")
return results

3
app/plugins/kuaidaili.py
View File

@@ -9,6 +9,7 @@ VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
class KuaiDaiLiPlugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "kuaidaili"
display_name = "快代理"
description = "从快代理网站爬取免费代理"
@@ -45,5 +46,5 @@ class KuaiDaiLiPlugin(BaseHTTPPlugin):
results.append(ProxyRaw(ip, int(port), protocol))
if results:
logger.info(f"{self.display_name} 解析完成，获得 {len(results)} 个潜在代理")
logger.info(f"{self.display_name} 解析完成，获取 {len(results)} 个潜在代理")
return results

1
app/plugins/proxylist_download.py
View File

@@ -5,6 +5,7 @@ from app.core.log import logger
class ProxyListDownloadPlugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "proxylist_download"
display_name = "ProxyListDownload"
description = "从 ProxyListDownload API 获取代理"

11
app/plugins/proxyscrape.py
View File

@@ -6,19 +6,20 @@ from app.core.log import logger
class ProxyScrapePlugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
"""
从 ProxyScrape 公开 API 获取代理。
覆盖 http/https/socks4/socks5 全协议，专门用于测试插件系统的可扩展性。
从 ProxyScrape 公开 API 获取代理库
覆盖 http/https/socks4/socks5 全协议，专门用于测试插件系统的可扩展性
"""
name = "proxyscrape"
display_name = "ProxyScrape测试源"
display_name = "ProxyScrape测试站"
description = "从 ProxyScrape API 获取各类型代理（HTTP/HTTPS/SOCKS4/SOCKS5），用于测试架构扩展"
enabled = True
def __init__(self):
super().__init__()
# 使用多个公开 GitHub 代理列表作为源，稳定性较高
# 使用多个公开 GitHub 代理列表作为源，稳定性较差
self.urls = [
("http", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt"),
("https", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/https.txt"),
@@ -71,5 +72,5 @@ class ProxyScrapePlugin(BaseHTTPPlugin):
ip = f"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}"
port = random.randint(1024, 65535)
test_proxies.append(ProxyRaw(ip, port, protocol))
logger.info(f"生成 {len(test_proxies)} 个测试代理: HTTP/HTTPS/SOCKS4/SOCKS5 各 3 个")
logger.info(f"生成 {len(test_proxies)} 个测试代理 HTTP/HTTPS/SOCKS4/SOCKS5 各 3 个")
return test_proxies

5
app/plugins/speedx.py
View File

@@ -6,8 +6,9 @@ from app.core.log import logger
class SpeedXPlugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "speedx"
display_name = "SpeedX代理源"
display_name = "SpeedX代理库"
description = "从 SpeedX GitHub 仓库获取 SOCKS 代理列表"
def __init__(self):
@@ -47,5 +48,5 @@ class SpeedXPlugin(BaseHTTPPlugin):
results.append(ProxyRaw(ip, int(port), protocol))
if results:
logger.info(f"{self.display_name} 解析完成，获得 {len(results)} 个潜在代理")
logger.info(f"{self.display_name} 解析完成，获取 {len(results)} 个潜在代理")
return results

3
app/plugins/yundaili.py
View File

@@ -9,6 +9,7 @@ VALID_PROTOCOLS = ("http", "https", "socks4", "socks5")
class YunDaiLiPlugin(BaseHTTPPlugin):
default_config = {"max_pages": 5}
name = "yundaili"
display_name = "云代理"
description = "从云代理网站爬取免费代理"
@@ -47,5 +48,5 @@ class YunDaiLiPlugin(BaseHTTPPlugin):
results.append(ProxyRaw(ip, int(port), protocol))
if results:
logger.info(f"{self.display_name} 解析完成，获得 {len(results)} 个潜在代理")
logger.info(f"{self.display_name} 解析完成，获取 {len(results)} 个潜在代理")
return results