si-meng-spec/ProxyPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation

Browse Source 
- Add Free_Proxy_Website-style fpw_* plugins and register them
- Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting
- Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore
- Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait
- Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys
- WebSocket dashboard stats; ProxyList pool_filter and API alignment
- POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404
- pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB
- .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import

Made-with: Cursor
This commit is contained in:
祀梦
2026-04-05 13:39:19 +08:00
parent 92c7fa19e2
commit 0131c8b408
63 changed files with 2331 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
tests/integration/test_settings_api.py
View File

@@ -12,7 +12,7 @@ class TestSettingsAPI:
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
assert "crawl_timeout" in data["data"]
assert "crawl_timeout" not in data["data"]
assert "validation_timeout" in data["data"]
assert "auto_validate" in data["data"]
@@ -21,17 +21,16 @@ class TestSettingsAPI:
"""测试 GET /api/settings 返回结构"""
response = await client.get("/api/settings")
data = response.json()["data"]
# 验证所有预期的设置项
expected_keys = [
"crawl_timeout",
"validation_timeout",
"max_retries",
"default_concurrency",
"min_proxy_score",
"proxy_expiry_days",
"auto_validate",
"auto_validate_after_crawl",
"validate_interval_minutes",
"validation_targets",
]
for key in expected_keys:
assert key in data, f"缺少设置项: {key}"
@@ -40,65 +39,45 @@ class TestSettingsAPI:
async def test_save_settings(self, client):
"""测试 POST /api/settings"""
settings = {
"crawl_timeout": 45,
"validation_timeout": 15,
"max_retries": 5,
"default_concurrency": 100,
"min_proxy_score": 10,
"proxy_expiry_days": 14,
"auto_validate": True,
"auto_validate_after_crawl": False,
"validate_interval_minutes": 60,
"validation_targets": [
"http://httpbin.org/ip",
],
}
response = await client.post("/api/settings", json=settings)
assert response.status_code == 200
data = response.json()
assert data["code"] == 200
# 验证返回的数据与提交的一致
for key, value in settings.items():
assert data["data"][key] == value
@pytest.mark.asyncio
async def test_save_settings_partial(self, client):
"""测试 POST /api/settings - 部分更新（实际上会替换所有）"""
# 先获取当前设置
response = await client.get("/api/settings")
current_settings = response.json()["data"]
# 修改部分设置
new_settings = current_settings.copy()
new_settings["crawl_timeout"] = 60
new_settings["validation_timeout"] = 25
new_settings["auto_validate"] = False
response = await client.post("/api/settings", json=new_settings)
assert response.status_code == 200
data = response.json()
assert data["data"]["crawl_timeout"] == 60
assert data["data"]["validation_timeout"] == 25
assert data["data"]["auto_validate"] is False
@pytest.mark.asyncio
async def test_save_settings_validation_error(self, client):
"""测试 POST /api/settings - 验证错误"""
# crawl_timeout 必须在 5-120 之间
invalid_settings = {
"crawl_timeout": 200, # 超出范围
"validation_timeout": 10,
"max_retries": 3,
"default_concurrency": 50,
"min_proxy_score": 0,
"proxy_expiry_days": 7,
"auto_validate": True,
"validate_interval_minutes": 30,
}
response = await client.post("/api/settings", json=invalid_settings)
assert response.status_code == 422 # 验证错误
@pytest.mark.asyncio
async def test_save_settings_invalid_type(self, client):
"""测试 POST /api/settings - 无效类型"""
invalid_settings = {
"crawl_timeout": "invalid", # 应该是整数
"validation_timeout": 10,
"max_retries": 3,
"validation_timeout": 100,
"default_concurrency": 50,
"min_proxy_score": 0,
"proxy_expiry_days": 7,
@@ -108,31 +87,62 @@ class TestSettingsAPI:
response = await client.post("/api/settings", json=invalid_settings)
assert response.status_code == 422
@pytest.mark.asyncio
async def test_save_settings_invalid_type(self, client):
"""测试 POST /api/settings - 无效类型"""
invalid_settings = {
"validation_timeout": 10,
"default_concurrency": "invalid",
"min_proxy_score": 0,
"proxy_expiry_days": 7,
"auto_validate": True,
"validate_interval_minutes": 30,
}
response = await client.post("/api/settings", json=invalid_settings)
assert response.status_code == 422
@pytest.mark.asyncio
async def test_save_settings_ignores_deprecated_crawl_timeout(self, client):
"""旧客户端若仍提交 crawl_timeout，应忽略且保存成功"""
response = await client.get("/api/settings")
base = response.json()["data"]
payload = {**base, "crawl_timeout": 999}
response = await client.post("/api/settings", json=payload)
assert response.status_code == 200
again = (await client.get("/api/settings")).json()["data"]
assert "crawl_timeout" not in again
@pytest.mark.asyncio
async def test_save_settings_ignores_obsolete_max_retries(self, client):
"""已移除的 max_retries 键若仍被提交，应忽略。"""
response = await client.get("/api/settings")
base = response.json()["data"]
payload = {**base, "max_retries": 9}
response = await client.post("/api/settings", json=payload)
assert response.status_code == 200
again = (await client.get("/api/settings")).json()["data"]
assert "max_retries" not in again
@pytest.mark.asyncio
async def test_settings_roundtrip(self, client):
"""测试设置读写一致性"""
# 生成随机但有效的设置
import random
test_settings = {
"crawl_timeout": random.randint(10, 60),
"validation_timeout": random.randint(5, 30),
"max_retries": random.randint(1, 5),
"default_concurrency": random.randint(20, 100),
"min_proxy_score": random.randint(0, 50),
"proxy_expiry_days": random.randint(1, 14),
"auto_validate": random.choice([True, False]),
"validate_interval_minutes": random.randint(10, 120),
}
# 写入设置
response = await client.post("/api/settings", json=test_settings)
assert response.status_code == 200
# 读取设置
response = await client.get("/api/settings")
saved_settings = response.json()["data"]
# 验证一致性
for key, value in test_settings.items():
assert saved_settings[key] == value, f"设置项 {key} 不一致"
@@ -140,9 +150,7 @@ class TestSettingsAPI:
async def test_settings_roundtrip_with_validation_targets(self, client):
"""测试设置读写一致性 - 包含数组类型的 validation_targets"""
test_settings = {
"crawl_timeout": 30,
"validation_timeout": 10,
"max_retries": 3,
"default_concurrency": 50,
"min_proxy_score": 0,
"proxy_expiry_days": 7,
@@ -154,13 +162,11 @@ class TestSettingsAPI:
],
}
# 写入设置
response = await client.post("/api/settings", json=test_settings)
assert response.status_code == 200
data = response.json()
assert data["data"]["validation_targets"] == test_settings["validation_targets"]
# 读取设置
response = await client.get("/api/settings")
saved_settings = response.json()["data"]
assert saved_settings["validation_targets"] == test_settings["validation_targets"]
@@ -179,7 +185,6 @@ class TestSettingsAPI:
data = response.json()
assert data["data"]["validation_targets"] == []
# 读取确认
response = await client.get("/api/settings")
saved_settings = response.json()["data"]
assert saved_settings["validation_targets"] == []