feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation

- Add Free_Proxy_Website-style fpw_* plugins and register them
- Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting
- Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore
- Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait
- Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys
- WebSocket dashboard stats; ProxyList pool_filter and API alignment
- POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404
- pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB
- .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import

Made-with: Cursor
This commit is contained in:
祀梦
2026-04-05 13:39:19 +08:00
parent 92c7fa19e2
commit 0131c8b408
63 changed files with 2331 additions and 531 deletions

View File

@@ -81,23 +81,25 @@ class TestProxyRepository:
@pytest.mark.asyncio
async def test_iter_batches(self, db, proxy_repo):
"""测试流式分批读取"""
# 插入 5 条测试数据
"""测试流式分批读取(与库内已有数据共存,只校验增量与分批形状)"""
async with db.execute("SELECT COUNT(*) FROM proxies") as c:
before = (await c.fetchone())[0]
for i in range(5):
await proxy_repo.insert_or_update(db, f"192.168.1.{i}", 8000 + i, "http", 10)
await proxy_repo.insert_or_update(db, f"192.168.99.{i}", 8000 + i, "http", 10)
async with db.execute("SELECT COUNT(*) FROM proxies") as c:
after = (await c.fetchone())[0]
assert after == before + 5
batches = []
async for batch in proxy_repo.iter_batches(db, batch_size=2):
batches.append(batch)
assert len(batches) == 3
assert len(batches[0]) == 2
assert len(batches[1]) == 2
assert len(batches[2]) == 1
# 清理
assert sum(len(b) for b in batches) == after
assert len(batches[-1]) in (1, 2)
assert all(len(b) <= 2 for b in batches)
for i in range(5):
await proxy_repo.delete(db, f"192.168.1.{i}", 8000 + i)
await proxy_repo.delete(db, f"192.168.99.{i}", 8000 + i)
@pytest.mark.asyncio
async def test_batch_delete(self, db, proxy_repo):
@@ -121,6 +123,38 @@ class TestProxyRepository:
"""测试获取统计信息"""
stats = await proxy_repo.get_stats(db)
assert "total" in stats
assert "pending" in stats
assert "available" in stats
assert "avg_score" in stats
assert "http_count" in stats
@pytest.mark.asyncio
async def test_get_today_new_count_only_validated_available(self, db, proxy_repo):
"""今日新增不计待验证;仅今日创建且 validated=1、score>0"""
base = await proxy_repo.get_today_new_count(db)
await proxy_repo.upsert_from_crawl(db, "192.168.88.20", 9020, "http", 0)
assert await proxy_repo.get_today_new_count(db) == base
await proxy_repo.insert_or_update(db, "192.168.88.21", 9021, "http", 55)
assert await proxy_repo.get_today_new_count(db) == base + 1
await proxy_repo.delete(db, "192.168.88.20", 9020)
await proxy_repo.delete(db, "192.168.88.21", 9021)
@pytest.mark.asyncio
async def test_upsert_many_from_crawl(self, db, proxy_repo):
from app.models.domain import ProxyRaw
raws = [
ProxyRaw("10.0.0.1", 18080, "http"),
ProxyRaw("10.0.0.2", 18081, "socks5"),
]
await proxy_repo.upsert_many_from_crawl(db, raws, 0)
await db.commit()
p1 = await proxy_repo.get_by_ip_port(db, "10.0.0.1", 18080)
assert p1 is not None
assert p1.validated == 0
p2 = await proxy_repo.get_by_ip_port(db, "10.0.0.2", 18081)
assert p2.protocol == "socks5"
await proxy_repo.delete(db, "10.0.0.1", 18080)
await proxy_repo.delete(db, "10.0.0.2", 18081)