feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation
- Add Free_Proxy_Website-style fpw_* plugins and register them - Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting - Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore - Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait - Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys - WebSocket dashboard stats; ProxyList pool_filter and API alignment - POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404 - pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB - .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import Made-with: Cursor
This commit is contained in:
@@ -81,23 +81,25 @@ class TestProxyRepository:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_batches(self, db, proxy_repo):
|
||||
"""测试流式分批读取"""
|
||||
# 插入 5 条测试数据
|
||||
"""测试流式分批读取(与库内已有数据共存,只校验增量与分批形状)"""
|
||||
async with db.execute("SELECT COUNT(*) FROM proxies") as c:
|
||||
before = (await c.fetchone())[0]
|
||||
for i in range(5):
|
||||
await proxy_repo.insert_or_update(db, f"192.168.1.{i}", 8000 + i, "http", 10)
|
||||
|
||||
await proxy_repo.insert_or_update(db, f"192.168.99.{i}", 8000 + i, "http", 10)
|
||||
async with db.execute("SELECT COUNT(*) FROM proxies") as c:
|
||||
after = (await c.fetchone())[0]
|
||||
assert after == before + 5
|
||||
|
||||
batches = []
|
||||
async for batch in proxy_repo.iter_batches(db, batch_size=2):
|
||||
batches.append(batch)
|
||||
|
||||
assert len(batches) == 3
|
||||
assert len(batches[0]) == 2
|
||||
assert len(batches[1]) == 2
|
||||
assert len(batches[2]) == 1
|
||||
|
||||
# 清理
|
||||
|
||||
assert sum(len(b) for b in batches) == after
|
||||
assert len(batches[-1]) in (1, 2)
|
||||
assert all(len(b) <= 2 for b in batches)
|
||||
|
||||
for i in range(5):
|
||||
await proxy_repo.delete(db, f"192.168.1.{i}", 8000 + i)
|
||||
await proxy_repo.delete(db, f"192.168.99.{i}", 8000 + i)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_delete(self, db, proxy_repo):
|
||||
@@ -121,6 +123,38 @@ class TestProxyRepository:
|
||||
"""测试获取统计信息"""
|
||||
stats = await proxy_repo.get_stats(db)
|
||||
assert "total" in stats
|
||||
assert "pending" in stats
|
||||
assert "available" in stats
|
||||
assert "avg_score" in stats
|
||||
assert "http_count" in stats
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_today_new_count_only_validated_available(self, db, proxy_repo):
|
||||
"""今日新增不计待验证;仅今日创建且 validated=1、score>0"""
|
||||
base = await proxy_repo.get_today_new_count(db)
|
||||
await proxy_repo.upsert_from_crawl(db, "192.168.88.20", 9020, "http", 0)
|
||||
assert await proxy_repo.get_today_new_count(db) == base
|
||||
|
||||
await proxy_repo.insert_or_update(db, "192.168.88.21", 9021, "http", 55)
|
||||
assert await proxy_repo.get_today_new_count(db) == base + 1
|
||||
|
||||
await proxy_repo.delete(db, "192.168.88.20", 9020)
|
||||
await proxy_repo.delete(db, "192.168.88.21", 9021)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_many_from_crawl(self, db, proxy_repo):
|
||||
from app.models.domain import ProxyRaw
|
||||
|
||||
raws = [
|
||||
ProxyRaw("10.0.0.1", 18080, "http"),
|
||||
ProxyRaw("10.0.0.2", 18081, "socks5"),
|
||||
]
|
||||
await proxy_repo.upsert_many_from_crawl(db, raws, 0)
|
||||
await db.commit()
|
||||
p1 = await proxy_repo.get_by_ip_port(db, "10.0.0.1", 18080)
|
||||
assert p1 is not None
|
||||
assert p1.validated == 0
|
||||
p2 = await proxy_repo.get_by_ip_port(db, "10.0.0.2", 18081)
|
||||
assert p2.protocol == "socks5"
|
||||
await proxy_repo.delete(db, "10.0.0.1", 18080)
|
||||
await proxy_repo.delete(db, "10.0.0.2", 18081)
|
||||
|
||||
Reference in New Issue
Block a user