feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation

- Add Free_Proxy_Website-style fpw_* plugins and register them
- Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting
- Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore
- Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait
- Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys
- WebSocket dashboard stats; ProxyList pool_filter and API alignment
- POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404
- pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB
- .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import

Made-with: Cursor
This commit is contained in:
祀梦
2026-04-05 13:39:19 +08:00
parent 92c7fa19e2
commit 0131c8b408
63 changed files with 2331 additions and 531 deletions

View File

@@ -54,10 +54,23 @@ async def init_db():
await db.execute("UPDATE proxies SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL")
logger.info("Migrated: added created_at column")
# 迁移validated 0=待验证 1=已验证入池(参与分数维护)
try:
await db.execute("SELECT validated FROM proxies LIMIT 1")
except Exception:
await db.execute(
"ALTER TABLE proxies ADD COLUMN validated INTEGER NOT NULL DEFAULT 0"
)
await db.execute(
"UPDATE proxies SET validated = 1 WHERE score > 0"
)
logger.info("Migrated: added validated column")
await db.execute("CREATE INDEX IF NOT EXISTS idx_score ON proxies(score)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_protocol ON proxies(protocol)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_last_check ON proxies(last_check)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_ip_port ON proxies(ip, port)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_validated ON proxies(validated)")
# 插件设置表
await db.execute("""
@@ -94,6 +107,10 @@ async def init_db():
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# 仅移除已废弃设置键,不碰 proxies 表数据
await db.execute(
"DELETE FROM settings WHERE key IN ('crawl_timeout', 'max_retries')"
)
await db.commit()
logger.info("Database initialized")
@@ -112,6 +129,19 @@ async def get_db() -> AsyncIterator[aiosqlite.Connection]:
await db.close()
@asynccontextmanager
async def get_db_connection() -> AsyncIterator[aiosqlite.Connection]:
"""单连接贯穿「读库 → await 网络 I/O → 写库」,减少验证 worker 每条代理两次 connect。"""
ensure_db_dir()
db = await aiosqlite.connect(DB_PATH)
try:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
yield db
finally:
await db.close()
@asynccontextmanager
async def transaction() -> AsyncIterator[aiosqlite.Connection]:
"""获取带有显式事务控制的数据库连接