feat: fpw plugins, validation/crawl perf, WS stats, test DB isolation
- Add Free_Proxy_Website-style fpw_* plugins and register them - Per-plugin crawl timeout (crawl_timeout_seconds=120); remove global crawl_timeout setting - Validator: fix connect vs total timeout on save; SOCKS session LRU cache; drop redundant semaphore - Validation handler uses single DB connection; batch upsert after crawl; WorkerPool put_nowait - Remove unused max_retries from settings API/UI; settings maintenance SQL + init_db cleanup of deprecated keys - WebSocket dashboard stats; ProxyList pool_filter and API alignment - POST /api/proxies/delete-one for IPv6-safe deletes; task poll stops on 404 - pytest uses PROXYPOOL_DB_PATH=db/proxies.test.sqlite so tests do not wipe production DB - .gitignore: explicit proxies.test.sqlite patterns; fix plugin_service ValidationException import Made-with: Cursor
This commit is contained in:
@@ -54,10 +54,23 @@ async def init_db():
|
||||
await db.execute("UPDATE proxies SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL")
|
||||
logger.info("Migrated: added created_at column")
|
||||
|
||||
# 迁移:validated 0=待验证 1=已验证入池(参与分数维护)
|
||||
try:
|
||||
await db.execute("SELECT validated FROM proxies LIMIT 1")
|
||||
except Exception:
|
||||
await db.execute(
|
||||
"ALTER TABLE proxies ADD COLUMN validated INTEGER NOT NULL DEFAULT 0"
|
||||
)
|
||||
await db.execute(
|
||||
"UPDATE proxies SET validated = 1 WHERE score > 0"
|
||||
)
|
||||
logger.info("Migrated: added validated column")
|
||||
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_score ON proxies(score)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_protocol ON proxies(protocol)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_last_check ON proxies(last_check)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_ip_port ON proxies(ip, port)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_validated ON proxies(validated)")
|
||||
|
||||
# 插件设置表
|
||||
await db.execute("""
|
||||
@@ -94,6 +107,10 @@ async def init_db():
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
# 仅移除已废弃设置键,不碰 proxies 表数据
|
||||
await db.execute(
|
||||
"DELETE FROM settings WHERE key IN ('crawl_timeout', 'max_retries')"
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
logger.info("Database initialized")
|
||||
@@ -112,6 +129,19 @@ async def get_db() -> AsyncIterator[aiosqlite.Connection]:
|
||||
await db.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_db_connection() -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""单连接贯穿「读库 → await 网络 I/O → 写库」,减少验证 worker 每条代理两次 connect。"""
|
||||
ensure_db_dir()
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
try:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
yield db
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction() -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""获取带有显式事务控制的数据库连接
|
||||
|
||||
Reference in New Issue
Block a user