si-meng-spec/ProxyPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sqlite): avoid validation database locked under high concurrency

Browse Source 
- Release DB connection during validator.validate(); serialize writes with asyncio.Lock
- Add busy_timeout and longer connect timeout on aiosqlite connections
- WAL writes no longer contend across 120 parallel workers holding long-lived connections

Made-with: Cursor
This commit is contained in:
祀梦
2026-04-05 13:43:18 +08:00
parent 46d84a0cdc
commit 2c98abaf91
2 changed files with 67 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
app/api/lifespan.py
View File

@@ -3,7 +3,7 @@ import asyncio
from contextlib import AsyncExitStack, asynccontextmanager
from fastapi import FastAPI
from app.core.db import init_db, get_db, get_db_connection
from app.core.db import init_db, get_db
from app.core.config import settings as app_settings
from app.core.log import logger
from app.core.execution import AsyncWorkerPool, JobExecutor
@@ -46,52 +46,58 @@ async def lifespan(app: FastAPI):
if db_settings.get("validation_targets") is not None:
validator.update_test_urls(db_settings["validation_targets"])
# 验证 WorkerPool
# 高并发验证时若多 worker 同时写 SQLite 会 database is locked；网络阶段不占连接，写路径串行化
validation_write_lock = asyncio.Lock()
async def validation_handler(proxy):
async with get_db_connection() as db:
async with get_db() as db:
existing = await proxy_repo.get_by_ip_port(db, proxy.ip, proxy.port)
is_valid, latency = await validator.validate(
proxy.ip, proxy.port, proxy.protocol
)
if not existing:
return
if existing.validated == 0:
if is_valid:
await proxy_repo.insert_or_update(
db,
proxy.ip,
proxy.port,
proxy.protocol,
score=app_settings.score_valid,
)
if latency:
await proxy_repo.update_response_time(
db, proxy.ip, proxy.port, latency
is_valid, latency = await validator.validate(
proxy.ip, proxy.port, proxy.protocol
)
async with validation_write_lock:
async with get_db() as db:
if not existing:
return
if existing.validated == 0:
if is_valid:
await proxy_repo.insert_or_update(
db,
proxy.ip,
proxy.port,
proxy.protocol,
score=app_settings.score_valid,
)
if latency:
await proxy_repo.update_response_time(
db, proxy.ip, proxy.port, latency
)
else:
await proxy_repo.delete(db, proxy.ip, proxy.port)
else:
await proxy_repo.delete(db, proxy.ip, proxy.port)
else:
if is_valid:
await proxy_repo.insert_or_update(
db,
proxy.ip,
proxy.port,
proxy.protocol,
score=app_settings.score_valid,
)
if latency:
await proxy_repo.update_response_time(
db, proxy.ip, proxy.port, latency
if is_valid:
await proxy_repo.insert_or_update(
db,
proxy.ip,
proxy.port,
proxy.protocol,
score=app_settings.score_valid,
)
if latency:
await proxy_repo.update_response_time(
db, proxy.ip, proxy.port, latency
)
else:
await proxy_repo.update_score(
db,
proxy.ip,
proxy.port,
app_settings.score_invalid,
app_settings.score_min,
app_settings.score_max,
)
else:
await proxy_repo.update_score(
db,
proxy.ip,
proxy.port,
app_settings.score_invalid,
app_settings.score_min,
app_settings.score_max,
)
worker_pool = AsyncWorkerPool(
worker_count=db_settings.get("default_concurrency", app_settings.validator_max_concurrency),