fix(sqlite): avoid validation database locked under high concurrency

- Release DB connection during validator.validate(); serialize writes with asyncio.Lock
- Add busy_timeout and longer connect timeout on aiosqlite connections
- WAL writes no longer contend across 120 parallel workers holding long-lived connections

Made-with: Cursor
This commit is contained in:
祀梦
2026-04-05 13:43:18 +08:00
parent 46d84a0cdc
commit 2c98abaf91
2 changed files with 67 additions and 55 deletions

View File

@@ -16,12 +16,21 @@ def ensure_db_dir():
os.makedirs(db_dir, exist_ok=True)
async def _apply_connection_pragmas(db: aiosqlite.Connection) -> None:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await db.execute("PRAGMA busy_timeout=30000")
# aiosqlite/sqlite3等待锁的最长时间与高并发验证写入配合
_SQLITE_CONNECT_TIMEOUT = 30.0
async def init_db():
"""初始化数据库表结构(支持迁移)"""
ensure_db_dir()
async with aiosqlite.connect(DB_PATH) as db:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
async with aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT) as db:
await _apply_connection_pragmas(db)
await db.execute("PRAGMA cache_size=-64000")
await db.execute("PRAGMA temp_store=MEMORY")
@@ -120,10 +129,9 @@ async def init_db():
async def get_db() -> AsyncIterator[aiosqlite.Connection]:
"""获取数据库连接的异步上下文管理器"""
ensure_db_dir()
db = await aiosqlite.connect(DB_PATH)
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
try:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await _apply_connection_pragmas(db)
yield db
finally:
await db.close()
@@ -131,12 +139,11 @@ async def get_db() -> AsyncIterator[aiosqlite.Connection]:
@asynccontextmanager
async def get_db_connection() -> AsyncIterator[aiosqlite.Connection]:
"""单连接贯穿「读库 → await 网络 I/O → 写库」,减少验证 worker 每条代理两次 connect"""
"""与 get_db 相同 pragma/超时;保留别名供需「长连接」语义处使用"""
ensure_db_dir()
db = await aiosqlite.connect(DB_PATH)
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
try:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await _apply_connection_pragmas(db)
yield db
finally:
await db.close()
@@ -152,10 +159,9 @@ async def transaction() -> AsyncIterator[aiosqlite.Connection]:
# 如果抛出异常,自动 rollback
"""
ensure_db_dir()
db = await aiosqlite.connect(DB_PATH)
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
try:
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await _apply_connection_pragmas(db)
await db.execute("BEGIN")
yield db
await db.commit()