fix(sqlite): avoid validation database locked under high concurrency
- Release DB connection during validator.validate(); serialize writes with asyncio.Lock - Add busy_timeout and longer connect timeout on aiosqlite connections - WAL writes no longer contend across 120 parallel workers holding long-lived connections Made-with: Cursor
This commit is contained in:
@@ -16,12 +16,21 @@ def ensure_db_dir():
|
||||
os.makedirs(db_dir, exist_ok=True)
|
||||
|
||||
|
||||
async def _apply_connection_pragmas(db: aiosqlite.Connection) -> None:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
await db.execute("PRAGMA busy_timeout=30000")
|
||||
|
||||
|
||||
# aiosqlite/sqlite3:等待锁的最长时间(秒),与高并发验证写入配合
|
||||
_SQLITE_CONNECT_TIMEOUT = 30.0
|
||||
|
||||
|
||||
async def init_db():
|
||||
"""初始化数据库表结构(支持迁移)"""
|
||||
ensure_db_dir()
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
async with aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT) as db:
|
||||
await _apply_connection_pragmas(db)
|
||||
await db.execute("PRAGMA cache_size=-64000")
|
||||
await db.execute("PRAGMA temp_store=MEMORY")
|
||||
|
||||
@@ -120,10 +129,9 @@ async def init_db():
|
||||
async def get_db() -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""获取数据库连接的异步上下文管理器"""
|
||||
ensure_db_dir()
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
|
||||
try:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
await _apply_connection_pragmas(db)
|
||||
yield db
|
||||
finally:
|
||||
await db.close()
|
||||
@@ -131,12 +139,11 @@ async def get_db() -> AsyncIterator[aiosqlite.Connection]:
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_db_connection() -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""单连接贯穿「读库 → await 网络 I/O → 写库」,减少验证 worker 每条代理两次 connect。"""
|
||||
"""与 get_db 相同 pragma/超时;保留别名供需「长连接」语义处使用。"""
|
||||
ensure_db_dir()
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
|
||||
try:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
await _apply_connection_pragmas(db)
|
||||
yield db
|
||||
finally:
|
||||
await db.close()
|
||||
@@ -152,10 +159,9 @@ async def transaction() -> AsyncIterator[aiosqlite.Connection]:
|
||||
# 如果抛出异常,自动 rollback
|
||||
"""
|
||||
ensure_db_dir()
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
db = await aiosqlite.connect(DB_PATH, timeout=_SQLITE_CONNECT_TIMEOUT)
|
||||
try:
|
||||
await db.execute("PRAGMA journal_mode=WAL")
|
||||
await db.execute("PRAGMA synchronous=NORMAL")
|
||||
await _apply_connection_pragmas(db)
|
||||
await db.execute("BEGIN")
|
||||
yield db
|
||||
await db.commit()
|
||||
|
||||
Reference in New Issue
Block a user