refactor(backend): optimize database safety, validator performance, and scheduler concurrency
- Fix SQL injection risks in proxy_repo and task_repo - Atomic acquire_pending with UPDATE ... RETURNING - Reuse aiohttp ClientSession in ValidatorService - Replace polling with asyncio.Event in SchedulerService - Optimize ValidationQueue.drain with asyncio.Condition - Concurrent plugin crawling with asyncio.gather - Unify ProxyRaw model import path - Fix test baseline and remove tracked __pycache__ files
This commit is contained in:
@@ -190,7 +190,12 @@ class ProxyRepository:
|
||||
params.append(max_score)
|
||||
|
||||
where_clause = " AND ".join(conditions)
|
||||
order_clause = f"{sort_by} {sort_order}"
|
||||
allowed_sort_by = {"ip", "port", "protocol", "score", "last_check"}
|
||||
allowed_sort_order = {"ASC", "DESC"}
|
||||
if sort_by not in allowed_sort_by or sort_order.upper() not in allowed_sort_order:
|
||||
order_clause = "last_check DESC"
|
||||
else:
|
||||
order_clause = f"{sort_by} {sort_order.upper()}"
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
count_query = f"SELECT COUNT(*) FROM proxies WHERE {where_clause}"
|
||||
@@ -268,7 +273,8 @@ class ProxyRepository:
|
||||
async def clean_expired(db: aiosqlite.Connection, days: int) -> int:
|
||||
try:
|
||||
await db.execute(
|
||||
"DELETE FROM proxies WHERE last_check < datetime('now', '-{} days')".format(days)
|
||||
"DELETE FROM proxies WHERE last_check < datetime('now', '-' || ? || ' days')",
|
||||
(days,),
|
||||
)
|
||||
await db.commit()
|
||||
return db.total_changes
|
||||
|
||||
@@ -33,22 +33,16 @@ class ValidationTaskRepository:
|
||||
try:
|
||||
async with db.execute(
|
||||
"""
|
||||
SELECT id, ip, port, protocol FROM validation_tasks
|
||||
WHERE status = 'pending'
|
||||
ORDER BY id ASC
|
||||
LIMIT 1
|
||||
UPDATE validation_tasks
|
||||
SET status = 'processing', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = (SELECT id FROM validation_tasks WHERE status = 'pending' ORDER BY id ASC LIMIT 1)
|
||||
RETURNING id, ip, port, protocol
|
||||
"""
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
task_id = row[0]
|
||||
await db.execute(
|
||||
"UPDATE validation_tasks SET status = 'processing', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
||||
(task_id,),
|
||||
)
|
||||
await db.commit()
|
||||
return {"id": task_id, "ip": row[1], "port": row[2], "protocol": row[3]}
|
||||
return {"id": row[0], "ip": row[1], "port": row[2], "protocol": row[3]}
|
||||
except Exception as e:
|
||||
logger.error(f"acquire_pending failed: {e}")
|
||||
return None
|
||||
@@ -126,7 +120,8 @@ class ValidationTaskRepository:
|
||||
async def cleanup_old(db: aiosqlite.Connection, days: int = 7) -> int:
|
||||
try:
|
||||
await db.execute(
|
||||
"DELETE FROM validation_tasks WHERE updated_at < datetime('now', '-{} days')".format(days)
|
||||
"DELETE FROM validation_tasks WHERE updated_at < datetime('now', '-' || ? || ' days')",
|
||||
(days,),
|
||||
)
|
||||
await db.commit()
|
||||
return db.total_changes
|
||||
|
||||
Reference in New Issue
Block a user