feat: external plugin loading, score threshold, expiry cleanup and more improvements

Made-with: Cursor

app/repositories/proxy_repo.py

@@ -3,6 +3,8 @@ import aiosqlite
 from datetime import datetime, timedelta
 from typing import List, Optional, Tuple, Union
 
+from app.core.config import settings as app_settings
+
 from app.models.domain import Proxy, ProxyRaw
 from app.core.log import logger
 
@@ -54,10 +56,12 @@ class ProxyRepository:
         ip: str,
         port: int,
         protocol: str = "http",
-        score: int = 10,
+        score: Optional[int] = None,
     ) -> bool:
         if protocol not in VALID_PROTOCOLS:
             protocol = "http"
+        if score is None:
+            score = int(app_settings.score_valid)
         try:
             await db.execute(
                 """
@@ -85,7 +89,7 @@ class ProxyRepository:
         protocol: str = "http",
         initial_score: int = 0,
     ) -> None:
-        """爬取入库：待验证状态（validated=0, score=0）；再次爬取同一条则重置为待验证。"""
+        """爬取入库：待验证（validated=0）；score 由 initial_score 决定（通常来自配置 score_valid）。"""
         if protocol not in VALID_PROTOCOLS:
             protocol = "http"
         await db.execute(
@@ -232,13 +236,17 @@ class ProxyRepository:
             return None
 
     @staticmethod
-    async def get_random(db: aiosqlite.Connection) -> Optional[Proxy]:
+    async def get_random(
+        db: aiosqlite.Connection, min_score: int = 1
+    ) -> Optional[Proxy]:
+        ms = max(1, int(min_score))
         async with db.execute(
             f"""
             SELECT {_SELECT_PROXY_COLS} FROM proxies
-            WHERE validated = 1 AND score > 0
+            WHERE validated = 1 AND score >= ?
             ORDER BY RANDOM() LIMIT 1
-            """
+            """,
+            (ms,),
         ) as cursor:
             row = await cursor.fetchone()
             if row:
@@ -306,12 +314,18 @@ class ProxyRepository:
         protocol: Optional[str] = None,
         batch_size: int = 1000,
         only_usable: bool = False,
+        usable_min_score: int = 1,
     ):
         """流式分批读取代理，避免一次性加载大量数据到内存"""
         offset = 0
         while True:
             batch = await ProxyRepository._list_batch_offset(
-                db, protocol, batch_size, offset, only_usable=only_usable
+                db,
+                protocol,
+                batch_size,
+                offset,
+                only_usable=only_usable,
+                usable_min_score=usable_min_score,
             )
             if not batch:
                 break
@@ -325,12 +339,15 @@ class ProxyRepository:
         batch_size: int,
         offset: int,
         only_usable: bool,
+        usable_min_score: int = 1,
     ) -> List[Proxy]:
         query = f"SELECT {_SELECT_PROXY_COLS} FROM proxies"
         params: List = []
         clauses = []
         if only_usable:
-            clauses.append("validated = 1 AND score > 0")
+            ms = max(1, int(usable_min_score))
+            clauses.append("validated = 1 AND score >= ?")
+            params.append(ms)
         if protocol:
             clauses.append("protocol = ?")
             params.append(protocol.lower())
@@ -396,12 +413,16 @@ class ProxyRepository:
         return proxies, total
 
     @staticmethod
-    async def get_stats(db: aiosqlite.Connection) -> dict:
+    async def get_stats(
+        db: aiosqlite.Connection, low_score_threshold: int = 0
+    ) -> dict:
         """统计快照。
 
         协议计数（http/https/socks*）仅含已验证且 score>0 的可用代理，供首页图表与「可用」口径一致。
         pending_* 为待验证池（validated=0）按协议分布。
+        invalid_count：已验证且 score<=0，或 score 低于系统「最低分」阈值（阈值>0 时）。
         """
+        thr = max(0, int(low_score_threshold))
         query = """
             SELECT
                 COUNT(*) as total,
@@ -416,12 +437,12 @@ class ProxyRepository:
                 COUNT(CASE WHEN validated = 0 AND protocol = 'https' THEN 1 END) as pending_https_count,
                 COUNT(CASE WHEN validated = 0 AND protocol = 'socks4' THEN 1 END) as pending_socks4_count,
                 COUNT(CASE WHEN validated = 0 AND protocol = 'socks5' THEN 1 END) as pending_socks5_count,
-                COUNT(CASE WHEN validated = 1 AND score <= 0 THEN 1 END) as invalid_count,
+                COUNT(CASE WHEN validated = 1 AND (score <= 0 OR (? > 0 AND score < ?)) THEN 1 END) as invalid_count,
                 (SELECT AVG(response_time_ms) FROM proxies WHERE validated = 1 AND score > 0
                     AND response_time_ms IS NOT NULL AND response_time_ms > 0) as avg_response_ms
             FROM proxies
         """
-        async with db.execute(query) as cursor:
+        async with db.execute(query, (thr, thr)) as cursor:
             row = await cursor.fetchone()
             if row:
                 avg_lat = row[13]
@@ -477,10 +498,19 @@ class ProxyRepository:
             return 0
 
     @staticmethod
-    async def clean_invalid(db: aiosqlite.Connection) -> int:
-        await db.execute(
-            "DELETE FROM proxies WHERE validated = 1 AND score <= 0"
-        )
+    async def clean_invalid(
+        db: aiosqlite.Connection, low_score_threshold: int = 0
+    ) -> int:
+        thr = max(0, int(low_score_threshold))
+        if thr > 0:
+            await db.execute(
+                "DELETE FROM proxies WHERE validated = 1 AND (score <= 0 OR score < ?)",
+                (thr,),
+            )
+        else:
+            await db.execute(
+                "DELETE FROM proxies WHERE validated = 1 AND score <= 0"
+            )
         await db.commit()
         return db.total_changes