feat: external plugin loading, score threshold, expiry cleanup and more improvements

Made-with: Cursor

app/api/lifespan.py

@@ -14,6 +14,7 @@ from app.services.validator_service import ValidatorService
 from app.services.proxy_scoring import compute_proxy_quality_score
 from app.services.plugin_runner import PluginRunner
 from app.services.scheduler_service import SchedulerService
+from app.services.proxy_service import ProxyService
 from app.api.ws_manager import ConnectionManager
 from app.api.realtime import stats_broadcaster_loop
 
@@ -80,10 +81,14 @@ async def lifespan(app: FastAPI):
                                 proxy.protocol,
                                 score=q_score,
                             )
-                            if latency:
-                                await proxy_repo.update_response_time(
-                                    db, proxy.ip, proxy.port, latency
-                                )
+                            rt_ms = (
+                                float(latency)
+                                if latency is not None and float(latency) > 0
+                                else float(app_settings.score_default_latency_ms)
+                            )
+                            await proxy_repo.update_response_time(
+                                db, proxy.ip, proxy.port, rt_ms
+                            )
                         else:
                             await proxy_repo.delete(db, proxy.ip, proxy.port)
                     else:
@@ -104,10 +109,14 @@ async def lifespan(app: FastAPI):
                                 proxy.protocol,
                                 score=q_score,
                             )
-                            if latency:
-                                await proxy_repo.update_response_time(
-                                    db, proxy.ip, proxy.port, latency
-                                )
+                            rt_ms = (
+                                float(latency)
+                                if latency is not None and float(latency) > 0
+                                else float(app_settings.score_default_latency_ms)
+                            )
+                            await proxy_repo.update_response_time(
+                                db, proxy.ip, proxy.port, rt_ms
+                            )
                         else:
                             await proxy_repo.update_score(
                                 db,
@@ -125,20 +134,25 @@ async def lifespan(app: FastAPI):
         )
         await stack.enter_async_context(worker_pool)
 
-        # Job 执行器：槽位需覆盖「全部爬取」时 N 个 CrawlJob + 聚合任务 + 全量验证等
+        # Job 执行器：并发槽位（crawler_max_queue_size 与插件数共同约束，避免 crawl-all 死锁）
         _n_plugins = len(registry.list_plugins())
-        _max_jobs = max(24, _n_plugins + 8)
+        _floor = max(24, _n_plugins + 8)
+        _max_jobs = max(_floor, app_settings.crawler_max_queue_size)
         executor = JobExecutor(worker_pool=worker_pool, max_concurrent_jobs=_max_jobs)
         await stack.enter_async_context(executor)
 
         # 插件运行器
         plugin_runner = PluginRunner()
 
+        proxy_service = ProxyService()
+
         # 调度器
         scheduler = SchedulerService(
             executor=executor,
             worker_pool=worker_pool,
             interval_minutes=db_settings.get("validate_interval_minutes", 30),
+            proxy_service=proxy_service,
+            settings_repo=settings_repo,
         )
 
         # 挂载到 app.state

app/api/routes/proxies.py

@@ -10,9 +10,12 @@ from app.models.schemas import ProxyListRequest, BatchDeleteRequest, ProxyDelete
 from app.api.deps import get_proxy_service, get_scheduler_service
 from app.api.common import success_response, format_proxy
 from app.core.exceptions import ProxyPoolException, ProxyNotFoundException
+from app.core.config import settings as app_settings
 
 router = APIRouter(prefix="/api/proxies", tags=["proxies"])
 
+_EXPORT_MAX = int(app_settings.export_max_records)
+
 
 @router.get("/stats")
 async def get_stats(
@@ -60,7 +63,7 @@ async def get_random_proxy(service: ProxyService = Depends(get_proxy_service)):
 async def export_proxies(
     fmt: str,
     protocol: Optional[str] = None,
-    limit: int = Query(default=10000, ge=1, le=100000),
+    limit: int = Query(default=_EXPORT_MAX, ge=1, le=_EXPORT_MAX),
     service: ProxyService = Depends(get_proxy_service),
 ):
     if fmt not in ("csv", "txt", "json"):