feat: external plugin loading, score threshold, expiry cleanup and more improvements

Made-with: Cursor

app/core/execution/job.py

@@ -102,14 +102,22 @@ class CrawlJob(Job):
         proxies: List[ProxyRaw] = result.proxies if result else []
 
         if proxies:
+            from app.core.config import settings as app_settings
             from app.core.db import transaction
             from app.repositories.proxy_repo import ProxyRepository
 
             try:
+                initial = max(
+                    app_settings.score_min,
+                    min(app_settings.score_max, int(app_settings.score_valid)),
+                )
                 async with transaction() as db:
-                    await ProxyRepository.upsert_many_from_crawl(db, proxies, 0)
+                    await ProxyRepository.upsert_many_from_crawl(
+                        db, proxies, initial
+                    )
                 logger.info(
-                    f"CrawlJob {self.id}: persisted {len(proxies)} crawled proxies as pending"
+                    f"CrawlJob {self.id}: persisted {len(proxies)} crawled proxies "
+                    f"as pending (initial score={initial})"
                 )
             except Exception as e:
                 logger.error(