feat: external plugin loading, score threshold, expiry cleanup and more improvements

Made-with: Cursor

app/core/plugin_system/registry.py

@@ -1,7 +1,10 @@
 """插件注册中心 - 显式注册，类型安全，测试友好"""
 import importlib
+import importlib.util
 import inspect
 import os
+import sys
+from pathlib import Path
 from typing import Dict, List, Type, Optional
 from app.core.plugin_system.base import BaseCrawlerPlugin
 from app.core.log import logger
@@ -77,6 +80,57 @@ class PluginRegistry:
                 except Exception as e:
                     logger.error(f"Failed to load module {module_name}: {e}")
 
+    def load_external_plugins_directory(self, directory: Path) -> int:
+        """从项目下任意目录加载 ``BaseCrawlerPlugin`` 子类（每个 ``.py`` 一个模块）。
+
+        与内置 ``app.plugins`` 并存；若 ``name`` 与已注册插件冲突则跳过并打日志。
+        """
+        directory = Path(directory).resolve()
+        if not directory.is_dir():
+            logger.info("外部插件目录不存在，已跳过: %s", directory)
+            return 0
+        loaded = 0
+        for path in sorted(directory.glob("*.py")):
+            if path.name.startswith("_"):
+                continue
+            mod_name = f"proxypool_ext_{path.stem}_{abs(hash(str(path))) % 10_000_000_000}"
+            try:
+                spec = importlib.util.spec_from_file_location(mod_name, path)
+                if spec is None or spec.loader is None:
+                    continue
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[mod_name] = module
+                spec.loader.exec_module(module)
+                for attr_name in dir(module):
+                    obj = getattr(module, attr_name)
+                    if (
+                        inspect.isclass(obj)
+                        and issubclass(obj, BaseCrawlerPlugin)
+                        and obj is not BaseCrawlerPlugin
+                        and obj not in self._plugins.values()
+                    ):
+                        if not getattr(obj, "name", None):
+                            logger.warning(
+                                "跳过外部插件类（缺少 name）: %s in %s",
+                                obj.__name__,
+                                path,
+                            )
+                            continue
+                        if obj.name in self._plugins:
+                            logger.warning(
+                                "外部插件 %s 与已注册插件重名，已跳过: %s",
+                                obj.name,
+                                path,
+                            )
+                            continue
+                        self.register(obj)
+                        loaded += 1
+            except Exception as e:
+                logger.error("加载外部插件失败 %s: %s", path, e, exc_info=True)
+        if loaded:
+            logger.info("从 %s 额外加载 %s 个插件", directory, loaded)
+        return loaded
+
 
 # 全局注册中心实例
 registry = PluginRegistry()