全面架构重构:建立分层架构与高度可扩展的插件系统

后端重构:
- 新增分层架构:API Routes -> Services -> Repositories -> Infrastructure
- 彻底移除全局单例,全面采用 FastAPI 依赖注入
- 新增 api/ 目录拆分路由(proxies, plugins, scheduler, settings, stats)
- 新增 services/ 业务逻辑层:ProxyService, PluginService, SchedulerService, ValidatorService, SettingsService
- 新增 repositories/ 数据访问层:ProxyRepository, SettingsRepository, PluginSettingsRepository
- 新增 models/ 层:Pydantic Schemas + Domain Models
- 重写 core/config.py:采用 Pydantic Settings 管理配置
- 新增 core/db.py:基于 asynccontextmanager 的连接管理,支持数据库迁移
- 新增 core/exceptions.py:统一业务异常体系

插件系统重构(核心):
- 新增 core/plugin_system/:BaseCrawlerPlugin + PluginRegistry
- 采用显式注册模式(装饰器 + plugins/__init__.py),类型安全、测试友好
- 新增 plugins/base.py:BaseHTTPPlugin 通用 HTTP 爬虫基类
- 迁移全部 7 个插件到新架构(fate0, proxylist_download, ip3366, ip89, kuaidaili, speedx, yundaili)
- 插件状态持久化到 plugin_settings 表

任务调度重构:
- 新增 core/tasks/queue.py:ValidationQueue + WorkerPool
- 解耦爬取与验证:爬虫只负责爬取,代理提交队列后由 Worker 异步验证
- 调度器定时从数据库拉取存量代理并分批投入验证队列

前端调整:
- 新增 frontend/src/services/ 层拆分 API 调用逻辑
- 调整 stores/ 和 views/ 使用 Service 层
- 保持 API 兼容性,页面无需大幅修改

其他:
- 新增 main.py 作为新入口
- 新增 DESIGN.md 架构设计文档
- 更新 requirements.txt 增加 pydantic-settings
This commit is contained in:
祀梦
2026-04-02 11:55:05 +08:00
parent a79f78b338
commit 209a744d94
56 changed files with 2891 additions and 2095 deletions

View File

@@ -0,0 +1,19 @@
import { pluginsAPI } from '../api'
export const pluginService = {
async getPlugins() {
return pluginsAPI.getPlugins()
},
async togglePlugin(pluginId, enabled) {
return pluginsAPI.togglePlugin(pluginId, enabled)
},
async crawlPlugin(pluginId) {
return pluginsAPI.crawlPlugin(pluginId)
},
async crawlAll() {
return pluginsAPI.crawlAll()
}
}

View File

@@ -0,0 +1,27 @@
import { statsAPI, proxiesAPI } from '../api'
export const proxyService = {
async getStats() {
return statsAPI.getStats()
},
async getProxies(params, signal) {
return proxiesAPI.getProxies(params, signal)
},
async deleteProxy(ip, port) {
return proxiesAPI.deleteProxy(ip, port)
},
async batchDelete(proxies) {
return proxiesAPI.batchDeleteProxies(proxies)
},
async cleanInvalid() {
return proxiesAPI.cleanInvalidProxies()
},
async export(format, protocol) {
return proxiesAPI.exportProxies(format, protocol)
}
}

View File

@@ -0,0 +1,19 @@
import { schedulerAPI } from '../api'
export const schedulerService = {
async start() {
return schedulerAPI.start()
},
async stop() {
return schedulerAPI.stop()
},
async validateNow() {
return schedulerAPI.validateNow()
},
async getStatus() {
return schedulerAPI.getStatus()
}
}

View File

@@ -0,0 +1,11 @@
import { settingsAPI } from '../api'
export const settingService = {
async getSettings() {
return settingsAPI.getSettings()
},
async saveSettings(data) {
return settingsAPI.saveSettings(data)
}
}

View File

@@ -1,6 +1,6 @@
import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import { pluginsAPI } from '../api'
import { pluginService } from '../services/pluginService'
/**
* Plugins Store
@@ -24,7 +24,7 @@ export const usePluginsStore = defineStore('plugins', () => {
async function fetchPlugins() {
loading.value = true
try {
const response = await pluginsAPI.getPlugins()
const response = await pluginService.getPlugins()
if (response.code === 200) {
plugins.value = response.data.plugins || []
return true
@@ -45,7 +45,7 @@ export const usePluginsStore = defineStore('plugins', () => {
*/
async function togglePlugin(pluginId, enabled) {
try {
const response = await pluginsAPI.togglePlugin(pluginId, enabled)
const response = await pluginService.togglePlugin(pluginId, enabled)
if (response.code === 200) {
const plugin = plugins.value.find(p => p.id === pluginId)
if (plugin) {
@@ -66,7 +66,7 @@ export const usePluginsStore = defineStore('plugins', () => {
*/
async function crawlPlugin(pluginId) {
try {
const response = await pluginsAPI.crawlPlugin(pluginId)
const response = await pluginService.crawlPlugin(pluginId)
return response.code === 200
} catch (error) {
console.error('触发插件爬取失败:', error)

View File

@@ -1,6 +1,6 @@
import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import { proxiesAPI, statsAPI } from '../api'
import { proxyService } from '../services/proxyService'
/**
* 判断是否为用户取消的错误
@@ -34,7 +34,7 @@ export const useProxyStore = defineStore('proxy', () => {
*/
async function fetchStats() {
try {
const response = await statsAPI.getStats()
const response = await proxyService.getStats()
if (response.code === 200) {
stats.value = response.data
return true
@@ -54,7 +54,7 @@ export const useProxyStore = defineStore('proxy', () => {
async function fetchProxies(params, signal) {
loading.value = true
try {
const response = await proxiesAPI.getProxies(params, signal)
const response = await proxyService.getProxies(params, signal)
if (response.code === 200) {
proxies.value = response.data.list
total.value = response.data.total
@@ -79,7 +79,7 @@ export const useProxyStore = defineStore('proxy', () => {
*/
async function deleteProxy(ip, port) {
try {
const response = await proxiesAPI.deleteProxy(ip, port)
const response = await proxyService.deleteProxy(ip, port)
return response.code === 200
} catch (error) {
console.error('删除代理失败:', error)
@@ -96,7 +96,7 @@ export const useProxyStore = defineStore('proxy', () => {
if (!proxyList?.length) return 0
try {
const response = await proxiesAPI.batchDeleteProxies(proxyList)
const response = await proxyService.batchDelete(proxyList)
if (response.code === 200) {
return response.data.deleted_count
}
@@ -112,7 +112,7 @@ export const useProxyStore = defineStore('proxy', () => {
*/
async function cleanInvalidProxies() {
try {
const response = await proxiesAPI.cleanInvalidProxies()
const response = await proxyService.cleanInvalid()
if (response.code === 200) {
return response.data.deleted_count
}
@@ -130,7 +130,7 @@ export const useProxyStore = defineStore('proxy', () => {
*/
async function exportProxies(format, protocol = null) {
try {
const response = await proxiesAPI.exportProxies(format, protocol)
const response = await proxyService.export(format, protocol)
// 创建下载链接
const url = window.URL.createObjectURL(new Blob([response]))

View File

@@ -138,7 +138,7 @@ import {
Box
} from '@element-plus/icons-vue'
import { usePluginsStore } from '../stores/plugins'
import { pluginsAPI } from '../api'
import { pluginService } from '../services/pluginService'
import { formatTime } from '../utils/format'
import PageHeader from '../components/PageHeader.vue'
@@ -168,7 +168,7 @@ async function handleCrawl(pluginId) {
crawlingPlugin.value = pluginId
lastCrawlResult.value = null
const response = await pluginsAPI.crawlPlugin(pluginId)
const response = await pluginService.crawlPlugin(pluginId)
if (response.code === 200) {
lastCrawlResult.value = {
@@ -216,7 +216,7 @@ async function handleCrawlAll() {
crawlingAll.value = true
lastCrawlResult.value = null
const response = await pluginsAPI.crawlAll()
const response = await pluginService.crawlAll()
if (response.code === 200) {
lastCrawlResult.value = {

View File

@@ -190,7 +190,8 @@ import {
VideoPause,
Refresh
} from '@element-plus/icons-vue'
import { settingsAPI, schedulerAPI } from '../api'
import { settingService } from '../services/settingService'
import { schedulerService } from '../services/schedulerService'
import PageHeader from '../components/PageHeader.vue'
// ==================== 状态 ====================
@@ -237,7 +238,7 @@ const formRules = {
async function fetchSettings() {
loading.value = true
try {
const response = await settingsAPI.getSettings()
const response = await settingService.getSettings()
if (response.code === 200) {
Object.assign(settings, response.data)
}
@@ -251,7 +252,7 @@ async function fetchSettings() {
async function fetchSchedulerStatus() {
try {
const response = await schedulerAPI.getStatus()
const response = await schedulerService.getStatus()
if (response.code === 200) {
schedulerRunning.value = response.data.running
}
@@ -264,7 +265,7 @@ async function fetchSchedulerStatus() {
async function handleStartScheduler() {
schedulerLoading.value = true
try {
const response = await schedulerAPI.start()
const response = await schedulerService.start()
if (response.code === 200) {
schedulerRunning.value = true
ElMessage.success('自动验证已启动')
@@ -282,7 +283,7 @@ async function handleStartScheduler() {
async function handleStopScheduler() {
schedulerLoading.value = true
try {
const response = await schedulerAPI.stop()
const response = await schedulerService.stop()
if (response.code === 200) {
schedulerRunning.value = false
ElMessage.success('自动验证已停止')
@@ -310,7 +311,7 @@ async function handleValidateNow() {
)
validating.value = true
const response = await schedulerAPI.validateNow()
const response = await schedulerService.validateNow()
if (response.code === 200) {
ElMessage.success('全量验证已启动,请在日志中查看进度')
} else {
@@ -333,7 +334,7 @@ async function handleSave() {
saving.value = true
try {
const response = await settingsAPI.saveSettings(settings)
const response = await settingService.saveSettings(settings)
if (response.code === 200) {
ElMessage.success('配置保存成功')