si-meng-spec/ProxyPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 全面重构核心架构，消除反复修改的根因

Browse Source 
- 删除 ValidationQueue 双轨持久化队列，替换为纯内存 AsyncWorkerPool
- 引入统一后台任务框架 JobExecutor（Job/CrawlJob/ValidateAllJob）
- 新增 PluginRunner 统一插件执行（超时、重试、健康检查、统计）
- 重构 SchedulerService 职责收敛为仅定时触发 ValidateAllJob
- 使用 AsyncExitStack 重构 lifespan，安全管理长生命周期资源
- 路由层瘦身 50%+，业务异常上抛由全局中间件统一处理
- 实现设置全热更新（WorkerPool 并发、Validator 超时即时生效）
- 前端 Store 强制写后重新拉取，消除乐观更新数据不同步
- 删除 queue.py / task_repo.py / task_service.py
- 新增 execution 单元测试，全部 85 个测试通过
This commit is contained in:
祀梦
2026-04-04 22:36:57 +08:00
parent 4ef7931941
commit b972b64616
33 changed files with 1168 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
WebUI/src/composables/useScheduler.js Normal file
View File

@@ -0,0 +1,73 @@
import { ref } from 'vue'
import { schedulerService } from '../services/schedulerService'
const schedulerRunning = ref(false)
const schedulerLoading = ref(false)
const validating = ref(false)
export function useScheduler() {
async function fetchStatus() {
try {
const response = await schedulerService.getStatus()
if (response.code === 200) {
schedulerRunning.value = response.data.running
}
} catch (error) {
console.error('获取调度器状态失败:', error)
}
}
async function startScheduler(onSuccess) {
schedulerLoading.value = true
try {
const response = await schedulerService.start()
if (response.code === 200) {
schedulerRunning.value = true
onSuccess?.('自动验证已启动')
}
} catch (error) {
console.error('启动调度器失败:', error)
} finally {
schedulerLoading.value = false
}
}
async function stopScheduler(onSuccess) {
schedulerLoading.value = true
try {
const response = await schedulerService.stop()
if (response.code === 200) {
schedulerRunning.value = false
onSuccess?.('自动验证已停止')
}
} catch (error) {
console.error('停止调度器失败:', error)
} finally {
schedulerLoading.value = false
}
}
async function validateNow(onSuccess) {
validating.value = true
try {
const response = await schedulerService.validateNow()
if (response.code === 200) {
onSuccess?.('全量验证已启动')
}
} catch (error) {
console.error('启动验证失败:', error)
} finally {
validating.value = false
}
}
return {
schedulerRunning,
schedulerLoading,
validating,
fetchStatus,
startScheduler,
stopScheduler,
validateNow,
}
}

28
WebUI/src/composables/useTaskPolling.js Normal file
View File

@@ -0,0 +1,28 @@
import { tasksAPI } from '../api'
const POLL_INTERVAL = 1000
const MAX_POLL_ATTEMPTS = 30
/**
* 轮询任务状态直到完成或失败
* @param {string} taskId
* @returns {Promise<object>}
*/
export async function pollTaskStatus(taskId) {
for (let i = 0; i < MAX_POLL_ATTEMPTS; i++) {
await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL))
const response = await tasksAPI.getTaskStatus(taskId)
if (response.code !== 200) {
continue
}
const status = response.data.status
if (status === 'completed' || status === 'failed') {
return response
}
}
return {
code: 200,
message: '任务进行中，请稍后刷新查看结果',
data: { task_id: taskId, status: 'running' }
}
}

39
WebUI/src/services/pluginService.js
View File

@@ -1,26 +1,5 @@
import { pluginsAPI, tasksAPI } from '../api'
const POLL_INTERVAL = 1000
const MAX_POLL_ATTEMPTS = 30
async function pollTaskStatus(taskId) {
for (let i = 0; i < MAX_POLL_ATTEMPTS; i++) {
await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL))
const response = await tasksAPI.getTaskStatus(taskId)
if (response.code !== 200) {
continue
}
const status = response.data.status
if (status === 'completed' || status === 'failed') {
return response
}
}
return {
code: 200,
message: '爬取任务进行中，请稍后刷新查看结果',
data: { task_id: taskId, status: 'running' }
}
}
import { pluginsAPI } from '../api'
import { pollTaskStatus } from '../composables/useTaskPolling'
export const pluginService = {
async getPlugins() {
@@ -54,14 +33,18 @@ export const pluginService = {
async crawlAll() {
const startRes = await pluginsAPI.crawlAll()
if (startRes.code !== 200 || !startRes.data?.task_id) {
if (startRes.code !== 200 || !startRes.data?.task_ids?.length) {
return startRes
}
const finalRes = await pollTaskStatus(startRes.data.task_id)
// 批量轮询所有任务，取最后一个完成的结果
const results = await Promise.all(
startRes.data.task_ids.map(tid => pollTaskStatus(tid))
)
const last = results[results.length - 1]
return {
code: finalRes.code,
message: finalRes.data?.message || finalRes.message,
data: finalRes.data?.data || finalRes.data
code: last.code,
message: last.data?.message || last.message,
data: last.data?.data || last.data
}
}
}

11
WebUI/src/stores/plugins.js
View File

@@ -40,7 +40,7 @@ export const usePluginsStore = defineStore('plugins', () => {
/**
* 切换插件启用状态
* @param {string|number} pluginId
* @param {string} pluginId
* @param {boolean} enabled
* @returns {Promise<boolean>}
*/
@@ -48,10 +48,7 @@ export const usePluginsStore = defineStore('plugins', () => {
try {
const response = await pluginService.togglePlugin(pluginId, enabled)
if (response.code === 200) {
const plugin = plugins.value.find(p => p.id === pluginId)
if (plugin) {
plugin.enabled = enabled
}
await fetchPlugins() // 强制重新拉取最新状态
return true
}
} catch (error) {
@@ -62,7 +59,7 @@ export const usePluginsStore = defineStore('plugins', () => {
/**
* 触发插件爬取
* @param {string|number} pluginId
* @param {string} pluginId
* @returns {Promise<boolean>}
*/
async function crawlPlugin(pluginId) {
@@ -77,7 +74,7 @@ export const usePluginsStore = defineStore('plugins', () => {
/**
* 根据 ID 获取插件
* @param {string|number} id
* @param {string} id
* @returns {object|undefined}
*/
function getPluginById(id) {

7
WebUI/src/stores/proxy.js
View File

@@ -80,7 +80,10 @@ export const useProxyStore = defineStore('proxy', () => {
async function deleteProxy(ip, port) {
try {
const response = await proxyService.deleteProxy(ip, port)
return response.code === 200
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
return true
}
} catch (error) {
console.error('删除代理失败:', error)
return false
@@ -98,6 +101,7 @@ export const useProxyStore = defineStore('proxy', () => {
try {
const response = await proxyService.batchDelete(proxyList)
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
return response.data.deleted_count
}
} catch (error) {
@@ -114,6 +118,7 @@ export const useProxyStore = defineStore('proxy', () => {
try {
const response = await proxyService.cleanInvalid()
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
return response.data.deleted_count
}
} catch (error) {

76
WebUI/src/views/Settings.vue
View File

@@ -191,18 +191,25 @@ import {
Refresh
} from '@element-plus/icons-vue'
import { settingService } from '../services/settingService'
import { schedulerService } from '../services/schedulerService'
import { useScheduler } from '../composables/useScheduler'
import PageHeader from '../components/PageHeader.vue'
// ==================== Composables ====================
const {
schedulerRunning,
schedulerLoading,
validating,
fetchStatus,
startScheduler,
stopScheduler,
validateNow
} = useScheduler()
// ==================== 状态 ====================
const loading = ref(false)
const saving = ref(false)
const formRef = ref(null)
const schedulerRunning = ref(false)
const schedulerLoading = ref(false)
const validating = ref(false)
const settings = reactive({
crawl_timeout: 30,
validation_timeout: 10,
@@ -250,52 +257,13 @@ async function fetchSettings() {
}
}
async function fetchSchedulerStatus() {
try {
const response = await schedulerService.getStatus()
if (response.code === 200) {
schedulerRunning.value = response.data.running
}
} catch (error) {
console.error('获取调度器状态失败:', error)
}
}
// ==================== 调度器控制 ====================
async function handleStartScheduler() {
schedulerLoading.value = true
try {
const response = await schedulerService.start()
if (response.code === 200) {
schedulerRunning.value = true
ElMessage.success('自动验证已启动')
} else {
ElMessage.error('启动失败')
}
} catch (error) {
console.error('启动调度器失败:', error)
ElMessage.error('启动失败')
} finally {
schedulerLoading.value = false
}
await startScheduler((msg) => ElMessage.success(msg))
}
async function handleStopScheduler() {
schedulerLoading.value = true
try {
const response = await schedulerService.stop()
if (response.code === 200) {
schedulerRunning.value = false
ElMessage.success('自动验证已停止')
} else {
ElMessage.error('停止失败')
}
} catch (error) {
console.error('停止调度器失败:', error)
ElMessage.error('停止失败')
} finally {
schedulerLoading.value = false
}
await stopScheduler((msg) => ElMessage.success(msg))
}
async function handleValidateNow() {
@@ -309,21 +277,12 @@ async function handleValidateNow() {
type: 'info'
}
)
validating.value = true
const response = await schedulerService.validateNow()
if (response.code === 200) {
ElMessage.success('全量验证已启动，请在日志中查看进度')
} else {
ElMessage.error('启动验证失败')
}
await validateNow((msg) => ElMessage.success(msg))
} catch (error) {
if (error !== 'cancel') {
console.error('启动验证失败:', error)
ElMessage.error('启动验证失败')
}
} finally {
validating.value = false
}
}
@@ -338,8 +297,7 @@ async function handleSave() {
if (response.code === 200) {
ElMessage.success('配置保存成功')
// 刷新调度器状态
await fetchSchedulerStatus()
await fetchStatus()
} else {
ElMessage.error('配置保存失败')
}
@@ -354,7 +312,7 @@ async function handleSave() {
// ==================== 生命周期 ====================
onMounted(() => {
fetchSettings()
fetchSchedulerStatus()
fetchStatus()
})
</script>