si-meng-spec/ProxyPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Round 3 fixes: cancelled polling, aggregator invalid_count, filter state, scheduler atomicity, HTTP exception handler, tests

Browse Source
This commit is contained in:
祀梦
2026-04-05 10:20:23 +08:00
parent 49e440cb41
commit dc5f050683
32 changed files with 321 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
WebUI/src/composables/useScheduler.js
View File

@@ -1,11 +1,10 @@
import { ref } from 'vue'
import { schedulerService } from '../services/schedulerService'
const schedulerRunning = ref(false)
const schedulerLoading = ref(false)
const validating = ref(false)
export function useScheduler() {
const schedulerRunning = ref(false)
const schedulerLoading = ref(false)
const validating = ref(false)
async function fetchStatus() {
try {
const response = await schedulerService.getStatus()

19
WebUI/src/composables/useTaskPolling.js
View File

@@ -11,13 +11,18 @@ const MAX_POLL_ATTEMPTS = 30
export async function pollTaskStatus(taskId) {
for (let i = 0; i < MAX_POLL_ATTEMPTS; i++) {
await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL))
const response = await tasksAPI.getTaskStatus(taskId)
if (response.code !== 200) {
continue
}
const status = response.data.status
if (status === 'completed' || status === 'failed') {
return response
try {
const response = await tasksAPI.getTaskStatus(taskId)
if (response.code !== 200) {
continue
}
const status = response.data.status
if (status === 'completed' || status === 'failed' || status === 'cancelled') {
return response
}
} catch (error) {
// 网络异常时继续轮询，不中断
console.warn('轮询任务状态失败:', error)
}
}
return {

18
WebUI/src/services/pluginService.js
View File

@@ -26,25 +26,21 @@ export const pluginService = {
const finalRes = await pollTaskStatus(startRes.data.task_id)
return {
code: finalRes.code,
message: finalRes.data?.message || finalRes.message,
data: finalRes.data?.data || finalRes.data
message: finalRes.message,
data: finalRes.data?.result
}
},
async crawlAll() {
const startRes = await pluginsAPI.crawlAll()
if (startRes.code !== 200 || !startRes.data?.task_ids?.length) {
if (startRes.code !== 200 || !startRes.data?.task_id) {
return startRes
}
// 批量轮询所有任务，取最后一个完成的结果
const results = await Promise.all(
startRes.data.task_ids.map(tid => pollTaskStatus(tid))
)
const last = results[results.length - 1]
const finalRes = await pollTaskStatus(startRes.data.task_id)
return {
code: last.code,
message: last.data?.message || last.message,
data: last.data?.data || last.data
code: finalRes.code,
message: finalRes.message,
data: finalRes.data?.result
}
}
}

16
WebUI/src/stores/proxy.js
View File

@@ -77,17 +77,17 @@ export const useProxyStore = defineStore('proxy', () => {
* @param {number|string} port
* @returns {Promise<boolean>}
*/
async function deleteProxy(ip, port) {
async function deleteProxy(ip, port, page = 1, pageSize = 20, filters = {}) {
try {
const response = await proxyService.deleteProxy(ip, port)
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
await fetchProxies({ page, page_size: pageSize, ...filters }) // 刷新列表
return true
}
} catch (error) {
console.error('删除代理失败:', error)
return false
}
return false
}
/**
@@ -95,13 +95,13 @@ export const useProxyStore = defineStore('proxy', () => {
* @param {Array<{ip: string, port: number}>} proxyList
* @returns {Promise<number>} 实际删除的数量
*/
async function batchDeleteProxies(proxyList) {
async function batchDeleteProxies(proxyList, page = 1, pageSize = 20, filters = {}) {
if (!proxyList?.length) return 0
try {
const response = await proxyService.batchDelete(proxyList)
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
await fetchProxies({ page, page_size: pageSize, ...filters }) // 刷新列表
return response.data.deleted_count
}
} catch (error) {
@@ -114,17 +114,17 @@ export const useProxyStore = defineStore('proxy', () => {
* 清理无效代理
* @returns {Promise<number>} 删除的数量
*/
async function cleanInvalidProxies() {
async function cleanInvalidProxies(page = 1, pageSize = 20, filters = {}) {
try {
const response = await proxyService.cleanInvalid()
if (response.code === 200) {
await fetchProxies({ page: 1, page_size: 20 }) // 刷新列表
await fetchProxies({ page, page_size: pageSize, ...filters }) // 刷新列表
return response.data.deleted_count
}
} catch (error) {
console.error('清理无效代理失败:', error)
}
return 0
return -1
}
/**

6
WebUI/src/views/Dashboard.vue
View File

@@ -149,9 +149,13 @@ async function handleClean() {
)
const deletedCount = await proxyStore.cleanInvalidProxies()
if (deletedCount >= 0) {
if (deletedCount > 0) {
ElMessage.success(`已清理 ${deletedCount} 个无效代理`)
await proxyStore.fetchStats()
} else if (deletedCount === 0) {
ElMessage.info('没有需要清理的无效代理')
} else if (deletedCount === -1) {
ElMessage.error('清理无效代理失败')
}
} catch {
// 用户取消

38
WebUI/src/views/Plugins.vue
View File

@@ -101,11 +101,11 @@
<el-icon v-if="crawlResults[row.id].type === 'success'" class="result-icon success"><CircleCheck /></el-icon>
<el-icon v-else class="result-icon failed"><CircleClose /></el-icon>
<span class="result-text">{{ crawlResults[row.id].message }}</span>
<span v-if="crawlResults[row.id].data?.valid_count !== undefined" class="result-count valid">
有效 {{ crawlResults[row.id].data.valid_count }}
<span v-if="crawlResults[row.id].data?.success_count !== undefined" class="result-count valid">
有效 {{ crawlResults[row.id].data.success_count }}
</span>
<span v-if="crawlResults[row.id].data?.invalid_count !== undefined" class="result-count invalid">
无效 {{ crawlResults[row.id].data.invalid_count }}
<span v-if="crawlResults[row.id].data?.failure_count !== undefined" class="result-count invalid">
无效 {{ crawlResults[row.id].data.failure_count }}
</span>
<el-icon class="result-close" @click="clearCrawlResult(row.id)"><Close /></el-icon>
</div>
@@ -134,9 +134,7 @@
<span v-if="allCrawlResult.data.total_crawled !== undefined">
爬取: {{ allCrawlResult.data.total_crawled }}
</span>
<span v-if="allCrawlResult.data.proxy_count !== undefined">
爬取: {{ allCrawlResult.data.proxy_count }}
</span>
<span v-if="allCrawlResult.data.valid_count !== undefined" class="valid-count">
有效: {{ allCrawlResult.data.valid_count }}
</span>
@@ -235,14 +233,18 @@ async function handleToggle(pluginId, enabled) {
}
async function handleOpenConfig(row) {
currentPlugin.value = row
const response = await pluginService.getPluginConfig(row.id)
if (response.code === 200) {
Object.keys(configForm).forEach(key => delete configForm[key])
Object.assign(configForm, response.data.config || {})
configDialogVisible.value = true
} else {
ElMessage.error('获取插件配置失败')
try {
currentPlugin.value = row
const response = await pluginService.getPluginConfig(row.id)
if (response.code === 200) {
Object.keys(configForm).forEach(key => delete configForm[key])
Object.assign(configForm, response.data.config || {})
configDialogVisible.value = true
} else {
ElMessage.error('获取插件配置失败')
}
} catch (error) {
ElMessage.error('获取插件配置出错')
}
}
@@ -321,11 +323,13 @@ async function handleCrawlAll() {
if (response.code === 200) {
allCrawlResult.value = {
type: 'success',
type: response.data?.cancelled ? 'info' : 'success',
message: response.message,
data: response.data
}
ElMessage.success('批量爬取完成')
if (!response.data?.cancelled) {
ElMessage.success('批量爬取完成')
}
} else {
allCrawlResult.value = {
type: 'error',

51
WebUI/src/views/ProxyList.vue
View File

@@ -190,18 +190,27 @@ async function fetchProxies() {
}
abortController = new AbortController()
const success = await proxyStore.fetchProxies({
page: currentPage.value,
page_size: pageSize.value,
protocol: filterForm.protocol || null,
min_score: filterForm.minScore,
sort_by: filterForm.sortBy,
sort_order: filterForm.sortOrder
}, abortController.signal)
abortController = null
if (!success) {
ElMessage.error('获取代理列表失败')
try {
const success = await proxyStore.fetchProxies({
page: currentPage.value,
page_size: pageSize.value,
protocol: filterForm.protocol || null,
min_score: filterForm.minScore,
sort_by: filterForm.sortBy,
sort_order: filterForm.sortOrder
}, abortController.signal)
if (!success) {
ElMessage.error('获取代理列表失败')
}
} catch (error) {
if (error.name === 'AbortError') {
// 用户主动取消，不提示错误
return
}
throw error
} finally {
abortController = null
}
}
@@ -223,10 +232,15 @@ async function handleDelete(proxy) {
const confirmed = await confirmDelete(`代理 ${proxy.ip}:${proxy.port}`)
if (!confirmed) return
const success = await proxyStore.deleteProxy(proxy.ip, proxy.port)
const filters = {
protocol: filterForm.protocol || null,
min_score: filterForm.minScore,
sort_by: filterForm.sortBy,
sort_order: filterForm.sortOrder
}
const success = await proxyStore.deleteProxy(proxy.ip, proxy.port, currentPage.value, pageSize.value, filters)
if (success) {
ElMessage.success('删除成功')
fetchProxies()
}
}
@@ -237,11 +251,16 @@ async function handleBatchDelete() {
const confirmed = await confirmBatchDelete(count, '代理')
if (!confirmed) return
const deletedCount = await proxyStore.batchDeleteProxies(selectedProxies.value)
const filters = {
protocol: filterForm.protocol || null,
min_score: filterForm.minScore,
sort_by: filterForm.sortBy,
sort_order: filterForm.sortOrder
}
const deletedCount = await proxyStore.batchDeleteProxies(selectedProxies.value, currentPage.value, pageSize.value, filters)
if (deletedCount > 0) {
ElMessage.success(`已删除 ${deletedCount} 个代理`)
selectedProxies.value = []
fetchProxies()
}
}