feat(crawl): browser-like headers, HTTP/2, curl_cffi TLS fingerprint fallback
- get_headers(url): Referer, Sec-Fetch-*, sec-ch-ua, API vs HTML Accept - httpx AsyncClient/ sync Client with optional HTTP/2 (h2 extra) - On 403/429/503/520-523/525/567 or request errors, retry via curl_cffi chrome124 impersonate - POST: Origin, Referer, Content-Type for form posts - kuaidaili/ip3366: forward get_headers(url=...) Made-with: Cursor
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import re
|
||||
from typing import List
|
||||
from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
from app.core.plugin_system import ProxyRaw
|
||||
from app.plugins.base import BaseHTTPPlugin
|
||||
@@ -26,8 +26,8 @@ class Ip3366Plugin(BaseHTTPPlugin):
|
||||
f"http://www.ip3366.net/free/?stype=2&page={i}" for i in range(1, max_pages + 1)
|
||||
]
|
||||
|
||||
def get_headers(self) -> dict:
|
||||
headers = super().get_headers()
|
||||
def get_headers(self, url: Optional[str] = None, **kwargs) -> dict:
|
||||
headers = super().get_headers(url=url, **kwargs)
|
||||
headers["Referer"] = "http://www.ip3366.net/free/"
|
||||
return headers
|
||||
|
||||
|
||||
Reference in New Issue
Block a user