imedinews.ge、rustavi2.ge、1tv.ge、bm.ge、armenpress.am
对比两次请求
import requests
cookies = { 'cf_clearance': '2xUhgmxJp0xlCxywZNp_p_qtv2zs7I8tax0EmixdD2Y-1784515876-1.2.1.1-qC.kt_LZczHbUg.3IxyjCNmWS5D_MUnyjEQC_0wruC5adJb6wEuBXjhIoS4gLRcX8LIqYGT.9UfR0Wyw9.dnfMa6GZHbcJ726RtgbM.jmNa4SsQsowc8Rv0h.rhckqjao3KaQylY.Ym1Ru1h.JR6N.mEP5VzRgVr3bz4QuVh75UFSB9lgriI54LmPzzOYoE7YsCpgQxHpuzug1I1hThxfiMPbhwbyuKfL.YrLbeESF12fNDVKw1tfHKUkE59R3WA9uWlHQbrDtD_FZMOg1tePCKgOCgpBG5hfvSm.hYzmjC_SJ8XsjbVpa85lJdbLyNea6iN3IfYNG0P1GGVTgLzw26MZbYPrBlp8MrIk7zOOtDnNDXYtNCN3lTwt7avkEIx63NHLe5Mfdqmy0ZM9krclc61IykXF35jnxNftkeWfSIfOk_F09OTbTOzld0CVpG6bx9hxEVZhX3gqk7ZVmpaFH9Du291D.kNxD1fLGhLfxLR49Nu77cSw30GCzM8OkFwtNSjw.q5ZTqCASzMExY6Gw',}
headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'pragma': 'no-cache', 'priority': 'u=0, i', 'sec-ch-ua': '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"', 'sec-ch-ua-arch': '"arm"', 'sec-ch-ua-bitness': '"64"', 'sec-ch-ua-full-version': '"150.0.7871.115"', 'sec-ch-ua-full-version-list': '"Not;A=Brand";v="8.0.0.0", "Chromium";v="150.0.7871.115", "Google Chrome";v="150.0.7871.115"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-model': '""', 'sec-ch-ua-platform': '"macOS"', 'sec-ch-ua-platform-version': '"26.5.2"', 'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', 'sec-fetch-user': '?1', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36',}
response = requests.get('https://nopecha.com/demo/cloudflare', cookies=cookies, headers=headers)
# print(response.text)print(response.status_code)请求结果返回403
import curl_cffi
cookies = { 'cf_clearance': '2xUhgmxJp0xlCxywZNp_p_qtv2zs7I8tax0EmixdD2Y-1784515876-1.2.1.1-qC.kt_LZczHbUg.3IxyjCNmWS5D_MUnyjEQC_0wruC5adJb6wEuBXjhIoS4gLRcX8LIqYGT.9UfR0Wyw9.dnfMa6GZHbcJ726RtgbM.jmNa4SsQsowc8Rv0h.rhckqjao3KaQylY.Ym1Ru1h.JR6N.mEP5VzRgVr3bz4QuVh75UFSB9lgriI54LmPzzOYoE7YsCpgQxHpuzug1I1hThxfiMPbhwbyuKfL.YrLbeESF12fNDVKw1tfHKUkE59R3WA9uWlHQbrDtD_FZMOg1tePCKgOCgpBG5hfvSm.hYzmjC_SJ8XsjbVpa85lJdbLyNea6iN3IfYNG0P1GGVTgLzw26MZbYPrBlp8MrIk7zOOtDnNDXYtNCN3lTwt7avkEIx63NHLe5Mfdqmy0ZM9krclc61IykXF35jnxNftkeWfSIfOk_F09OTbTOzld0CVpG6bx9hxEVZhX3gqk7ZVmpaFH9Du291D.kNxD1fLGhLfxLR49Nu77cSw30GCzM8OkFwtNSjw.q5ZTqCASzMExY6Gw',}
headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'pragma': 'no-cache', 'priority': 'u=0, i', 'sec-ch-ua': '"Not;A=Brand";v="8", "Chromium";v="150", "Google Chrome";v="150"', 'sec-ch-ua-arch': '"arm"', 'sec-ch-ua-bitness': '"64"', 'sec-ch-ua-full-version': '"150.0.7871.115"', 'sec-ch-ua-full-version-list': '"Not;A=Brand";v="8.0.0.0", "Chromium";v="150.0.7871.115", "Google Chrome";v="150.0.7871.115"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-model': '""', 'sec-ch-ua-platform': '"macOS"', 'sec-ch-ua-platform-version': '"26.5.2"', 'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', 'sec-fetch-user': '?1', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36', # 'cookie': 'cf_clearance=2xUhgmxJp0xlCxywZNp_p_qtv2zs7I8tax0EmixdD2Y-1784515876-1.2.1.1-qC.kt_LZczHbUg.3IxyjCNmWS5D_MUnyjEQC_0wruC5adJb6wEuBXjhIoS4gLRcX8LIqYGT.9UfR0Wyw9.dnfMa6GZHbcJ726RtgbM.jmNa4SsQsowc8Rv0h.rhckqjao3KaQylY.Ym1Ru1h.JR6N.mEP5VzRgVr3bz4QuVh75UFSB9lgriI54LmPzzOYoE7YsCpgQxHpuzug1I1hThxfiMPbhwbyuKfL.YrLbeESF12fNDVKw1tfHKUkE59R3WA9uWlHQbrDtD_FZMOg1tePCKgOCgpBG5hfvSm.hYzmjC_SJ8XsjbVpa85lJdbLyNea6iN3IfYNG0P1GGVTgLzw26MZbYPrBlp8MrIk7zOOtDnNDXYtNCN3lTwt7avkEIx63NHLe5Mfdqmy0ZM9krclc61IykXF35jnxNftkeWfSIfOk_F09OTbTOzld0CVpG6bx9hxEVZhX3gqk7ZVmpaFH9Du291D.kNxD1fLGhLfxLR49Nu77cSw30GCzM8OkFwtNSjw.q5ZTqCASzMExY6Gw',}
response = curl_cffi.get('https://nopecha.com/demo/cloudflare', cookies=cookies, headers=headers, impersonate="chrome124")
print(response.text)print(response.status_code)请求结果返回200
curl_cffi可以模拟浏览器的 TLS/JA3 和 HTTP/2 指纹。
对比指纹
import requestsimport curl_cffi
response = requests.get("https://tls.browserleaks.com/json")print(response.text)response = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome124")print(response.text)输出如下:

对比请求
import requestsimport curl_cfficookies = { 'cf_clearance': '2xUhgmxJp0xlCxywZNp_p_qtv2zs7I8tax0EmixdD2Y-1784515876-1.2.1.1-qC.kt_LZczHbUg.3IxyjCNmWS5D_MUnyjEQC_0wruC5adJb6wEuBXjhIoS4gLRcX8LIqYGT.9UfR0Wyw9.dnfMa6GZHbcJ726RtgbM.jmNa4SsQsowc8Rv0h.rhckqjao3KaQylY.Ym1Ru1h.JR6N.mEP5VzRgVr3bz4QuVh75UFSB9lgriI54LmPzzOYoE7YsCpgQxHpuzug1I1hThxfiMPbhwbyuKfL.YrLbeESF12fNDVKw1tfHKUkE59R3WA9uWlHQbrDtD_FZMOg1tePCKgOCgpBG5hfvSm.hYzmjC_SJ8XsjbVpa85lJdbLyNea6iN3IfYNG0P1GGVTgLzw26MZbYPrBlp8MrIk7zOOtDnNDXYtNCN3lTwt7avkEIx63NHLe5Mfdqmy0ZM9krclc61IykXF35jnxNftkeWfSIfOk_F09OTbTOzld0CVpG6bx9hxEVZhX3gqk7ZVmpaFH9Du291D.kNxD1fLGhLfxLR49Nu77cSw30GCzM8OkFwtNSjw.q5ZTqCASzMExY6Gw',}
headers = { # 同上请求头,省略 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36'}response = requests.get("https://httpbin.org/anything", headers=headers, cookies=cookies)print(response.text)response = curl_cffi.get("https://httpbin.org/anything", headers=headers, cookies=cookies, impersonate="chrome124")print(response.text)当不给入请求头时,requests会携带默认UA请求头。
仅携带user-agent请求头时,结果如下:
curl_cffi还会额外加入其他的请求头,这些都是浏览器自动添加的安全相关请求头,开发者无法通过 JS 修改(Sec- 前缀是 “Security” 的缩写),主要用于防止 CSRF 攻击和帮助服务器识别请求上下文。

当给入完整的请求头时,由于没有给入Accept-Encoding,所以不同的请求库给了不一样的默认值。
X-Amzn-Trace-Id 是 AWS 的请求链路追踪 header。非请求相关的,不影响结果。

综上,在使用相同的url、headers、cookies下,使用了不同的请求库得到了不一样的结果,原因是因为cloudflare五秒盾检查了TLS/JA3指纹的
如何获取cf_clearance值呢
cf_clearance是在浏览器上第一次访问被保护的网站通过验证时生成的值,默认有效期30分钟
| Challenges 总览 | https://developers.cloudflare.com/cloudflare-challenges/ |
|---|---|
| Under Attack 模式 | https://developers.cloudflare.com/fundamentals/reference/under-attack-mode/ |
| Security Level(安全级别) | https://developers.cloudflare.com/waf/tools/security-level/ |
主要是请求一些混淆JS检查是否真实浏览器环境,而非脚本。
所以可以通过使用浏览器自动化去获取cf_clearance, 以绕过检查。
https://github.com/sarperavci/CloudflareBypassForScraping
由一个真正的隐身浏览器提供支持(通过 CloakBrowser 修补的 Chromium)。
https://github.com/CloakHQ/cloakbrowser
能够通过所有机器人检测测试的隐身Chrome。即插即用的Playwright替代品,带源级指纹补丁。30/30测试通过。
https://github.com/ultrafunkamsterdam/undetected-chromedriver
定制化 Selenium Chromedriver | 无需配置 | 通过所有反爬虫系统(如 Distil / Imperva/ Datadadome / CloudFlare IUAM)
https://github.com/ultrafunkamsterdam/nodriver
Undetected-Chromedriver 的继任者。它提供了一个速度极快的框架,用于网页自动化、网页抓取、机器人程序以及其他任何通常会受到 Captcha、CloudFlare、Imperva 或 hCaptcha 等烦人的反机器人系统阻碍的创意应用。
或许还可以试试项目https://github.com/VeNoMouS/cloudscraper,但是对我测试的demo并不起作用。
先尝试一些常见的浏览器自动化框架,即使人工点击,也是无法通过的,因为浏览器环境已经被检测到了。
还有无头模式下,更容易被检测为机器人。
from playwright.sync_api import sync_playwright
with sync_playwright() as p: browser = p.chromium.launch( headless=False ) page = browser.new_page() page.goto("https://nopecha.com/demo/cloudflare",wait_until="domcontentloaded",timeout=30000) print(page.title()) page.wait_for_timeout(10000) page.pause() # pause状态下始终无法通过check browser.close()尝试使用playwright-stealth等隐藏浏览器自动化特征
比如以下:
https://pypi.org/project/undetected-playwright/
https://pypi.org/project/playwright-stealth/
Drissionpage 默认下的自动化特征相比playwright隐藏更好。
还有https://github.com/D4Vinci/Scrapling也自带隐藏。
其实这里人工点击也能通过cloudflare检测,但是无法实现自动化点击通过。
from playwright.sync_api import sync_playwright
# uv add undetected-playwright# https://pypi.org/project/undetected-playwright/from undetected_playwright import Tarnished
# https://pypi.org/project/playwright-stealth/# from playwright_stealth import Stealth
with sync_playwright() as p: browser = p.chromium.launch( headless=False, args=[ "--disable-blink-features=AutomationControlled", # 禁用自动化特征 "--no-sandbox", # 提升兼容性 "--disable-dev-shm-usage", # 避免共享内存问题 ], ignore_default_args=["--enable-automation"], # 移除默认自动化参数 ) context = browser.new_context() # 配合参数绕过瑞数检测 Tarnished.apply_stealth(context) page = context.new_page() page.goto("https://nopecha.com/demo/cloudflare",wait_until="domcontentloaded",timeout=30000) print(page.title()) page.wait_for_timeout(10000) # 等页面加载 page.pause() browser.close()CloakBrowser
安装后第一次运行程序时,会自动下载chrome浏览器二进制文件,但是默认的下载地址下载速度极慢。
找到下载路径:https://cloakbrowser.dev/chromium-v145.0.7632.109.2/cloakbrowser-darwin-arm64.tar.gz
这是默认的,还有备用从GitHub下载,地址如下:https://github.com/CloakHQ/CloakBrowser/releases/download/chromium-v145.0.7632.109.2/cloakbrowser-darwin-arm64.tar.gz
使用下载加速:https://gh-proxy.com/ ,获取一个代理下载地址下载
下载后解压到目录:默认在_~/.cloakbrowser/chromium-145.0.7632.109.2/Chromium.app _路径下
解除验证:xattr -cr ~/.cloakbrowser/chromium-*/Chromium.app
现在可以运行了,想进行自动化点击,但是却发现如何都选取不到那个checkbox元素。这是因为受限于#shadow-root (closed),playwright API 无法选取到iframe内部元素。#shadow-root (closed) 是 Shadow DOM 技术,是一种封装机制,故意阻止外部访问内部元素。
不过可以通过在页面加载前注入JS,将 closed shadow dom 变为 open,让playwright API 选取到iframe中的checkbox元素。
但是注入该JS会导致验证不可通过,即使人工点击,估计被检测到了浏览器环境不正常。(不注入时人工点击是可以通过的),与最终目的相矛盾了,该手段便不可取。
page = browser.new_page()
# 在页面加载前注入,将 closed shadow dom 变为 open,让playwright API 选取到iframe中的checkbox元素。# 但是注入该JS会导致验证不可通过,即使人工点击,估计被检测到了浏览器环境不正常。(不注入时人工点击是可以通过的)page.add_init_script(""" const originalAttachShadow = Element.prototype.attachShadow; Element.prototype.attachShadow = function(init) { // 强制将 closed 改为 open return originalAttachShadow.call(this, { ...init, mode: 'open' }); };""")page.goto("https://nopecha.com/demo/cloudflare",wait_until="domcontentloaded",timeout=30000)page.wait_for_timeout(8000) # 等页面加载
def find_turnstile_checkbox(page): # 受限于#shadow-root (closed),playwright API 无法选取到iframe中的checkbox元素。 # #shadow-root (closed) 是 Shadow DOM 技术,是一种封装机制,故意阻止外部访问内部元素。 for iframe in page.frames: if iframe.url.startswith("https://challenges.cloudflare.com"): checkbox = iframe.locator('input[type="checkbox"]') # checkbox.click() # checkbox.check() # 验证通过会 POST https://nopecha.com/demo/cloudflare 返回 302 return checkbox # child中也存在一个可以点击的,目前不知道有啥区别 for iframe in page.frames: # 这里body中嵌套了一个内部iframe,使用js动态创建的。 for child in iframe.child_frames: try: # query_selector 比 count 更稳定 print("child:", child) if checkbox := child.query_selector('input[type="checkbox"]'): return checkbox except Exception as e: print(f"跳过: {e}") continue最后使用JS获取元素坐标,然后模拟鼠标点击坐标的方式通过验证。代码参考https://github.com/sarperavci/CloudflareBypassForScraping/blob/120f8fdad225b4ee06b4d7095443c8ba09785539/cf_bypasser/core/bypasser.py 进行简化,便于理解,很多边缘情况未处理。
import logging
# 在 import cloakbrowser 之前或之后执行均可logging.basicConfig( level=logging.INFO, format='%(asctime)s [%(name)s] %(levelname)s: %(message)s',)logger = logging.getLogger("cloudflare_bypass")from cloakbrowser import launch # Logger 自身级别设为最低
# https://cloakbrowser.dev/chromium-v145.0.7632.109.2/cloakbrowser-darwin-arm64.tar.gz# https://github.com/CloakHQ/CloakBrowser/releases/download/chromium-v145.0.7632.109.2/cloakbrowser-darwin-arm64.tar.gz# 下载加速:https://gh-proxy.com/# 解除验证:xattr -cr ~/.cloakbrowser/chromium-*/Chromium.app# 默认在~/.cloakbrowser/chromium-145.0.7632.109.2/Chromium.app 路径下## 使用 Playwright 时应始终使用 page.locator(selector).* 或 page.click(selector) 等方式,# 避免使用 page.query_selector() 返回的 ElementHandle,因为后者会绕过 humanize 管道,导致鼠标瞬移、键盘无时序
# 在 Turnstile iframe 内运行的 JS:遍历打开和关闭的 shadow root,并返回复选框中心相对于 iframe 视口的坐标。_FIND_CHECKBOX_JS = """() => { function find(root){ if(!root) return null; const direct = root.querySelector && root.querySelector('input[type=checkbox]'); if(direct) return direct; for(const el of (root.querySelectorAll ? root.querySelectorAll('*') : [])){ const sr = el.fakeShadowRoot || el.shadowRoot; if(sr){ const r = find(sr); if(r) return r; } } return null; } const cb = find(document); if(!cb) return {found:false}; const r = cb.getBoundingClientRect(); return {found:true, checked:cb.checked, x:r.x+r.width/2, y:r.y+r.height/2, w:r.width};}"""
def is_bypassed(page) -> bool: """Check if the Cloudflare challenge has been cleared (and not a block page).""" try: title = page.title() if "just a moment" in title.lower(): return False html_content = page.content() lowered = html_content.lower() if "please complete the captcha" in lowered: return False return True except Exception as e: logger.info(f"Error checking bypass status: {e}") return False
def get_cookies_and_user_agent(context, page): try: cookies = context.cookies() cookie_dict = {c["name"]: c["value"] for c in cookies} user_agent = page.evaluate("navigator.userAgent") return {"cookies": cookie_dict, "user_agent": user_agent} except Exception as e: logger.info(f"Error getting cookies and user agent: {e}") return None
def click_turnstile_checkbox(page): cf_frames = [f for f in page.frames if "challenges.cloudflare" in (f.url or "")] for frame in cf_frames: try: # 调试用:确认 fakeShadowRoot 是否生效 # has_fake = frame.evaluate("() => { const d = document.createElement('div'); d.attachShadow({mode:'closed'}); return 'fakeShadowRoot' in d; }") # print("fakeShadowRoot available:", has_fake) info = frame.evaluate(_FIND_CHECKBOX_JS) if not info.get("found") or info.get("w", 0) <= 0 or info.get("checked"): continue frame_el = frame.frame_element() box = frame_el.bounding_box() if not box: continue # checkbox coords are iframe-relative; offset by the iframe's page position page.mouse.click(box["x"] + info["x"], box["y"] + info["y"]) # only count it as clicked if the box became checked or disappeared after = frame.evaluate(_FIND_CHECKBOX_JS) if (not after.get("found")) or after.get("checked"): logger.info("Clicked Turnstile checkbox via fakeShadowRoot") return True logger.info("Turnstile checkbox click did not register, retrying") except Exception as e: logger.info(f"Checkbox click attempt failed: {e}") return False
browser = launch( # 优先使用socket5协议 # proxy="http://user:pass@residential-proxy:port", # residential IP, not datacenter # geoip=True, # match timezone + locale to proxy IP headless=False, # some sites detect headless even with C++ patches humanize=True, # human-like mouse, keyboard, scroll # 通过修补的Chromium实现原生封闭shadow-root访问,使我们无需注入attachShadow补丁即可读取和点击Cloudflare验证码复选框。 args=["--enable-blink-features=FakeShadowRoot"])print("browser launched")context = browser.new_context()page = context.new_page()response = page.goto("https://nopecha.com/demo/cloudflare",wait_until="domcontentloaded", timeout=30000)
print("状态码:", response.status)page.wait_for_timeout(8000) # 等页面加载
res = click_turnstile_checkbox(page)print("res:", res)if res: page.wait_for_timeout(8000) if is_bypassed(page): print("bypassed") data = get_cookies_and_user_agent(context, page) # cf_clearance cookies and user agent print("data:", data)
page.pause()browser.close()