demo: https://www.ishumei.com/trial/captcha.html
“点击图中最小的蓝色长方体”

思路:裁剪图中的所有目标,VLM模型做逻辑推理选择,依据目标定位坐标。
YOLO模型定位裁剪图中的所有目标并获得坐标,VLM模型做逻辑推理判断“是否为蓝色,是否为长方体”,对裁剪的几何体图片做尺寸判断,找其中最小的。
先写一个采集图片的程序,采集图片。 https://captcha1.fengkongcloud.cn/ca/v1/register 接口没什么加密需要逆向的。
使用现有的模型批量标注,再人工调整缺漏,提升效率。

训练集88张,验证集20张,小模型yolo26n, Epochs 200 训练结果如下

使用新图片测试模型结果

VLM大语言模型存在几个问题,第一,成本比较高;还有,对坐标的感知不是很准确。还有几何体的判断,对空间几何的感知能力 GePBench, 最强 VLM 在“识别几何形状”上也只有约 71–81%
针对当前实际情况优化,实际100次刷新问题测试下来,可以缩小范围优化。
尺寸只有”最小”,可以直接计算裁剪后的图片面积。 颜色只有”红黄蓝绿”,可以 HSV 色相判断。 几何体判断,有”圆柱体,长方体,球体,圆锥,三棱锥,三棱柱,六棱柱,圆柱体” 7类,可以微调一个小 CNN(ResNet/EfficientNet) 做闭集分类。
颜色判断
import argparseimport csvimport mathimport shutilimport sysfrom pathlib import Path
import cv2import numpy as np
IMG_EXTS = {".jpg", ".jpeg", ".png"}COLORS = ("红色", "黄色", "绿色", "蓝色")
def hue_word(h: float) -> str: if h >= 170 or h < 12: return "红色" if 15 <= h < 38: # 下界 15:渲染域的"黄色"球体实测中心色相 16~17(12~18 为红黄过渡带) return "黄色" if 38 <= h < 90: return "绿色" if 90 <= h < 135: return "蓝色" return "其他"
def center_color(bgr: np.ndarray, patch: int = 11, s_min: int = 50, v_min: int = 50) -> str: """中心 patch 逐像素分箱投票;高光/阴影像素剔除;票数不足外扩一档""" h, w = bgr.shape[:2] hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV) cy, cx = h // 2, w // 2 for p in (patch, patch * 2 + 1, patch * 4 + 1): r = p // 2 roi = hsv[max(0, cy - r):cy + r + 1, max(0, cx - r):cx + r + 1] Hh, Ss, Vv = roi[..., 0].astype(int), roi[..., 1].astype(int), roi[..., 2].astype(int) ok = (Ss >= s_min) & (Vv >= v_min) if ok.sum() < max(3, (p * p) // 4): # 中心被高光/阴影占据,外扩 continue votes = {} for word in COLORS + ("其他",): votes[word] = 0 for hh in Hh[ok]: votes[hue_word(int(hh))] += 1 word, cnt = max(votes.items(), key=lambda kv: kv[1]) return word if cnt >= 0.5 * int(ok.sum()) else "其他" return "其他"
几何体判断
无监督聚类、再用题目文案当真值锚点归并
对裁剪出的几何体小图做无监督聚类:同一种几何体归入同一文件夹(不识别具体名称)
思路(核心难点是颜色干扰——同形状不同颜色必须归同簇,故先灰度化再提特征):
- 保持纵横比贴到白底 128×128 画布,转灰度(消除颜色维度)
- ImageNet 预训练 ResNet18 提 512 维形状嵌入,L2 归一化
- PCA 降维后,KMeans 与层次聚类(Agglomerative/cosine) 各扫一遍 k∈[kmin,kmax], 取轮廓系数(silhouette, cosine) 最高的组合作为最终簇数与划分
import argparseimport csvimport mathimport shutilimport sysfrom pathlib import Path
import cv2import numpy as npimport torchimport torch.nn.functional as Fimport torchvisionfrom sklearn.cluster import AgglomerativeClustering, KMeansfrom sklearn.decomposition import PCAfrom sklearn.metrics import silhouette_score
IMG_EXTS = {".jpg", ".jpeg", ".png"}CANVAS = 128
def load_prep(path: Path): """保持纵横比贴白底画布 + 灰度化,输出 ImageNet 归一化张量;读不出来返回 None""" img = cv2.imread(str(path)) if img is None: return None h, w = img.shape[:2] scale = (CANVAS - 8) / max(h, w) nw, nh = max(1, round(w * scale)), max(1, round(h * scale)) rs = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_AREA) canvas = np.full((CANVAS, CANVAS, 3), 255, np.uint8) y0, x0 = (CANVAS - nh) // 2, (CANVAS - nw) // 2 canvas[y0:y0 + nh, x0:x0 + nw] = rs gray = cv2.cvtColor(canvas, cv2.COLOR_BGR2GRAY) # 关键:去掉颜色 rgb = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB) x = torch.from_numpy(rgb).permute(2, 0, 1).float().div_(255) mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1) std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1) return (x - mean) / std
@torch.no_grad()def embed_all(paths: list, device) -> np.ndarray: weights = torchvision.models.ResNet18_Weights.DEFAULT backbone = torchvision.models.resnet18(weights=weights) feat = torch.nn.Sequential(*list(backbone.children())[:-1]).to(device).eval() out = [] B = 256 for i in range(0, len(paths), B): batch = torch.stack([load_prep(p) for p in paths[i:i + B]]).to(device) out.append(F.normalize(feat(batch).flatten(1), dim=1).cpu()) return torch.cat(out).numpy()
def choose_clustering(X: np.ndarray, kmin: int, kmax: int): """KMeans / Agglomerative × k 扫描,silhouette(cosine) 最高者胜出""" best = (-2.0, None, None) # (score, name, labels) for k in range(kmin, kmax + 1): candidates = [ ("kmeans", KMeans(n_clusters=k, n_init=20, random_state=0).fit_predict(X)), ("agglo", AgglomerativeClustering( n_clusters=k, metric="cosine", linkage="average").fit_predict(X)), ] for name, labels in candidates: s = silhouette_score(X, labels, metric="cosine") print(f" k={k:2d} {name:6s} silhouette={s:.4f}") if s > best[0]: best = (s, name, labels) return best得到下面分类的结果,每一类下都是相同的几何体。只是有些没有被归为同一类。

人工整理完成正确的7个几何体分类,并标注对应的几何体名称。

已分类的各组样本嵌入取质心,固化为 7 个”形状原型”(原型 ↔ 名称由groups/mapping.json 固定)。之后对任意新裁剪图:嵌入 → 与 7 原型算余弦相似度 →
最近者即其几何体名称,同时给出 top1-top2 余量(margin)作置信度。构建原型shape_prototypes.npz
import argparseimport csvimport jsonimport mathimport shutilimport sysfrom pathlib import Path
import cv2import numpy as npimport torchimport torch.nn.functional as F
from cluster_shapes import embed_all, load_prep, contact_sheet, IMG_EXTS # 复用嵌入管线
HERE = Path("ishumei")PROTOS = HERE / "images/spatial_select_crops/shape_prototypes.npz"
def build(groups_dir: Path) -> None: mapping = json.loads((groups_dir / "mapping.json").read_text(encoding="utf-8")) device = "mps" if torch.backends.mps.is_available() else "cpu" names, cents = [], [] for gdir in sorted(mapping): files = sorted(p for p in (groups_dir / gdir).iterdir() if p.suffix.lower() in IMG_EXTS) if not files: print(f"跳过空组 {gdir}", file=sys.stderr) continue feats = embed_all(files, device) c = feats.mean(axis=0) cents.append(c / np.linalg.norm(c)) names.append(mapping[gdir]) print(f"{gdir} [{mapping[gdir]}]: {len(files)} 样本 -> 质心") np.savez(PROTOS, names=np.array(names), centroids=np.stack(cents)) print(f"\n原型已保存: {PROTOS} ({len(names)} 类)")整合所有的判断识别,标注判断准确率。
def solve_once(session: requests.Session, yolo, shape_clf: ShapeClassifier, run_dir: Path, conf: float, submit: bool = True) -> dict: """解一道验证码,落盘全部中间产物;submit=True 时把答案提交 fverify 实测判定""" run_dir.mkdir(parents=True, exist_ok=True)
# 1. 刷新 + 2. 拉图 uuid = gen_captcha_uuid() data = register(session, "spatial_select", uuid) start_ms = int(time.time() * 1000) # gt 耗时从验证码下发时刻起算 detail = data["detail"] url, content = download_bg(session, detail) scene_path = run_dir / "scene.jpg" scene_path.write_bytes(content) img = cv2.imread(str(scene_path)) h, w = img.shape[:2]
# 3. 拆题 question = (detail.get("order") or [""])[0] size, color, shape = parse_order(question) shape = SHAPE_ALIASES.get(shape, shape) result = { "rid": detail["rid"], "captchaUuid": uuid, "question": question, "size": size, "color": color, "shape": shape, "scene": str(scene_path), "url": url, "image_size": [w, h], "objects": [], } if not (size and color and shape): result.update(ok=False, reason=f"题目无法完整拆解: {question}") return result
# 4. YOLO 检测 + 裁剪 res = yolo.predict(source=img, conf=conf, verbose=False)[0] for i, box in enumerate(res.boxes): x1, y1, x2, y2 = (int(v) for v in box.xyxy[0]) cx, cy = (x1 + x2) // 2, (y1 + y2) // 2 crop_path = run_dir / f"obj_{i}_{cx}_{cy}.jpg" cv2.imwrite(str(crop_path), img[y1:y2, x1:x2]) result["objects"].append({ "idx": i, "bbox": [x1, y1, x2, y2], "center": [cx, cy], "area": (x2 - x1) * (y2 - y1), "conf": round(float(box.conf), 3), "crop": str(crop_path), }) if not result["objects"]: result.update(ok=False, reason="YOLO 未检出任何几何体") return result
# 5. 子图识别:几何体(最近原型) + 颜色(中心点 HSV) for o, (shp, score, margin) in zip( result["objects"], shape_clf.classify_paths([o["crop"] for o in result["objects"]])): o["shape"] = shp o["shape_score"] = round(score, 4) o["shape_margin"] = round(margin, 4) o["color"] = center_color(cv2.imread(str(o["crop"])))
# 6. 双匹配候选中按尺寸词选答案 candidates = [o for o in result["objects"] if o["color"] == color and o["shape"] == shape] result["candidates"] = [o["idx"] for o in candidates] answer = pick_by_size(candidates, size) if answer is None: got = {(o["color"], o["shape"]) for o in result["objects"]} result.update(ok=False, reason=f"无 颜色+几何体 匹配的候选(题目要 {color}{shape},图中有 {got})") return result
# 7. 标注落盘 result.update(ok=True, answer=answer) answer_crop = run_dir / f"answer_{answer['center'][0]}_{answer['center'][1]}.jpg" shutil.copy2(answer["crop"], answer_crop) result["answer_crop"] = str(answer_crop) result["marked"] = str(run_dir / "marked.jpg") annotate(img, result["objects"], answer, Path(result["marked"]), question=question) return result
协议对 captcha-sdk v1.0.4 的逆向
除 organization/rid/captchaUuid/ostype/protocol/act.os/版本号外,
其余字段均为 DES-ECB + ZeroPadding + Base64,密钥为 8 字节 ASCII 硬编码。
空间推理模式与图标点选共用同一 SDK 与字段结构,sp 只有一个点击点。
测试了100次通过率100%

代码省略了部分方法,避免过于混乱。只保留核心的思路,可能并不能直接运行。 大部分由Zcode+GLM5.3 flash完成。