1950 字
10 分钟
数美空间推理验证码

demo: https://www.ishumei.com/trial/captcha.html

“点击图中最小的蓝色长方体” image.png

思路:裁剪图中的所有目标,VLM模型做逻辑推理选择,依据目标定位坐标。

YOLO模型定位裁剪图中的所有目标并获得坐标,VLM模型做逻辑推理判断“是否为蓝色,是否为长方体”,对裁剪的几何体图片做尺寸判断,找其中最小的。

先写一个采集图片的程序,采集图片。 https://captcha1.fengkongcloud.cn/ca/v1/register 接口没什么加密需要逆向的。

使用现有的模型批量标注,再人工调整缺漏,提升效率。 image.png

训练集88张,验证集20张,小模型yolo26n, Epochs 200 训练结果如下

image.png

使用新图片测试模型结果

VLM大语言模型存在几个问题,第一,成本比较高;还有,对坐标的感知不是很准确。还有几何体的判断,对空间几何的感知能力 GePBench, 最强 VLM 在“识别几何形状”上也只有约 71–81%

针对当前实际情况优化,实际100次刷新问题测试下来,可以缩小范围优化。

尺寸只有”最小”,可以直接计算裁剪后的图片面积。 颜色只有”红黄蓝绿”,可以 HSV 色相判断。 几何体判断,有”圆柱体,长方体,球体,圆锥,三棱锥,三棱柱,六棱柱,圆柱体” 7类,可以微调一个小 CNN(ResNet/EfficientNet) 做闭集分类。

颜色判断#

import argparse
import csv
import math
import shutil
import sys
from pathlib import Path
import cv2
import numpy as np
IMG_EXTS = {".jpg", ".jpeg", ".png"}
COLORS = ("红色", "黄色", "绿色", "蓝色")
def hue_word(h: float) -> str:
if h >= 170 or h < 12:
return "红色"
if 15 <= h < 38: # 下界 15:渲染域的"黄色"球体实测中心色相 16~17(12~18 为红黄过渡带)
return "黄色"
if 38 <= h < 90:
return "绿色"
if 90 <= h < 135:
return "蓝色"
return "其他"
def center_color(bgr: np.ndarray, patch: int = 11, s_min: int = 50, v_min: int = 50) -> str:
"""中心 patch 逐像素分箱投票;高光/阴影像素剔除;票数不足外扩一档"""
h, w = bgr.shape[:2]
hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV)
cy, cx = h // 2, w // 2
for p in (patch, patch * 2 + 1, patch * 4 + 1):
r = p // 2
roi = hsv[max(0, cy - r):cy + r + 1, max(0, cx - r):cx + r + 1]
Hh, Ss, Vv = roi[..., 0].astype(int), roi[..., 1].astype(int), roi[..., 2].astype(int)
ok = (Ss >= s_min) & (Vv >= v_min)
if ok.sum() < max(3, (p * p) // 4): # 中心被高光/阴影占据,外扩
continue
votes = {}
for word in COLORS + ("其他",):
votes[word] = 0
for hh in Hh[ok]:
votes[hue_word(int(hh))] += 1
word, cnt = max(votes.items(), key=lambda kv: kv[1])
return word if cnt >= 0.5 * int(ok.sum()) else "其他"
return "其他"

image.png

几何体判断#

无监督聚类、再用题目文案当真值锚点归并

对裁剪出的几何体小图做无监督聚类:同一种几何体归入同一文件夹(不识别具体名称)

思路(核心难点是颜色干扰——同形状不同颜色必须归同簇,故先灰度化再提特征):

  1. 保持纵横比贴到白底 128×128 画布,转灰度(消除颜色维度)
  2. ImageNet 预训练 ResNet18 提 512 维形状嵌入,L2 归一化
  3. PCA 降维后,KMeans 与层次聚类(Agglomerative/cosine) 各扫一遍 k∈[kmin,kmax], 取轮廓系数(silhouette, cosine) 最高的组合作为最终簇数与划分
cluster_shapes.py
import argparse
import csv
import math
import shutil
import sys
from pathlib import Path
import cv2
import numpy as np
import torch
import torch.nn.functional as F
import torchvision
from sklearn.cluster import AgglomerativeClustering, KMeans
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_score
IMG_EXTS = {".jpg", ".jpeg", ".png"}
CANVAS = 128
def load_prep(path: Path):
"""保持纵横比贴白底画布 + 灰度化,输出 ImageNet 归一化张量;读不出来返回 None"""
img = cv2.imread(str(path))
if img is None:
return None
h, w = img.shape[:2]
scale = (CANVAS - 8) / max(h, w)
nw, nh = max(1, round(w * scale)), max(1, round(h * scale))
rs = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_AREA)
canvas = np.full((CANVAS, CANVAS, 3), 255, np.uint8)
y0, x0 = (CANVAS - nh) // 2, (CANVAS - nw) // 2
canvas[y0:y0 + nh, x0:x0 + nw] = rs
gray = cv2.cvtColor(canvas, cv2.COLOR_BGR2GRAY) # 关键:去掉颜色
rgb = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB)
x = torch.from_numpy(rgb).permute(2, 0, 1).float().div_(255)
mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
return (x - mean) / std
@torch.no_grad()
def embed_all(paths: list, device) -> np.ndarray:
weights = torchvision.models.ResNet18_Weights.DEFAULT
backbone = torchvision.models.resnet18(weights=weights)
feat = torch.nn.Sequential(*list(backbone.children())[:-1]).to(device).eval()
out = []
B = 256
for i in range(0, len(paths), B):
batch = torch.stack([load_prep(p) for p in paths[i:i + B]]).to(device)
out.append(F.normalize(feat(batch).flatten(1), dim=1).cpu())
return torch.cat(out).numpy()
def choose_clustering(X: np.ndarray, kmin: int, kmax: int):
"""KMeans / Agglomerative × k 扫描,silhouette(cosine) 最高者胜出"""
best = (-2.0, None, None) # (score, name, labels)
for k in range(kmin, kmax + 1):
candidates = [
("kmeans", KMeans(n_clusters=k, n_init=20, random_state=0).fit_predict(X)),
("agglo", AgglomerativeClustering(
n_clusters=k, metric="cosine", linkage="average").fit_predict(X)),
] for name, labels in candidates:
s = silhouette_score(X, labels, metric="cosine")
print(f" k={k:2d} {name:6s} silhouette={s:.4f}")
if s > best[0]:
best = (s, name, labels)
return best

得到下面分类的结果,每一类下都是相同的几何体。只是有些没有被归为同一类。 image.png

人工整理完成正确的7个几何体分类,并标注对应的几何体名称。

image.png

已分类的各组样本嵌入取质心,固化为 7 个”形状原型”(原型 ↔ 名称由groups/mapping.json 固定)。之后对任意新裁剪图:嵌入 → 与 7 原型算余弦相似度 →
最近者即其几何体名称,同时给出 top1-top2 余量(margin)作置信度。构建原型shape_prototypes.npz

import argparse
import csv
import json
import math
import shutil
import sys
from pathlib import Path
import cv2
import numpy as np
import torch
import torch.nn.functional as F
from cluster_shapes import embed_all, load_prep, contact_sheet, IMG_EXTS # 复用嵌入管线
HERE = Path("ishumei")
PROTOS = HERE / "images/spatial_select_crops/shape_prototypes.npz"
def build(groups_dir: Path) -> None:
mapping = json.loads((groups_dir / "mapping.json").read_text(encoding="utf-8"))
device = "mps" if torch.backends.mps.is_available() else "cpu"
names, cents = [], []
for gdir in sorted(mapping):
files = sorted(p for p in (groups_dir / gdir).iterdir()
if p.suffix.lower() in IMG_EXTS)
if not files:
print(f"跳过空组 {gdir}", file=sys.stderr)
continue
feats = embed_all(files, device)
c = feats.mean(axis=0)
cents.append(c / np.linalg.norm(c))
names.append(mapping[gdir])
print(f"{gdir} [{mapping[gdir]}]: {len(files)} 样本 -> 质心")
np.savez(PROTOS, names=np.array(names), centroids=np.stack(cents))
print(f"\n原型已保存: {PROTOS} ({len(names)} 类)")

整合所有的判断识别,标注判断准确率。

def solve_once(session: requests.Session, yolo, shape_clf: ShapeClassifier,
run_dir: Path, conf: float, submit: bool = True) -> dict:
"""解一道验证码,落盘全部中间产物;submit=True 时把答案提交 fverify 实测判定"""
run_dir.mkdir(parents=True, exist_ok=True)
# 1. 刷新 + 2. 拉图
uuid = gen_captcha_uuid()
data = register(session, "spatial_select", uuid)
start_ms = int(time.time() * 1000) # gt 耗时从验证码下发时刻起算
detail = data["detail"]
url, content = download_bg(session, detail)
scene_path = run_dir / "scene.jpg"
scene_path.write_bytes(content)
img = cv2.imread(str(scene_path))
h, w = img.shape[:2]
# 3. 拆题
question = (detail.get("order") or [""])[0]
size, color, shape = parse_order(question)
shape = SHAPE_ALIASES.get(shape, shape)
result = {
"rid": detail["rid"], "captchaUuid": uuid, "question": question,
"size": size, "color": color, "shape": shape,
"scene": str(scene_path), "url": url, "image_size": [w, h], "objects": [],
} if not (size and color and shape):
result.update(ok=False, reason=f"题目无法完整拆解: {question}")
return result
# 4. YOLO 检测 + 裁剪
res = yolo.predict(source=img, conf=conf, verbose=False)[0]
for i, box in enumerate(res.boxes):
x1, y1, x2, y2 = (int(v) for v in box.xyxy[0])
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
crop_path = run_dir / f"obj_{i}_{cx}_{cy}.jpg"
cv2.imwrite(str(crop_path), img[y1:y2, x1:x2])
result["objects"].append({
"idx": i, "bbox": [x1, y1, x2, y2], "center": [cx, cy],
"area": (x2 - x1) * (y2 - y1), "conf": round(float(box.conf), 3),
"crop": str(crop_path),
}) if not result["objects"]:
result.update(ok=False, reason="YOLO 未检出任何几何体")
return result
# 5. 子图识别:几何体(最近原型) + 颜色(中心点 HSV)
for o, (shp, score, margin) in zip(
result["objects"], shape_clf.classify_paths([o["crop"] for o in result["objects"]])):
o["shape"] = shp
o["shape_score"] = round(score, 4)
o["shape_margin"] = round(margin, 4)
o["color"] = center_color(cv2.imread(str(o["crop"])))
# 6. 双匹配候选中按尺寸词选答案
candidates = [o for o in result["objects"] if o["color"] == color and o["shape"] == shape]
result["candidates"] = [o["idx"] for o in candidates]
answer = pick_by_size(candidates, size)
if answer is None:
got = {(o["color"], o["shape"]) for o in result["objects"]}
result.update(ok=False, reason=f"无 颜色+几何体 匹配的候选(题目要 {color}{shape},图中有 {got})")
return result
# 7. 标注落盘
result.update(ok=True, answer=answer)
answer_crop = run_dir / f"answer_{answer['center'][0]}_{answer['center'][1]}.jpg"
shutil.copy2(answer["crop"], answer_crop)
result["answer_crop"] = str(answer_crop)
result["marked"] = str(run_dir / "marked.jpg")
annotate(img, result["objects"], answer, Path(result["marked"]), question=question)
return result

image.png

协议对 captcha-sdk v1.0.4 的逆向
除 organization/rid/captchaUuid/ostype/protocol/act.os/版本号外,
其余字段均为 DES-ECB + ZeroPadding + Base64,密钥为 8 字节 ASCII 硬编码。
空间推理模式与图标点选共用同一 SDK 与字段结构,sp 只有一个点击点。
测试了100次通过率100%

image.png

代码省略了部分方法,避免过于混乱。只保留核心的思路,可能并不能直接运行。 大部分由Zcode+GLM5.3 flash完成。

数美空间推理验证码
https://blog.tiamix.cn/posts/数美空间推理验证码/
作者
Tiam
发布于
2026-10-02
许可协议
CC BY-NC-SA 4.0