ccf8eeb21d
Camera capture, corner detection/refinement, and preview crop pipeline for trading cards. Co-authored-by: Cursor <cursoragent@cursor.com>
485 lines
14 KiB
Dart
485 lines
14 KiB
Dart
import 'dart:math' as math;
|
||
import 'dart:ui';
|
||
|
||
import 'package:image/image.dart' as img;
|
||
|
||
/// Refine an axis-aligned detection into a tilted card quadrilateral.
|
||
///
|
||
/// Returns corners ordered TL → TR → BR → BL in full-image pixel coords,
|
||
/// or `null` if refinement fails (caller should keep the AABB).
|
||
class CardCornerRefiner {
|
||
/// [aabb] is the ML Kit box in the same pixel space as [image].
|
||
static List<Offset>? refine(img.Image image, Rect aabb) {
|
||
final pad = math.max(aabb.width, aabb.height) * 0.12;
|
||
final roi = Rect.fromLTRB(
|
||
(aabb.left - pad).clamp(0, image.width - 1.0),
|
||
(aabb.top - pad).clamp(0, image.height - 1.0),
|
||
(aabb.right + pad).clamp(1.0, image.width.toDouble()),
|
||
(aabb.bottom + pad).clamp(1.0, image.height.toDouble()),
|
||
);
|
||
|
||
final x0 = roi.left.floor();
|
||
final y0 = roi.top.floor();
|
||
final rw = math.max(1, roi.width.ceil());
|
||
final rh = math.max(1, roi.height.ceil());
|
||
if (rw < 24 || rh < 24) return null;
|
||
|
||
final gray = List<int>.filled(rw * rh, 0);
|
||
for (var y = 0; y < rh; y++) {
|
||
final sy = (y0 + y).clamp(0, image.height - 1);
|
||
for (var x = 0; x < rw; x++) {
|
||
final sx = (x0 + x).clamp(0, image.width - 1);
|
||
final p = image.getPixel(sx, sy);
|
||
gray[y * rw + x] = (0.299 * p.r + 0.587 * p.g + 0.114 * p.b).round();
|
||
}
|
||
}
|
||
|
||
// Light blur to stabilize edges.
|
||
final blurred = _boxBlur3(gray, rw, rh);
|
||
final thr = _otsu(blurred);
|
||
final mask = List<bool>.filled(rw * rh, false);
|
||
for (var i = 0; i < blurred.length; i++) {
|
||
mask[i] = blurred[i] >= thr;
|
||
}
|
||
|
||
// Prefer polarity where the center of the AABB is foreground.
|
||
final cx = ((aabb.center.dx - x0).round()).clamp(0, rw - 1);
|
||
final cy = ((aabb.center.dy - y0).round()).clamp(0, rh - 1);
|
||
if (!mask[cy * rw + cx]) {
|
||
for (var i = 0; i < mask.length; i++) {
|
||
mask[i] = !mask[i];
|
||
}
|
||
}
|
||
|
||
final component = _largestComponent(mask, rw, rh, cx, cy);
|
||
if (component == null || component.length < 40) return null;
|
||
|
||
final hull = _convexHull(component);
|
||
if (hull.length < 3) return null;
|
||
|
||
final quad = _minAreaRect(hull);
|
||
if (quad == null) return null;
|
||
|
||
// Map back to full-image coords and order TL, TR, BR, BL.
|
||
var ordered = _orderCorners([
|
||
for (final p in quad) Offset(x0 + p.dx, y0 + p.dy),
|
||
]);
|
||
|
||
// Reject half-card blobs (e.g. Energy art only) before padding.
|
||
if (!_looksLikeFullCard(ordered, aabb)) return null;
|
||
|
||
// Light outward pad so the physical border isn't clipped (avoid big gaps).
|
||
ordered = _padQuad(
|
||
ordered,
|
||
scale: 1.05,
|
||
topExtra: 0.03,
|
||
imageW: image.width.toDouble(),
|
||
imageH: image.height.toDouble(),
|
||
);
|
||
|
||
// Only pull top up if it still sits clearly below the ML box top.
|
||
ordered = _coverAabbTop(
|
||
ordered,
|
||
aabb,
|
||
image.width.toDouble(),
|
||
image.height.toDouble(),
|
||
);
|
||
|
||
if (!_looksLikeFullCard(ordered, aabb)) return null;
|
||
|
||
return ordered;
|
||
}
|
||
|
||
/// TCG portrait ~63:88. Reject quads that only cover artwork / half the AABB.
|
||
static bool _looksLikeFullCard(List<Offset> q, Rect aabb) {
|
||
final topW = (q[1] - q[0]).distance;
|
||
final botW = (q[2] - q[3]).distance;
|
||
final leftH = (q[3] - q[0]).distance;
|
||
final rightH = (q[2] - q[1]).distance;
|
||
final w = (topW + botW) / 2;
|
||
final h = (leftH + rightH) / 2;
|
||
if (w < 8 || h < 8) return false;
|
||
|
||
final short = math.min(w, h);
|
||
final long = math.max(w, h);
|
||
final ratio = short / long;
|
||
// Portrait card ≈ 0.72; half-art blobs are often nearer square (~0.9–1.0).
|
||
if (ratio < 0.55 || ratio > 0.88) return false;
|
||
|
||
final aabbArea = aabb.width * aabb.height;
|
||
if (aabbArea < 1) return false;
|
||
final area = _quadArea(q);
|
||
if (area < aabbArea * 0.55 || area > aabbArea * 1.55) return false;
|
||
|
||
var minY = q[0].dy, maxY = q[0].dy;
|
||
var minX = q[0].dx, maxX = q[0].dx;
|
||
for (final p in q) {
|
||
if (p.dy < minY) minY = p.dy;
|
||
if (p.dy > maxY) maxY = p.dy;
|
||
if (p.dx < minX) minX = p.dx;
|
||
if (p.dx > maxX) maxX = p.dx;
|
||
}
|
||
final quadBounds = Rect.fromLTRB(minX, minY, maxX, maxY);
|
||
final inter = quadBounds.intersect(aabb);
|
||
final cover =
|
||
(inter.width * inter.height) / aabbArea;
|
||
if (cover < 0.65) return false;
|
||
|
||
// Must span most of the detection height (blocks upper-half-only locks).
|
||
if ((maxY - minY) < aabb.height * 0.72) return false;
|
||
|
||
return true;
|
||
}
|
||
|
||
/// Extend TL/TR when ML Kit's box still reaches above the refined top.
|
||
static List<Offset> _coverAabbTop(
|
||
List<Offset> q,
|
||
Rect aabb,
|
||
double imageW,
|
||
double imageH,
|
||
) {
|
||
final topY = (q[0].dy + q[1].dy) / 2;
|
||
final targetTop = aabb.top;
|
||
final need = topY - targetTop;
|
||
// Ignore tiny gaps; only fix real header clipping.
|
||
if (need < aabb.height * 0.02) return q;
|
||
|
||
final topMid = Offset((q[0].dx + q[1].dx) / 2, (q[0].dy + q[1].dy) / 2);
|
||
final botMid = Offset((q[2].dx + q[3].dx) / 2, (q[2].dy + q[3].dy) / 2);
|
||
final up = topMid - botMid;
|
||
final len = up.distance;
|
||
if (len < 1) return q;
|
||
final bump = up * (need / len);
|
||
|
||
Offset clamp(Offset p) => Offset(
|
||
p.dx.clamp(0.0, imageW - 1),
|
||
p.dy.clamp(0.0, imageH - 1),
|
||
);
|
||
return [
|
||
clamp(q[0] + bump),
|
||
clamp(q[1] + bump),
|
||
q[2],
|
||
q[3],
|
||
];
|
||
}
|
||
|
||
/// Grow quad from its center; [topExtra] adds more margin on TL/TR.
|
||
static List<Offset> _padQuad(
|
||
List<Offset> q, {
|
||
required double scale,
|
||
required double topExtra,
|
||
required double imageW,
|
||
required double imageH,
|
||
}) {
|
||
final c = Offset(
|
||
(q[0].dx + q[1].dx + q[2].dx + q[3].dx) / 4,
|
||
(q[0].dy + q[1].dy + q[2].dy + q[3].dy) / 4,
|
||
);
|
||
final expanded = [
|
||
for (final p in q)
|
||
Offset(
|
||
c.dx + (p.dx - c.dx) * scale,
|
||
c.dy + (p.dy - c.dy) * scale,
|
||
),
|
||
];
|
||
|
||
// Push top edge further along card height (away from bottom).
|
||
final topMid = Offset(
|
||
(expanded[0].dx + expanded[1].dx) / 2,
|
||
(expanded[0].dy + expanded[1].dy) / 2,
|
||
);
|
||
final botMid = Offset(
|
||
(expanded[2].dx + expanded[3].dx) / 2,
|
||
(expanded[2].dy + expanded[3].dy) / 2,
|
||
);
|
||
final up = topMid - botMid;
|
||
final bump = Offset(up.dx * topExtra, up.dy * topExtra);
|
||
expanded[0] = expanded[0] + bump;
|
||
expanded[1] = expanded[1] + bump;
|
||
|
||
Offset clamp(Offset p) => Offset(
|
||
p.dx.clamp(0.0, imageW - 1),
|
||
p.dy.clamp(0.0, imageH - 1),
|
||
);
|
||
return expanded.map(clamp).toList(growable: false);
|
||
}
|
||
|
||
static List<int> _boxBlur3(List<int> src, int w, int h) {
|
||
final tmp = List<int>.filled(w * h, 0);
|
||
final out = List<int>.filled(w * h, 0);
|
||
for (var y = 0; y < h; y++) {
|
||
for (var x = 0; x < w; x++) {
|
||
var s = 0;
|
||
var n = 0;
|
||
for (var dx = -1; dx <= 1; dx++) {
|
||
final xx = x + dx;
|
||
if (xx < 0 || xx >= w) continue;
|
||
s += src[y * w + xx];
|
||
n++;
|
||
}
|
||
tmp[y * w + x] = s ~/ n;
|
||
}
|
||
}
|
||
for (var y = 0; y < h; y++) {
|
||
for (var x = 0; x < w; x++) {
|
||
var s = 0;
|
||
var n = 0;
|
||
for (var dy = -1; dy <= 1; dy++) {
|
||
final yy = y + dy;
|
||
if (yy < 0 || yy >= h) continue;
|
||
s += tmp[yy * w + x];
|
||
n++;
|
||
}
|
||
out[y * w + x] = s ~/ n;
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
static int _otsu(List<int> gray) {
|
||
final hist = List<int>.filled(256, 0);
|
||
for (final v in gray) {
|
||
hist[v.clamp(0, 255)]++;
|
||
}
|
||
final total = gray.length;
|
||
var sum = 0;
|
||
for (var i = 0; i < 256; i++) {
|
||
sum += i * hist[i];
|
||
}
|
||
var sumB = 0;
|
||
var wB = 0;
|
||
var best = 0.0;
|
||
var thr = 128;
|
||
for (var t = 0; t < 256; t++) {
|
||
wB += hist[t];
|
||
if (wB == 0) continue;
|
||
final wF = total - wB;
|
||
if (wF == 0) break;
|
||
sumB += t * hist[t];
|
||
final mB = sumB / wB;
|
||
final mF = (sum - sumB) / wF;
|
||
final between = wB * wF * (mB - mF) * (mB - mF);
|
||
if (between > best) {
|
||
best = between;
|
||
thr = t;
|
||
}
|
||
}
|
||
return thr;
|
||
}
|
||
|
||
/// BFS largest component; prefer one covering seed if large enough.
|
||
static List<Offset>? _largestComponent(
|
||
List<bool> mask,
|
||
int w,
|
||
int h,
|
||
int seedX,
|
||
int seedY,
|
||
) {
|
||
final seen = List<bool>.filled(w * h, false);
|
||
List<Offset>? bestEdge;
|
||
var bestCount = 0;
|
||
List<Offset>? seededEdge;
|
||
var seededCount = 0;
|
||
final qx = List<int>.filled(w * h, 0);
|
||
final qy = List<int>.filled(w * h, 0);
|
||
|
||
for (var sy = 0; sy < h; sy++) {
|
||
for (var sx = 0; sx < w; sx++) {
|
||
final start = sy * w + sx;
|
||
if (!mask[start] || seen[start]) continue;
|
||
var head = 0;
|
||
var tail = 0;
|
||
qx[tail] = sx;
|
||
qy[tail] = sy;
|
||
tail++;
|
||
seen[start] = true;
|
||
final edgePts = <Offset>[];
|
||
var containsSeed = false;
|
||
var count = 0;
|
||
while (head < tail) {
|
||
final x = qx[head];
|
||
final y = qy[head];
|
||
head++;
|
||
count++;
|
||
var onEdge = false;
|
||
for (final d in const [
|
||
[-1, 0],
|
||
[1, 0],
|
||
[0, -1],
|
||
[0, 1],
|
||
]) {
|
||
final nx = x + d[0];
|
||
final ny = y + d[1];
|
||
if (nx < 0 ||
|
||
ny < 0 ||
|
||
nx >= w ||
|
||
ny >= h ||
|
||
!mask[ny * w + nx]) {
|
||
onEdge = true;
|
||
break;
|
||
}
|
||
}
|
||
if (onEdge) edgePts.add(Offset(x.toDouble(), y.toDouble()));
|
||
if (x == seedX && y == seedY) containsSeed = true;
|
||
for (final d in const [
|
||
[-1, 0],
|
||
[1, 0],
|
||
[0, -1],
|
||
[0, 1],
|
||
]) {
|
||
final nx = x + d[0];
|
||
final ny = y + d[1];
|
||
if (nx < 0 || ny < 0 || nx >= w || ny >= h) continue;
|
||
final ni = ny * w + nx;
|
||
if (!mask[ni] || seen[ni]) continue;
|
||
seen[ni] = true;
|
||
qx[tail] = nx;
|
||
qy[tail] = ny;
|
||
tail++;
|
||
}
|
||
}
|
||
if (edgePts.length < 8) continue;
|
||
if (count > bestCount) {
|
||
bestCount = count;
|
||
bestEdge = edgePts;
|
||
}
|
||
if (containsSeed) {
|
||
seededCount = count;
|
||
seededEdge = edgePts;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (seededEdge != null && seededCount >= bestCount * 0.35) {
|
||
return seededEdge;
|
||
}
|
||
return bestEdge;
|
||
}
|
||
|
||
static List<Offset> _convexHull(List<Offset> pts) {
|
||
if (pts.length <= 2) return List.of(pts);
|
||
final sorted = [...pts]..sort((a, b) {
|
||
final c = a.dx.compareTo(b.dx);
|
||
return c != 0 ? c : a.dy.compareTo(b.dy);
|
||
});
|
||
// Deduplicate densely: subsample for speed on large blobs.
|
||
final sample = <Offset>[];
|
||
final step = math.max(1, sorted.length ~/ 800);
|
||
for (var i = 0; i < sorted.length; i += step) {
|
||
sample.add(sorted[i]);
|
||
}
|
||
if (sample.last != sorted.last) sample.add(sorted.last);
|
||
|
||
double cross(Offset o, Offset a, Offset b) =>
|
||
(a.dx - o.dx) * (b.dy - o.dy) - (a.dy - o.dy) * (b.dx - o.dx);
|
||
|
||
final lower = <Offset>[];
|
||
for (final p in sample) {
|
||
while (lower.length >= 2 &&
|
||
cross(lower[lower.length - 2], lower[lower.length - 1], p) <= 0) {
|
||
lower.removeLast();
|
||
}
|
||
lower.add(p);
|
||
}
|
||
final upper = <Offset>[];
|
||
for (final p in sample.reversed) {
|
||
while (upper.length >= 2 &&
|
||
cross(upper[upper.length - 2], upper[upper.length - 1], p) <= 0) {
|
||
upper.removeLast();
|
||
}
|
||
upper.add(p);
|
||
}
|
||
lower.removeLast();
|
||
upper.removeLast();
|
||
return [...lower, ...upper];
|
||
}
|
||
|
||
/// Rotating-calipers style min-area rectangle from convex hull.
|
||
static List<Offset>? _minAreaRect(List<Offset> hull) {
|
||
if (hull.length < 3) return null;
|
||
var bestArea = double.infinity;
|
||
List<Offset>? best;
|
||
|
||
for (var i = 0; i < hull.length; i++) {
|
||
final a = hull[i];
|
||
final b = hull[(i + 1) % hull.length];
|
||
final edge = b - a;
|
||
final len = edge.distance;
|
||
if (len < 1e-6) continue;
|
||
final ux = edge.dx / len;
|
||
final uy = edge.dy / len;
|
||
final vx = -uy;
|
||
final vy = ux;
|
||
|
||
var minU = double.infinity, maxU = -double.infinity;
|
||
var minV = double.infinity, maxV = -double.infinity;
|
||
for (final p in hull) {
|
||
final u = (p.dx - a.dx) * ux + (p.dy - a.dy) * uy;
|
||
final v = (p.dx - a.dx) * vx + (p.dy - a.dy) * vy;
|
||
if (u < minU) minU = u;
|
||
if (u > maxU) maxU = u;
|
||
if (v < minV) minV = v;
|
||
if (v > maxV) maxV = v;
|
||
}
|
||
final area = (maxU - minU) * (maxV - minV);
|
||
if (area < bestArea) {
|
||
bestArea = area;
|
||
// Corners in edge-aligned space → image space.
|
||
Offset corner(double u, double v) => Offset(
|
||
a.dx + u * ux + v * vx,
|
||
a.dy + u * uy + v * vy,
|
||
);
|
||
best = [
|
||
corner(minU, minV),
|
||
corner(maxU, minV),
|
||
corner(maxU, maxV),
|
||
corner(minU, maxV),
|
||
];
|
||
}
|
||
}
|
||
return best;
|
||
}
|
||
|
||
static List<Offset> _orderCorners(List<Offset> corners) {
|
||
assert(corners.length == 4);
|
||
final c = Offset(
|
||
(corners[0].dx + corners[1].dx + corners[2].dx + corners[3].dx) / 4,
|
||
(corners[0].dy + corners[1].dy + corners[2].dy + corners[3].dy) / 4,
|
||
);
|
||
final sorted = [...corners]
|
||
..sort((a, b) {
|
||
final aa = math.atan2(a.dy - c.dy, a.dx - c.dx);
|
||
final bb = math.atan2(b.dy - c.dy, b.dx - c.dx);
|
||
return aa.compareTo(bb);
|
||
});
|
||
|
||
// After atan2 sort (CCW from +x), pick the top-left-most as start.
|
||
var start = 0;
|
||
var bestScore = double.infinity;
|
||
for (var i = 0; i < 4; i++) {
|
||
final score = sorted[i].dx + sorted[i].dy;
|
||
if (score < bestScore) {
|
||
bestScore = score;
|
||
start = i;
|
||
}
|
||
}
|
||
return [
|
||
sorted[start],
|
||
sorted[(start + 1) % 4],
|
||
sorted[(start + 2) % 4],
|
||
sorted[(start + 3) % 4],
|
||
];
|
||
}
|
||
|
||
static double _quadArea(List<Offset> q) {
|
||
// Shoelace.
|
||
var s = 0.0;
|
||
for (var i = 0; i < 4; i++) {
|
||
final a = q[i];
|
||
final b = q[(i + 1) % 4];
|
||
s += a.dx * b.dy - b.dx * a.dy;
|
||
}
|
||
return s.abs() / 2;
|
||
}
|
||
}
|