ccf8eeb21d
Camera capture, corner detection/refinement, and preview crop pipeline for trading cards. Co-authored-by: Cursor <cursoragent@cursor.com>
514 lines
14 KiB
Markdown
514 lines
14 KiB
Markdown
# CardDex 卡牌检测 Demo — Flutter 开发方案
|
||
|
||
> **目标**：复刻 CardDex 本地 TF Lite 目标检测 + 卡片定位能力
|
||
> **周期**：~3.5 天（单人）
|
||
> **依赖**：CardDex APK 内提取的 `detect.tflite`（COCO 通用检测模型，4MB）
|
||
|
||
---
|
||
|
||
## 一、架构概览
|
||
|
||
```
|
||
┌──────────────────────────────────────┐
|
||
│ Flutter UI │
|
||
│ camera_preview + detection_overlay │
|
||
├──────────────────────────────────────┤
|
||
│ TFLite Detector Service │
|
||
│ ├─ 模型加载 (detect.tflite) │
|
||
│ ├─ 图像预处理 (300×300 RGB) │
|
||
│ ├─ 推理 (输出 boxes/classes/scores) │
|
||
│ └─ NMS 后处理 (IoU 去重) │
|
||
├──────────────────────────────────────┤
|
||
│ Camera Plugin (拍照) │
|
||
└──────────────────────────────────────┘
|
||
```
|
||
|
||
**核心流程**：拍照 → 缩放到 300×300 → TF Lite 推理 → NMS 去重 → 画框展示
|
||
|
||
---
|
||
|
||
## 二、项目结构
|
||
|
||
```
|
||
carddex_demo/
|
||
├── pubspec.yaml
|
||
├── lib/
|
||
│ ├── main.dart # 入口 + 相机预览
|
||
│ ├── services/
|
||
│ │ ├── detector_service.dart # TF Lite 加载/推理
|
||
│ │ └── image_processor.dart # 图像预处理
|
||
│ ├── utils/
|
||
│ │ └── nms.dart # NMS 非极大值抑制
|
||
│ └── widgets/
|
||
│ └── detection_painter.dart # 检测框叠加渲染
|
||
└── assets/
|
||
└── ml/
|
||
├── detect.tflite # 从 CardDex APK 提取
|
||
└── labelmap.txt # COCO 90 类标签
|
||
```
|
||
|
||
---
|
||
|
||
## 三、依赖清单
|
||
|
||
```yaml
|
||
# pubspec.yaml
|
||
dependencies:
|
||
flutter:
|
||
sdk: flutter
|
||
camera: ^0.10.5 # 相机采集
|
||
tflite_flutter: ^0.11.0 # TF Lite 推理
|
||
image: ^4.1.0 # Dart 侧图像处理
|
||
|
||
flutter:
|
||
assets:
|
||
- assets/ml/detect.tflite
|
||
- assets/ml/labelmap.txt
|
||
```
|
||
|
||
---
|
||
|
||
## 四、核心代码骨架
|
||
|
||
### 4.1 主入口 — `main.dart`
|
||
|
||
```dart
|
||
import 'package:flutter/material.dart';
|
||
import 'package:camera/camera.dart';
|
||
import 'services/detector_service.dart';
|
||
import 'widgets/detection_painter.dart';
|
||
|
||
late List<CameraDescription> _cameras;
|
||
|
||
void main() async {
|
||
WidgetsFlutterBinding.ensureInitialized();
|
||
_cameras = await availableCameras();
|
||
await DetectorService.instance.initialize();
|
||
runApp(const CardDexDemo());
|
||
}
|
||
|
||
class CardDexDemo extends StatelessWidget {
|
||
const CardDexDemo({super.key});
|
||
|
||
@override
|
||
Widget build(BuildContext context) {
|
||
return MaterialApp(
|
||
home: ScannerScreen(camera: _cameras.first),
|
||
);
|
||
}
|
||
}
|
||
|
||
class ScannerScreen extends StatefulWidget {
|
||
final CameraDescription camera;
|
||
const ScannerScreen({super.key, required this.camera});
|
||
|
||
@override
|
||
State<ScannerScreen> createState() => _ScannerScreenState();
|
||
}
|
||
|
||
class _ScannerScreenState extends State<ScannerScreen> {
|
||
late CameraController _controller;
|
||
List<Detection> _detections = [];
|
||
bool _isProcessing = false;
|
||
|
||
@override
|
||
void initState() {
|
||
super.initState();
|
||
_controller = CameraController(
|
||
widget.camera,
|
||
ResolutionPreset.high,
|
||
);
|
||
_controller.initialize().then((_) => setState(() {}));
|
||
}
|
||
|
||
Future<void> _takePicture() async {
|
||
if (_isProcessing) return;
|
||
setState(() => _isProcessing = true);
|
||
|
||
final xFile = await _controller.takePicture();
|
||
final bytes = await xFile.readAsBytes();
|
||
|
||
// 推理
|
||
final detections = await DetectorService.instance.detect(bytes);
|
||
|
||
setState(() {
|
||
_detections = detections;
|
||
_isProcessing = false;
|
||
});
|
||
}
|
||
|
||
@override
|
||
Widget build(BuildContext context) {
|
||
if (!_controller.value.isInitialized) {
|
||
return const Center(child: CircularProgressIndicator());
|
||
}
|
||
|
||
return Scaffold(
|
||
body: Stack(
|
||
fit: StackFit.expand,
|
||
children: [
|
||
CameraPreview(_controller),
|
||
if (_detections.isNotEmpty)
|
||
CustomPaint(
|
||
painter: DetectionPainter(_detections),
|
||
),
|
||
Positioned(
|
||
bottom: 40,
|
||
left: 0,
|
||
right: 0,
|
||
child: Center(
|
||
child: FloatingActionButton(
|
||
onPressed: _isProcessing ? null : _takePicture,
|
||
child: _isProcessing
|
||
? const CircularProgressIndicator(color: Colors.white)
|
||
: const Icon(Icons.camera),
|
||
),
|
||
),
|
||
),
|
||
],
|
||
),
|
||
);
|
||
}
|
||
|
||
@override
|
||
void dispose() {
|
||
_controller.dispose();
|
||
super.dispose();
|
||
}
|
||
}
|
||
```
|
||
|
||
### 4.2 检测服务 — `detector_service.dart`
|
||
|
||
```dart
|
||
import 'dart:io';
|
||
import 'package:tflite_flutter/tflite_flutter.dart';
|
||
import 'package:image/image.dart' as img;
|
||
import '../utils/nms.dart';
|
||
import 'image_processor.dart';
|
||
|
||
class Detection {
|
||
final Rect boundingBox;
|
||
final String label;
|
||
final double confidence;
|
||
|
||
Detection({
|
||
required this.boundingBox,
|
||
required this.label,
|
||
required this.confidence,
|
||
});
|
||
}
|
||
|
||
class DetectorService {
|
||
static final DetectorService instance = DetectorService._();
|
||
DetectorService._();
|
||
|
||
Interpreter? _interpreter;
|
||
List<String> _labels = [];
|
||
bool _isInitialized = false;
|
||
|
||
// 模型输入尺寸（COCO SSD MobileNet 标准）
|
||
static const int inputSize = 300;
|
||
static const double confidenceThreshold = 0.5;
|
||
|
||
Future<void> initialize() async {
|
||
if (_isInitialized) return;
|
||
|
||
// 加载模型
|
||
_interpreter = await Interpreter.fromAsset('assets/ml/detect.tflite');
|
||
|
||
// 加载标签
|
||
final labelData = await rootBundle.loadString('assets/ml/labelmap.txt');
|
||
_labels = labelData
|
||
.split('\n')
|
||
.map((l) => l.trim())
|
||
.where((l) => l.isNotEmpty)
|
||
.toList();
|
||
|
||
_isInitialized = true;
|
||
}
|
||
|
||
Future<List<Detection>> detect(Uint8List imageBytes) async {
|
||
if (!_isInitialized) throw Exception('Detector not initialized');
|
||
|
||
// 1. 图像预处理
|
||
final input = ImageProcessor.preprocess(imageBytes, inputSize);
|
||
|
||
// 2. 分配输出张量
|
||
// SSD MobileNet 输出: [1, num_detections, 4] boxes, [1, num_detections] classes, [1, num_detections] scores
|
||
final outputBoxes = List.filled(1 * 10 * 4, 0.0).reshape([1, 10, 4]);
|
||
final outputClasses = List.filled(1 * 10, 0.0).reshape([1, 10]);
|
||
final outputScores = List.filled(1 * 10, 0.0).reshape([1, 10]);
|
||
final numDetections = List.filled(1, 0.0).reshape([1]);
|
||
|
||
// 3. 推理
|
||
_interpreter!.runForMultipleInputs([input], {
|
||
0: outputBoxes,
|
||
1: outputClasses,
|
||
2: outputScores,
|
||
3: numDetections,
|
||
});
|
||
|
||
// 4. 解析结果
|
||
final boxes = <List<double>>[];
|
||
final classes = <int>[];
|
||
final scores = <double>[];
|
||
|
||
final count = numDetections[0][0].toInt().clamp(0, 10);
|
||
for (int i = 0; i < count; i++) {
|
||
if (outputScores[0][i] >= confidenceThreshold) {
|
||
boxes.add([
|
||
outputBoxes[0][i][1], // ymin
|
||
outputBoxes[0][i][0], // xmin
|
||
outputBoxes[0][i][3], // ymax
|
||
outputBoxes[0][i][2], // xmax
|
||
]);
|
||
classes.add(outputClasses[0][i].toInt());
|
||
scores.add(outputScores[0][i]);
|
||
}
|
||
}
|
||
|
||
// 5. NMS 去重
|
||
final indices = NMS.suppress(boxes, scores, iouThreshold: 0.5);
|
||
|
||
// 6. 生成检测结果
|
||
return indices.map((i) {
|
||
final box = boxes[i];
|
||
return Detection(
|
||
boundingBox: Rect.fromLTRB(
|
||
box[1] * inputSize, // xmin * width
|
||
box[0] * inputSize, // ymin * height
|
||
box[3] * inputSize, // xmax * width
|
||
box[2] * inputSize, // ymax * height
|
||
),
|
||
label: classes[i] < _labels.length ? _labels[classes[i]] : 'unknown',
|
||
confidence: scores[i],
|
||
);
|
||
}).toList();
|
||
}
|
||
}
|
||
```
|
||
|
||
### 4.3 图像预处理 — `image_processor.dart`
|
||
|
||
```dart
|
||
import 'dart:typed_data';
|
||
import 'package:image/image.dart' as img;
|
||
|
||
class ImageProcessor {
|
||
/// 将拍照得到的 JPEG bytes 预处理为 TF Lite 输入张量
|
||
/// 返回 [1, 300, 300, 3] 的 Float32List（归一化到 [0,1]）
|
||
static Float32List preprocess(Uint8List jpegBytes, int targetSize) {
|
||
// 1. 解码 JPEG
|
||
final image = img.decodeJpg(jpegBytes);
|
||
if (image == null) throw Exception('Failed to decode image');
|
||
|
||
// 2. 缩放到 300×300
|
||
final resized = img.copyResize(image, width: targetSize, height: targetSize);
|
||
|
||
// 3. RGB → Float32List，归一化到 [0, 1]
|
||
final input = Float32List(1 * targetSize * targetSize * 3);
|
||
int pixelIndex = 0;
|
||
for (int y = 0; y < targetSize; y++) {
|
||
for (int x = 0; x < targetSize; x++) {
|
||
final pixel = resized.getPixel(x, y);
|
||
input[pixelIndex++] = pixel.r / 255.0;
|
||
input[pixelIndex++] = pixel.g / 255.0;
|
||
input[pixelIndex++] = pixel.b / 255.0;
|
||
}
|
||
}
|
||
|
||
return input;
|
||
}
|
||
}
|
||
```
|
||
|
||
### 4.4 NMS 去重 — `nms.dart`
|
||
|
||
```dart
|
||
class NMS {
|
||
/// 贪心 NMS，按 score 降序，保留 IoU < threshold 的框
|
||
static List<int> suppress(
|
||
List<List<double>> boxes,
|
||
List<double> scores, {
|
||
double iouThreshold = 0.5,
|
||
}) {
|
||
// 按分数排序
|
||
final indices = List.generate(scores.length, (i) => i);
|
||
indices.sort((a, b) => scores[b].compareTo(scores[a]));
|
||
|
||
final selected = <int>[];
|
||
final suppressed = List.filled(boxes.length, false);
|
||
|
||
for (final idx in indices) {
|
||
if (suppressed[idx]) continue;
|
||
selected.add(idx);
|
||
|
||
for (int j = 0; j < boxes.length; j++) {
|
||
if (j == idx || suppressed[j]) continue;
|
||
if (_iou(boxes[idx], boxes[j]) > iouThreshold) {
|
||
suppressed[j] = true;
|
||
}
|
||
}
|
||
}
|
||
|
||
return selected;
|
||
}
|
||
|
||
/// 计算两个框的 IoU
|
||
static double _iou(List<double> a, List<double> b) {
|
||
final x1 = max(a[1], b[1]);
|
||
final y1 = max(a[0], b[0]);
|
||
final x2 = min(a[3], b[3]);
|
||
final y2 = min(a[2], b[2]);
|
||
|
||
if (x2 <= x1 || y2 <= y1) return 0.0;
|
||
|
||
final interArea = (x2 - x1) * (y2 - y1);
|
||
final areaA = (a[3] - a[1]) * (a[2] - a[0]);
|
||
final areaB = (b[3] - b[1]) * (b[2] - b[0]);
|
||
|
||
return interArea / (areaA + areaB - interArea);
|
||
}
|
||
}
|
||
```
|
||
|
||
### 4.5 检测框渲染 — `detection_painter.dart`
|
||
|
||
```dart
|
||
import 'package:flutter/material.dart';
|
||
import '../services/detector_service.dart';
|
||
|
||
class DetectionPainter extends CustomPainter {
|
||
final List<Detection> detections;
|
||
|
||
DetectionPainter(this.detections);
|
||
|
||
@override
|
||
void paint(Canvas canvas, Size size) {
|
||
final paint = Paint()
|
||
..color = Colors.green
|
||
..style = PaintingStyle.stroke
|
||
..strokeWidth = 3.0;
|
||
|
||
final textStyle = TextStyle(
|
||
color: Colors.white,
|
||
fontSize: 14,
|
||
backgroundColor: Colors.green.withOpacity(0.8),
|
||
);
|
||
|
||
for (final det in detections) {
|
||
// 画框
|
||
canvas.drawRect(det.boundingBox, paint);
|
||
|
||
// 画标签
|
||
final tp = TextPainter(
|
||
text: TextSpan(
|
||
text: '${det.label} ${(det.confidence * 100).toStringAsFixed(0)}%',
|
||
style: textStyle,
|
||
),
|
||
textDirection: TextDirection.ltr,
|
||
);
|
||
tp.layout();
|
||
tp.paint(
|
||
canvas,
|
||
Offset(det.boundingBox.left, det.boundingBox.top - tp.height),
|
||
);
|
||
}
|
||
}
|
||
|
||
@override
|
||
bool shouldRepaint(covariant CustomPainter oldDelegate) => true;
|
||
}
|
||
```
|
||
|
||
---
|
||
|
||
## 五、模型提取
|
||
|
||
```bash
|
||
# 从 CardDex APK 提取模型和标签
|
||
cd apk_analysis/recognize/apk_extracted_carddex
|
||
unzip com.card.dex.identifier.detect.apk \
|
||
"assets/flutter_assets/assets/ml/detect.tflite" \
|
||
"assets/flutter_assets/assets/ml/labelmap.txt" \
|
||
-d ../../../carddex_demo/assets/ml/
|
||
```
|
||
|
||
---
|
||
|
||
## 六、构建与运行
|
||
|
||
```bash
|
||
# 1. 创建 Flutter 项目
|
||
flutter create carddex_demo
|
||
cd carddex_demo
|
||
|
||
# 2. 添加依赖（编辑 pubspec.yaml）
|
||
|
||
# 3. 复制模型文件到 assets/ml/
|
||
|
||
# 4. 替换 lib/ 下的代码
|
||
|
||
# 5. 运行（需要真机，模拟器无相机）
|
||
flutter run
|
||
```
|
||
|
||
---
|
||
|
||
## 七、关键注意事项
|
||
|
||
### 7.1 模型输出格式适配
|
||
|
||
CardDex 用的是 SSD MobileNet，输出张量取决于具体的 TFLite 模型。如果上面的输出索引不对，需要先用工具查看：
|
||
|
||
```bash
|
||
# 查看模型输入输出信息
|
||
python3 -c "
|
||
import tensorflow as tf
|
||
interpreter = tf.lite.Interpreter(model_path='assets/ml/detect.tflite')
|
||
print('Input:', interpreter.get_input_details())
|
||
print('Output:', interpreter.get_output_details())
|
||
"
|
||
```
|
||
|
||
### 7.2 平台配置
|
||
|
||
**iOS** (`ios/Podfile`)：确保最低版本 ≥ 12.0，并添加相机权限描述。
|
||
**Android** (`android/app/build.gradle`)：`minSdkVersion` ≥ 21。
|
||
|
||
### 7.3 性能
|
||
|
||
- `detect.tflite` ~4MB，推理耗时约 50-200ms（取决于设备）
|
||
- 拍照后单次推理，不做实时帧，性能无忧
|
||
- GPU delegate 可选，配置方法：`Interpreter.fromAsset('detect.tflite', options: InterpreterOptions()..useGpuDelegateWhenAvailable = true)`
|
||
|
||
### 7.4 局限
|
||
|
||
| 局限 | 说明 | 后续方向 |
|
||
|------|------|---------|
|
||
| 只能定位，不能分类 | COCO 标签里没有"卡牌"，检测框内的物体不知道是什么卡 | 自训卡牌分类模型或接 API |
|
||
| 模型不针对卡牌优化 | 卡片是矩形平面物体，COCO 模型对此场景精度一般 | 用卡牌数据 fine-tune |
|
||
| 多卡同框效果差 | SSD 对小目标和密集目标检测能力有限 | 换 YOLO-NAS 或 RT-DETR |
|
||
|
||
---
|
||
|
||
## 八、Demo 验收标准
|
||
|
||
- [ ] 打开 App 显示相机预览
|
||
- [ ] 对准一张卡牌拍照
|
||
- [ ] 画面出现绿色检测框 + 标签 + 置信度
|
||
- [ ] 对准其他物体（杯子、手机等）也能检测
|
||
- [ ] 无明显崩溃或 ANR
|
||
|
||
---
|
||
|
||
## 九、进阶方向（Demo 后）
|
||
|
||
| 阶段 | 内容 | 预估 |
|
||
|------|------|------|
|
||
| **Phase 2** | 替换为卡牌专用检测模型（自训练） | 1 周 |
|
||
| **Phase 3** | 接入服务端 API 做卡牌分类 | 3 天 |
|
||
| **Phase 4** | 实时帧推理 + GPU 加速 | 3 天 |
|
||
| **Phase 5** | 多卡同框检测 + 批量识别 | 1 周 |
|