immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-10-11 22:49:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(mobile): select OCR text across lines in the asset viewer

Browse Source
This commit is contained in:
LeLunZ
2026-10-10 21:08:58 +02:00
parent 2685234ed3
commit 48d36a046d
2 changed files with 199 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobile/lib/presentation/widgets/asset_viewer/ocr_overlay.widget.dart
+97 -200
View File
@@ -1,12 +1,11 @@
import 'dart:async';
import 'dart:math' as math;
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/data/store.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/domain/models/ocr.model.dart';
import 'package:immich_mobile/extensions/build_context_extensions.dart';
import 'package:immich_mobile/widgets/photo_view/photo_view.dart';
class OcrOverlay extends ConsumerStatefulWidget {
@@ -28,8 +27,6 @@ class OcrOverlay extends ConsumerStatefulWidget {
}
class _OcrOverlayState extends ConsumerState<OcrOverlay> {
int? _selectedBoxIndex;
// Current transform read from the PhotoView controller.
// Null until the controller has emitted at least one real event or until
// we can seed a reliable value from controller.value on init.
@@ -87,25 +84,25 @@ class _OcrOverlayState extends ConsumerState<OcrOverlay> {
@override
Widget build(BuildContext context) {
if (widget.asset is! RemoteAsset) {
final asset = widget.asset;
if (asset is! RemoteAsset) {
return const SizedBox.shrink();
}
final ocrData = ref.watch(Store.ocr.forAsset((widget.asset as RemoteAsset).id));
final ocrData = ref.watch(Store.ocr.forAsset(asset.id));
return ocrData.when(
data: (data) {
if (data.isEmpty) {
return const SizedBox.shrink();
}
return _OcrBoxes(
return OcrSelectionLayer(
key: ValueKey(asset.id),
ocrData: data,
controller: widget.controller,
imageSize: widget.imageSize,
viewportSize: widget.viewportSize,
controllerValue: _controllerValue,
selectedBoxIndex: _selectedBoxIndex,
onSelectionChanged: (index) => setState(() => _selectedBoxIndex = index),
);
},
loading: () => const SizedBox.shrink(),
@@ -114,25 +111,38 @@ class _OcrOverlayState extends ConsumerState<OcrOverlay> {
}
}
class _OcrBoxes extends StatelessWidget {
/// Lays an invisible, selectable line of text over every recognized text box,
/// so the text can be selected directly on the image.
@visibleForTesting
class OcrSelectionLayer extends StatefulWidget {
final List<Ocr> ocrData;
final PhotoViewControllerBase? controller;
final Size imageSize;
final Size viewportSize;
final PhotoViewControllerValue? controllerValue;
final int? selectedBoxIndex;
final ValueChanged<int?> onSelectionChanged;
const _OcrBoxes({
const OcrSelectionLayer({
super.key,
required this.ocrData,
required this.controller,
required this.imageSize,
required this.viewportSize,
required this.controllerValue,
required this.selectedBoxIndex,
required this.onSelectionChanged,
this.controller,
this.controllerValue,
});
@override
State<OcrSelectionLayer> createState() => _OcrSelectionLayerState();
}
class _OcrSelectionLayerState extends State<OcrSelectionLayer> {
final _selectionDelegate = _OcrSelectionDelegate();
@override
void dispose() {
_selectionDelegate.dispose();
super.dispose();
}
@override
Widget build(BuildContext context) {
// Use the actual decoded image size from PhotoView's scaleBoundaries when
@@ -140,12 +150,13 @@ class _OcrBoxes extends StatelessWidget {
// serves a ~1440px preview for large originals), so the decoded dimensions
// can differ significantly from the stored asset dimensions. Using the wrong
// size would scale every coordinate by the ratio between the two resolutions.
final resolvedImageSize = controller?.scaleBoundaries?.childSize ?? imageSize;
final resolvedImageSize = widget.controller?.scaleBoundaries?.childSize ?? widget.imageSize;
final viewportSize = widget.viewportSize;
final scale =
controllerValue?.scale ??
widget.controllerValue?.scale ??
math.min(viewportSize.width / resolvedImageSize.width, viewportSize.height / resolvedImageSize.height);
final position = controllerValue?.position ?? Offset.zero;
final position = widget.controllerValue?.position ?? Offset.zero;
final imageWidth = resolvedImageSize.width;
final imageHeight = resolvedImageSize.height;
@@ -157,12 +168,9 @@ class _OcrBoxes extends StatelessWidget {
final cy = viewportHeight / 2 + position.dy;
final quads = <List<Offset>>[];
final boxes = <Widget>[];
for (final entry in ocrData.asMap().entries) {
final index = entry.key;
final ocr = entry.value;
final lines = <Widget>[];
for (final ocr in widget.ocrData) {
// Map normalized image coords (0–1) to viewport space
final x1 = cx + (ocr.x1 - 0.5) * imageWidth * scale;
final y1 = cy + (ocr.y1 - 0.5) * imageHeight * scale;
@@ -173,146 +181,52 @@ class _OcrBoxes extends StatelessWidget {
final x4 = cx + (ocr.x4 - 0.5) * imageWidth * scale;
final y4 = cy + (ocr.y4 - 0.5) * imageHeight * scale;
// Bounding rectangle for hit testing and Positioned placement
final minX = [x1, x2, x3, x4].reduce((a, b) => a < b ? a : b);
final maxX = [x1, x2, x3, x4].reduce((a, b) => a > b ? a : b);
final minY = [y1, y2, y3, y4].reduce((a, b) => a < b ? a : b);
final maxY = [y1, y2, y3, y4].reduce((a, b) => a > b ? a : b);
final width = Offset(x2 - x1, y2 - y1).distance;
final height = Offset(x4 - x1, y4 - y1).distance;
if (width <= 0 || height <= 0 || ocr.text.isEmpty) {
continue;
}
quads.add([Offset(x1, y1), Offset(x2, y2), Offset(x3, y3), Offset(x4, y4)]);
boxes.add(
_OcrBoxItem(
key: ValueKey(index),
ocr: ocr,
index: index,
isSelected: selectedBoxIndex == index,
points: [
Offset(x1 - minX, y1 - minY),
Offset(x2 - minX, y2 - minY),
Offset(x3 - minX, y3 - minY),
Offset(x4 - minX, y4 - minY),
],
left: minX,
top: minY,
width: maxX - minX,
height: maxY - minY,
angle: math.atan2(y2 - y1, x2 - x1),
labelDx: (minX + maxX) / 2 - minX,
labelDy: (minY + maxY) / 2 - minY,
onSelectionChanged: onSelectionChanged,
lines.add(
Positioned(
key: ValueKey(ocr.id),
left: x1,
top: y1,
width: width,
height: height,
child: Transform.rotate(
angle: math.atan2(y2 - y1, x2 - x1),
alignment: Alignment.topLeft,
child: FittedBox(
fit: BoxFit.fill,
child: Text(
ocr.text.replaceAll('\n', ' '),
maxLines: 1,
softWrap: false,
textScaler: TextScaler.noScaling,
style: const TextStyle(color: Colors.transparent, fontSize: 16, height: 1.0),
),
),
),
),
);
}
return GestureDetector(
behavior: HitTestBehavior.translucent,
onTap: () => onSelectionChanged(null),
child: ClipRect(
child: Stack(
children: [
// Fills the viewport so taps outside boxes deselect
SizedBox(width: viewportWidth, height: viewportHeight),
// Dark scrim with the text boxes punched out
Positioned.fill(
child: IgnorePointer(
child: CustomPaint(painter: _OcrScrimPainter(quads: quads)),
),
),
...boxes,
],
),
),
);
}
}
class _OcrBoxItem extends StatelessWidget {
final Ocr ocr;
final int index;
final bool isSelected;
final List<Offset> points;
final double left;
final double top;
final double width;
final double height;
final double angle;
final double labelDx;
final double labelDy;
final ValueChanged<int?> onSelectionChanged;
const _OcrBoxItem({
super.key,
required this.ocr,
required this.index,
required this.isSelected,
required this.points,
required this.left,
required this.top,
required this.width,
required this.height,
required this.angle,
required this.labelDx,
required this.labelDy,
required this.onSelectionChanged,
});
@override
Widget build(BuildContext context) {
return Positioned(
left: left,
top: top,
child: GestureDetector(
onTap: () => onSelectionChanged(isSelected ? null : index),
behavior: HitTestBehavior.translucent,
child: SizedBox(
width: width,
height: height,
return ClipRect(
child: SelectionArea(
child: SelectionContainer(
delegate: _selectionDelegate,
child: Stack(
children: [
CustomPaint(
painter: _OcrBoxPainter(
points: points,
isSelected: isSelected,
colorScheme: context.themeData.colorScheme,
// Dark scrim with the text boxes punched out
Positioned.fill(
child: IgnorePointer(
child: CustomPaint(painter: _OcrBoxesPainter(quads: quads)),
),
size: Size(width, height),
),
if (isSelected)
Positioned(
left: labelDx,
top: labelDy,
child: FractionalTranslation(
translation: const Offset(-0.5, -0.5),
child: Transform.rotate(
angle: angle,
alignment: Alignment.center,
child: Container(
margin: const EdgeInsets.all(2),
padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2),
decoration: BoxDecoration(
color: Colors.grey[800]?.withValues(alpha: 0.4),
borderRadius: const BorderRadius.all(Radius.circular(4)),
),
child: ConstrainedBox(
constraints: BoxConstraints(maxWidth: math.max(50, width), maxHeight: math.max(20, height)),
child: FittedBox(
fit: BoxFit.scaleDown,
child: SelectableText(
ocr.text,
style: TextStyle(
color: Colors.white,
fontSize: math.max(12, height * 0.6),
fontWeight: FontWeight.bold,
),
textAlign: TextAlign.center,
),
),
),
),
),
),
),
...lines,
],
),
),
@@ -321,66 +235,49 @@ class _OcrBoxItem extends StatelessWidget {
}
}
class _OcrScrimPainter extends CustomPainter {
class _OcrSelectionDelegate extends StaticSelectionContainerDelegate {
@override
SelectedContent? getSelectedContent() {
final lines = [
for (final selectable in selectables)
if (selectable.getSelectedContent() case final SelectedContent content) content.plainText,
];
if (lines.isEmpty) {
return null;
}
return SelectedContent(plainText: lines.join('\n'));
}
}
class _OcrBoxesPainter extends CustomPainter {
final List<List<Offset>> quads;
const _OcrScrimPainter({required this.quads});
const _OcrBoxesPainter({required this.quads});
@override
void paint(Canvas canvas, Size size) {
// Fill the whole viewport, then subtract each text quad using the even-odd
// rule so the original image shows through the boxes.
final path = Path()
final scrim = Path()
..fillType = PathFillType.evenOdd
..addRect(Offset.zero & size);
final boxes = Path();
for (final quad in quads) {
path
..moveTo(quad[0].dx, quad[0].dy)
..lineTo(quad[1].dx, quad[1].dy)
..lineTo(quad[2].dx, quad[2].dy)
..lineTo(quad[3].dx, quad[3].dy)
..close();
scrim.addPolygon(quad, true);
boxes.addPolygon(quad, true);
}
canvas.drawPath(path, Paint()..color = Colors.black54);
canvas.drawPath(scrim, Paint()..color = Colors.black54);
canvas.drawPath(
boxes,
Paint()
..color = Colors.white.withValues(alpha: 0.8)
..style = PaintingStyle.stroke
..strokeWidth = 1.5,
);
}
@override
bool shouldRepaint(_OcrScrimPainter oldDelegate) => true;
}
class _OcrBoxPainter extends CustomPainter {
final List<Offset> points;
final bool isSelected;
final ColorScheme colorScheme;
const _OcrBoxPainter({required this.points, required this.isSelected, required this.colorScheme});
@override
void paint(Canvas canvas, Size size) {
final paint = Paint()
..color = isSelected ? colorScheme.primary : colorScheme.secondary
..style = PaintingStyle.stroke
..strokeWidth = 2.0;
final fillPaint = Paint()
..color = isSelected ? colorScheme.primary.withValues(alpha: 0.45) : Colors.transparent
..style = PaintingStyle.fill;
final path = Path()
..moveTo(points[0].dx, points[0].dy)
..lineTo(points[1].dx, points[1].dy)
..lineTo(points[2].dx, points[2].dy)
..lineTo(points[3].dx, points[3].dy)
..close();
canvas.drawPath(path, fillPaint);
canvas.drawPath(path, paint);
}
@override
bool shouldRepaint(_OcrBoxPainter oldDelegate) {
return oldDelegate.isSelected != isSelected || !listEquals(oldDelegate.points, points);
}
bool shouldRepaint(_OcrBoxesPainter oldDelegate) => true;
}
mobile/test/presentation/widgets/asset_viewer/ocr_overlay_test.dart
+102
View File
@@ -0,0 +1,102 @@
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/domain/models/ocr.model.dart';
import 'package:immich_mobile/presentation/widgets/asset_viewer/ocr_overlay.widget.dart';
// A 400x300 image centered in a 400x600 viewport: the image starts at y = 150.
const _viewportSize = Size(400, 600);
const _imageSize = Size(400, 300);
// Viewport rect (40, 180) - (240, 210)
const _firstLine = Offset(100, 195);
// Viewport rect (40, 240) - (240, 270)
const _secondLine = Offset(100, 255);
Ocr _line(String id, String text, double top) => Ocr(
id: id,
assetId: 'asset',
x1: 0.1,
y1: top,
x2: 0.6,
y2: top,
x3: 0.6,
y3: top + 0.1,
x4: 0.1,
y4: top + 0.1,
boxScore: 1,
textScore: 1,
text: text,
isVisible: true,
);
void main() {
final ocrData = [_line('1', 'hello world', 0.1), _line('2', 'second line', 0.3)];
String? clipboard;
setUp(() {
clipboard = null;
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler(
SystemChannels.platform,
(call) async {
if (call.method == 'Clipboard.setData') {
clipboard = (call.arguments as Map)['text'] as String?;
}
return null;
},
);
});
tearDown(() {
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler(
SystemChannels.platform,
null,
);
});
Future<void> pumpLayer(WidgetTester tester) async {
await tester.pumpWidget(
MaterialApp(
home: Align(
alignment: Alignment.topLeft,
child: SizedBox.fromSize(
size: _viewportSize,
child: OcrSelectionLayer(ocrData: ocrData, imageSize: _imageSize, viewportSize: _viewportSize),
),
),
),
);
// The lines take a few frames to register as selectable
await tester.pumpAndSettle();
}
const mobile = TargetPlatformVariant({TargetPlatform.iOS, TargetPlatform.android});
testWidgets('long press on text selects it', (tester) async {
await pumpLayer(tester);
await tester.longPressAt(_firstLine);
await tester.pumpAndSettle();
expect(find.text('Copy'), findsOneWidget);
await tester.tap(find.text('Copy'));
await tester.pumpAndSettle();
expect(clipboard, anyOf('hello', 'world'));
}, variant: mobile);
testWidgets('copies lines separated by line breaks', (tester) async {
await pumpLayer(tester);
await tester.longPressAt(_secondLine);
await tester.pumpAndSettle();
await tester.tap(find.text(defaultTargetPlatform == TargetPlatform.iOS ? 'Select All' : 'Select all'));
await tester.pumpAndSettle();
await tester.tap(find.text('Copy'));
await tester.pumpAndSettle();
expect(clipboard, 'hello world\nsecond line');
}, variant: mobile);
}