feat: add screenshot OCR extraction
This commit is contained in:
+34
-11
@@ -16,6 +16,7 @@ extern void nativeOverlayCopy(int x, int y, int w, int h, const char *annotation
|
||||
extern void nativeOverlaySave(int x, int y, int w, int h, const char *annotationsJSON, const char *dir);
|
||||
extern void nativeOverlaySaveRemote(int x, int y, int w, int h, const char *annotationsJSON);
|
||||
extern void nativeOverlaySummarize(int x, int y, int w, int h, const char *annotationsJSON);
|
||||
extern void nativeOverlayOCR(int x, int y, int w, int h, const char *annotationsJSON);
|
||||
extern void nativeOverlayCancel(void);
|
||||
|
||||
static NSWindow *nativeOverlayWindow = nil;
|
||||
@@ -103,6 +104,7 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
@property(strong) NSButton *clipboardButton;
|
||||
@property(strong) NSButton *saveButton;
|
||||
@property(strong) NSButton *saveRemoteButton;
|
||||
@property(strong) NSButton *ocrButton;
|
||||
@property(strong) NSButton *summaryButton;
|
||||
@property(strong) NSButton *uploadButton;
|
||||
@property(strong) NSButton *penButton;
|
||||
@@ -124,6 +126,7 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
- (void)copySelection;
|
||||
- (void)saveSelection;
|
||||
- (void)saveRemoteSelection;
|
||||
- (void)ocrSelection;
|
||||
- (void)summarizeSelection;
|
||||
- (void)cancelSelection;
|
||||
- (void)beginTextAnnotationAtPoint:(NSPoint)localPoint;
|
||||
@@ -146,13 +149,14 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
_activeColor = [NSColor colorWithCalibratedRed:239.0/255.0 green:68.0/255.0 blue:68.0/255.0 alpha:1.0];
|
||||
_annotations = [NSMutableArray array];
|
||||
|
||||
// Use SnipHoverButton for the 5 action buttons so we get hover-color
|
||||
// Use SnipHoverButton for the action buttons so we get hover-color
|
||||
// animation. Annotation buttons stay as plain NSButton because they
|
||||
// already have explicit on/off "active" styling.
|
||||
_cancelButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(cancelSelection)];
|
||||
_clipboardButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(copySelection)];
|
||||
_saveButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveSelection)];
|
||||
_saveRemoteButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveRemoteSelection)];
|
||||
_ocrButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(ocrSelection)];
|
||||
_summaryButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(summarizeSelection)];
|
||||
_uploadButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(confirmSelection)];
|
||||
// Tooltip strings shown on hover for each action button.
|
||||
@@ -161,6 +165,7 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
[_clipboardButton setToolTip:@"复制图片"];
|
||||
[_saveButton setToolTip:@"保存本地"];
|
||||
[_saveRemoteButton setToolTip:@"保存远端"];
|
||||
[_ocrButton setToolTip:@"提取文字"];
|
||||
[_summaryButton setToolTip:@"复制总结"];
|
||||
[_uploadButton setToolTip:@"上传云端"];
|
||||
_penButton = [NSButton buttonWithTitle:@"" target:self action:@selector(selectPen)];
|
||||
@@ -183,10 +188,10 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
// Add the action toolbar background BEFORE the buttons so it sits
|
||||
// behind them in the view hierarchy (AppKit z-order = subview order).
|
||||
[self addSubview:_actionToolbarBg];
|
||||
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) {
|
||||
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _ocrButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) {
|
||||
[self addSubview:view];
|
||||
}
|
||||
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) {
|
||||
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _ocrButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) {
|
||||
[view setHidden:YES];
|
||||
}
|
||||
[_sizeLabel setHidden:YES];
|
||||
@@ -249,8 +254,8 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
}
|
||||
|
||||
- (void)styleControls {
|
||||
// Cancel / Copy / Save / Save-remote / Upload — square icon buttons
|
||||
// rendered from the user-supplied SVG assets. All five default to white
|
||||
// Cancel / Copy / Save / Save-remote / OCR / Summary / Upload — square icon buttons
|
||||
// rendered from the user-supplied SVG assets. All actions default to white
|
||||
// and animate to a per-action accent color on hover (cancel = red,
|
||||
// others = blue). The base "white default" replaces the previous blue
|
||||
// upload tint so the toolbar reads as a uniform icon row.
|
||||
@@ -260,6 +265,7 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
// save-remote.svg — provided by the user; its content is the same arrow
|
||||
// icon as the previous upload, so we reuse it verbatim here.
|
||||
NSImage *saveRemoteIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M481.749333 353.792c16.682667-16.64 43.690667-16.64 60.373334 0l120.917333 120.832 3.541333 4.010667a42.666667 42.666667 0 0 1-63.872 56.32L554.666667 486.954667V896a42.666667 42.666667 0 0 1-85.333334 0v-409.173333l-48.170666 48.128a42.666667 42.666667 0 0 1-60.330667 0l-3.541333-4.010667a42.666667 42.666667 0 0 1 3.541333-56.32zM512 85.333333c122.538667 0 227.84 73.813333 273.92 179.370667A213.333333 213.333333 0 0 1 725.333333 682.666667a42.666667 42.666667 0 0 1-4.992-85.034667L725.333333 597.333333a128 128 0 0 0 36.394667-250.794666l-38.144-11.264-15.914667-36.437334a213.376 213.376 0 0 0-407.296 58.026667l-7.381333 58.368-57.173333 13.824A85.418667 85.418667 0 0 0 256 597.333333h42.666667a42.666667 42.666667 0 0 1 0 85.333334H256a170.666667 170.666667 0 0 1-40.277333-336.554667A298.709333 298.709333 0 0 1 512 85.333333z' fill='white'/></svg>"];
|
||||
NSImage *ocrIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M192 160h192v64H256v128h-64V160zM640 160h192v192h-64V224H640v-64zM256 672v128h128v64H192V672h64zM832 672v192H640v-64h128V672h64zM469.333333 288h85.333334l170.666666 448h-78.933333l-39.253333-106.666667H416.853333L377.6 736H298.666667l170.666666-448zM439.466667 565.333333h144.896L512 368.981333 439.466667 565.333333z' fill='white'/></svg>"];
|
||||
NSImage *summaryIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M256 128h448l192 192v576H256V128z m64 64v640h512V352H672V192H320z m416 45.248V288h50.752L736 237.248zM384 448h384v64H384v-64z m0 128h384v64H384v-64z m0 128h256v64H384v-64zM192 256v704h512v64H128V256h64z' fill='white'/></svg>"];
|
||||
// upload.svg — newly replaced "send/cloud" icon supplied by the user.
|
||||
NSImage *uploadIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M550.528 544.128L640 539.776V678.4l178.005333-178.005333L640 322.389333v129.024l-72.618667 10.922667c-155.52 23.424-251.733333 73.557333-312.874666 164.437333 84.906667-50.602667 178.261333-76.928 296.021333-82.645333zM554.666667 629.376c-27.392 1.322667-53.034667 3.968-77.141334 7.808l-8.192 1.365333c-136.704 23.637333-224.810667 87.168-310.954666 176.128-26.154667 27.008-51.882667 59.648-51.882667 59.648-14.848 18.176-24.021333 14.037333-20.352-9.173333 0 0 4.394667-31.402667 11.690667-65.792C144.938667 577.066667 250.581333 423.68 554.666667 377.941333V209.066667a38.4 38.4 0 0 1 65.536-27.136l291.328 291.285333a38.4 38.4 0 0 1 0 54.314667l-291.328 291.285333A38.4 38.4 0 0 1 554.666667 791.637333v-162.261333z' fill='white'/></svg>"];
|
||||
@@ -268,10 +274,11 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
[self styleIconButton:_clipboardButton image:copyIcon];
|
||||
[self styleIconButton:_saveButton image:saveIcon];
|
||||
[self styleIconButton:_saveRemoteButton image:saveRemoteIcon];
|
||||
[self styleIconButton:_ocrButton image:ocrIcon];
|
||||
[self styleIconButton:_summaryButton image:summaryIcon];
|
||||
[self styleIconButton:_uploadButton image:uploadIcon];
|
||||
|
||||
// Configure hover colors for the 5 action buttons. White is the shared
|
||||
// Configure hover colors for the action buttons. White is the shared
|
||||
// base; cancel turns red on hover and the rest turn blue, providing a
|
||||
// glance-able cue for destructive vs. constructive actions.
|
||||
NSColor *baseWhite = [NSColor whiteColor];
|
||||
@@ -281,12 +288,14 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
SnipHoverButton *clipboardHB = (SnipHoverButton *)_clipboardButton;
|
||||
SnipHoverButton *saveHB = (SnipHoverButton *)_saveButton;
|
||||
SnipHoverButton *saveRemoteHB = (SnipHoverButton *)_saveRemoteButton;
|
||||
SnipHoverButton *ocrHB = (SnipHoverButton *)_ocrButton;
|
||||
SnipHoverButton *summaryHB = (SnipHoverButton *)_summaryButton;
|
||||
SnipHoverButton *uploadHB = (SnipHoverButton *)_uploadButton;
|
||||
cancelHB.baseColor = baseWhite; cancelHB.hoverColor = hoverRed;
|
||||
clipboardHB.baseColor = baseWhite; clipboardHB.hoverColor = hoverBlue;
|
||||
saveHB.baseColor = baseWhite; saveHB.hoverColor = hoverBlue;
|
||||
saveRemoteHB.baseColor = baseWhite; saveRemoteHB.hoverColor = hoverBlue;
|
||||
ocrHB.baseColor = baseWhite; ocrHB.hoverColor = hoverBlue;
|
||||
summaryHB.baseColor = baseWhite; summaryHB.hoverColor = hoverBlue;
|
||||
uploadHB.baseColor = baseWhite; uploadHB.hoverColor = hoverBlue;
|
||||
|
||||
@@ -646,6 +655,7 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
[_clipboardButton setHidden:!visible];
|
||||
[_saveButton setHidden:!visible];
|
||||
[_saveRemoteButton setHidden:!visible];
|
||||
[_ocrButton setHidden:!visible];
|
||||
[_summaryButton setHidden:!visible];
|
||||
[_uploadButton setHidden:!visible];
|
||||
[_actionToolbarBg setHidden:!visible];
|
||||
@@ -665,12 +675,12 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
[_sizeLabel setStringValue:[NSString stringWithFormat:@"%.0f × %.0f", _selection.size.width, _selection.size.height]];
|
||||
[_sizeLabel setFrame:NSMakeRect(_selection.origin.x, MAX(0, _selection.origin.y - 26), 110, 22)];
|
||||
|
||||
// Action toolbar layout — 6 icon buttons, each 28x28, separated by 8px,
|
||||
// with 4px outer padding. Total = 6*28 + 5*8 + 2*4 = 216px wide.
|
||||
// Action toolbar layout — 7 icon buttons, each 28x28, separated by 8px,
|
||||
// with 4px outer padding. Total = 7*28 + 6*8 + 2*4 = 252px wide.
|
||||
CGFloat actionBtnSize = 28;
|
||||
CGFloat actionGap = 8;
|
||||
CGFloat actionPad = 4;
|
||||
NSInteger actionCount = 6;
|
||||
NSInteger actionCount = 7;
|
||||
CGFloat toolbarW = actionPad * 2 + actionBtnSize * actionCount + actionGap * (actionCount - 1);
|
||||
CGFloat toolbarH = 40;
|
||||
CGFloat x = _selection.origin.x + _selection.size.width - toolbarW;
|
||||
@@ -688,8 +698,9 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
[_clipboardButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 1, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_saveButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 2, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_saveRemoteButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 3, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_summaryButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 5, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_ocrButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_summaryButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 5, btnY, actionBtnSize, actionBtnSize)];
|
||||
[_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 6, btnY, actionBtnSize, actionBtnSize)];
|
||||
|
||||
// Mark toolbar sits to the LEFT of the action toolbar (same row) with an
|
||||
// 8px gap between the two groups, so they never overlap on tiny selections.
|
||||
@@ -930,6 +941,18 @@ static id nativeOverlayKeyMonitor = nil;
|
||||
nativeOverlaySaveRemote((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]);
|
||||
}
|
||||
|
||||
// `ocrSelection` sends the screenshot to the configured OCR provider and
|
||||
// copies the extracted text.
|
||||
- (void)ocrSelection {
|
||||
if (!_hasSelection) {
|
||||
return;
|
||||
}
|
||||
NSRect r = [self globalRectForSelection:_selection];
|
||||
[self closeOverlayWindow];
|
||||
NSString *json = [self annotationsJSON];
|
||||
nativeOverlayOCR((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]);
|
||||
}
|
||||
|
||||
// `summarizeSelection` uploads the screenshot to S3, sends it to the
|
||||
// configured multimodal model, and copies the generated summary.
|
||||
- (void)summarizeSelection {
|
||||
|
||||
Reference in New Issue
Block a user