From baec01610ced4947ab50dc7ee6b57690be9df641 Mon Sep 17 00:00:00 2001 From: mamamiyear Date: Wed, 8 Jul 2026 00:42:49 +0800 Subject: [PATCH] feat: add screenshot OCR extraction --- app.go | 99 ++++ frontend/src/App.vue | 29 +- frontend/src/views/CaptureOverlay.vue | 30 +- frontend/src/views/SettingsView.vue | 164 +++++- frontend/wailsjs/go/main/App.d.ts | 4 + frontend/wailsjs/go/main/App.js | 8 + frontend/wailsjs/go/models.ts | 124 +++-- internal/application/capture_ocr.go | 57 +++ internal/application/capture_ocr_test.go | 50 ++ internal/domain/config.go | 126 +++++ internal/infrastructure/ocr/client.go | 568 +++++++++++++++++++++ internal/infrastructure/ocr/client_test.go | 150 ++++++ native_overlay_callbacks_darwin.go | 12 + native_overlay_darwin.go | 45 +- 14 files changed, 1420 insertions(+), 46 deletions(-) create mode 100644 internal/application/capture_ocr.go create mode 100644 internal/application/capture_ocr_test.go create mode 100644 internal/infrastructure/ocr/client.go create mode 100644 internal/infrastructure/ocr/client_test.go diff --git a/app.go b/app.go index 36a5c7e..ff57d8c 100644 --- a/app.go +++ b/app.go @@ -23,6 +23,7 @@ import ( "github.com/mmmy/snapgo/internal/infrastructure/display" "github.com/mmmy/snapgo/internal/infrastructure/hotkey" llmpkg "github.com/mmmy/snapgo/internal/infrastructure/llm" + ocrpkg "github.com/mmmy/snapgo/internal/infrastructure/ocr" "github.com/mmmy/snapgo/internal/infrastructure/oss" "github.com/mmmy/snapgo/internal/infrastructure/screencapture" sshpkg "github.com/mmmy/snapgo/internal/infrastructure/ssh" @@ -499,6 +500,48 @@ func (a *App) finishSummary(svc *application.CaptureSummaryService, summary stri return nil } +func (a *App) runOCRPipeline(pngBytes []byte) error { + a.mu.RLock() + cfg := a.cfg + a.mu.RUnlock() + + if !cfg.IsOCRConfigured() { + err := fmt.Errorf("请先在文字提取配置页填写 OCR Provider 的 AccessKey ID 和 AccessKey Secret") + a.emitOperationStatus("ocr", "需要配置 OCR", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + + providerID, ocrCfg, _ := cfg.ActiveOCRProvider() + recognizer, err := ocrpkg.NewClient(providerID, ocrCfg) + if err != nil { + a.emitOperationStatus("ocr", "OCR 配置错误", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + + svc := &application.CaptureOCRService{ + Recognizer: recognizer, + Clipboard: a.clip, + } + + a.emitOperationStatus("ocr", "识别中", "正在提取截图文字", "running") + text, err := svc.Recognize(a.ctx, pngBytes) + if err != nil { + a.emitOperationStatus("ocr", "识别失败", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + if err := svc.CopyText(a.ctx, text); err != nil { + a.emitOperationStatus("ocr", "复制失败", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + a.emitOperationStatus("ocr", "提取完成", "文字已复制到剪贴板", "success") + wruntime.EventsEmit(a.ctx, "upload:success", "ocr text copied to clipboard") + return nil +} + func (a *App) consumePendingCapture() (*pendingCapture, error) { a.pendingMu.Lock() pc := a.pending @@ -876,6 +919,35 @@ func (a *App) SummarizeRegion(result CaptureResult) error { return a.runSummaryPipeline(cropped) } +// ExtractTextRegion sends the selected screenshot to the configured OCR +// provider and copies the extracted text to the clipboard. +func (a *App) ExtractTextRegion(result CaptureResult) error { + pc, err := a.consumePendingCapture() + if err != nil { + slog.Warn("ExtractTextRegion: no pending capture", "err", err) + return err + } + defer func() { + a.capturing.Store(false) + a.dismissOverlay() + }() + + a.dismissOverlay() + flushFrame() + + slog.Info("ExtractTextRegion: capturing region", + "x", result.Rect.X, "y", result.Rect.Y, + "w", result.Rect.W, "h", result.Rect.H, + "annotations", len(result.Annotations)) + cropped, err := a.captureSelectedPNG(result, pc) + if err != nil { + slog.Error("ExtractTextRegion: capture failed", "err", err) + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + return a.runOCRPipeline(cropped) +} + // SaveNativeRegionToRemote is the macOS-native overlay equivalent of // SaveRegionToRemote. The AppKit panel is already closed by the time this // runs (see saveRemoteSelection in native_overlay_darwin.go), so we only @@ -933,6 +1005,33 @@ func (a *App) SummarizeNativeRegion(result CaptureResult) error { return a.runSummaryPipeline(cropped) } +// ExtractTextNativeRegion is the macOS-native overlay equivalent of +// ExtractTextRegion. The AppKit panel is already closed before this runs. +func (a *App) ExtractTextNativeRegion(result CaptureResult) error { + pc, err := a.consumePendingCapture() + if err != nil { + slog.Warn("ExtractTextNativeRegion: no pending capture", "err", err) + return err + } + defer func() { + a.capturing.Store(false) + hideDockIcon() + }() + + flushFrame() + slog.Info("ExtractTextNativeRegion: capturing region", + "x", result.Rect.X, "y", result.Rect.Y, + "w", result.Rect.W, "h", result.Rect.H, + "annotations", len(result.Annotations)) + cropped, err := a.captureSelectedPNG(result, pc) + if err != nil { + slog.Error("ExtractTextNativeRegion: capture failed", "err", err) + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + return a.runOCRPipeline(cropped) +} + func parseNativeAnnotations(raw string) []application.Annotation { if raw == "" { return nil diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 0858d66..4260a6d 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -23,6 +23,7 @@ import { SaveRegionImage, SaveRegionToRemote, SummarizeRegion, + ExtractTextRegion, CancelRegion, GetConfig, } from '../wailsjs/go/main/App' @@ -58,7 +59,7 @@ async function loadThemePreference() { // `SettingsTab` is hoisted to the App shell so the sidebar (which lives // here) and the inner SettingsView (which renders the matching card) can // share a single source of truth without an event bus. -type SettingsTab = 'general' | 's3' | 'ssh' | 'llm' +type SettingsTab = 'general' | 's3' | 'ssh' | 'llm' | 'ocr' const activeTab = ref('general') // Sidebar entries are declarative so adding a destination type later is @@ -68,6 +69,7 @@ const sidebarItems: Array<{ id: SettingsTab; label: string }> = [ { id: 's3', label: '对象存储' }, { id: 'ssh', label: '远程主机' }, { id: 'llm', label: '智能识图' }, + { id: 'ocr', label: '文字提取' }, ] interface OverlayPayload { @@ -233,6 +235,28 @@ async function onOverlaySummarize(rect: { } } +async function onOverlayOCR(rect: { + rect: { + x: number + y: number + w: number + h: number + } + annotations: Array<{ + tool: string + color: string + points: Array<{ x: number; y: number }> + }> +}) { + mode.value = 'settings' + overlayPayload.value = null + try { + await ExtractTextRegion(rect as any) + } catch { + /* Surfaced via upload:failure */ + } +} + async function onOverlayCancel() { mode.value = 'settings' overlayPayload.value = null @@ -264,6 +288,8 @@ onMounted(() => { 'success', url === 'summary copied to clipboard' ? 'Summary copied to clipboard' + : url === 'ocr text copied to clipboard' + ? 'OCR text copied to clipboard' : `Copied: ${url}` ) }) @@ -303,6 +329,7 @@ onUnmounted(() => { @save="onOverlaySave" @save-remote="onOverlaySaveRemote" @summarize="onOverlaySummarize" + @ocr="onOverlayOCR" @cancel="onOverlayCancel" /> diff --git a/frontend/src/views/CaptureOverlay.vue b/frontend/src/views/CaptureOverlay.vue index 7889b06..ca0b60f 100644 --- a/frontend/src/views/CaptureOverlay.vue +++ b/frontend/src/views/CaptureOverlay.vue @@ -56,6 +56,10 @@ const emit = defineEmits<{ e: 'summarize', payload: { rect: Rect; annotations: Annotation[] } ): void + ( + e: 'ocr', + payload: { rect: Rect; annotations: Annotation[] } + ): void (e: 'cancel'): void }>() @@ -119,7 +123,7 @@ const sizeLabel = computed(() => { }) const MARK_TOOLBAR_W = 222 -const ACTION_TOOLBAR_W = 216 +const ACTION_TOOLBAR_W = 252 const TOOLBAR_GROUP_GAP = 8 const rightToolbarPos = computed(() => { @@ -379,7 +383,11 @@ function onSummarize() { emitAction('summarize') } -function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote' | 'summarize') { +function onOCR() { + emitAction('ocr') +} + +function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote' | 'summarize' | 'ocr') { if (!rect.value) return commitTextDraft() const payload = { @@ -391,6 +399,7 @@ function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote' | 'summa if (action === 'save') emit('save', payload) if (action === 'save-remote') emit('save-remote', payload) if (action === 'summarize') emit('summarize', payload) + if (action === 'ocr') emit('ocr', payload) } function onCancel() { @@ -716,6 +725,21 @@ onUnmounted(() => { @click="onSaveRemote" v-html="saveRemoteIcon" /> + + + +
; export function CopyRegionImage(arg1:main.CaptureResult):Promise; +export function ExtractTextNativeRegion(arg1:main.CaptureResult):Promise; + +export function ExtractTextRegion(arg1:main.CaptureResult):Promise; + export function GetConfig():Promise; export function QuitApp():Promise; diff --git a/frontend/wailsjs/go/main/App.js b/frontend/wailsjs/go/main/App.js index d51dce7..2da5701 100755 --- a/frontend/wailsjs/go/main/App.js +++ b/frontend/wailsjs/go/main/App.js @@ -30,6 +30,14 @@ export function CopyRegionImage(arg1) { return window['go']['main']['App']['CopyRegionImage'](arg1); } +export function ExtractTextNativeRegion(arg1) { + return window['go']['main']['App']['ExtractTextNativeRegion'](arg1); +} + +export function ExtractTextRegion(arg1) { + return window['go']['main']['App']['ExtractTextRegion'](arg1); +} + export function GetConfig() { return window['go']['main']['App']['GetConfig'](); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 118f622..e0dabaf 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -1,13 +1,13 @@ export namespace application { - + export class Point { x: number; y: number; - + static createFrom(source: any = {}) { return new Point(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.x = source["x"]; @@ -19,11 +19,11 @@ export namespace application { color: string; points: Point[]; text?: string; - + static createFrom(source: any = {}) { return new Annotation(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.tool = source["tool"]; @@ -31,7 +31,7 @@ export namespace application { this.points = this.convertValues(source["points"], Point); this.text = source["text"]; } - + convertValues(a: any, classs: any, asMap: boolean = false): any { if (!a) { return a; @@ -54,7 +54,67 @@ export namespace application { } export namespace domain { - + + export class OCRProviderConfig { + label: string; + endpoint: string; + accessKeyId: string; + accessKeySecret: string; + region: string; + service: string; + action: string; + version: string; + timeoutSecs: number; + + static createFrom(source: any = {}) { + return new OCRProviderConfig(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.label = source["label"]; + this.endpoint = source["endpoint"]; + this.accessKeyId = source["accessKeyId"]; + this.accessKeySecret = source["accessKeySecret"]; + this.region = source["region"]; + this.service = source["service"]; + this.action = source["action"]; + this.version = source["version"]; + this.timeoutSecs = source["timeoutSecs"]; + } + } + export class OCRConfig { + activeProvider: string; + providers: Record; + + static createFrom(source: any = {}) { + return new OCRConfig(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.activeProvider = source["activeProvider"]; + this.providers = this.convertValues(source["providers"], OCRProviderConfig, true); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } export class LLMProviderConfig { label: string; baseUrl: string; @@ -64,11 +124,11 @@ export namespace domain { temperature: number; timeoutSecs: number; maxInlineBytes: number; - + static createFrom(source: any = {}) { return new LLMProviderConfig(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.label = source["label"]; @@ -85,18 +145,18 @@ export namespace domain { activeProvider: string; prompt: string; providers: Record; - + static createFrom(source: any = {}) { return new LLMConfig(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.activeProvider = source["activeProvider"]; this.prompt = source["prompt"]; this.providers = this.convertValues(source["providers"], LLMProviderConfig, true); } - + convertValues(a: any, classs: any, asMap: boolean = false): any { if (!a) { return a; @@ -125,11 +185,11 @@ export namespace domain { strictHostKey: boolean; knownHostsPath: string; connectTimeoutSecs: number; - + static createFrom(source: any = {}) { return new SSHConfig(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.host = source["host"]; @@ -152,11 +212,11 @@ export namespace domain { pathPrefix: string; publicUrlBase: string; usePathStyle: boolean; - + static createFrom(source: any = {}) { return new S3Config(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.endpoint = source["endpoint"]; @@ -175,11 +235,12 @@ export namespace domain { s3: S3Config; ssh: SSHConfig; llm: LLMConfig; - + ocr: OCRConfig; + static createFrom(source: any = {}) { return new AppConfig(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.hotkey = source["hotkey"]; @@ -187,8 +248,9 @@ export namespace domain { this.s3 = this.convertValues(source["s3"], S3Config); this.ssh = this.convertValues(source["ssh"], SSHConfig); this.llm = this.convertValues(source["llm"], LLMConfig); + this.ocr = this.convertValues(source["ocr"], OCRConfig); } - + convertValues(a: any, classs: any, asMap: boolean = false): any { if (!a) { return a; @@ -207,22 +269,24 @@ export namespace domain { return a; } } - - - + + + + + } export namespace main { - + export class CaptureActionResult { completed: boolean; path?: string; - + static createFrom(source: any = {}) { return new CaptureActionResult(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.completed = source["completed"]; @@ -234,11 +298,11 @@ export namespace main { y: number; w: number; h: number; - + static createFrom(source: any = {}) { return new RegionRect(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.x = source["x"]; @@ -250,17 +314,17 @@ export namespace main { export class CaptureResult { rect: RegionRect; annotations: application.Annotation[]; - + static createFrom(source: any = {}) { return new CaptureResult(source); } - + constructor(source: any = {}) { if ('string' === typeof source) source = JSON.parse(source); this.rect = this.convertValues(source["rect"], RegionRect); this.annotations = this.convertValues(source["annotations"], application.Annotation); } - + convertValues(a: any, classs: any, asMap: boolean = false): any { if (!a) { return a; diff --git a/internal/application/capture_ocr.go b/internal/application/capture_ocr.go new file mode 100644 index 0000000..4dd572f --- /dev/null +++ b/internal/application/capture_ocr.go @@ -0,0 +1,57 @@ +package application + +import ( + "context" + "fmt" + "strings" + + "github.com/mmmy/snapgo/internal/infrastructure/clipboard" +) + +// OCRRecognizer describes a provider capable of extracting text from a PNG +// screenshot. +type OCRRecognizer interface { + RecognizeText(ctx context.Context, pngBytes []byte) (string, error) +} + +// CaptureOCRService wires screenshot bytes -> OCR provider -> clipboard. It +// leaves progress reporting to the caller so native and web overlays can share +// the same status HUD. +type CaptureOCRService struct { + Recognizer OCRRecognizer + Clipboard clipboard.Writer +} + +// Recognize asks the configured OCR provider to extract text from the PNG. +func (s *CaptureOCRService) Recognize(ctx context.Context, pngBytes []byte) (string, error) { + if s.Recognizer == nil { + return "", fmt.Errorf("ocr is not configured") + } + if len(pngBytes) == 0 { + return "", fmt.Errorf("empty screenshot") + } + text, err := s.Recognizer.RecognizeText(ctx, pngBytes) + if err != nil { + return "", err + } + text = strings.TrimSpace(text) + if text == "" { + return "", fmt.Errorf("ocr result is empty") + } + return text, nil +} + +// CopyText writes the extracted text to the clipboard. +func (s *CaptureOCRService) CopyText(_ context.Context, text string) error { + text = strings.TrimSpace(text) + if text == "" { + return fmt.Errorf("empty ocr text") + } + if s.Clipboard == nil { + return fmt.Errorf("clipboard is not configured") + } + if err := s.Clipboard.WriteText(text); err != nil { + return fmt.Errorf("clipboard write failed: %w", err) + } + return nil +} diff --git a/internal/application/capture_ocr_test.go b/internal/application/capture_ocr_test.go new file mode 100644 index 0000000..000b651 --- /dev/null +++ b/internal/application/capture_ocr_test.go @@ -0,0 +1,50 @@ +package application + +import ( + "context" + "testing" +) + +type fakeOCRRecognizer struct { + image []byte + text string +} + +func (f *fakeOCRRecognizer) RecognizeText(_ context.Context, pngBytes []byte) (string, error) { + f.image = append([]byte(nil), pngBytes...) + if f.text == "" { + return "extracted text", nil + } + return f.text, nil +} + +func TestCaptureOCRServiceRecognizesAndCopies(t *testing.T) { + ctx := context.Background() + recognizer := &fakeOCRRecognizer{} + clip := &fakeClipboard{} + svc := &CaptureOCRService{Recognizer: recognizer, Clipboard: clip} + + text, err := svc.Recognize(ctx, []byte("png")) + if err != nil { + t.Fatalf("recognize: %v", err) + } + if string(recognizer.image) != "png" { + t.Fatalf("expected screenshot bytes forwarded, got %q", string(recognizer.image)) + } + if text != "extracted text" { + t.Fatalf("expected extracted text, got %q", text) + } + if err := svc.CopyText(ctx, text); err != nil { + t.Fatalf("copy text: %v", err) + } + if clip.text != "extracted text" { + t.Fatalf("expected ocr text copied, got %q", clip.text) + } +} + +func TestCaptureOCRServiceRejectsEmptyResult(t *testing.T) { + svc := &CaptureOCRService{Recognizer: &fakeOCRRecognizer{text: " \n "}} + if _, err := svc.Recognize(context.Background(), []byte("png")); err == nil { + t.Fatalf("expected empty OCR result to fail") + } +} diff --git a/internal/domain/config.go b/internal/domain/config.go index 30f8482..8fe421e 100644 --- a/internal/domain/config.go +++ b/internal/domain/config.go @@ -107,6 +107,29 @@ type LLMConfig struct { Providers map[string]LLMProviderConfig `json:"providers"` } +// OCRProviderConfig stores one cloud OCR endpoint that accepts a screenshot +// image and returns extracted text. Aliyun and Volcengine both authenticate +// with an AccessKey pair, but their signing schemes use slightly different +// endpoint metadata; keeping Region/Service/Action/Version configurable lets +// the presets track official API changes without a schema rewrite. +type OCRProviderConfig struct { + Label string `json:"label"` + Endpoint string `json:"endpoint"` + AccessKeyID string `json:"accessKeyId"` + AccessKeySecret string `json:"accessKeySecret"` + Region string `json:"region"` + Service string `json:"service"` + Action string `json:"action"` + Version string `json:"version"` + TimeoutSecs int `json:"timeoutSecs"` +} + +// OCRConfig controls screenshot text extraction. +type OCRConfig struct { + ActiveProvider string `json:"activeProvider"` + Providers map[string]OCRProviderConfig `json:"providers"` +} + // SSH authentication method identifiers stored in SSHConfig.AuthMethod. const ( // SSHAuthBuiltin is the legacy combined method (password → agent → key @@ -129,6 +152,12 @@ const ( LLMProviderOpenAI = "openai" ) +// Built-in OCR provider identifiers. +const ( + OCRProviderAliyun = "aliyun" + OCRProviderVolcengine = "volcengine" +) + // Theme preference identifiers stored in AppConfig.Theme. const ( ThemeAuto = "auto" @@ -172,6 +201,9 @@ type AppConfig struct { // LLM holds provider settings for the "copy summary" screenshot action. LLM LLMConfig `json:"llm"` + + // OCR holds provider settings for the "extract text" screenshot action. + OCR OCRConfig `json:"ocr"` } // DefaultAppConfig returns sane zero-value defaults used on first launch. @@ -190,6 +222,7 @@ func DefaultAppConfig() AppConfig { StrictHostKey: false, }, LLM: DefaultLLMConfig(), + OCR: DefaultOCRConfig(), } cfg.Normalize() return cfg @@ -234,6 +267,33 @@ func DefaultLLMConfig() LLMConfig { } } +// DefaultOCRConfig returns the built-in OCR provider presets. Users only need +// to fill AccessKey credentials for the common regions/actions, while advanced +// accounts can still override endpoint metadata from the settings UI. +func DefaultOCRConfig() OCRConfig { + return OCRConfig{ + ActiveProvider: OCRProviderAliyun, + Providers: map[string]OCRProviderConfig{ + OCRProviderAliyun: { + Label: "阿里云文字识别", + Endpoint: "https://ocr-api.cn-hangzhou.aliyuncs.com", + Action: "RecognizeGeneral", + Version: "2021-07-07", + TimeoutSecs: 30, + }, + OCRProviderVolcengine: { + Label: "火山引擎文字识别", + Endpoint: "https://visual.volcengineapi.com", + Region: "cn-north-1", + Service: "cv", + Action: "OCRNormal", + Version: "2020-08-26", + TimeoutSecs: 30, + }, + }, + } +} + // Normalize fills defaults into configs written by older app versions while // preserving any user-provided provider fields. func (c *AppConfig) Normalize() { @@ -295,6 +355,43 @@ func (c *AppConfig) Normalize() { } c.LLM.Providers[id] = current } + + defaultOCR := DefaultOCRConfig() + if c.OCR.ActiveProvider == "" { + c.OCR.ActiveProvider = defaultOCR.ActiveProvider + } + if c.OCR.Providers == nil { + c.OCR.Providers = map[string]OCRProviderConfig{} + } + for id, def := range defaultOCR.Providers { + current, ok := c.OCR.Providers[id] + if !ok { + c.OCR.Providers[id] = def + continue + } + if current.Label == "" { + current.Label = def.Label + } + if current.Endpoint == "" { + current.Endpoint = def.Endpoint + } + if current.Region == "" { + current.Region = def.Region + } + if current.Service == "" { + current.Service = def.Service + } + if current.Action == "" { + current.Action = def.Action + } + if current.Version == "" { + current.Version = def.Version + } + if current.TimeoutSecs == 0 { + current.TimeoutSecs = def.TimeoutSecs + } + c.OCR.Providers[id] = current + } } // IsS3Configured reports whether the user has filled the mandatory S3 fields. @@ -329,3 +426,32 @@ func (c AppConfig) IsLLMConfigured() bool { _, provider, ok := c.ActiveLLMProvider() return ok && provider.BaseURL != "" && provider.APIKey != "" && provider.Model != "" } + +// ActiveOCRProvider returns the selected OCR provider config plus a boolean +// indicating whether the selection exists. +func (c AppConfig) ActiveOCRProvider() (string, OCRProviderConfig, bool) { + id := c.OCR.ActiveProvider + if id == "" { + id = OCRProviderAliyun + } + provider, ok := c.OCR.Providers[id] + return id, provider, ok +} + +// IsOCRConfigured reports whether the selected OCR provider has the fields +// required to sign and send a request. +func (c AppConfig) IsOCRConfigured() bool { + id, provider, ok := c.ActiveOCRProvider() + if !ok || + provider.Endpoint == "" || + provider.AccessKeyID == "" || + provider.AccessKeySecret == "" || + provider.Action == "" || + provider.Version == "" { + return false + } + if id == OCRProviderVolcengine && (provider.Region == "" || provider.Service == "") { + return false + } + return true +} diff --git a/internal/infrastructure/ocr/client.go b/internal/infrastructure/ocr/client.go new file mode 100644 index 0000000..61248fe --- /dev/null +++ b/internal/infrastructure/ocr/client.go @@ -0,0 +1,568 @@ +// Package ocr contains cloud OCR adapters. +package ocr + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/mmmy/snapgo/internal/application" + "github.com/mmmy/snapgo/internal/domain" +) + +// Client calls one configured OCR provider. +type Client struct { + providerID string + cfg domain.OCRProviderConfig + httpClient *http.Client + now func() time.Time + nonce func() string +} + +var _ application.OCRRecognizer = (*Client)(nil) + +// NewClient validates cfg and returns a reusable OCR client. +func NewClient(providerID string, cfg domain.OCRProviderConfig) (*Client, error) { + if strings.TrimSpace(cfg.Endpoint) == "" { + return nil, fmt.Errorf("ocr endpoint is required") + } + if strings.TrimSpace(cfg.AccessKeyID) == "" { + return nil, fmt.Errorf("ocr access key id is required") + } + if strings.TrimSpace(cfg.AccessKeySecret) == "" { + return nil, fmt.Errorf("ocr access key secret is required") + } + if strings.TrimSpace(cfg.Action) == "" { + return nil, fmt.Errorf("ocr action is required") + } + if strings.TrimSpace(cfg.Version) == "" { + return nil, fmt.Errorf("ocr version is required") + } + switch providerID { + case domain.OCRProviderAliyun: + case domain.OCRProviderVolcengine: + if strings.TrimSpace(cfg.Region) == "" { + return nil, fmt.Errorf("volcengine ocr region is required") + } + if strings.TrimSpace(cfg.Service) == "" { + return nil, fmt.Errorf("volcengine ocr service is required") + } + default: + return nil, fmt.Errorf("unsupported ocr provider %q", providerID) + } + timeout := cfg.TimeoutSecs + if timeout <= 0 { + timeout = 30 + } + return &Client{ + providerID: providerID, + cfg: cfg, + httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second}, + now: time.Now, + nonce: randomNonce, + }, nil +} + +// RecognizeText extracts visible text from a PNG screenshot. +func (c *Client) RecognizeText(ctx context.Context, pngBytes []byte) (string, error) { + if len(pngBytes) == 0 { + return "", fmt.Errorf("empty screenshot") + } + switch c.providerID { + case domain.OCRProviderAliyun: + return c.recognizeAliyun(ctx, pngBytes) + case domain.OCRProviderVolcengine: + return c.recognizeVolcengine(ctx, pngBytes) + default: + return "", fmt.Errorf("unsupported ocr provider %q", c.providerID) + } +} + +func (c *Client) recognizeAliyun(ctx context.Context, pngBytes []byte) (string, error) { + endpoint, err := parseEndpoint(c.cfg.Endpoint) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), bytes.NewReader(pngBytes)) + if err != nil { + return "", fmt.Errorf("build aliyun ocr request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/octet-stream") + c.signAliyun(req, pngBytes) + return c.doOCRRequest(req, "aliyun ocr") +} + +func (c *Client) recognizeVolcengine(ctx context.Context, pngBytes []byte) (string, error) { + endpoint, err := parseEndpoint(c.cfg.Endpoint) + if err != nil { + return "", err + } + query := endpoint.Query() + query.Set("Action", c.cfg.Action) + query.Set("Version", c.cfg.Version) + endpoint.RawQuery = query.Encode() + + body := map[string]string{ + "image_base64": base64.StdEncoding.EncodeToString(pngBytes), + } + payload, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("marshal volcengine ocr request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("build volcengine ocr request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + c.signVolcengine(req, payload) + return c.doOCRRequest(req, "volcengine ocr") +} + +func (c *Client) doOCRRequest(req *http.Request, label string) (string, error) { + resp, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("%s request: %w", label, err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("%s status %d: %s", label, resp.StatusCode, compactBody(data)) + } + text, err := parseOCRText(data) + if err != nil { + return "", fmt.Errorf("%s response: %w", label, err) + } + return text, nil +} + +func (c *Client) signAliyun(req *http.Request, payload []byte) { + now := c.now().UTC() + payloadHash := sha256Hex(payload) + nonce := c.nonce() + + req.Header.Set("x-acs-action", c.cfg.Action) + req.Header.Set("x-acs-version", c.cfg.Version) + req.Header.Set("x-acs-date", now.Format("2006-01-02T15:04:05Z")) + req.Header.Set("x-acs-signature-nonce", nonce) + req.Header.Set("x-acs-content-sha256", payloadHash) + + signedHeaders := []string{ + "content-type", + "host", + "x-acs-action", + "x-acs-content-sha256", + "x-acs-date", + "x-acs-signature-nonce", + "x-acs-version", + } + canonicalHeaders := canonicalHeaders(req, signedHeaders) + canonicalRequest := strings.Join([]string{ + req.Method, + canonicalURI(req.URL), + canonicalQuery(req.URL), + canonicalHeaders, + strings.Join(signedHeaders, ";"), + payloadHash, + }, "\n") + stringToSign := "ACS3-HMAC-SHA256\n" + sha256Hex([]byte(canonicalRequest)) + signature := hmacSHA256Hex([]byte(c.cfg.AccessKeySecret), []byte(stringToSign)) + req.Header.Set( + "Authorization", + fmt.Sprintf("ACS3-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + c.cfg.AccessKeyID, + strings.Join(signedHeaders, ";"), + signature, + ), + ) +} + +func (c *Client) signVolcengine(req *http.Request, payload []byte) { + now := c.now().UTC() + xDate := now.Format("20060102T150405Z") + shortDate := now.Format("20060102") + payloadHash := sha256Hex(payload) + + req.Header.Set("X-Date", xDate) + req.Header.Set("X-Content-Sha256", payloadHash) + + signedHeaders := []string{ + "content-type", + "host", + "x-content-sha256", + "x-date", + } + canonicalHeaders := canonicalHeaders(req, signedHeaders) + canonicalRequest := strings.Join([]string{ + req.Method, + canonicalURI(req.URL), + canonicalQuery(req.URL), + canonicalHeaders, + strings.Join(signedHeaders, ";"), + payloadHash, + }, "\n") + scope := strings.Join([]string{shortDate, c.cfg.Region, c.cfg.Service, "request"}, "/") + stringToSign := strings.Join([]string{ + "HMAC-SHA256", + xDate, + scope, + sha256Hex([]byte(canonicalRequest)), + }, "\n") + signingKey := volcengineSigningKey(c.cfg.AccessKeySecret, shortDate, c.cfg.Region, c.cfg.Service) + signature := hmacSHA256Hex(signingKey, []byte(stringToSign)) + req.Header.Set( + "Authorization", + fmt.Sprintf("HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + c.cfg.AccessKeyID, + scope, + strings.Join(signedHeaders, ";"), + signature, + ), + ) +} + +func parseEndpoint(raw string) (*url.URL, error) { + endpoint := strings.TrimSpace(raw) + if endpoint == "" { + return nil, fmt.Errorf("ocr endpoint is required") + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + parsed, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("parse ocr endpoint: %w", err) + } + if parsed.Scheme == "" || parsed.Host == "" { + return nil, fmt.Errorf("invalid ocr endpoint %q", raw) + } + if parsed.Path == "" { + parsed.Path = "/" + } + return parsed, nil +} + +func parseOCRText(data []byte) (string, error) { + var root any + if err := json.Unmarshal(data, &root); err != nil { + return "", fmt.Errorf("decode json: %w", err) + } + if msg := responseError(root); msg != "" { + return "", errors.New(msg) + } + parts := collectOCRParts(root, "") + if len(parts) == 0 { + return "", fmt.Errorf("no text found") + } + return strings.Join(dedupeNonEmpty(parts), "\n"), nil +} + +func responseError(v any) string { + obj, ok := v.(map[string]any) + if !ok { + return "" + } + if meta, ok := obj["ResponseMetadata"].(map[string]any); ok { + if errObj, ok := meta["Error"].(map[string]any); ok { + if msg := stringValue(errObj["Message"]); msg != "" { + return msg + } + if code := stringValue(errObj["Code"]); code != "" { + return code + } + } + } + if errObj, ok := obj["Error"].(map[string]any); ok { + if msg := stringValue(errObj["Message"]); msg != "" { + return msg + } + if msg := stringValue(errObj["message"]); msg != "" { + return msg + } + } + if code, ok := numericCode(obj["code"]); ok && code != 0 && code != 10000 { + if msg := stringValue(obj["message"]); msg != "" { + return msg + } + if msg := stringValue(obj["msg"]); msg != "" { + return msg + } + return fmt.Sprintf("provider code %.0f", code) + } + if code := stringValue(obj["Code"]); code != "" && !isSuccessCode(code) { + if msg := stringValue(obj["Message"]); msg != "" { + return msg + } + return code + } + return "" +} + +func collectOCRParts(v any, parentKey string) []string { + switch value := v.(type) { + case string: + text := strings.TrimSpace(value) + if text == "" { + return nil + } + if looksLikeJSON(text) { + var nested any + if err := json.Unmarshal([]byte(text), &nested); err == nil { + return collectOCRParts(nested, parentKey) + } + } + if isTextKey(parentKey) || isContainerTextKey(parentKey) { + return []string{text} + } + return nil + case []any: + parts := make([]string, 0, len(value)) + for _, item := range value { + parts = append(parts, collectOCRParts(item, parentKey)...) + } + return parts + case map[string]any: + parts := make([]string, 0) + for _, key := range priorityOCRKeys() { + if child, ok := value[key]; ok { + parts = append(parts, collectOCRParts(child, key)...) + } + } + keys := make([]string, 0, len(value)) + for key := range value { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if isPriorityOCRKey(key) || isMetadataKey(key) { + continue + } + parts = append(parts, collectOCRParts(value[key], key)...) + } + return parts + default: + return nil + } +} + +func priorityOCRKeys() []string { + return []string{ + "Data", + "data", + "Result", + "result", + "content", + "Content", + "text", + "Text", + "DetectedText", + "detected_text", + "line_text", + "LineText", + "line_texts", + "LineTexts", + "word", + "Word", + "words", + "Words", + "words_result", + "WordsResult", + "prism_wordsInfo", + "prism_words_info", + "ocr_infos", + "OCRInfos", + "items", + "Items", + "blocks", + "Blocks", + "regions", + "Regions", + } +} + +func isPriorityOCRKey(key string) bool { + for _, item := range priorityOCRKeys() { + if item == key { + return true + } + } + return false +} + +func isTextKey(key string) bool { + switch normalizeKey(key) { + case "content", "text", "detectedtext", "linetext", "word", "words", "value": + return true + default: + return false + } +} + +func isContainerTextKey(key string) bool { + switch normalizeKey(key) { + case "data", "result", "linetexts", "texts": + return true + default: + return false + } +} + +func isMetadataKey(key string) bool { + switch normalizeKey(key) { + case "requestid", "request", "code", "status", "statuscode", "success", + "error", "errors", "message", "msg", "cost", "angle", "probability", + "confidence", "height", "width", "left", "top", "right", "bottom", + "x", "y", "responsemetadata": + return true + default: + return false + } +} + +func normalizeKey(key string) string { + key = strings.ToLower(key) + key = strings.ReplaceAll(key, "_", "") + key = strings.ReplaceAll(key, "-", "") + return key +} + +func dedupeNonEmpty(parts []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(parts)) + for _, part := range parts { + for _, line := range strings.Split(part, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if _, ok := seen[line]; ok { + continue + } + seen[line] = struct{}{} + out = append(out, line) + } + } + return out +} + +func looksLikeJSON(text string) bool { + return strings.HasPrefix(text, "{") || strings.HasPrefix(text, "[") +} + +func stringValue(v any) string { + if s, ok := v.(string); ok { + return strings.TrimSpace(s) + } + return "" +} + +func numericCode(v any) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case int: + return float64(n), true + case json.Number: + f, err := n.Float64() + return f, err == nil + default: + return 0, false + } +} + +func isSuccessCode(code string) bool { + switch strings.ToLower(strings.TrimSpace(code)) { + case "", "ok", "success", "200", "10000": + return true + default: + return false + } +} + +func canonicalURI(u *url.URL) string { + if u == nil || u.EscapedPath() == "" { + return "/" + } + return u.EscapedPath() +} + +func canonicalQuery(u *url.URL) string { + if u == nil || u.RawQuery == "" { + return "" + } + values, _ := url.ParseQuery(u.RawQuery) + return values.Encode() +} + +func canonicalHeaders(req *http.Request, signedHeaders []string) string { + lines := make([]string, 0, len(signedHeaders)) + for _, key := range signedHeaders { + var value string + if key == "host" { + value = req.URL.Host + } else { + value = req.Header.Get(key) + } + lines = append(lines, key+":"+normalizeHeaderValue(value)+"\n") + } + return strings.Join(lines, "") +} + +func normalizeHeaderValue(value string) string { + return strings.Join(strings.Fields(strings.TrimSpace(value)), " ") +} + +func volcengineSigningKey(secret, date, region, service string) []byte { + kDate := hmacSHA256([]byte(secret), []byte(date)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + return hmacSHA256(kService, []byte("request")) +} + +func hmacSHA256(key, data []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write(data) + return mac.Sum(nil) +} + +func hmacSHA256Hex(key, data []byte) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func randomNonce() string { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("%d", time.Now().UnixNano()) + } + return hex.EncodeToString(buf) +} + +func compactBody(data []byte) string { + text := strings.TrimSpace(string(data)) + if text == "" { + return "" + } + if len(text) > 800 { + return text[:800] + "..." + } + return text +} diff --git a/internal/infrastructure/ocr/client_test.go b/internal/infrastructure/ocr/client_test.go new file mode 100644 index 0000000..14c7307 --- /dev/null +++ b/internal/infrastructure/ocr/client_test.go @@ -0,0 +1,150 @@ +package ocr + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/mmmy/snapgo/internal/domain" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestAliyunRecognizeTextSignsRawPNGRequest(t *testing.T) { + var requestBody []byte + transport := roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "ocr-api.cn-hangzhou.aliyuncs.com" { + t.Fatalf("unexpected host %s", r.URL.Host) + } + if got := r.Header.Get("x-acs-action"); got != "RecognizeGeneral" { + t.Fatalf("unexpected action %q", got) + } + if got := r.Header.Get("x-acs-version"); got != "2021-07-07" { + t.Fatalf("unexpected version %q", got) + } + if got := r.Header.Get("x-acs-date"); got != "2026-07-08T01:02:03Z" { + t.Fatalf("unexpected date %q", got) + } + if got := r.Header.Get("Authorization"); !strings.Contains(got, "ACS3-HMAC-SHA256 Credential=ak") { + t.Fatalf("unexpected auth header %q", got) + } + var err error + requestBody, err = io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewBufferString( + `{"Data":"{\"content\":\"第一行\\n第二行\"}"}`, + )), + }, nil + }) + + client, err := NewClient(domain.OCRProviderAliyun, domain.OCRProviderConfig{ + Endpoint: "https://ocr-api.cn-hangzhou.aliyuncs.com", + AccessKeyID: "ak", + AccessKeySecret: "sk", + Action: "RecognizeGeneral", + Version: "2021-07-07", + }) + if err != nil { + t.Fatalf("new client: %v", err) + } + client.httpClient.Transport = transport + client.now = func() time.Time { return time.Date(2026, 7, 8, 1, 2, 3, 0, time.UTC) } + client.nonce = func() string { return "nonce" } + + text, err := client.RecognizeText(context.Background(), []byte("png")) + if err != nil { + t.Fatalf("recognize text: %v", err) + } + if string(requestBody) != "png" { + t.Fatalf("expected raw png body, got %q", string(requestBody)) + } + if text != "第一行\n第二行" { + t.Fatalf("unexpected OCR text %q", text) + } +} + +func TestVolcengineRecognizeTextSignsBase64JSONRequest(t *testing.T) { + var requestBody map[string]string + transport := roundTripFunc(func(r *http.Request) (*http.Response, error) { + if got := r.URL.Query().Get("Action"); got != "OCRNormal" { + t.Fatalf("unexpected action %q", got) + } + if got := r.URL.Query().Get("Version"); got != "2020-08-26" { + t.Fatalf("unexpected version %q", got) + } + if got := r.Header.Get("X-Date"); got != "20260708T010203Z" { + t.Fatalf("unexpected x-date %q", got) + } + if got := r.Header.Get("Authorization"); !strings.Contains(got, "HMAC-SHA256 Credential=ak/20260708/cn-north-1/cv/request") { + t.Fatalf("unexpected auth header %q", got) + } + if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil { + t.Fatalf("decode request: %v", err) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewBufferString( + `{"ResponseMetadata":{"RequestId":"r"},"Result":{"LineTexts":["你好","世界"]}}`, + )), + }, nil + }) + + client, err := NewClient(domain.OCRProviderVolcengine, domain.OCRProviderConfig{ + Endpoint: "https://visual.volcengineapi.com", + AccessKeyID: "ak", + AccessKeySecret: "sk", + Region: "cn-north-1", + Service: "cv", + Action: "OCRNormal", + Version: "2020-08-26", + }) + if err != nil { + t.Fatalf("new client: %v", err) + } + client.httpClient.Transport = transport + client.now = func() time.Time { return time.Date(2026, 7, 8, 1, 2, 3, 0, time.UTC) } + + text, err := client.RecognizeText(context.Background(), []byte("png")) + if err != nil { + t.Fatalf("recognize text: %v", err) + } + if requestBody["image_base64"] != base64.StdEncoding.EncodeToString([]byte("png")) { + t.Fatalf("expected base64 image body, got %#v", requestBody) + } + if text != "你好\n世界" { + t.Fatalf("unexpected OCR text %q", text) + } +} + +func TestParseOCRTextSupportsWordsResult(t *testing.T) { + text, err := parseOCRText([]byte(`{"data":{"words_result":[{"words":"foo"},{"words":"bar"}]}}`)) + if err != nil { + t.Fatalf("parse ocr text: %v", err) + } + if text != "foo\nbar" { + t.Fatalf("unexpected OCR text %q", text) + } +} + +func TestParseOCRTextReturnsProviderError(t *testing.T) { + _, err := parseOCRText([]byte(`{"ResponseMetadata":{"Error":{"Code":"BadRequest","Message":"bad image"}}}`)) + if err == nil || !strings.Contains(err.Error(), "bad image") { + t.Fatalf("expected provider error, got %v", err) + } +} diff --git a/native_overlay_callbacks_darwin.go b/native_overlay_callbacks_darwin.go index d9ef0f7..d3048ab 100644 --- a/native_overlay_callbacks_darwin.go +++ b/native_overlay_callbacks_darwin.go @@ -86,6 +86,18 @@ func nativeOverlaySummarize(x, y, w, h C.int, annotationsJSON *C.char) { }() } +//export nativeOverlayOCR +func nativeOverlayOCR(x, y, w, h C.int, annotationsJSON *C.char) { + app := consumeNativeOverlayApp() + if app == nil { + return + } + result := nativeCaptureResult(x, y, w, h, annotationsJSON) + go func() { + _ = app.ExtractTextNativeRegion(result) + }() +} + //export nativeOverlayCancel func nativeOverlayCancel() { app := consumeNativeOverlayApp() diff --git a/native_overlay_darwin.go b/native_overlay_darwin.go index f5beaf2..29e81f1 100644 --- a/native_overlay_darwin.go +++ b/native_overlay_darwin.go @@ -16,6 +16,7 @@ extern void nativeOverlayCopy(int x, int y, int w, int h, const char *annotation extern void nativeOverlaySave(int x, int y, int w, int h, const char *annotationsJSON, const char *dir); extern void nativeOverlaySaveRemote(int x, int y, int w, int h, const char *annotationsJSON); extern void nativeOverlaySummarize(int x, int y, int w, int h, const char *annotationsJSON); +extern void nativeOverlayOCR(int x, int y, int w, int h, const char *annotationsJSON); extern void nativeOverlayCancel(void); static NSWindow *nativeOverlayWindow = nil; @@ -103,6 +104,7 @@ static id nativeOverlayKeyMonitor = nil; @property(strong) NSButton *clipboardButton; @property(strong) NSButton *saveButton; @property(strong) NSButton *saveRemoteButton; +@property(strong) NSButton *ocrButton; @property(strong) NSButton *summaryButton; @property(strong) NSButton *uploadButton; @property(strong) NSButton *penButton; @@ -124,6 +126,7 @@ static id nativeOverlayKeyMonitor = nil; - (void)copySelection; - (void)saveSelection; - (void)saveRemoteSelection; +- (void)ocrSelection; - (void)summarizeSelection; - (void)cancelSelection; - (void)beginTextAnnotationAtPoint:(NSPoint)localPoint; @@ -146,13 +149,14 @@ static id nativeOverlayKeyMonitor = nil; _activeColor = [NSColor colorWithCalibratedRed:239.0/255.0 green:68.0/255.0 blue:68.0/255.0 alpha:1.0]; _annotations = [NSMutableArray array]; - // Use SnipHoverButton for the 5 action buttons so we get hover-color + // Use SnipHoverButton for the action buttons so we get hover-color // animation. Annotation buttons stay as plain NSButton because they // already have explicit on/off "active" styling. _cancelButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(cancelSelection)]; _clipboardButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(copySelection)]; _saveButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveSelection)]; _saveRemoteButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveRemoteSelection)]; + _ocrButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(ocrSelection)]; _summaryButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(summarizeSelection)]; _uploadButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(confirmSelection)]; // Tooltip strings shown on hover for each action button. @@ -161,6 +165,7 @@ static id nativeOverlayKeyMonitor = nil; [_clipboardButton setToolTip:@"复制图片"]; [_saveButton setToolTip:@"保存本地"]; [_saveRemoteButton setToolTip:@"保存远端"]; + [_ocrButton setToolTip:@"提取文字"]; [_summaryButton setToolTip:@"复制总结"]; [_uploadButton setToolTip:@"上传云端"]; _penButton = [NSButton buttonWithTitle:@"" target:self action:@selector(selectPen)]; @@ -183,10 +188,10 @@ static id nativeOverlayKeyMonitor = nil; // Add the action toolbar background BEFORE the buttons so it sits // behind them in the view hierarchy (AppKit z-order = subview order). [self addSubview:_actionToolbarBg]; - for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) { + for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _ocrButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) { [self addSubview:view]; } - for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) { + for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _ocrButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _textButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) { [view setHidden:YES]; } [_sizeLabel setHidden:YES]; @@ -249,8 +254,8 @@ static id nativeOverlayKeyMonitor = nil; } - (void)styleControls { - // Cancel / Copy / Save / Save-remote / Upload — square icon buttons - // rendered from the user-supplied SVG assets. All five default to white + // Cancel / Copy / Save / Save-remote / OCR / Summary / Upload — square icon buttons + // rendered from the user-supplied SVG assets. All actions default to white // and animate to a per-action accent color on hover (cancel = red, // others = blue). The base "white default" replaces the previous blue // upload tint so the toolbar reads as a uniform icon row. @@ -260,6 +265,7 @@ static id nativeOverlayKeyMonitor = nil; // save-remote.svg — provided by the user; its content is the same arrow // icon as the previous upload, so we reuse it verbatim here. NSImage *saveRemoteIcon = [self iconFromSVG:@""]; + NSImage *ocrIcon = [self iconFromSVG:@""]; NSImage *summaryIcon = [self iconFromSVG:@""]; // upload.svg — newly replaced "send/cloud" icon supplied by the user. NSImage *uploadIcon = [self iconFromSVG:@""]; @@ -268,10 +274,11 @@ static id nativeOverlayKeyMonitor = nil; [self styleIconButton:_clipboardButton image:copyIcon]; [self styleIconButton:_saveButton image:saveIcon]; [self styleIconButton:_saveRemoteButton image:saveRemoteIcon]; + [self styleIconButton:_ocrButton image:ocrIcon]; [self styleIconButton:_summaryButton image:summaryIcon]; [self styleIconButton:_uploadButton image:uploadIcon]; - // Configure hover colors for the 5 action buttons. White is the shared + // Configure hover colors for the action buttons. White is the shared // base; cancel turns red on hover and the rest turn blue, providing a // glance-able cue for destructive vs. constructive actions. NSColor *baseWhite = [NSColor whiteColor]; @@ -281,12 +288,14 @@ static id nativeOverlayKeyMonitor = nil; SnipHoverButton *clipboardHB = (SnipHoverButton *)_clipboardButton; SnipHoverButton *saveHB = (SnipHoverButton *)_saveButton; SnipHoverButton *saveRemoteHB = (SnipHoverButton *)_saveRemoteButton; + SnipHoverButton *ocrHB = (SnipHoverButton *)_ocrButton; SnipHoverButton *summaryHB = (SnipHoverButton *)_summaryButton; SnipHoverButton *uploadHB = (SnipHoverButton *)_uploadButton; cancelHB.baseColor = baseWhite; cancelHB.hoverColor = hoverRed; clipboardHB.baseColor = baseWhite; clipboardHB.hoverColor = hoverBlue; saveHB.baseColor = baseWhite; saveHB.hoverColor = hoverBlue; saveRemoteHB.baseColor = baseWhite; saveRemoteHB.hoverColor = hoverBlue; + ocrHB.baseColor = baseWhite; ocrHB.hoverColor = hoverBlue; summaryHB.baseColor = baseWhite; summaryHB.hoverColor = hoverBlue; uploadHB.baseColor = baseWhite; uploadHB.hoverColor = hoverBlue; @@ -646,6 +655,7 @@ static id nativeOverlayKeyMonitor = nil; [_clipboardButton setHidden:!visible]; [_saveButton setHidden:!visible]; [_saveRemoteButton setHidden:!visible]; + [_ocrButton setHidden:!visible]; [_summaryButton setHidden:!visible]; [_uploadButton setHidden:!visible]; [_actionToolbarBg setHidden:!visible]; @@ -665,12 +675,12 @@ static id nativeOverlayKeyMonitor = nil; [_sizeLabel setStringValue:[NSString stringWithFormat:@"%.0f × %.0f", _selection.size.width, _selection.size.height]]; [_sizeLabel setFrame:NSMakeRect(_selection.origin.x, MAX(0, _selection.origin.y - 26), 110, 22)]; - // Action toolbar layout — 6 icon buttons, each 28x28, separated by 8px, - // with 4px outer padding. Total = 6*28 + 5*8 + 2*4 = 216px wide. + // Action toolbar layout — 7 icon buttons, each 28x28, separated by 8px, + // with 4px outer padding. Total = 7*28 + 6*8 + 2*4 = 252px wide. CGFloat actionBtnSize = 28; CGFloat actionGap = 8; CGFloat actionPad = 4; - NSInteger actionCount = 6; + NSInteger actionCount = 7; CGFloat toolbarW = actionPad * 2 + actionBtnSize * actionCount + actionGap * (actionCount - 1); CGFloat toolbarH = 40; CGFloat x = _selection.origin.x + _selection.size.width - toolbarW; @@ -688,8 +698,9 @@ static id nativeOverlayKeyMonitor = nil; [_clipboardButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 1, btnY, actionBtnSize, actionBtnSize)]; [_saveButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 2, btnY, actionBtnSize, actionBtnSize)]; [_saveRemoteButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 3, btnY, actionBtnSize, actionBtnSize)]; - [_summaryButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)]; - [_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 5, btnY, actionBtnSize, actionBtnSize)]; + [_ocrButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)]; + [_summaryButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 5, btnY, actionBtnSize, actionBtnSize)]; + [_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 6, btnY, actionBtnSize, actionBtnSize)]; // Mark toolbar sits to the LEFT of the action toolbar (same row) with an // 8px gap between the two groups, so they never overlap on tiny selections. @@ -930,6 +941,18 @@ static id nativeOverlayKeyMonitor = nil; nativeOverlaySaveRemote((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]); } +// `ocrSelection` sends the screenshot to the configured OCR provider and +// copies the extracted text. +- (void)ocrSelection { + if (!_hasSelection) { + return; + } + NSRect r = [self globalRectForSelection:_selection]; + [self closeOverlayWindow]; + NSString *json = [self annotationsJSON]; + nativeOverlayOCR((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]); +} + // `summarizeSelection` uploads the screenshot to S3, sends it to the // configured multimodal model, and copies the generated summary. - (void)summarizeSelection {