diff --git a/app.go b/app.go index e41075b..3fac457 100644 --- a/app.go +++ b/app.go @@ -387,12 +387,6 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error { cfg := a.cfg a.mu.RUnlock() - if !cfg.IsS3Configured() { - err := fmt.Errorf("请先在 S3 配置页填写可用的对象存储配置") - a.emitOperationStatus("summary", "需要配置 S3", err.Error(), "error") - wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) - return err - } if !cfg.IsLLMConfigured() { err := fmt.Errorf("请先在 LLM 配置页填写 API Key 和模型") a.emitOperationStatus("summary", "需要配置 LLM", err.Error(), "error") @@ -400,28 +394,52 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error { return err } + providerID, llmCfg, _ := cfg.ActiveLLMProvider() + visionClient, err := llmpkg.NewVisionClient(providerID, llmCfg) + if err != nil { + a.emitOperationStatus("summary", "LLM 配置错误", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + + svc := &application.CaptureSummaryService{ + Summarizer: visionClient, + Clipboard: a.clip, + Prompt: cfg.LLM.Prompt, + PathPrefix: cfg.S3.PathPrefix, + MaxInlineBytes: llmCfg.MaxInlineBytes, + } + + // Preferred path: send the screenshot inline as a base64 data URL so the + // summary never depends on S3 being configured or publicly reachable. + if svc.CanInline(pngBytes) { + a.emitOperationStatus("summary", "识别中", "正在请求多模态模型（本地内联图片）", "running") + summary, err := svc.Summarize(a.ctx, svc.InlineDataURL(pngBytes)) + if err != nil { + a.emitOperationStatus("summary", "识别失败", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + return a.finishSummary(svc, summary) + } + + // Oversized image: fall back to uploading to S3 and passing a public URL. + if !cfg.IsS3Configured() { + err := fmt.Errorf("截图尺寸超过内联上限（%d 字节），请缩小截图区域，或在 S3 配置页填写公网可达的对象存储后重试", llmCfg.MaxInlineBytes) + a.emitOperationStatus("summary", "尺寸超限", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + s3Provider, err := oss.NewS3Provider(cfg.S3) if err != nil { a.emitOperationStatus("summary", "S3 配置错误", err.Error(), "error") wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) return err } - providerID, llmCfg, _ := cfg.ActiveLLMProvider() - visionClient, err := llmpkg.NewVisionClient(providerID, llmCfg) - if err != nil { - a.emitOperationStatus("summary", "LLM 配置错误", err.Error(), "error") - wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) - return err - } - svc := &application.CaptureSummaryService{ - Provider: s3Provider, - Summarizer: visionClient, - Clipboard: a.clip, - Prompt: cfg.LLM.Prompt, - PathPrefix: cfg.S3.PathPrefix, - } + svc.Provider = s3Provider - a.emitOperationStatus("summary", "上传中", "正在上传截图供模型读取", "running") + a.emitOperationStatus("summary", "上传中", "截图较大，正在上传供模型读取", "running") uploaded, err := svc.UploadImage(a.ctx, pngBytes) if err != nil { a.emitOperationStatus("summary", "上传失败", err.Error(), "error") @@ -429,6 +447,20 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error { return err } + // The remote model fetches the URL itself, so verify it is reachable and + // clean up the temporary object once we are done with it either way. + defer func() { + if derr := svc.DeleteUploaded(a.ctx, uploaded.Key); derr != nil { + slog.Warn("summary: failed to delete temporary object", "key", uploaded.Key, "err", derr) + } + }() + + if err := svc.EnsureReachable(a.ctx, uploaded.URL); err != nil { + a.emitOperationStatus("summary", "链接不可达", err.Error(), "error") + wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) + return err + } + a.emitOperationStatus("summary", "识别中", "图片已上传，正在请求多模态模型", "running") summary, err := svc.Summarize(a.ctx, uploaded.URL) if err != nil { @@ -436,12 +468,17 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error { wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) return err } + return a.finishSummary(svc, summary) +} + +// finishSummary copies the summary to the clipboard and emits the terminal +// success/failure UI state shared by the inline and S3 paths. +func (a *App) finishSummary(svc *application.CaptureSummaryService, summary string) error { if err := svc.CopySummary(a.ctx, summary); err != nil { a.emitOperationStatus("summary", "复制失败", err.Error(), "error") wruntime.EventsEmit(a.ctx, "upload:failure", err.Error()) return err } - a.emitOperationStatus("summary", "识别完成", "总结已复制到剪贴板", "success") wruntime.EventsEmit(a.ctx, "upload:success", "summary copied to clipboard") return nil diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 338f2d9..c124d7c 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -61,6 +61,7 @@ export namespace domain { maxTokens: number; temperature: number; timeoutSecs: number; + maxInlineBytes: number; static createFrom(source: any = {}) { return new LLMProviderConfig(source); @@ -75,6 +76,7 @@ export namespace domain { this.maxTokens = source["maxTokens"]; this.temperature = source["temperature"]; this.timeoutSecs = source["timeoutSecs"]; + this.maxInlineBytes = source["maxInlineBytes"]; } } export class LLMConfig { diff --git a/internal/application/capture_summary.go b/internal/application/capture_summary.go index e360557..5b71a9f 100644 --- a/internal/application/capture_summary.go +++ b/internal/application/capture_summary.go @@ -2,32 +2,65 @@ package application import ( "context" + "encoding/base64" "fmt" + "net/http" "time" "github.com/mmmy/snapgo/internal/domain" "github.com/mmmy/snapgo/internal/infrastructure/clipboard" ) -// VisionSummarizer describes a multimodal model capable of reading a public -// image URL and returning a textual summary. +// VisionSummarizer describes a multimodal model capable of reading an image +// reference (either a public https URL or an inline base64 data URL) and +// returning a textual summary. type VisionSummarizer interface { SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) } -// CaptureSummaryService wires screenshot bytes -> S3 public URL -> LLM -// summary -> clipboard. It intentionally does not notify UI state itself: -// the caller emits fine-grained "uploading / recognizing / done" progress. +// ObjectDeleter is an optional capability an OSSProvider may implement so the +// summary pipeline can delete the temporary screenshot it uploaded purely so a +// remote model could fetch it. It is kept separate from domain.OSSProvider to +// avoid forcing every provider (or test fake) to implement deletion. +type ObjectDeleter interface { + Delete(ctx context.Context, key string) error +} + +// CaptureSummaryService wires screenshot bytes -> multimodal LLM summary -> +// clipboard. It prefers sending the image inline as a base64 data URL and only +// falls back to an S3 upload (public URL) when the payload exceeds the +// provider's inline size limit. It intentionally does not notify UI state +// itself: the caller emits fine-grained "uploading / recognizing / done" +// progress. type CaptureSummaryService struct { Provider domain.OSSProvider Summarizer VisionSummarizer Clipboard clipboard.Writer Prompt string PathPrefix string + // MaxInlineBytes caps the PNG size eligible for inline base64. Payloads + // larger than this must go through S3. A value <= 0 disables the inline + // path entirely (always upload). + MaxInlineBytes int + // HTTPClient is used for the public-URL reachability probe. Defaults to a + // short-timeout client when nil. + HTTPClient *http.Client +} + +// CanInline reports whether the screenshot is small enough to be embedded +// directly in the request as a base64 data URL. +func (s *CaptureSummaryService) CanInline(pngBytes []byte) bool { + return s.MaxInlineBytes > 0 && len(pngBytes) <= s.MaxInlineBytes +} + +// InlineDataURL encodes the PNG as an RFC 2397 data URL suitable for the +// image_url field of an OpenAI-compatible request. +func (s *CaptureSummaryService) InlineDataURL(pngBytes []byte) string { + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngBytes) } // UploadImage uploads the screenshot and returns the public URL visible to -// the multimodal provider. +// the multimodal provider. Used only when the image is too large to inline. func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte) (*domain.UploadResult, error) { if s.Provider == nil { return nil, fmt.Errorf("s3 is not configured") @@ -49,7 +82,60 @@ func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte }, nil } -// Summarize asks the configured model to summarize the uploaded image URL. +// EnsureReachable verifies the public URL can actually be fetched, so we can +// surface a clear "link not reachable" message instead of an opaque provider +// error when the bucket is private or fronted by an inaccessible CDN. +func (s *CaptureSummaryService) EnsureReachable(ctx context.Context, imageURL string) error { + client := s.HTTPClient + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + probe := func(method string) (int, error) { + req, err := http.NewRequestWithContext(ctx, method, imageURL, nil) + if err != nil { + return 0, err + } + resp, err := client.Do(req) + if err != nil { + return 0, err + } + resp.Body.Close() + return resp.StatusCode, nil + } + status, err := probe(http.MethodHead) + // Some S3-compatible endpoints reject HEAD; retry with GET before failing. + if err != nil || status == http.StatusMethodNotAllowed || status == http.StatusForbidden { + if s, gerr := probe(http.MethodGet); gerr == nil { + status, err = s, nil + } else if err == nil { + err = gerr + } + } + if err != nil { + return fmt.Errorf("图片链接不可达（模型无法读取上传的截图）：%w", err) + } + if status < 200 || status >= 300 { + return fmt.Errorf("图片链接不可达：HTTP %d，请确认对象存储 bucket 公网可读或已正确配置 PublicURLBase", status) + } + return nil +} + +// DeleteUploaded removes the temporary object from S3 if the provider supports +// deletion. It is best-effort: a nil return means either success or that the +// provider cannot delete. +func (s *CaptureSummaryService) DeleteUploaded(ctx context.Context, key string) error { + if key == "" { + return nil + } + deleter, ok := s.Provider.(ObjectDeleter) + if !ok { + return nil + } + return deleter.Delete(ctx, key) +} + +// Summarize asks the configured model to summarize the image reference, which +// may be a public URL or an inline base64 data URL. func (s *CaptureSummaryService) Summarize(ctx context.Context, imageURL string) (string, error) { if s.Summarizer == nil { return "", fmt.Errorf("llm is not configured") diff --git a/internal/application/capture_summary_test.go b/internal/application/capture_summary_test.go index a808bd3..7c69e45 100644 --- a/internal/application/capture_summary_test.go +++ b/internal/application/capture_summary_test.go @@ -2,6 +2,7 @@ package application import ( "context" + "strings" "testing" ) @@ -10,6 +11,7 @@ type fakeOSSProvider struct { data []byte contentType string url string + deletedKey string } func (f *fakeOSSProvider) Upload(_ context.Context, key string, data []byte, contentType string) (string, error) { @@ -24,6 +26,11 @@ func (f *fakeOSSProvider) Upload(_ context.Context, key string, data []byte, con func (f *fakeOSSProvider) Name() string { return "fake" } +func (f *fakeOSSProvider) Delete(_ context.Context, key string) error { + f.deletedKey = key + return nil +} + type fakeSummarizer struct { prompt string url string @@ -84,3 +91,50 @@ func TestCaptureSummaryServiceUploadsSummarizesAndCopies(t *testing.T) { t.Fatalf("expected summary copied, got %q", clip.text) } } + +func TestCaptureSummaryServiceInlineDataURL(t *testing.T) { + svc := &CaptureSummaryService{MaxInlineBytes: 10} + + if !svc.CanInline([]byte("small")) { + t.Fatalf("expected 5-byte payload to be inlinable under 10-byte cap") + } + if svc.CanInline([]byte("this is definitely too big")) { + t.Fatalf("expected oversized payload to be rejected for inlining") + } + + url := svc.InlineDataURL([]byte("png")) + if !strings.HasPrefix(url, "data:image/png;base64,") { + t.Fatalf("expected base64 data URL prefix, got %q", url) + } + if strings.Contains(url, "https://") { + t.Fatalf("inline URL must not be a remote URL, got %q", url) + } +} + +func TestCaptureSummaryServiceDeleteUploadedUsesDeleter(t *testing.T) { + provider := &fakeOSSProvider{} + svc := &CaptureSummaryService{Provider: provider} + + if err := svc.DeleteUploaded(context.Background(), "snapgo/a.png"); err != nil { + t.Fatalf("delete uploaded: %v", err) + } + if provider.deletedKey != "snapgo/a.png" { + t.Fatalf("expected object deleted, got %q", provider.deletedKey) + } +} + +func TestCaptureSummaryServiceDeleteUploadedNoDeleterIsNoop(t *testing.T) { + // A provider without Delete must not error when cleanup is requested. + svc := &CaptureSummaryService{Provider: nonDeletingProvider{}} + if err := svc.DeleteUploaded(context.Background(), "snapgo/a.png"); err != nil { + t.Fatalf("expected no-op delete, got %v", err) + } +} + +type nonDeletingProvider struct{} + +func (nonDeletingProvider) Upload(_ context.Context, _ string, _ []byte, _ string) (string, error) { + return "https://cdn.example.com/x.png", nil +} + +func (nonDeletingProvider) Name() string { return "no-delete" } diff --git a/internal/domain/config.go b/internal/domain/config.go index da36bca..30f8482 100644 --- a/internal/domain/config.go +++ b/internal/domain/config.go @@ -86,8 +86,20 @@ type LLMProviderConfig struct { MaxTokens int `json:"maxTokens"` Temperature float64 `json:"temperature"` TimeoutSecs int `json:"timeoutSecs"` + // MaxInlineBytes caps the screenshot size (in bytes of the PNG payload) + // that may be sent inline as a base64 data URL. Above this threshold the + // summary pipeline falls back to uploading the image to S3 and passing a + // public URL instead, because most providers reject oversized inline + // images. A value <= 0 means "use the built-in default". + MaxInlineBytes int `json:"maxInlineBytes"` } +// DefaultMaxInlineBytes is the fallback inline-image cap (~4 MiB) used when a +// provider config does not specify MaxInlineBytes. It is a conservative bound +// that keeps a single base64 data URL within the request-size limits accepted +// by the common OpenAI-compatible multimodal endpoints. +const DefaultMaxInlineBytes = 4 << 20 + // LLMConfig controls screenshot summarisation. type LLMConfig struct { ActiveProvider string `json:"activeProvider"` @@ -192,28 +204,31 @@ func DefaultLLMConfig() LLMConfig { Prompt: DefaultSummaryPrompt, Providers: map[string]LLMProviderConfig{ LLMProviderQwen: { - Label: "阿里千问多模态", - BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", - Model: "qwen-vl-plus", - MaxTokens: 600, - Temperature: 0.2, - TimeoutSecs: 60, + Label: "阿里千问多模态", + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", + Model: "qwen-vl-plus", + MaxTokens: 600, + Temperature: 0.2, + TimeoutSecs: 60, + MaxInlineBytes: DefaultMaxInlineBytes, }, LLMProviderDoubao: { - Label: "火山方舟豆包多模态", - BaseURL: "https://ark.cn-beijing.volces.com/api/v3", - Model: "", - MaxTokens: 600, - Temperature: 0.2, - TimeoutSecs: 60, + Label: "火山方舟豆包多模态", + BaseURL: "https://ark.cn-beijing.volces.com/api/v3", + Model: "", + MaxTokens: 600, + Temperature: 0.2, + TimeoutSecs: 60, + MaxInlineBytes: DefaultMaxInlineBytes, }, LLMProviderOpenAI: { - Label: "ChatGPT / OpenAI-compatible", - BaseURL: "https://api.openai.com/v1", - Model: "gpt-5.5", - MaxTokens: 600, - Temperature: 0.2, - TimeoutSecs: 60, + Label: "ChatGPT / OpenAI-compatible", + BaseURL: "https://api.openai.com/v1", + Model: "gpt-5.5", + MaxTokens: 600, + Temperature: 0.2, + TimeoutSecs: 60, + MaxInlineBytes: DefaultMaxInlineBytes, }, }, } @@ -275,6 +290,9 @@ func (c *AppConfig) Normalize() { if current.TimeoutSecs == 0 { current.TimeoutSecs = def.TimeoutSecs } + if current.MaxInlineBytes <= 0 { + current.MaxInlineBytes = DefaultMaxInlineBytes + } c.LLM.Providers[id] = current } } diff --git a/internal/infrastructure/oss/s3.go b/internal/infrastructure/oss/s3.go index 0109991..028e3bd 100644 --- a/internal/infrastructure/oss/s3.go +++ b/internal/infrastructure/oss/s3.go @@ -101,6 +101,20 @@ func (p *S3Provider) BuildPublicURL(key string) string { return endpoint + "/" + p.cfg.Bucket + "/" + key } +// Delete removes a previously uploaded object by key. +// +// The summary pipeline uses this to clean up the temporary screenshot it had +// to upload only so a remote multimodal model could fetch it via public URL. +func (p *S3Provider) Delete(ctx context.Context, key string) error { + if _, err := p.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: awsv2.String(p.cfg.Bucket), + Key: awsv2.String(key), + }); err != nil { + return fmt.Errorf("s3 delete object: %w", err) + } + return nil +} + // TestConnection performs a small put + delete to verify credentials and // bucket reachability. Used by the "Test connection" button in settings. func (p *S3Provider) TestConnection(ctx context.Context) error {