feat: prefer base64 inline image for summary, fall back to S3

Send the screenshot to the multimodal model inline as a base64 data URL
so summarisation no longer requires S3 to be configured or publicly
reachable. Only when the PNG exceeds the provider MaxInlineBytes limit
do we upload to S3, verify the public URL is reachable, and delete the
temporary object once done. Oversized images without S3 now surface a
clear hint instead of a hard S3 requirement.
This commit is contained in:
2026-07-07 13:39:23 +08:00
parent 7c53c0f184
commit 87874e64cf
6 changed files with 258 additions and 47 deletions
+93 -7
View File
@@ -2,32 +2,65 @@ package application
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"time"
"github.com/mmmy/snapgo/internal/domain"
"github.com/mmmy/snapgo/internal/infrastructure/clipboard"
)
// VisionSummarizer describes a multimodal model capable of reading a public
// image URL and returning a textual summary.
// VisionSummarizer describes a multimodal model capable of reading an image
// reference (either a public https URL or an inline base64 data URL) and
// returning a textual summary.
type VisionSummarizer interface {
SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error)
}
// CaptureSummaryService wires screenshot bytes -> S3 public URL -> LLM
// summary -> clipboard. It intentionally does not notify UI state itself:
// the caller emits fine-grained "uploading / recognizing / done" progress.
// ObjectDeleter is an optional capability an OSSProvider may implement so the
// summary pipeline can delete the temporary screenshot it uploaded purely so a
// remote model could fetch it. It is kept separate from domain.OSSProvider to
// avoid forcing every provider (or test fake) to implement deletion.
type ObjectDeleter interface {
Delete(ctx context.Context, key string) error
}
// CaptureSummaryService wires screenshot bytes -> multimodal LLM summary ->
// clipboard. It prefers sending the image inline as a base64 data URL and only
// falls back to an S3 upload (public URL) when the payload exceeds the
// provider's inline size limit. It intentionally does not notify UI state
// itself: the caller emits fine-grained "uploading / recognizing / done"
// progress.
type CaptureSummaryService struct {
Provider domain.OSSProvider
Summarizer VisionSummarizer
Clipboard clipboard.Writer
Prompt string
PathPrefix string
// MaxInlineBytes caps the PNG size eligible for inline base64. Payloads
// larger than this must go through S3. A value <= 0 disables the inline
// path entirely (always upload).
MaxInlineBytes int
// HTTPClient is used for the public-URL reachability probe. Defaults to a
// short-timeout client when nil.
HTTPClient *http.Client
}
// CanInline reports whether the screenshot is small enough to be embedded
// directly in the request as a base64 data URL.
func (s *CaptureSummaryService) CanInline(pngBytes []byte) bool {
return s.MaxInlineBytes > 0 && len(pngBytes) <= s.MaxInlineBytes
}
// InlineDataURL encodes the PNG as an RFC 2397 data URL suitable for the
// image_url field of an OpenAI-compatible request.
func (s *CaptureSummaryService) InlineDataURL(pngBytes []byte) string {
return "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngBytes)
}
// UploadImage uploads the screenshot and returns the public URL visible to
// the multimodal provider.
// the multimodal provider. Used only when the image is too large to inline.
func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte) (*domain.UploadResult, error) {
if s.Provider == nil {
return nil, fmt.Errorf("s3 is not configured")
@@ -49,7 +82,60 @@ func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte
}, nil
}
// Summarize asks the configured model to summarize the uploaded image URL.
// EnsureReachable verifies the public URL can actually be fetched, so we can
// surface a clear "link not reachable" message instead of an opaque provider
// error when the bucket is private or fronted by an inaccessible CDN.
func (s *CaptureSummaryService) EnsureReachable(ctx context.Context, imageURL string) error {
client := s.HTTPClient
if client == nil {
client = &http.Client{Timeout: 10 * time.Second}
}
probe := func(method string) (int, error) {
req, err := http.NewRequestWithContext(ctx, method, imageURL, nil)
if err != nil {
return 0, err
}
resp, err := client.Do(req)
if err != nil {
return 0, err
}
resp.Body.Close()
return resp.StatusCode, nil
}
status, err := probe(http.MethodHead)
// Some S3-compatible endpoints reject HEAD; retry with GET before failing.
if err != nil || status == http.StatusMethodNotAllowed || status == http.StatusForbidden {
if s, gerr := probe(http.MethodGet); gerr == nil {
status, err = s, nil
} else if err == nil {
err = gerr
}
}
if err != nil {
return fmt.Errorf("图片链接不可达(模型无法读取上传的截图):%w", err)
}
if status < 200 || status >= 300 {
return fmt.Errorf("图片链接不可达:HTTP %d,请确认对象存储 bucket 公网可读或已正确配置 PublicURLBase", status)
}
return nil
}
// DeleteUploaded removes the temporary object from S3 if the provider supports
// deletion. It is best-effort: a nil return means either success or that the
// provider cannot delete.
func (s *CaptureSummaryService) DeleteUploaded(ctx context.Context, key string) error {
if key == "" {
return nil
}
deleter, ok := s.Provider.(ObjectDeleter)
if !ok {
return nil
}
return deleter.Delete(ctx, key)
}
// Summarize asks the configured model to summarize the image reference, which
// may be a public URL or an inline base64 data URL.
func (s *CaptureSummaryService) Summarize(ctx context.Context, imageURL string) (string, error) {
if s.Summarizer == nil {
return "", fmt.Errorf("llm is not configured")