feat: prefer base64 inline image for summary, fall back to S3

Send the screenshot to the multimodal model inline as a base64 data URL
so summarisation no longer requires S3 to be configured or publicly
reachable. Only when the PNG exceeds the provider MaxInlineBytes limit
do we upload to S3, verify the public URL is reachable, and delete the
temporary object once done. Oversized images without S3 now surface a
clear hint instead of a hard S3 requirement.
This commit is contained in:
2026-07-07 13:39:23 +08:00
parent 7c53c0f184
commit 87874e64cf
6 changed files with 258 additions and 47 deletions
+59 -22
View File
@@ -387,12 +387,6 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error {
cfg := a.cfg
a.mu.RUnlock()
if !cfg.IsS3Configured() {
err := fmt.Errorf("请先在 S3 配置页填写可用的对象存储配置")
a.emitOperationStatus("summary", "需要配置 S3", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
if !cfg.IsLLMConfigured() {
err := fmt.Errorf("请先在 LLM 配置页填写 API Key 和模型")
a.emitOperationStatus("summary", "需要配置 LLM", err.Error(), "error")
@@ -400,28 +394,52 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error {
return err
}
providerID, llmCfg, _ := cfg.ActiveLLMProvider()
visionClient, err := llmpkg.NewVisionClient(providerID, llmCfg)
if err != nil {
a.emitOperationStatus("summary", "LLM 配置错误", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
svc := &application.CaptureSummaryService{
Summarizer: visionClient,
Clipboard: a.clip,
Prompt: cfg.LLM.Prompt,
PathPrefix: cfg.S3.PathPrefix,
MaxInlineBytes: llmCfg.MaxInlineBytes,
}
// Preferred path: send the screenshot inline as a base64 data URL so the
// summary never depends on S3 being configured or publicly reachable.
if svc.CanInline(pngBytes) {
a.emitOperationStatus("summary", "识别中", "正在请求多模态模型(本地内联图片)", "running")
summary, err := svc.Summarize(a.ctx, svc.InlineDataURL(pngBytes))
if err != nil {
a.emitOperationStatus("summary", "识别失败", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
return a.finishSummary(svc, summary)
}
// Oversized image: fall back to uploading to S3 and passing a public URL.
if !cfg.IsS3Configured() {
err := fmt.Errorf("截图尺寸超过内联上限(%d 字节),请缩小截图区域,或在 S3 配置页填写公网可达的对象存储后重试", llmCfg.MaxInlineBytes)
a.emitOperationStatus("summary", "尺寸超限", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
s3Provider, err := oss.NewS3Provider(cfg.S3)
if err != nil {
a.emitOperationStatus("summary", "S3 配置错误", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
providerID, llmCfg, _ := cfg.ActiveLLMProvider()
visionClient, err := llmpkg.NewVisionClient(providerID, llmCfg)
if err != nil {
a.emitOperationStatus("summary", "LLM 配置错误", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
svc := &application.CaptureSummaryService{
Provider: s3Provider,
Summarizer: visionClient,
Clipboard: a.clip,
Prompt: cfg.LLM.Prompt,
PathPrefix: cfg.S3.PathPrefix,
}
svc.Provider = s3Provider
a.emitOperationStatus("summary", "上传中", "正在上传截图供模型读取", "running")
a.emitOperationStatus("summary", "上传中", "截图较大,正在上传供模型读取", "running")
uploaded, err := svc.UploadImage(a.ctx, pngBytes)
if err != nil {
a.emitOperationStatus("summary", "上传失败", err.Error(), "error")
@@ -429,6 +447,20 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error {
return err
}
// The remote model fetches the URL itself, so verify it is reachable and
// clean up the temporary object once we are done with it either way.
defer func() {
if derr := svc.DeleteUploaded(a.ctx, uploaded.Key); derr != nil {
slog.Warn("summary: failed to delete temporary object", "key", uploaded.Key, "err", derr)
}
}()
if err := svc.EnsureReachable(a.ctx, uploaded.URL); err != nil {
a.emitOperationStatus("summary", "链接不可达", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
a.emitOperationStatus("summary", "识别中", "图片已上传,正在请求多模态模型", "running")
summary, err := svc.Summarize(a.ctx, uploaded.URL)
if err != nil {
@@ -436,12 +468,17 @@ func (a *App) runSummaryPipeline(pngBytes []byte) error {
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
return a.finishSummary(svc, summary)
}
// finishSummary copies the summary to the clipboard and emits the terminal
// success/failure UI state shared by the inline and S3 paths.
func (a *App) finishSummary(svc *application.CaptureSummaryService, summary string) error {
if err := svc.CopySummary(a.ctx, summary); err != nil {
a.emitOperationStatus("summary", "复制失败", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
a.emitOperationStatus("summary", "识别完成", "总结已复制到剪贴板", "success")
wruntime.EventsEmit(a.ctx, "upload:success", "summary copied to clipboard")
return nil