feat: prefer base64 inline image for summary, fall back to S3

Send the screenshot to the multimodal model inline as a base64 data URL
so summarisation no longer requires S3 to be configured or publicly
reachable. Only when the PNG exceeds the provider MaxInlineBytes limit
do we upload to S3, verify the public URL is reachable, and delete the
temporary object once done. Oversized images without S3 now surface a
clear hint instead of a hard S3 requirement.
This commit is contained in:
2026-07-07 13:39:23 +08:00
parent 7c53c0f184
commit 87874e64cf
6 changed files with 258 additions and 47 deletions
+36 -18
View File
@@ -86,8 +86,20 @@ type LLMProviderConfig struct {
MaxTokens int `json:"maxTokens"`
Temperature float64 `json:"temperature"`
TimeoutSecs int `json:"timeoutSecs"`
// MaxInlineBytes caps the screenshot size (in bytes of the PNG payload)
// that may be sent inline as a base64 data URL. Above this threshold the
// summary pipeline falls back to uploading the image to S3 and passing a
// public URL instead, because most providers reject oversized inline
// images. A value <= 0 means "use the built-in default".
MaxInlineBytes int `json:"maxInlineBytes"`
}
// DefaultMaxInlineBytes is the fallback inline-image cap (~4 MiB) used when a
// provider config does not specify MaxInlineBytes. It is a conservative bound
// that keeps a single base64 data URL within the request-size limits accepted
// by the common OpenAI-compatible multimodal endpoints.
const DefaultMaxInlineBytes = 4 << 20
// LLMConfig controls screenshot summarisation.
type LLMConfig struct {
ActiveProvider string `json:"activeProvider"`
@@ -192,28 +204,31 @@ func DefaultLLMConfig() LLMConfig {
Prompt: DefaultSummaryPrompt,
Providers: map[string]LLMProviderConfig{
LLMProviderQwen: {
Label: "阿里千问多模态",
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
Model: "qwen-vl-plus",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
Label: "阿里千问多模态",
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
Model: "qwen-vl-plus",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
MaxInlineBytes: DefaultMaxInlineBytes,
},
LLMProviderDoubao: {
Label: "火山方舟豆包多模态",
BaseURL: "https://ark.cn-beijing.volces.com/api/v3",
Model: "",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
Label: "火山方舟豆包多模态",
BaseURL: "https://ark.cn-beijing.volces.com/api/v3",
Model: "",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
MaxInlineBytes: DefaultMaxInlineBytes,
},
LLMProviderOpenAI: {
Label: "ChatGPT / OpenAI-compatible",
BaseURL: "https://api.openai.com/v1",
Model: "gpt-5.5",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
Label: "ChatGPT / OpenAI-compatible",
BaseURL: "https://api.openai.com/v1",
Model: "gpt-5.5",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
MaxInlineBytes: DefaultMaxInlineBytes,
},
},
}
@@ -275,6 +290,9 @@ func (c *AppConfig) Normalize() {
if current.TimeoutSecs == 0 {
current.TimeoutSecs = def.TimeoutSecs
}
if current.MaxInlineBytes <= 0 {
current.MaxInlineBytes = DefaultMaxInlineBytes
}
c.LLM.Providers[id] = current
}
}