feat: you can summary screen shot directly
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/mmmy/snapgo/internal/domain"
|
||||
"github.com/mmmy/snapgo/internal/infrastructure/clipboard"
|
||||
)
|
||||
|
||||
// VisionSummarizer describes a multimodal model capable of reading a public
|
||||
// image URL and returning a textual summary.
|
||||
type VisionSummarizer interface {
|
||||
SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error)
|
||||
}
|
||||
|
||||
// CaptureSummaryService wires screenshot bytes -> S3 public URL -> LLM
|
||||
// summary -> clipboard. It intentionally does not notify UI state itself:
|
||||
// the caller emits fine-grained "uploading / recognizing / done" progress.
|
||||
type CaptureSummaryService struct {
|
||||
Provider domain.OSSProvider
|
||||
Summarizer VisionSummarizer
|
||||
Clipboard clipboard.Writer
|
||||
Prompt string
|
||||
PathPrefix string
|
||||
}
|
||||
|
||||
// UploadImage uploads the screenshot and returns the public URL visible to
|
||||
// the multimodal provider.
|
||||
func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte) (*domain.UploadResult, error) {
|
||||
if s.Provider == nil {
|
||||
return nil, fmt.Errorf("s3 is not configured")
|
||||
}
|
||||
if len(pngBytes) == 0 {
|
||||
return nil, fmt.Errorf("empty screenshot")
|
||||
}
|
||||
key := buildObjectKey(s.PathPrefix)
|
||||
start := time.Now()
|
||||
url, err := s.Provider.Upload(ctx, key, pngBytes, "image/png")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3 put object: %w", err)
|
||||
}
|
||||
return &domain.UploadResult{
|
||||
URL: url,
|
||||
Key: key,
|
||||
Provider: s.Provider.Name(),
|
||||
Elapsed: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Summarize asks the configured model to summarize the uploaded image URL.
|
||||
func (s *CaptureSummaryService) Summarize(ctx context.Context, imageURL string) (string, error) {
|
||||
if s.Summarizer == nil {
|
||||
return "", fmt.Errorf("llm is not configured")
|
||||
}
|
||||
return s.Summarizer.SummarizeImage(ctx, s.Prompt, imageURL)
|
||||
}
|
||||
|
||||
// CopySummary writes the final summary text to the clipboard.
|
||||
func (s *CaptureSummaryService) CopySummary(_ context.Context, summary string) error {
|
||||
if summary == "" {
|
||||
return fmt.Errorf("empty summary")
|
||||
}
|
||||
if s.Clipboard == nil {
|
||||
return fmt.Errorf("clipboard is not configured")
|
||||
}
|
||||
if err := s.Clipboard.WriteText(summary); err != nil {
|
||||
return fmt.Errorf("clipboard write failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeOSSProvider struct {
|
||||
key string
|
||||
data []byte
|
||||
contentType string
|
||||
url string
|
||||
}
|
||||
|
||||
func (f *fakeOSSProvider) Upload(_ context.Context, key string, data []byte, contentType string) (string, error) {
|
||||
f.key = key
|
||||
f.data = append([]byte(nil), data...)
|
||||
f.contentType = contentType
|
||||
if f.url == "" {
|
||||
return "https://cdn.example.com/" + key, nil
|
||||
}
|
||||
return f.url, nil
|
||||
}
|
||||
|
||||
func (f *fakeOSSProvider) Name() string { return "fake" }
|
||||
|
||||
type fakeSummarizer struct {
|
||||
prompt string
|
||||
url string
|
||||
text string
|
||||
}
|
||||
|
||||
func (f *fakeSummarizer) SummarizeImage(_ context.Context, prompt string, imageURL string) (string, error) {
|
||||
f.prompt = prompt
|
||||
f.url = imageURL
|
||||
if f.text == "" {
|
||||
return "summary text", nil
|
||||
}
|
||||
return f.text, nil
|
||||
}
|
||||
|
||||
func TestCaptureSummaryServiceUploadsSummarizesAndCopies(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
provider := &fakeOSSProvider{}
|
||||
summarizer := &fakeSummarizer{}
|
||||
clip := &fakeClipboard{}
|
||||
svc := &CaptureSummaryService{
|
||||
Provider: provider,
|
||||
Summarizer: summarizer,
|
||||
Clipboard: clip,
|
||||
Prompt: "describe screenshot",
|
||||
PathPrefix: "snapgo/",
|
||||
}
|
||||
|
||||
uploaded, err := svc.UploadImage(ctx, []byte("png"))
|
||||
if err != nil {
|
||||
t.Fatalf("upload image: %v", err)
|
||||
}
|
||||
if provider.contentType != "image/png" {
|
||||
t.Fatalf("expected image/png content type, got %q", provider.contentType)
|
||||
}
|
||||
if string(provider.data) != "png" {
|
||||
t.Fatalf("expected uploaded png bytes, got %q", string(provider.data))
|
||||
}
|
||||
|
||||
summary, err := svc.Summarize(ctx, uploaded.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize: %v", err)
|
||||
}
|
||||
if summarizer.prompt != "describe screenshot" {
|
||||
t.Fatalf("expected prompt forwarded, got %q", summarizer.prompt)
|
||||
}
|
||||
if summarizer.url != uploaded.URL {
|
||||
t.Fatalf("expected uploaded url forwarded, got %q", summarizer.url)
|
||||
}
|
||||
if summary != "summary text" {
|
||||
t.Fatalf("expected summary text, got %q", summary)
|
||||
}
|
||||
|
||||
if err := svc.CopySummary(ctx, summary); err != nil {
|
||||
t.Fatalf("copy summary: %v", err)
|
||||
}
|
||||
if clip.text != "summary text" {
|
||||
t.Fatalf("expected summary copied, got %q", clip.text)
|
||||
}
|
||||
}
|
||||
+174
-18
@@ -10,12 +10,12 @@ package domain
|
||||
// endpoint, etc.).
|
||||
//
|
||||
// Field design notes:
|
||||
// - PathPrefix supports object name templating so users can group screenshots
|
||||
// by date.
|
||||
// - PublicURLBase is optional to support CDN-fronted buckets; otherwise the
|
||||
// adapter falls back to "{Endpoint}/{Bucket}/{Key}" (path-style).
|
||||
// - UsePathStyle defaults to true because many self-hosted MinIO / R2
|
||||
// deployments do not support virtual-hosted-style addressing.
|
||||
// - PathPrefix supports object name templating so users can group screenshots
|
||||
// by date.
|
||||
// - PublicURLBase is optional to support CDN-fronted buckets; otherwise the
|
||||
// adapter falls back to "{Endpoint}/{Bucket}/{Key}" (path-style).
|
||||
// - UsePathStyle defaults to true because many self-hosted MinIO / R2
|
||||
// deployments do not support virtual-hosted-style addressing.
|
||||
type S3Config struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
@@ -34,17 +34,17 @@ type S3Config struct {
|
||||
// - Host / Port form the destination endpoint. Port defaults to 22 when 0.
|
||||
// - User is the SSH login name.
|
||||
// - AuthMethod selects how the connection authenticates:
|
||||
// "password" → password only, via the in-process Go SSH client.
|
||||
// "key" → ssh-agent + ~/.ssh/id_* key files (no password),
|
||||
// via the in-process Go SSH client.
|
||||
// "kerberos" → delegate to the system /usr/bin/ssh binary so the
|
||||
// existing Kerberos (GSSAPI) credential cache from
|
||||
// `kinit` is reused. Native GSSAPI is required here
|
||||
// because macOS stores tickets in an API: ccache
|
||||
// that pure-Go SSH libraries cannot read.
|
||||
// "" / "builtin" → legacy combined behaviour (password → agent → key
|
||||
// files) kept only for backward compatibility with
|
||||
// configs written before the method was split.
|
||||
// "password" → password only, via the in-process Go SSH client.
|
||||
// "key" → ssh-agent + ~/.ssh/id_* key files (no password),
|
||||
// via the in-process Go SSH client.
|
||||
// "kerberos" → delegate to the system /usr/bin/ssh binary so the
|
||||
// existing Kerberos (GSSAPI) credential cache from
|
||||
// `kinit` is reused. Native GSSAPI is required here
|
||||
// because macOS stores tickets in an API: ccache
|
||||
// that pure-Go SSH libraries cannot read.
|
||||
// "" / "builtin" → legacy combined behaviour (password → agent → key
|
||||
// files) kept only for backward compatibility with
|
||||
// configs written before the method was split.
|
||||
// - Password is optional; when empty the implementation falls back to the
|
||||
// local SSH agent or ~/.ssh/id_* keys (i.e. the user is responsible for
|
||||
// having password-less access already configured on this machine).
|
||||
@@ -72,6 +72,29 @@ type SSHConfig struct {
|
||||
ConnectTimeoutSecs int `json:"connectTimeoutSecs"`
|
||||
}
|
||||
|
||||
// LLMProviderConfig stores one OpenAI-compatible multimodal chat endpoint.
|
||||
//
|
||||
// The three built-in providers (Qwen, Doubao/Ark, OpenAI-compatible) all use
|
||||
// the same request shape: chat/completions with a user content array that
|
||||
// contains text plus an image_url item. Keeping them as provider records lets
|
||||
// users switch between credentials/models without retyping every field.
|
||||
type LLMProviderConfig struct {
|
||||
Label string `json:"label"`
|
||||
BaseURL string `json:"baseUrl"`
|
||||
APIKey string `json:"apiKey"`
|
||||
Model string `json:"model"`
|
||||
MaxTokens int `json:"maxTokens"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
TimeoutSecs int `json:"timeoutSecs"`
|
||||
}
|
||||
|
||||
// LLMConfig controls screenshot summarisation.
|
||||
type LLMConfig struct {
|
||||
ActiveProvider string `json:"activeProvider"`
|
||||
Prompt string `json:"prompt"`
|
||||
Providers map[string]LLMProviderConfig `json:"providers"`
|
||||
}
|
||||
|
||||
// SSH authentication method identifiers stored in SSHConfig.AuthMethod.
|
||||
const (
|
||||
// SSHAuthBuiltin is the legacy combined method (password → agent → key
|
||||
@@ -87,6 +110,22 @@ const (
|
||||
SSHAuthKerberos = "kerberos"
|
||||
)
|
||||
|
||||
// Built-in LLM provider identifiers.
|
||||
const (
|
||||
LLMProviderQwen = "qwen"
|
||||
LLMProviderDoubao = "doubao"
|
||||
LLMProviderOpenAI = "openai"
|
||||
)
|
||||
|
||||
const DefaultSummaryPrompt = `你是一个截图内容总结助手。请阅读截图，并用中文输出一段适合直接粘贴到聊天、工单、Issue 或 PR 中的说明。
|
||||
|
||||
要求：
|
||||
- 先说明截图中最重要的信息和用户可能想表达的意图。
|
||||
- 如果截图包含报错、异常状态、表格、代码、界面控件或关键数字，请准确提取。
|
||||
- 如果截图内容不足以判断，不要编造；直接说明不确定点。
|
||||
- 输出尽量简洁，默认 3 到 6 条要点。
|
||||
- 不要输出 Markdown 标题，不要寒暄。`
|
||||
|
||||
// IsKerberos reports whether this config requests Kerberos/GSSAPI auth.
|
||||
func (c SSHConfig) IsKerberos() bool {
|
||||
return c.AuthMethod == SSHAuthKerberos
|
||||
@@ -108,11 +147,14 @@ type AppConfig struct {
|
||||
// SSH holds the configuration for the optional "save to remote via scp"
|
||||
// destination triggered by the save-remote toolbar button.
|
||||
SSH SSHConfig `json:"ssh"`
|
||||
|
||||
// LLM holds provider settings for the "copy summary" screenshot action.
|
||||
LLM LLMConfig `json:"llm"`
|
||||
}
|
||||
|
||||
// DefaultAppConfig returns sane zero-value defaults used on first launch.
|
||||
func DefaultAppConfig() AppConfig {
|
||||
return AppConfig{
|
||||
cfg := AppConfig{
|
||||
Hotkey: "cmd+shift+a",
|
||||
S3: S3Config{
|
||||
PathPrefix: "snapgo/",
|
||||
@@ -124,6 +166,102 @@ func DefaultAppConfig() AppConfig {
|
||||
ConnectTimeoutSecs: 10,
|
||||
StrictHostKey: false,
|
||||
},
|
||||
LLM: DefaultLLMConfig(),
|
||||
}
|
||||
cfg.Normalize()
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultLLMConfig returns the built-in provider presets. API keys and exact
|
||||
// models remain user-editable because each provider account may expose
|
||||
// different model names / endpoint IDs.
|
||||
func DefaultLLMConfig() LLMConfig {
|
||||
return LLMConfig{
|
||||
ActiveProvider: LLMProviderOpenAI,
|
||||
Prompt: DefaultSummaryPrompt,
|
||||
Providers: map[string]LLMProviderConfig{
|
||||
LLMProviderQwen: {
|
||||
Label: "阿里千问多模态",
|
||||
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
Model: "qwen-vl-plus",
|
||||
MaxTokens: 600,
|
||||
Temperature: 0.2,
|
||||
TimeoutSecs: 60,
|
||||
},
|
||||
LLMProviderDoubao: {
|
||||
Label: "火山方舟豆包多模态",
|
||||
BaseURL: "https://ark.cn-beijing.volces.com/api/v3",
|
||||
Model: "",
|
||||
MaxTokens: 600,
|
||||
Temperature: 0.2,
|
||||
TimeoutSecs: 60,
|
||||
},
|
||||
LLMProviderOpenAI: {
|
||||
Label: "ChatGPT / OpenAI-compatible",
|
||||
BaseURL: "https://api.openai.com/v1",
|
||||
Model: "gpt-5.5",
|
||||
MaxTokens: 600,
|
||||
Temperature: 0.2,
|
||||
TimeoutSecs: 60,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize fills defaults into configs written by older app versions while
|
||||
// preserving any user-provided provider fields.
|
||||
func (c *AppConfig) Normalize() {
|
||||
if c.Hotkey == "" {
|
||||
c.Hotkey = "cmd+shift+a"
|
||||
}
|
||||
if c.S3.PathPrefix == "" {
|
||||
c.S3.PathPrefix = "snapgo/"
|
||||
}
|
||||
if c.SSH.Port == 0 {
|
||||
c.SSH.Port = 22
|
||||
}
|
||||
if c.SSH.PathPrefix == "" {
|
||||
c.SSH.PathPrefix = "snapgo/"
|
||||
}
|
||||
if c.SSH.ConnectTimeoutSecs == 0 {
|
||||
c.SSH.ConnectTimeoutSecs = 10
|
||||
}
|
||||
|
||||
defaultLLM := DefaultLLMConfig()
|
||||
if c.LLM.ActiveProvider == "" {
|
||||
c.LLM.ActiveProvider = defaultLLM.ActiveProvider
|
||||
}
|
||||
if c.LLM.Prompt == "" {
|
||||
c.LLM.Prompt = defaultLLM.Prompt
|
||||
}
|
||||
if c.LLM.Providers == nil {
|
||||
c.LLM.Providers = map[string]LLMProviderConfig{}
|
||||
}
|
||||
for id, def := range defaultLLM.Providers {
|
||||
current, ok := c.LLM.Providers[id]
|
||||
if !ok {
|
||||
c.LLM.Providers[id] = def
|
||||
continue
|
||||
}
|
||||
if current.Label == "" {
|
||||
current.Label = def.Label
|
||||
}
|
||||
if current.BaseURL == "" {
|
||||
current.BaseURL = def.BaseURL
|
||||
}
|
||||
if current.Model == "" && id != LLMProviderDoubao {
|
||||
current.Model = def.Model
|
||||
}
|
||||
if current.MaxTokens == 0 {
|
||||
current.MaxTokens = def.MaxTokens
|
||||
}
|
||||
if current.Temperature < 0 {
|
||||
current.Temperature = def.Temperature
|
||||
}
|
||||
if current.TimeoutSecs == 0 {
|
||||
current.TimeoutSecs = def.TimeoutSecs
|
||||
}
|
||||
c.LLM.Providers[id] = current
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,3 +279,21 @@ func (c AppConfig) IsS3Configured() bool {
|
||||
func (c AppConfig) IsSSHConfigured() bool {
|
||||
return c.SSH.Host != "" && c.SSH.User != ""
|
||||
}
|
||||
|
||||
// ActiveLLMProvider returns the selected provider config plus a boolean
|
||||
// indicating whether the selection exists.
|
||||
func (c AppConfig) ActiveLLMProvider() (string, LLMProviderConfig, bool) {
|
||||
id := c.LLM.ActiveProvider
|
||||
if id == "" {
|
||||
id = LLMProviderOpenAI
|
||||
}
|
||||
provider, ok := c.LLM.Providers[id]
|
||||
return id, provider, ok
|
||||
}
|
||||
|
||||
// IsLLMConfigured reports whether the selected multimodal provider has the
|
||||
// fields required to make a request.
|
||||
func (c AppConfig) IsLLMConfigured() bool {
|
||||
_, provider, ok := c.ActiveLLMProvider()
|
||||
return ok && provider.BaseURL != "" && provider.APIKey != "" && provider.Model != ""
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
// Package config provides JSON-file backed persistence for AppConfig.
|
||||
//
|
||||
// Design rationale:
|
||||
// - JSON keeps the file human-readable, which is useful while the MVP has
|
||||
// no settings UI for every option.
|
||||
// - Storage location follows os.UserConfigDir() so each platform places it
|
||||
// in the canonical spot (macOS: ~/Library/Application Support/SnapGo).
|
||||
// - File mode 0600 / dir mode 0700 mitigate accidental leakage of access
|
||||
// keys before we wire up the OS keychain (deferred to a later spec).
|
||||
// - JSON keeps the file human-readable, which is useful while the MVP has
|
||||
// no settings UI for every option.
|
||||
// - Storage location follows os.UserConfigDir() so each platform places it
|
||||
// in the canonical spot (macOS: ~/Library/Application Support/SnapGo).
|
||||
// - File mode 0600 / dir mode 0700 mitigate accidental leakage of access
|
||||
// keys before we wire up the OS keychain (deferred to a later spec).
|
||||
package config
|
||||
|
||||
import (
|
||||
@@ -65,6 +65,7 @@ func (s *FileStore) Load() (domain.AppConfig, error) {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return domain.DefaultAppConfig(), fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
cfg.Normalize()
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -74,6 +75,7 @@ func (s *FileStore) Save(cfg domain.AppConfig) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
cfg.Normalize()
|
||||
data, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal config: %w", err)
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package llm contains multimodal LLM adapters.
|
||||
//
|
||||
// The built-in providers are intentionally implemented through the
|
||||
// OpenAI-compatible chat/completions shape. Qwen DashScope, Volcengine Ark,
|
||||
// and OpenAI-compatible gateways all accept the same "text + image_url"
|
||||
// message content pattern, so one small HTTP adapter is enough here.
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mmmy/snapgo/internal/domain"
|
||||
)
|
||||
|
||||
// VisionClient calls one configured multimodal chat endpoint.
|
||||
type VisionClient struct {
|
||||
providerID string
|
||||
cfg domain.LLMProviderConfig
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewVisionClient validates cfg and returns a reusable client.
|
||||
func NewVisionClient(providerID string, cfg domain.LLMProviderConfig) (*VisionClient, error) {
|
||||
if strings.TrimSpace(cfg.BaseURL) == "" {
|
||||
return nil, fmt.Errorf("llm base url is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return nil, fmt.Errorf("llm api key is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return nil, fmt.Errorf("llm model is required")
|
||||
}
|
||||
timeout := cfg.TimeoutSecs
|
||||
if timeout <= 0 {
|
||||
timeout = 60
|
||||
}
|
||||
return &VisionClient{
|
||||
providerID: providerID,
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SummarizeImage asks the model to summarise an image available at imageURL.
|
||||
func (c *VisionClient) SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) {
|
||||
prompt = strings.TrimSpace(prompt)
|
||||
if prompt == "" {
|
||||
return "", fmt.Errorf("llm prompt is empty")
|
||||
}
|
||||
imageURL = strings.TrimSpace(imageURL)
|
||||
if imageURL == "" {
|
||||
return "", fmt.Errorf("image url is empty")
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"model": c.cfg.Model,
|
||||
"messages": []map[string]any{
|
||||
{
|
||||
"role": "user",
|
||||
"content": []map[string]any{
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": map[string]any{"url": imageURL}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if c.cfg.MaxTokens > 0 {
|
||||
if c.providerID == domain.LLMProviderOpenAI {
|
||||
body["max_completion_tokens"] = c.cfg.MaxTokens
|
||||
} else {
|
||||
body["max_tokens"] = c.cfg.MaxTokens
|
||||
}
|
||||
}
|
||||
if c.cfg.Temperature >= 0 {
|
||||
body["temperature"] = c.cfg.Temperature
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal llm request: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionsURL(c.cfg.BaseURL), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build llm request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("llm request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return "", fmt.Errorf("llm status %d: %s", resp.StatusCode, compactBody(data))
|
||||
}
|
||||
|
||||
var decoded chatCompletionResponse
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
return "", fmt.Errorf("decode llm response: %w", err)
|
||||
}
|
||||
if decoded.Error != nil && decoded.Error.Message != "" {
|
||||
return "", fmt.Errorf("llm error: %s", decoded.Error.Message)
|
||||
}
|
||||
if len(decoded.Choices) == 0 {
|
||||
return "", fmt.Errorf("llm response has no choices")
|
||||
}
|
||||
text := strings.TrimSpace(contentText(decoded.Choices[0].Message.Content))
|
||||
if text == "" {
|
||||
return "", fmt.Errorf("llm response is empty")
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func chatCompletionsURL(baseURL string) string {
|
||||
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
if strings.HasSuffix(base, "/chat/completions") {
|
||||
return base
|
||||
}
|
||||
return base + "/chat/completions"
|
||||
}
|
||||
|
||||
type chatCompletionResponse struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content any `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code any `json:"code"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
func contentText(content any) string {
|
||||
switch v := content.(type) {
|
||||
case string:
|
||||
return v
|
||||
case []any:
|
||||
parts := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
obj, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if text, ok := obj["text"].(string); ok {
|
||||
parts = append(parts, text)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func compactBody(data []byte) string {
|
||||
text := strings.TrimSpace(string(data))
|
||||
if text == "" {
|
||||
return "<empty body>"
|
||||
}
|
||||
if len(text) > 800 {
|
||||
return text[:800] + "..."
|
||||
}
|
||||
return text
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/mmmy/snapgo/internal/domain"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func TestVisionClientSummarizeImageUsesChatCompletionImageURL(t *testing.T) {
|
||||
var requestBody map[string]any
|
||||
transport := roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
t.Fatalf("expected /v1/chat/completions, got %s", r.URL.Path)
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
|
||||
t.Fatalf("unexpected auth header %q", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"choices":[{"message":{"content":"summary ok"}}]}`)),
|
||||
}, nil
|
||||
})
|
||||
|
||||
client, err := NewVisionClient(domain.LLMProviderQwen, domain.LLMProviderConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
APIKey: "test-key",
|
||||
Model: "qwen-vl-plus",
|
||||
MaxTokens: 321,
|
||||
Temperature: 0.2,
|
||||
TimeoutSecs: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
client.httpClient.Transport = transport
|
||||
|
||||
text, err := client.SummarizeImage(context.Background(), "describe", "https://cdn.example.com/a.png")
|
||||
if err != nil {
|
||||
t.Fatalf("summarize image: %v", err)
|
||||
}
|
||||
if text != "summary ok" {
|
||||
t.Fatalf("expected response text, got %q", text)
|
||||
}
|
||||
if requestBody["model"] != "qwen-vl-plus" {
|
||||
t.Fatalf("expected model forwarded, got %#v", requestBody["model"])
|
||||
}
|
||||
if requestBody["max_tokens"] != float64(321) {
|
||||
t.Fatalf("expected max_tokens forwarded, got %#v", requestBody["max_tokens"])
|
||||
}
|
||||
|
||||
messages := requestBody["messages"].([]any)
|
||||
content := messages[0].(map[string]any)["content"].([]any)
|
||||
if content[0].(map[string]any)["text"] != "describe" {
|
||||
t.Fatalf("expected prompt content, got %#v", content[0])
|
||||
}
|
||||
image := content[1].(map[string]any)
|
||||
if image["type"] != "image_url" {
|
||||
t.Fatalf("expected image_url item, got %#v", image)
|
||||
}
|
||||
url := image["image_url"].(map[string]any)["url"]
|
||||
if url != "https://cdn.example.com/a.png" {
|
||||
t.Fatalf("expected image url forwarded, got %#v", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsURLAcceptsFullEndpoint(t *testing.T) {
|
||||
full := "https://example.com/v1/chat/completions"
|
||||
if got := chatCompletionsURL(full); got != full {
|
||||
t.Fatalf("expected full endpoint unchanged, got %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user