// Package llm contains multimodal LLM adapters. // // The built-in providers are intentionally implemented through the // OpenAI-compatible chat/completions shape. Qwen DashScope, Volcengine Ark, // and OpenAI-compatible gateways all accept the same "text + image_url" // message content pattern, so one small HTTP adapter is enough here. package llm import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "strings" "time" "github.com/mmmy/snapgo/internal/domain" ) // VisionClient calls one configured multimodal chat endpoint. type VisionClient struct { providerID string cfg domain.LLMProviderConfig httpClient *http.Client } // NewVisionClient validates cfg and returns a reusable client. func NewVisionClient(providerID string, cfg domain.LLMProviderConfig) (*VisionClient, error) { if strings.TrimSpace(cfg.BaseURL) == "" { return nil, fmt.Errorf("llm base url is required") } if strings.TrimSpace(cfg.APIKey) == "" { return nil, fmt.Errorf("llm api key is required") } if strings.TrimSpace(cfg.Model) == "" { return nil, fmt.Errorf("llm model is required") } timeout := cfg.TimeoutSecs if timeout <= 0 { timeout = 60 } return &VisionClient{ providerID: providerID, cfg: cfg, httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second}, }, nil } // SummarizeImage asks the model to summarise an image available at imageURL. func (c *VisionClient) SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) { prompt = strings.TrimSpace(prompt) if prompt == "" { return "", fmt.Errorf("llm prompt is empty") } imageURL = strings.TrimSpace(imageURL) if imageURL == "" { return "", fmt.Errorf("image url is empty") } body := map[string]any{ "model": c.cfg.Model, "messages": []map[string]any{ { "role": "user", "content": []map[string]any{ {"type": "text", "text": prompt}, {"type": "image_url", "image_url": map[string]any{"url": imageURL}}, }, }, }, } if c.cfg.MaxTokens > 0 { if c.providerID == domain.LLMProviderOpenAI { body["max_completion_tokens"] = c.cfg.MaxTokens } else { body["max_tokens"] = c.cfg.MaxTokens } } if c.cfg.Temperature >= 0 { body["temperature"] = c.cfg.Temperature } payload, err := json.Marshal(body) if err != nil { return "", fmt.Errorf("marshal llm request: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionsURL(c.cfg.BaseURL), bytes.NewReader(payload)) if err != nil { return "", fmt.Errorf("build llm request: %w", err) } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) resp, err := c.httpClient.Do(req) if err != nil { return "", fmt.Errorf("llm request: %w", err) } defer resp.Body.Close() data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) if resp.StatusCode < 200 || resp.StatusCode >= 300 { return "", fmt.Errorf("llm status %d: %s", resp.StatusCode, compactBody(data)) } var decoded chatCompletionResponse if err := json.Unmarshal(data, &decoded); err != nil { return "", fmt.Errorf("decode llm response: %w", err) } if decoded.Error != nil && decoded.Error.Message != "" { return "", fmt.Errorf("llm error: %s", decoded.Error.Message) } if len(decoded.Choices) == 0 { return "", fmt.Errorf("llm response has no choices") } text := strings.TrimSpace(contentText(decoded.Choices[0].Message.Content)) if text == "" { return "", fmt.Errorf("llm response is empty") } return text, nil } func chatCompletionsURL(baseURL string) string { base := strings.TrimRight(strings.TrimSpace(baseURL), "/") if strings.HasSuffix(base, "/chat/completions") { return base } return base + "/chat/completions" } type chatCompletionResponse struct { Choices []struct { Message struct { Content any `json:"content"` } `json:"message"` } `json:"choices"` Error *struct { Message string `json:"message"` Type string `json:"type"` Code any `json:"code"` } `json:"error"` } func contentText(content any) string { switch v := content.(type) { case string: return v case []any: parts := make([]string, 0, len(v)) for _, item := range v { obj, ok := item.(map[string]any) if !ok { continue } if text, ok := obj["text"].(string); ok { parts = append(parts, text) } } return strings.Join(parts, "\n") default: return "" } } func compactBody(data []byte) string { text := strings.TrimSpace(string(data)) if text == "" { return "" } if len(text) > 800 { return text[:800] + "..." } return text }