feat: you can summary screen shot directly

2026-07-07 00:59:26 +08:00
parent 39f0aaae02
commit 712a2cbb6a
17 changed files with 1432 additions and 51 deletions
@@ -0,0 +1,174 @@
+// Package llm contains multimodal LLM adapters.
+//
+// The built-in providers are intentionally implemented through the
+// OpenAI-compatible chat/completions shape. Qwen DashScope, Volcengine Ark,
+// and OpenAI-compatible gateways all accept the same "text + image_url"
+// message content pattern, so one small HTTP adapter is enough here.
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/mmmy/snapgo/internal/domain"
+)
+
+// VisionClient calls one configured multimodal chat endpoint.
+type VisionClient struct {
+	providerID string
+	cfg        domain.LLMProviderConfig
+	httpClient *http.Client
+}
+
+// NewVisionClient validates cfg and returns a reusable client.
+func NewVisionClient(providerID string, cfg domain.LLMProviderConfig) (*VisionClient, error) {
+	if strings.TrimSpace(cfg.BaseURL) == "" {
+		return nil, fmt.Errorf("llm base url is required")
+	}
+	if strings.TrimSpace(cfg.APIKey) == "" {
+		return nil, fmt.Errorf("llm api key is required")
+	}
+	if strings.TrimSpace(cfg.Model) == "" {
+		return nil, fmt.Errorf("llm model is required")
+	}
+	timeout := cfg.TimeoutSecs
+	if timeout <= 0 {
+		timeout = 60
+	}
+	return &VisionClient{
+		providerID: providerID,
+		cfg:        cfg,
+		httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second},
+	}, nil
+}
+
+// SummarizeImage asks the model to summarise an image available at imageURL.
+func (c *VisionClient) SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) {
+	prompt = strings.TrimSpace(prompt)
+	if prompt == "" {
+		return "", fmt.Errorf("llm prompt is empty")
+	}
+	imageURL = strings.TrimSpace(imageURL)
+	if imageURL == "" {
+		return "", fmt.Errorf("image url is empty")
+	}
+
+	body := map[string]any{
+		"model": c.cfg.Model,
+		"messages": []map[string]any{
+			{
+				"role": "user",
+				"content": []map[string]any{
+					{"type": "text", "text": prompt},
+					{"type": "image_url", "image_url": map[string]any{"url": imageURL}},
+				},
+			},
+		},
+	}
+	if c.cfg.MaxTokens > 0 {
+		if c.providerID == domain.LLMProviderOpenAI {
+			body["max_completion_tokens"] = c.cfg.MaxTokens
+		} else {
+			body["max_tokens"] = c.cfg.MaxTokens
+		}
+	}
+	if c.cfg.Temperature >= 0 {
+		body["temperature"] = c.cfg.Temperature
+	}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return "", fmt.Errorf("marshal llm request: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionsURL(c.cfg.BaseURL), bytes.NewReader(payload))
+	if err != nil {
+		return "", fmt.Errorf("build llm request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("llm request: %w", err)
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("llm status %d: %s", resp.StatusCode, compactBody(data))
+	}
+
+	var decoded chatCompletionResponse
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return "", fmt.Errorf("decode llm response: %w", err)
+	}
+	if decoded.Error != nil && decoded.Error.Message != "" {
+		return "", fmt.Errorf("llm error: %s", decoded.Error.Message)
+	}
+	if len(decoded.Choices) == 0 {
+		return "", fmt.Errorf("llm response has no choices")
+	}
+	text := strings.TrimSpace(contentText(decoded.Choices[0].Message.Content))
+	if text == "" {
+		return "", fmt.Errorf("llm response is empty")
+	}
+	return text, nil
+}
+
+func chatCompletionsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/chat/completions") {
+		return base
+	}
+	return base + "/chat/completions"
+}
+
+type chatCompletionResponse struct {
+	Choices []struct {
+		Message struct {
+			Content any `json:"content"`
+		} `json:"message"`
+	} `json:"choices"`
+	Error *struct {
+		Message string `json:"message"`
+		Type    string `json:"type"`
+		Code    any    `json:"code"`
+	} `json:"error"`
+}
+
+func contentText(content any) string {
+	switch v := content.(type) {
+	case string:
+		return v
+	case []any:
+		parts := make([]string, 0, len(v))
+		for _, item := range v {
+			obj, ok := item.(map[string]any)
+			if !ok {
+				continue
+			}
+			if text, ok := obj["text"].(string); ok {
+				parts = append(parts, text)
+			}
+		}
+		return strings.Join(parts, "\n")
+	default:
+		return ""
+	}
+}
+
+func compactBody(data []byte) string {
+	text := strings.TrimSpace(string(data))
+	if text == "" {
+		return "<empty body>"
+	}
+	if len(text) > 800 {
+		return text[:800] + "..."
+	}
+	return text
+}