feat: you can summary screen shot directly

2026-07-07 00:59:26 +08:00
parent 39f0aaae02
commit 712a2cbb6a
17 changed files with 1432 additions and 51 deletions
@@ -0,0 +1,174 @@
+// Package llm contains multimodal LLM adapters.
+//
+// The built-in providers are intentionally implemented through the
+// OpenAI-compatible chat/completions shape. Qwen DashScope, Volcengine Ark,
+// and OpenAI-compatible gateways all accept the same "text + image_url"
+// message content pattern, so one small HTTP adapter is enough here.
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/mmmy/snapgo/internal/domain"
+)
+
+// VisionClient calls one configured multimodal chat endpoint.
+type VisionClient struct {
+	providerID string
+	cfg        domain.LLMProviderConfig
+	httpClient *http.Client
+}
+
+// NewVisionClient validates cfg and returns a reusable client.
+func NewVisionClient(providerID string, cfg domain.LLMProviderConfig) (*VisionClient, error) {
+	if strings.TrimSpace(cfg.BaseURL) == "" {
+		return nil, fmt.Errorf("llm base url is required")
+	}
+	if strings.TrimSpace(cfg.APIKey) == "" {
+		return nil, fmt.Errorf("llm api key is required")
+	}
+	if strings.TrimSpace(cfg.Model) == "" {
+		return nil, fmt.Errorf("llm model is required")
+	}
+	timeout := cfg.TimeoutSecs
+	if timeout <= 0 {
+		timeout = 60
+	}
+	return &VisionClient{
+		providerID: providerID,
+		cfg:        cfg,
+		httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second},
+	}, nil
+}
+
+// SummarizeImage asks the model to summarise an image available at imageURL.
+func (c *VisionClient) SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) {
+	prompt = strings.TrimSpace(prompt)
+	if prompt == "" {
+		return "", fmt.Errorf("llm prompt is empty")
+	}
+	imageURL = strings.TrimSpace(imageURL)
+	if imageURL == "" {
+		return "", fmt.Errorf("image url is empty")
+	}
+
+	body := map[string]any{
+		"model": c.cfg.Model,
+		"messages": []map[string]any{
+			{
+				"role": "user",
+				"content": []map[string]any{
+					{"type": "text", "text": prompt},
+					{"type": "image_url", "image_url": map[string]any{"url": imageURL}},
+				},
+			},
+		},
+	}
+	if c.cfg.MaxTokens > 0 {
+		if c.providerID == domain.LLMProviderOpenAI {
+			body["max_completion_tokens"] = c.cfg.MaxTokens
+		} else {
+			body["max_tokens"] = c.cfg.MaxTokens
+		}
+	}
+	if c.cfg.Temperature >= 0 {
+		body["temperature"] = c.cfg.Temperature
+	}
+
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return "", fmt.Errorf("marshal llm request: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionsURL(c.cfg.BaseURL), bytes.NewReader(payload))
+	if err != nil {
+		return "", fmt.Errorf("build llm request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("llm request: %w", err)
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("llm status %d: %s", resp.StatusCode, compactBody(data))
+	}
+
+	var decoded chatCompletionResponse
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return "", fmt.Errorf("decode llm response: %w", err)
+	}
+	if decoded.Error != nil && decoded.Error.Message != "" {
+		return "", fmt.Errorf("llm error: %s", decoded.Error.Message)
+	}
+	if len(decoded.Choices) == 0 {
+		return "", fmt.Errorf("llm response has no choices")
+	}
+	text := strings.TrimSpace(contentText(decoded.Choices[0].Message.Content))
+	if text == "" {
+		return "", fmt.Errorf("llm response is empty")
+	}
+	return text, nil
+}
+
+func chatCompletionsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/chat/completions") {
+		return base
+	}
+	return base + "/chat/completions"
+}
+
+type chatCompletionResponse struct {
+	Choices []struct {
+		Message struct {
+			Content any `json:"content"`
+		} `json:"message"`
+	} `json:"choices"`
+	Error *struct {
+		Message string `json:"message"`
+		Type    string `json:"type"`
+		Code    any    `json:"code"`
+	} `json:"error"`
+}
+
+func contentText(content any) string {
+	switch v := content.(type) {
+	case string:
+		return v
+	case []any:
+		parts := make([]string, 0, len(v))
+		for _, item := range v {
+			obj, ok := item.(map[string]any)
+			if !ok {
+				continue
+			}
+			if text, ok := obj["text"].(string); ok {
+				parts = append(parts, text)
+			}
+		}
+		return strings.Join(parts, "\n")
+	default:
+		return ""
+	}
+}
+
+func compactBody(data []byte) string {
+	text := strings.TrimSpace(string(data))
+	if text == "" {
+		return "<empty body>"
+	}
+	if len(text) > 800 {
+		return text[:800] + "..."
+	}
+	return text
+}
@@ -0,0 +1,86 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/mmmy/snapgo/internal/domain"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func TestVisionClientSummarizeImageUsesChatCompletionImageURL(t *testing.T) {
+	var requestBody map[string]any
+	transport := roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		if r.URL.Path != "/v1/chat/completions" {
+			t.Fatalf("expected /v1/chat/completions, got %s", r.URL.Path)
+		}
+		if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
+			t.Fatalf("unexpected auth header %q", got)
+		}
+		if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		return &http.Response{
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Content-Type": []string{"application/json"}},
+			Body:       io.NopCloser(bytes.NewBufferString(`{"choices":[{"message":{"content":"summary ok"}}]}`)),
+		}, nil
+	})
+
+	client, err := NewVisionClient(domain.LLMProviderQwen, domain.LLMProviderConfig{
+		BaseURL:     "https://example.test/v1",
+		APIKey:      "test-key",
+		Model:       "qwen-vl-plus",
+		MaxTokens:   321,
+		Temperature: 0.2,
+		TimeoutSecs: 5,
+	})
+	if err != nil {
+		t.Fatalf("new client: %v", err)
+	}
+	client.httpClient.Transport = transport
+
+	text, err := client.SummarizeImage(context.Background(), "describe", "https://cdn.example.com/a.png")
+	if err != nil {
+		t.Fatalf("summarize image: %v", err)
+	}
+	if text != "summary ok" {
+		t.Fatalf("expected response text, got %q", text)
+	}
+	if requestBody["model"] != "qwen-vl-plus" {
+		t.Fatalf("expected model forwarded, got %#v", requestBody["model"])
+	}
+	if requestBody["max_tokens"] != float64(321) {
+		t.Fatalf("expected max_tokens forwarded, got %#v", requestBody["max_tokens"])
+	}
+
+	messages := requestBody["messages"].([]any)
+	content := messages[0].(map[string]any)["content"].([]any)
+	if content[0].(map[string]any)["text"] != "describe" {
+		t.Fatalf("expected prompt content, got %#v", content[0])
+	}
+	image := content[1].(map[string]any)
+	if image["type"] != "image_url" {
+		t.Fatalf("expected image_url item, got %#v", image)
+	}
+	url := image["image_url"].(map[string]any)["url"]
+	if url != "https://cdn.example.com/a.png" {
+		t.Fatalf("expected image url forwarded, got %#v", url)
+	}
+}
+
+func TestChatCompletionsURLAcceptsFullEndpoint(t *testing.T) {
+	full := "https://example.com/v1/chat/completions"
+	if got := chatCompletionsURL(full); got != full {
+		t.Fatalf("expected full endpoint unchanged, got %q", got)
+	}
+}