feat: you can summary screen shot directly

This commit is contained in:
2026-07-07 00:59:26 +08:00
parent 39f0aaae02
commit 712a2cbb6a
17 changed files with 1432 additions and 51 deletions
+169 -2
View File
@@ -22,6 +22,7 @@ import (
"github.com/mmmy/snapgo/internal/infrastructure/config"
"github.com/mmmy/snapgo/internal/infrastructure/display"
"github.com/mmmy/snapgo/internal/infrastructure/hotkey"
llmpkg "github.com/mmmy/snapgo/internal/infrastructure/llm"
"github.com/mmmy/snapgo/internal/infrastructure/oss"
"github.com/mmmy/snapgo/internal/infrastructure/screencapture"
sshpkg "github.com/mmmy/snapgo/internal/infrastructure/ssh"
@@ -96,6 +97,15 @@ type CaptureActionResult struct {
Path string `json:"path,omitempty"`
}
// OperationStatusPayload drives the frontend status HUD and mirrors the
// native macOS HUD state.
type OperationStatusPayload struct {
Operation string `json:"operation"`
Phase string `json:"phase"`
Message string `json:"message"`
State string `json:"state"`
}
// NewApp creates a new App with collaborators already initialised.
func NewApp() *App {
store, err := config.NewFileStore()
@@ -110,6 +120,7 @@ func NewApp() *App {
slog.Warn("config load failed, using defaults", "err", lerr)
}
}
cfg.Normalize()
return &App{
cfg: cfg,
configFile: store,
@@ -297,7 +308,13 @@ func (a *App) runUploadPipeline(provider domain.OSSProvider, pngBytes []byte) er
FallbackDir: filepath.Join(userPicturesDir(), "SnapGo"),
PathPrefix: cfg.S3.PathPrefix,
}
return svc.ExecuteWithBytes(a.ctx, pngBytes)
a.emitOperationStatus("upload", "上传中", "正在上传截图到 S3", "running")
if err := svc.ExecuteWithBytes(a.ctx, pngBytes); err != nil {
a.emitOperationStatus("upload", "上传失败", err.Error(), "error")
return err
}
a.emitOperationStatus("upload", "上传完成", "链接已复制到剪贴板", "success")
return nil
}
func (a *App) runCopyImagePipeline(pngBytes []byte) error {
@@ -332,6 +349,7 @@ func (a *App) runSaveRemotePipeline(pngBytes []byte) error {
err := fmt.Errorf("SSH host/user is not configured")
slog.Warn("save-remote rejected: ssh not configured",
"host", cfg.SSH.Host, "user", cfg.SSH.User)
a.emitOperationStatus("save-remote", "需要配置 SSH", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
@@ -355,7 +373,78 @@ func (a *App) runSaveRemotePipeline(pngBytes []byte) error {
Notifier: &runtimeNotifier{ctx: a.ctx},
Cfg: cfg.SSH,
}
return svc.ExecuteWithBytes(a.ctx, pngBytes)
a.emitOperationStatus("save-remote", "保存中", "正在保存截图到远端 SSH", "running")
if err := svc.ExecuteWithBytes(a.ctx, pngBytes); err != nil {
a.emitOperationStatus("save-remote", "保存失败", err.Error(), "error")
return err
}
a.emitOperationStatus("save-remote", "保存完成", "远端路径已复制到剪贴板", "success")
return nil
}
func (a *App) runSummaryPipeline(pngBytes []byte) error {
a.mu.RLock()
cfg := a.cfg
a.mu.RUnlock()
if !cfg.IsS3Configured() {
err := fmt.Errorf("请先在 S3 配置页填写可用的对象存储配置")
a.emitOperationStatus("summary", "需要配置 S3", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
if !cfg.IsLLMConfigured() {
err := fmt.Errorf("请先在 LLM 配置页填写 API Key 和模型")
a.emitOperationStatus("summary", "需要配置 LLM", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
s3Provider, err := oss.NewS3Provider(cfg.S3)
if err != nil {
a.emitOperationStatus("summary", "S3 配置错误", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
providerID, llmCfg, _ := cfg.ActiveLLMProvider()
visionClient, err := llmpkg.NewVisionClient(providerID, llmCfg)
if err != nil {
a.emitOperationStatus("summary", "LLM 配置错误", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
svc := &application.CaptureSummaryService{
Provider: s3Provider,
Summarizer: visionClient,
Clipboard: a.clip,
Prompt: cfg.LLM.Prompt,
PathPrefix: cfg.S3.PathPrefix,
}
a.emitOperationStatus("summary", "上传中", "正在上传截图供模型读取", "running")
uploaded, err := svc.UploadImage(a.ctx, pngBytes)
if err != nil {
a.emitOperationStatus("summary", "上传失败", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
a.emitOperationStatus("summary", "识别中", "图片已上传,正在请求多模态模型", "running")
summary, err := svc.Summarize(a.ctx, uploaded.URL)
if err != nil {
a.emitOperationStatus("summary", "识别失败", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
if err := svc.CopySummary(a.ctx, summary); err != nil {
a.emitOperationStatus("summary", "复制失败", err.Error(), "error")
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
a.emitOperationStatus("summary", "识别完成", "总结已复制到剪贴板", "success")
wruntime.EventsEmit(a.ctx, "upload:success", "summary copied to clipboard")
return nil
}
func (a *App) consumePendingCapture() (*pendingCapture, error) {
@@ -408,6 +497,7 @@ func (a *App) GetConfig() domain.AppConfig {
// SaveConfig persists the supplied configuration and re-registers the hotkey
// if it changed.
func (a *App) SaveConfig(cfg domain.AppConfig) error {
cfg.Normalize()
a.mu.Lock()
prev := a.cfg
a.cfg = cfg
@@ -705,6 +795,35 @@ func (a *App) SaveRegionToRemote(result CaptureResult) error {
return a.runSaveRemotePipeline(cropped)
}
// SummarizeRegion uploads the selected screenshot to S3, sends the public URL
// to the configured multimodal LLM, and copies the resulting summary.
func (a *App) SummarizeRegion(result CaptureResult) error {
pc, err := a.consumePendingCapture()
if err != nil {
slog.Warn("SummarizeRegion: no pending capture", "err", err)
return err
}
defer func() {
a.capturing.Store(false)
a.dismissOverlay()
}()
a.dismissOverlay()
flushFrame()
slog.Info("SummarizeRegion: capturing region",
"x", result.Rect.X, "y", result.Rect.Y,
"w", result.Rect.W, "h", result.Rect.H,
"annotations", len(result.Annotations))
cropped, err := a.captureSelectedPNG(result, pc)
if err != nil {
slog.Error("SummarizeRegion: capture failed", "err", err)
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
return a.runSummaryPipeline(cropped)
}
// SaveNativeRegionToRemote is the macOS-native overlay equivalent of
// SaveRegionToRemote. The AppKit panel is already closed by the time this
// runs (see saveRemoteSelection in native_overlay_darwin.go), so we only
@@ -735,6 +854,33 @@ func (a *App) SaveNativeRegionToRemote(result CaptureResult) error {
return a.runSaveRemotePipeline(cropped)
}
// SummarizeNativeRegion is the macOS-native overlay equivalent of
// SummarizeRegion. The AppKit panel is already closed before this runs.
func (a *App) SummarizeNativeRegion(result CaptureResult) error {
pc, err := a.consumePendingCapture()
if err != nil {
slog.Warn("SummarizeNativeRegion: no pending capture", "err", err)
return err
}
defer func() {
a.capturing.Store(false)
hideDockIcon()
}()
flushFrame()
slog.Info("SummarizeNativeRegion: capturing region",
"x", result.Rect.X, "y", result.Rect.Y,
"w", result.Rect.W, "h", result.Rect.H,
"annotations", len(result.Annotations))
cropped, err := a.captureSelectedPNG(result, pc)
if err != nil {
slog.Error("SummarizeNativeRegion: capture failed", "err", err)
wruntime.EventsEmit(a.ctx, "upload:failure", err.Error())
return err
}
return a.runSummaryPipeline(cropped)
}
func parseNativeAnnotations(raw string) []application.Annotation {
if raw == "" {
return nil
@@ -784,6 +930,27 @@ func (a *App) QuitApp() {
os.Exit(0)
}
func (a *App) emitOperationStatus(operation, phase, message, state string) {
if a.ctx != nil {
wruntime.EventsEmit(a.ctx, "operation:status", OperationStatusPayload{
Operation: operation,
Phase: phase,
Message: message,
State: state,
})
}
switch state {
case "success":
showOperationStatus(phase, message, operationStatusSuccess)
hideOperationStatusAfter(1400 * time.Millisecond)
case "error":
showOperationStatus(phase, message, operationStatusError)
hideOperationStatusAfter(2600 * time.Millisecond)
default:
showOperationStatus(phase, message, operationStatusRunning)
}
}
// runtimeNotifier emits success / failure events via the Wails runtime.
type runtimeNotifier struct{ ctx context.Context }
+132 -5
View File
@@ -3,9 +3,7 @@
* App shell — switches between two distinct UI modes that share the same
* Wails window:
*
* • "settings" : full settings UI (self-drawn title bar + sidebar + form).
* Self-drawn because the window is now Frameless to make
* the overlay paint edge-to-edge.
* • "settings" : full settings UI (native title bar + sidebar + form).
* • "overlay" : Snipaste-style region picker that fills the whole
* primary display.
*
@@ -24,6 +22,7 @@ import {
CopyRegionImage,
SaveRegionImage,
SaveRegionToRemote,
SummarizeRegion,
CancelRegion,
} from '../wailsjs/go/main/App'
@@ -39,7 +38,7 @@ const mode = ref<Mode>('settings')
// `SettingsTab` is hoisted to the App shell so the sidebar (which lives
// here) and the inner SettingsView (which renders the matching card) can
// share a single source of truth without an event bus.
type SettingsTab = 'general' | 's3' | 'ssh'
type SettingsTab = 'general' | 's3' | 'ssh' | 'llm'
const activeTab = ref<SettingsTab>('general')
// Sidebar entries are declarative so adding a destination type later is
@@ -48,6 +47,7 @@ const sidebarItems: Array<{ id: SettingsTab; label: string }> = [
{ id: 'general', label: 'General' },
{ id: 's3', label: 'S3' },
{ id: 'ssh', label: 'SSH' },
{ id: 'llm', label: 'LLM' },
]
interface OverlayPayload {
@@ -62,6 +62,14 @@ const capturing = ref(false)
const toast = ref<{ kind: 'success' | 'error'; text: string } | null>(null)
let toastTimer: number | undefined
const operationStatus = ref<{
operation: string
phase: string
message: string
state: 'running' | 'success' | 'error'
} | null>(null)
let operationTimer: number | undefined
function showToast(kind: 'success' | 'error', text: string) {
toast.value = { kind, text }
window.clearTimeout(toastTimer)
@@ -70,6 +78,21 @@ function showToast(kind: 'success' | 'error', text: string) {
}, 3000)
}
function updateOperationStatus(payload: {
operation: string
phase: string
message: string
state: 'running' | 'success' | 'error'
}) {
operationStatus.value = payload
window.clearTimeout(operationTimer)
if (payload.state !== 'running') {
operationTimer = window.setTimeout(() => {
operationStatus.value = null
}, payload.state === 'success' ? 1600 : 3200)
}
}
async function retryHotkey() {
try {
await RetryRegisterHotkey()
@@ -168,6 +191,28 @@ async function onOverlaySaveRemote(rect: {
}
}
async function onOverlaySummarize(rect: {
rect: {
x: number
y: number
w: number
h: number
}
annotations: Array<{
tool: string
color: string
points: Array<{ x: number; y: number }>
}>
}) {
mode.value = 'settings'
overlayPayload.value = null
try {
await SummarizeRegion(rect as any)
} catch {
/* Surfaced via upload:failure */
}
}
async function onOverlayCancel() {
mode.value = 'settings'
overlayPayload.value = null
@@ -194,7 +239,12 @@ onMounted(() => {
capturing.value = false
})
EventsOn('upload:success', (url: string) => {
showToast('success', `Copied: ${url}`)
showToast(
'success',
url === 'summary copied to clipboard'
? 'Summary copied to clipboard'
: `Copied: ${url}`
)
})
EventsOn('upload:failure', (reason: string) => {
showToast('error', reason)
@@ -205,6 +255,7 @@ onMounted(() => {
EventsOn('hotkey:error', (reason: string) => {
hotkeyStatus.value = { state: 'error', reason }
})
EventsOn('operation:status', updateOperationStatus)
})
onUnmounted(() => {
@@ -216,6 +267,7 @@ onUnmounted(() => {
EventsOff('upload:failure')
EventsOff('hotkey:ready')
EventsOff('hotkey:error')
EventsOff('operation:status')
})
</script>
@@ -229,6 +281,7 @@ onUnmounted(() => {
@copy="onOverlayCopy"
@save="onOverlaySave"
@save-remote="onOverlaySaveRemote"
@summarize="onOverlaySummarize"
@cancel="onOverlayCancel"
/>
@@ -264,6 +317,20 @@ onUnmounted(() => {
</main>
<Toast v-if="toast" :kind="toast.kind" :text="toast.text" />
<div
v-if="operationStatus"
class="operation-hud"
:class="operationStatus.state"
>
<span v-if="operationStatus.state === 'running'" class="spinner" />
<span v-else class="status-mark">
{{ operationStatus.state === 'success' ? 'OK' : '!' }}
</span>
<div>
<strong>{{ operationStatus.phase }}</strong>
<p>{{ operationStatus.message }}</p>
</div>
</div>
</div>
</template>
@@ -371,6 +438,66 @@ onUnmounted(() => {
min-width: 0;
}
.operation-hud {
position: fixed;
right: 18px;
bottom: 18px;
z-index: 20;
display: flex;
align-items: center;
gap: 12px;
min-width: 260px;
max-width: 360px;
padding: 12px 14px;
color: #f9fafb;
background: rgba(17, 24, 39, 0.94);
border-radius: 8px;
box-shadow: 0 12px 34px rgba(15, 23, 42, 0.28);
}
.operation-hud strong {
display: block;
margin-bottom: 2px;
font-size: 13px;
}
.operation-hud p {
margin: 0;
color: #d1d5db;
font-size: 12px;
line-height: 1.35;
}
.spinner {
width: 20px;
height: 20px;
border: 2px solid rgba(255, 255, 255, 0.25);
border-top-color: #60a5fa;
border-radius: 50%;
animation: spin 0.75s linear infinite;
flex: 0 0 auto;
}
.status-mark {
display: grid;
place-items: center;
width: 24px;
height: 24px;
border-radius: 50%;
font-size: 11px;
font-weight: 700;
flex: 0 0 auto;
}
.operation-hud.success .status-mark {
color: #052e16;
background: #86efac;
}
.operation-hud.error .status-mark {
color: #450a0a;
background: #fca5a5;
}
@keyframes spin {
to {
transform: rotate(360deg);
}
}
@media (prefers-color-scheme: dark) {
.app-root {
background: #1c1d22;
+37 -5
View File
@@ -51,6 +51,10 @@ const emit = defineEmits<{
e: 'save-remote',
payload: { rect: Rect; annotations: Annotation[] }
): void
(
e: 'summarize',
payload: { rect: Rect; annotations: Annotation[] }
): void
(e: 'cancel'): void
}>()
@@ -112,7 +116,7 @@ const sizeLabel = computed(() => {
})
const MARK_TOOLBAR_W = 190
const ACTION_TOOLBAR_W = 180
const ACTION_TOOLBAR_W = 216
const TOOLBAR_GROUP_GAP = 8
const rightToolbarPos = computed(() => {
@@ -363,7 +367,11 @@ function onSaveRemote() {
emitAction('save-remote')
}
function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote') {
function onSummarize() {
emitAction('summarize')
}
function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote' | 'summarize') {
if (!rect.value) return
const payload = {
rect: { ...rect.value },
@@ -373,6 +381,7 @@ function emitAction(action: 'confirm' | 'copy' | 'save' | 'save-remote') {
if (action === 'copy') emit('copy', payload)
if (action === 'save') emit('save', payload)
if (action === 'save-remote') emit('save-remote', payload)
if (action === 'summarize') emit('summarize', payload)
}
function onCancel() {
@@ -593,8 +602,8 @@ onUnmounted(() => {
/>
<button
class="action-btn"
data-tip="复制图"
aria-label="复制图"
data-tip="复制图"
aria-label="复制图"
@click="onCopy"
v-html="copyIcon"
/>
@@ -612,6 +621,19 @@ onUnmounted(() => {
@click="onSaveRemote"
v-html="saveRemoteIcon"
/>
<button
class="action-btn"
data-tip="复制总结"
aria-label="复制总结"
@click="onSummarize"
>
<svg viewBox="0 0 24 24" aria-hidden="true">
<path d="M6 3h8l4 4v14H6z" />
<path d="M14 3v5h4" />
<path d="M9 12h6" />
<path d="M9 16h5" />
</svg>
</button>
<button
class="action-btn primary"
data-tip="上传云端"
@@ -706,7 +728,7 @@ onUnmounted(() => {
width: 190px;
}
.action-toolbar {
width: 180px;
width: 216px;
}
.icon-btn,
@@ -762,6 +784,16 @@ onUnmounted(() => {
fill: currentColor;
pointer-events: none;
}
.action-btn > svg {
width: 18px;
height: 18px;
fill: none;
stroke: currentColor;
stroke-width: 2;
stroke-linecap: round;
stroke-linejoin: round;
pointer-events: none;
}
.action-btn::after {
content: attr(data-tip);
position: absolute;
+201 -7
View File
@@ -8,6 +8,7 @@
* • "general" — the global hotkey / capture parameters.
* • "s3" — S3-compatible object-storage credentials.
* • "ssh" — SSH/SCP destination for the "save remote" button.
* • "llm" — multimodal screenshot summary provider settings.
*
* State flow: load() pulls config from Go on mount, save() pushes back.
*/
@@ -22,7 +23,7 @@ import {
// Tab discriminator shared with the App shell. Kept as a string union so
// the parent can pass the value without importing a type from this view.
type TabId = 'general' | 's3' | 'ssh'
type TabId = 'general' | 's3' | 'ssh' | 'llm'
const props = defineProps<{
tab: TabId
@@ -51,12 +52,44 @@ interface SSHConfig {
connectTimeoutSecs: number
}
interface LLMProviderConfig {
label: string
baseUrl: string
apiKey: string
model: string
maxTokens: number
temperature: number
timeoutSecs: number
}
interface LLMConfig {
activeProvider: string
prompt: string
providers: Record<string, LLMProviderConfig>
}
interface AppConfig {
hotkey: string
s3: S3Config
ssh: SSHConfig
llm: LLMConfig
}
const llmProviderOptions = [
{ id: 'qwen', label: '阿里千问多模态' },
{ id: 'doubao', label: '火山方舟豆包多模态' },
{ id: 'openai', label: 'ChatGPT / OpenAI-compatible' },
]
const DEFAULT_SUMMARY_PROMPT = `你是一个截图内容总结助手。请阅读截图,并用中文输出一段适合直接粘贴到聊天、工单、Issue 或 PR 中的说明。
要求:
- 先说明截图中最重要的信息和用户可能想表达的意图。
- 如果截图包含报错、异常状态、表格、代码、界面控件或关键数字,请准确提取。
- 如果截图内容不足以判断,不要编造;直接说明不确定点。
- 输出尽量简洁,默认 3 到 6 条要点。
- 不要输出 Markdown 标题,不要寒暄。`
// `defaultConfig` keeps the initial render in sync with the Go-side
// DefaultAppConfig so the UI never flashes empty fields before load()
// completes. Centralising the defaults also avoids subtle drift between
@@ -85,6 +118,39 @@ function defaultConfig(): AppConfig {
knownHostsPath: '',
connectTimeoutSecs: 10,
},
llm: {
activeProvider: 'openai',
prompt: DEFAULT_SUMMARY_PROMPT,
providers: {
qwen: {
label: '阿里千问多模态',
baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
apiKey: '',
model: 'qwen-vl-plus',
maxTokens: 600,
temperature: 0.2,
timeoutSecs: 60,
},
doubao: {
label: '火山方舟豆包多模态',
baseUrl: 'https://ark.cn-beijing.volces.com/api/v3',
apiKey: '',
model: '',
maxTokens: 600,
temperature: 0.2,
timeoutSecs: 60,
},
openai: {
label: 'ChatGPT / OpenAI-compatible',
baseUrl: 'https://api.openai.com/v1',
apiKey: '',
model: 'gpt-5.5',
maxTokens: 600,
temperature: 0.2,
timeoutSecs: 60,
},
},
},
}
}
@@ -109,6 +175,15 @@ const sshPathDisplay = computed({
},
})
const activeLLMProvider = computed(() => {
const id = config.value.llm.activeProvider || 'openai'
if (!config.value.llm.providers[id]) {
config.value.llm.activeProvider = 'openai'
return config.value.llm.providers.openai
}
return config.value.llm.providers[id]
})
function flash(kind: 'ok' | 'err', text: string) {
message.value = { kind, text }
window.setTimeout(() => {
@@ -125,6 +200,11 @@ async function load() {
Object.assign(merged, cfg)
merged.s3 = { ...merged.s3, ...(cfg as any).s3 }
merged.ssh = { ...merged.ssh, ...(cfg as any).ssh }
merged.llm = { ...merged.llm, ...(cfg as any).llm }
merged.llm.providers = {
...defaultConfig().llm.providers,
...((cfg as any).llm?.providers ?? {}),
}
config.value = merged
}
}
@@ -368,6 +448,103 @@ onMounted(load)
</div>
</section>
<!-- LLM tab multimodal screenshot summary provider settings. -->
<section v-if="props.tab === 'llm'" class="card">
<h2>LLM screenshot summary</h2>
<div class="grid">
<label class="field full">
<span>Provider</span>
<select v-model="config.llm.activeProvider">
<option
v-for="provider in llmProviderOptions"
:key="provider.id"
:value="provider.id"
>
{{ provider.label }}
</option>
</select>
</label>
<label class="field full">
<span>Base URL *</span>
<input
v-model="activeLLMProvider.baseUrl"
placeholder="https://api.openai.com/v1"
spellcheck="false"
/>
</label>
<label class="field">
<span>API Key *</span>
<input v-model="activeLLMProvider.apiKey" type="password" />
</label>
<label class="field">
<span>Model *</span>
<input
v-model="activeLLMProvider.model"
placeholder="gpt-5.5 / qwen-vl-plus / Ark endpoint ID"
spellcheck="false"
/>
</label>
<label class="field">
<span>Max tokens</span>
<input
v-model.number="activeLLMProvider.maxTokens"
type="number"
min="64"
max="4096"
placeholder="600"
/>
</label>
<label class="field">
<span>Temperature</span>
<input
v-model.number="activeLLMProvider.temperature"
type="number"
min="0"
max="2"
step="0.1"
placeholder="0.2"
/>
</label>
<label class="field">
<span>Timeout (sec)</span>
<input
v-model.number="activeLLMProvider.timeoutSecs"
type="number"
min="10"
max="300"
placeholder="60"
/>
</label>
<label class="field full">
<span>Prompt</span>
<textarea
v-model="config.llm.prompt"
class="prompt-textarea"
spellcheck="false"
/>
</label>
</div>
<p class="hint">
The "copy summary" screenshot action first uploads the image through the
S3 configuration, then sends the public image URL to this multimodal
provider. Keep S3 configured before using it.
</p>
<p v-if="config.llm.activeProvider === 'doubao'" class="hint">
Volcengine Ark usually uses the inference endpoint ID as the model
value; paste the endpoint/model identifier from your Ark console.
</p>
<p v-else-if="config.llm.activeProvider === 'qwen'" class="hint">
DashScope OpenAI-compatible mode is used here. If your workspace uses a
regional endpoint, replace Base URL with that compatible-mode URL.
</p>
<div class="actions">
<button class="btn primary" :disabled="saving" @click="save">
{{ saving ? 'Saving' : 'Save' }}
</button>
</div>
</section>
<transition name="fade">
<div
v-if="message"
@@ -460,13 +637,13 @@ kbd {
.field input[type='text'],
.field input[type='number'],
.field input:not([type='checkbox']),
.field select {
.field select,
.field textarea {
border: 1px solid #d1d5db;
border-radius: 6px;
padding: 7px 10px;
font-size: 13px;
line-height: 1.4;
height: 36px;
background: #fff;
color: inherit;
outline: none;
@@ -474,8 +651,20 @@ kbd {
width: 100%;
box-sizing: border-box;
}
.field input[type='text'],
.field input[type='number'],
.field input:not([type='checkbox']),
.field select {
height: 36px;
}
.field textarea {
min-height: 168px;
resize: vertical;
font-family: inherit;
}
.field input:focus,
.field select:focus {
.field select:focus,
.field textarea:focus {
border-color: #3b82f6;
box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.18);
}
@@ -598,15 +787,20 @@ kbd {
background: #1f2937;
border-color: #374151;
}
.field input {
.field input,
.field textarea {
background: #111827;
border-color: #374151;
color: #e5e7eb;
color: #fff;
}
.field select {
background: #111827;
border-color: #374151;
color: #e5e7eb;
color: #fff;
}
.field input::placeholder,
.field textarea::placeholder {
color: #9ca3af;
}
.field input:disabled {
background: #1f2937;
+4
View File
@@ -37,6 +37,10 @@ export function SaveRegionToRemote(arg1:main.CaptureResult):Promise<void>;
export function ShowWindow():Promise<void>;
export function SummarizeNativeRegion(arg1:main.CaptureResult):Promise<void>;
export function SummarizeRegion(arg1:main.CaptureResult):Promise<void>;
export function TestConnection(arg1:domain.S3Config):Promise<void>;
export function TestSSHConnection(arg1:domain.SSHConfig):Promise<void>;
+8
View File
@@ -70,6 +70,14 @@ export function ShowWindow() {
return window['go']['main']['App']['ShowWindow']();
}
export function SummarizeNativeRegion(arg1) {
return window['go']['main']['App']['SummarizeNativeRegion'](arg1);
}
export function SummarizeRegion(arg1) {
return window['go']['main']['App']['SummarizeRegion'](arg1);
}
export function TestConnection(arg1) {
return window['go']['main']['App']['TestConnection'](arg1);
}
+62
View File
@@ -53,6 +53,64 @@ export namespace application {
export namespace domain {
export class LLMProviderConfig {
label: string;
baseUrl: string;
apiKey: string;
model: string;
maxTokens: number;
temperature: number;
timeoutSecs: number;
static createFrom(source: any = {}) {
return new LLMProviderConfig(source);
}
constructor(source: any = {}) {
if ('string' === typeof source) source = JSON.parse(source);
this.label = source["label"];
this.baseUrl = source["baseUrl"];
this.apiKey = source["apiKey"];
this.model = source["model"];
this.maxTokens = source["maxTokens"];
this.temperature = source["temperature"];
this.timeoutSecs = source["timeoutSecs"];
}
}
export class LLMConfig {
activeProvider: string;
prompt: string;
providers: Record<string, LLMProviderConfig>;
static createFrom(source: any = {}) {
return new LLMConfig(source);
}
constructor(source: any = {}) {
if ('string' === typeof source) source = JSON.parse(source);
this.activeProvider = source["activeProvider"];
this.prompt = source["prompt"];
this.providers = this.convertValues(source["providers"], LLMProviderConfig, true);
}
convertValues(a: any, classs: any, asMap: boolean = false): any {
if (!a) {
return a;
}
if (a.slice && a.map) {
return (a as any[]).map(elem => this.convertValues(elem, classs));
} else if ("object" === typeof a) {
if (asMap) {
for (const key of Object.keys(a)) {
a[key] = new classs(a[key]);
}
return a;
}
return new classs(a);
}
return a;
}
}
export class SSHConfig {
host: string;
port: number;
@@ -111,6 +169,7 @@ export namespace domain {
hotkey: string;
s3: S3Config;
ssh: SSHConfig;
llm: LLMConfig;
static createFrom(source: any = {}) {
return new AppConfig(source);
@@ -121,6 +180,7 @@ export namespace domain {
this.hotkey = source["hotkey"];
this.s3 = this.convertValues(source["s3"], S3Config);
this.ssh = this.convertValues(source["ssh"], SSHConfig);
this.llm = this.convertValues(source["llm"], LLMConfig);
}
convertValues(a: any, classs: any, asMap: boolean = false): any {
@@ -142,6 +202,8 @@ export namespace domain {
}
}
}
+72
View File
@@ -0,0 +1,72 @@
package application
import (
"context"
"fmt"
"time"
"github.com/mmmy/snapgo/internal/domain"
"github.com/mmmy/snapgo/internal/infrastructure/clipboard"
)
// VisionSummarizer describes a multimodal model capable of reading a public
// image URL and returning a textual summary.
type VisionSummarizer interface {
SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error)
}
// CaptureSummaryService wires screenshot bytes -> S3 public URL -> LLM
// summary -> clipboard. It intentionally does not notify UI state itself:
// the caller emits fine-grained "uploading / recognizing / done" progress.
type CaptureSummaryService struct {
Provider domain.OSSProvider
Summarizer VisionSummarizer
Clipboard clipboard.Writer
Prompt string
PathPrefix string
}
// UploadImage uploads the screenshot and returns the public URL visible to
// the multimodal provider.
func (s *CaptureSummaryService) UploadImage(ctx context.Context, pngBytes []byte) (*domain.UploadResult, error) {
if s.Provider == nil {
return nil, fmt.Errorf("s3 is not configured")
}
if len(pngBytes) == 0 {
return nil, fmt.Errorf("empty screenshot")
}
key := buildObjectKey(s.PathPrefix)
start := time.Now()
url, err := s.Provider.Upload(ctx, key, pngBytes, "image/png")
if err != nil {
return nil, fmt.Errorf("s3 put object: %w", err)
}
return &domain.UploadResult{
URL: url,
Key: key,
Provider: s.Provider.Name(),
Elapsed: time.Since(start),
}, nil
}
// Summarize asks the configured model to summarize the uploaded image URL.
func (s *CaptureSummaryService) Summarize(ctx context.Context, imageURL string) (string, error) {
if s.Summarizer == nil {
return "", fmt.Errorf("llm is not configured")
}
return s.Summarizer.SummarizeImage(ctx, s.Prompt, imageURL)
}
// CopySummary writes the final summary text to the clipboard.
func (s *CaptureSummaryService) CopySummary(_ context.Context, summary string) error {
if summary == "" {
return fmt.Errorf("empty summary")
}
if s.Clipboard == nil {
return fmt.Errorf("clipboard is not configured")
}
if err := s.Clipboard.WriteText(summary); err != nil {
return fmt.Errorf("clipboard write failed: %w", err)
}
return nil
}
@@ -0,0 +1,86 @@
package application
import (
"context"
"testing"
)
type fakeOSSProvider struct {
key string
data []byte
contentType string
url string
}
func (f *fakeOSSProvider) Upload(_ context.Context, key string, data []byte, contentType string) (string, error) {
f.key = key
f.data = append([]byte(nil), data...)
f.contentType = contentType
if f.url == "" {
return "https://cdn.example.com/" + key, nil
}
return f.url, nil
}
func (f *fakeOSSProvider) Name() string { return "fake" }
type fakeSummarizer struct {
prompt string
url string
text string
}
func (f *fakeSummarizer) SummarizeImage(_ context.Context, prompt string, imageURL string) (string, error) {
f.prompt = prompt
f.url = imageURL
if f.text == "" {
return "summary text", nil
}
return f.text, nil
}
func TestCaptureSummaryServiceUploadsSummarizesAndCopies(t *testing.T) {
ctx := context.Background()
provider := &fakeOSSProvider{}
summarizer := &fakeSummarizer{}
clip := &fakeClipboard{}
svc := &CaptureSummaryService{
Provider: provider,
Summarizer: summarizer,
Clipboard: clip,
Prompt: "describe screenshot",
PathPrefix: "snapgo/",
}
uploaded, err := svc.UploadImage(ctx, []byte("png"))
if err != nil {
t.Fatalf("upload image: %v", err)
}
if provider.contentType != "image/png" {
t.Fatalf("expected image/png content type, got %q", provider.contentType)
}
if string(provider.data) != "png" {
t.Fatalf("expected uploaded png bytes, got %q", string(provider.data))
}
summary, err := svc.Summarize(ctx, uploaded.URL)
if err != nil {
t.Fatalf("summarize: %v", err)
}
if summarizer.prompt != "describe screenshot" {
t.Fatalf("expected prompt forwarded, got %q", summarizer.prompt)
}
if summarizer.url != uploaded.URL {
t.Fatalf("expected uploaded url forwarded, got %q", summarizer.url)
}
if summary != "summary text" {
t.Fatalf("expected summary text, got %q", summary)
}
if err := svc.CopySummary(ctx, summary); err != nil {
t.Fatalf("copy summary: %v", err)
}
if clip.text != "summary text" {
t.Fatalf("expected summary copied, got %q", clip.text)
}
}
+174 -18
View File
@@ -10,12 +10,12 @@ package domain
// endpoint, etc.).
//
// Field design notes:
// - PathPrefix supports object name templating so users can group screenshots
// by date.
// - PublicURLBase is optional to support CDN-fronted buckets; otherwise the
// adapter falls back to "{Endpoint}/{Bucket}/{Key}" (path-style).
// - UsePathStyle defaults to true because many self-hosted MinIO / R2
// deployments do not support virtual-hosted-style addressing.
// - PathPrefix supports object name templating so users can group screenshots
// by date.
// - PublicURLBase is optional to support CDN-fronted buckets; otherwise the
// adapter falls back to "{Endpoint}/{Bucket}/{Key}" (path-style).
// - UsePathStyle defaults to true because many self-hosted MinIO / R2
// deployments do not support virtual-hosted-style addressing.
type S3Config struct {
Endpoint string `json:"endpoint"`
Region string `json:"region"`
@@ -34,17 +34,17 @@ type S3Config struct {
// - Host / Port form the destination endpoint. Port defaults to 22 when 0.
// - User is the SSH login name.
// - AuthMethod selects how the connection authenticates:
// "password" → password only, via the in-process Go SSH client.
// "key" → ssh-agent + ~/.ssh/id_* key files (no password),
// via the in-process Go SSH client.
// "kerberos" → delegate to the system /usr/bin/ssh binary so the
// existing Kerberos (GSSAPI) credential cache from
// `kinit` is reused. Native GSSAPI is required here
// because macOS stores tickets in an API: ccache
// that pure-Go SSH libraries cannot read.
// "" / "builtin" → legacy combined behaviour (password → agent → key
// files) kept only for backward compatibility with
// configs written before the method was split.
// "password" → password only, via the in-process Go SSH client.
// "key" → ssh-agent + ~/.ssh/id_* key files (no password),
// via the in-process Go SSH client.
// "kerberos" → delegate to the system /usr/bin/ssh binary so the
// existing Kerberos (GSSAPI) credential cache from
// `kinit` is reused. Native GSSAPI is required here
// because macOS stores tickets in an API: ccache
// that pure-Go SSH libraries cannot read.
// "" / "builtin" → legacy combined behaviour (password → agent → key
// files) kept only for backward compatibility with
// configs written before the method was split.
// - Password is optional; when empty the implementation falls back to the
// local SSH agent or ~/.ssh/id_* keys (i.e. the user is responsible for
// having password-less access already configured on this machine).
@@ -72,6 +72,29 @@ type SSHConfig struct {
ConnectTimeoutSecs int `json:"connectTimeoutSecs"`
}
// LLMProviderConfig stores one OpenAI-compatible multimodal chat endpoint.
//
// The three built-in providers (Qwen, Doubao/Ark, OpenAI-compatible) all use
// the same request shape: chat/completions with a user content array that
// contains text plus an image_url item. Keeping them as provider records lets
// users switch between credentials/models without retyping every field.
type LLMProviderConfig struct {
Label string `json:"label"`
BaseURL string `json:"baseUrl"`
APIKey string `json:"apiKey"`
Model string `json:"model"`
MaxTokens int `json:"maxTokens"`
Temperature float64 `json:"temperature"`
TimeoutSecs int `json:"timeoutSecs"`
}
// LLMConfig controls screenshot summarisation.
type LLMConfig struct {
ActiveProvider string `json:"activeProvider"`
Prompt string `json:"prompt"`
Providers map[string]LLMProviderConfig `json:"providers"`
}
// SSH authentication method identifiers stored in SSHConfig.AuthMethod.
const (
// SSHAuthBuiltin is the legacy combined method (password → agent → key
@@ -87,6 +110,22 @@ const (
SSHAuthKerberos = "kerberos"
)
// Built-in LLM provider identifiers.
const (
LLMProviderQwen = "qwen"
LLMProviderDoubao = "doubao"
LLMProviderOpenAI = "openai"
)
const DefaultSummaryPrompt = `你是一个截图内容总结助手。请阅读截图,并用中文输出一段适合直接粘贴到聊天、工单、Issue 或 PR 中的说明。
要求:
- 先说明截图中最重要的信息和用户可能想表达的意图。
- 如果截图包含报错、异常状态、表格、代码、界面控件或关键数字,请准确提取。
- 如果截图内容不足以判断,不要编造;直接说明不确定点。
- 输出尽量简洁,默认 3 到 6 条要点。
- 不要输出 Markdown 标题,不要寒暄。`
// IsKerberos reports whether this config requests Kerberos/GSSAPI auth.
func (c SSHConfig) IsKerberos() bool {
return c.AuthMethod == SSHAuthKerberos
@@ -108,11 +147,14 @@ type AppConfig struct {
// SSH holds the configuration for the optional "save to remote via scp"
// destination triggered by the save-remote toolbar button.
SSH SSHConfig `json:"ssh"`
// LLM holds provider settings for the "copy summary" screenshot action.
LLM LLMConfig `json:"llm"`
}
// DefaultAppConfig returns sane zero-value defaults used on first launch.
func DefaultAppConfig() AppConfig {
return AppConfig{
cfg := AppConfig{
Hotkey: "cmd+shift+a",
S3: S3Config{
PathPrefix: "snapgo/",
@@ -124,6 +166,102 @@ func DefaultAppConfig() AppConfig {
ConnectTimeoutSecs: 10,
StrictHostKey: false,
},
LLM: DefaultLLMConfig(),
}
cfg.Normalize()
return cfg
}
// DefaultLLMConfig returns the built-in provider presets. API keys and exact
// models remain user-editable because each provider account may expose
// different model names / endpoint IDs.
func DefaultLLMConfig() LLMConfig {
return LLMConfig{
ActiveProvider: LLMProviderOpenAI,
Prompt: DefaultSummaryPrompt,
Providers: map[string]LLMProviderConfig{
LLMProviderQwen: {
Label: "阿里千问多模态",
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
Model: "qwen-vl-plus",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
},
LLMProviderDoubao: {
Label: "火山方舟豆包多模态",
BaseURL: "https://ark.cn-beijing.volces.com/api/v3",
Model: "",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
},
LLMProviderOpenAI: {
Label: "ChatGPT / OpenAI-compatible",
BaseURL: "https://api.openai.com/v1",
Model: "gpt-5.5",
MaxTokens: 600,
Temperature: 0.2,
TimeoutSecs: 60,
},
},
}
}
// Normalize fills defaults into configs written by older app versions while
// preserving any user-provided provider fields.
func (c *AppConfig) Normalize() {
if c.Hotkey == "" {
c.Hotkey = "cmd+shift+a"
}
if c.S3.PathPrefix == "" {
c.S3.PathPrefix = "snapgo/"
}
if c.SSH.Port == 0 {
c.SSH.Port = 22
}
if c.SSH.PathPrefix == "" {
c.SSH.PathPrefix = "snapgo/"
}
if c.SSH.ConnectTimeoutSecs == 0 {
c.SSH.ConnectTimeoutSecs = 10
}
defaultLLM := DefaultLLMConfig()
if c.LLM.ActiveProvider == "" {
c.LLM.ActiveProvider = defaultLLM.ActiveProvider
}
if c.LLM.Prompt == "" {
c.LLM.Prompt = defaultLLM.Prompt
}
if c.LLM.Providers == nil {
c.LLM.Providers = map[string]LLMProviderConfig{}
}
for id, def := range defaultLLM.Providers {
current, ok := c.LLM.Providers[id]
if !ok {
c.LLM.Providers[id] = def
continue
}
if current.Label == "" {
current.Label = def.Label
}
if current.BaseURL == "" {
current.BaseURL = def.BaseURL
}
if current.Model == "" && id != LLMProviderDoubao {
current.Model = def.Model
}
if current.MaxTokens == 0 {
current.MaxTokens = def.MaxTokens
}
if current.Temperature < 0 {
current.Temperature = def.Temperature
}
if current.TimeoutSecs == 0 {
current.TimeoutSecs = def.TimeoutSecs
}
c.LLM.Providers[id] = current
}
}
@@ -141,3 +279,21 @@ func (c AppConfig) IsS3Configured() bool {
func (c AppConfig) IsSSHConfigured() bool {
return c.SSH.Host != "" && c.SSH.User != ""
}
// ActiveLLMProvider returns the selected provider config plus a boolean
// indicating whether the selection exists.
func (c AppConfig) ActiveLLMProvider() (string, LLMProviderConfig, bool) {
id := c.LLM.ActiveProvider
if id == "" {
id = LLMProviderOpenAI
}
provider, ok := c.LLM.Providers[id]
return id, provider, ok
}
// IsLLMConfigured reports whether the selected multimodal provider has the
// fields required to make a request.
func (c AppConfig) IsLLMConfigured() bool {
_, provider, ok := c.ActiveLLMProvider()
return ok && provider.BaseURL != "" && provider.APIKey != "" && provider.Model != ""
}
+8 -6
View File
@@ -1,12 +1,12 @@
// Package config provides JSON-file backed persistence for AppConfig.
//
// Design rationale:
// - JSON keeps the file human-readable, which is useful while the MVP has
// no settings UI for every option.
// - Storage location follows os.UserConfigDir() so each platform places it
// in the canonical spot (macOS: ~/Library/Application Support/SnapGo).
// - File mode 0600 / dir mode 0700 mitigate accidental leakage of access
// keys before we wire up the OS keychain (deferred to a later spec).
// - JSON keeps the file human-readable, which is useful while the MVP has
// no settings UI for every option.
// - Storage location follows os.UserConfigDir() so each platform places it
// in the canonical spot (macOS: ~/Library/Application Support/SnapGo).
// - File mode 0600 / dir mode 0700 mitigate accidental leakage of access
// keys before we wire up the OS keychain (deferred to a later spec).
package config
import (
@@ -65,6 +65,7 @@ func (s *FileStore) Load() (domain.AppConfig, error) {
if err := json.Unmarshal(data, &cfg); err != nil {
return domain.DefaultAppConfig(), fmt.Errorf("parse config: %w", err)
}
cfg.Normalize()
return cfg, nil
}
@@ -74,6 +75,7 @@ func (s *FileStore) Save(cfg domain.AppConfig) error {
s.mu.Lock()
defer s.mu.Unlock()
cfg.Normalize()
data, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("marshal config: %w", err)
@@ -0,0 +1,174 @@
// Package llm contains multimodal LLM adapters.
//
// The built-in providers are intentionally implemented through the
// OpenAI-compatible chat/completions shape. Qwen DashScope, Volcengine Ark,
// and OpenAI-compatible gateways all accept the same "text + image_url"
// message content pattern, so one small HTTP adapter is enough here.
package llm
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/mmmy/snapgo/internal/domain"
)
// VisionClient calls one configured multimodal chat endpoint.
type VisionClient struct {
providerID string
cfg domain.LLMProviderConfig
httpClient *http.Client
}
// NewVisionClient validates cfg and returns a reusable client.
func NewVisionClient(providerID string, cfg domain.LLMProviderConfig) (*VisionClient, error) {
if strings.TrimSpace(cfg.BaseURL) == "" {
return nil, fmt.Errorf("llm base url is required")
}
if strings.TrimSpace(cfg.APIKey) == "" {
return nil, fmt.Errorf("llm api key is required")
}
if strings.TrimSpace(cfg.Model) == "" {
return nil, fmt.Errorf("llm model is required")
}
timeout := cfg.TimeoutSecs
if timeout <= 0 {
timeout = 60
}
return &VisionClient{
providerID: providerID,
cfg: cfg,
httpClient: &http.Client{Timeout: time.Duration(timeout) * time.Second},
}, nil
}
// SummarizeImage asks the model to summarise an image available at imageURL.
func (c *VisionClient) SummarizeImage(ctx context.Context, prompt string, imageURL string) (string, error) {
prompt = strings.TrimSpace(prompt)
if prompt == "" {
return "", fmt.Errorf("llm prompt is empty")
}
imageURL = strings.TrimSpace(imageURL)
if imageURL == "" {
return "", fmt.Errorf("image url is empty")
}
body := map[string]any{
"model": c.cfg.Model,
"messages": []map[string]any{
{
"role": "user",
"content": []map[string]any{
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": map[string]any{"url": imageURL}},
},
},
},
}
if c.cfg.MaxTokens > 0 {
if c.providerID == domain.LLMProviderOpenAI {
body["max_completion_tokens"] = c.cfg.MaxTokens
} else {
body["max_tokens"] = c.cfg.MaxTokens
}
}
if c.cfg.Temperature >= 0 {
body["temperature"] = c.cfg.Temperature
}
payload, err := json.Marshal(body)
if err != nil {
return "", fmt.Errorf("marshal llm request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatCompletionsURL(c.cfg.BaseURL), bytes.NewReader(payload))
if err != nil {
return "", fmt.Errorf("build llm request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
resp, err := c.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("llm request: %w", err)
}
defer resp.Body.Close()
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("llm status %d: %s", resp.StatusCode, compactBody(data))
}
var decoded chatCompletionResponse
if err := json.Unmarshal(data, &decoded); err != nil {
return "", fmt.Errorf("decode llm response: %w", err)
}
if decoded.Error != nil && decoded.Error.Message != "" {
return "", fmt.Errorf("llm error: %s", decoded.Error.Message)
}
if len(decoded.Choices) == 0 {
return "", fmt.Errorf("llm response has no choices")
}
text := strings.TrimSpace(contentText(decoded.Choices[0].Message.Content))
if text == "" {
return "", fmt.Errorf("llm response is empty")
}
return text, nil
}
func chatCompletionsURL(baseURL string) string {
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
if strings.HasSuffix(base, "/chat/completions") {
return base
}
return base + "/chat/completions"
}
type chatCompletionResponse struct {
Choices []struct {
Message struct {
Content any `json:"content"`
} `json:"message"`
} `json:"choices"`
Error *struct {
Message string `json:"message"`
Type string `json:"type"`
Code any `json:"code"`
} `json:"error"`
}
func contentText(content any) string {
switch v := content.(type) {
case string:
return v
case []any:
parts := make([]string, 0, len(v))
for _, item := range v {
obj, ok := item.(map[string]any)
if !ok {
continue
}
if text, ok := obj["text"].(string); ok {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n")
default:
return ""
}
}
func compactBody(data []byte) string {
text := strings.TrimSpace(string(data))
if text == "" {
return "<empty body>"
}
if len(text) > 800 {
return text[:800] + "..."
}
return text
}
@@ -0,0 +1,86 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"testing"
"github.com/mmmy/snapgo/internal/domain"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func TestVisionClientSummarizeImageUsesChatCompletionImageURL(t *testing.T) {
var requestBody map[string]any
transport := roundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.URL.Path != "/v1/chat/completions" {
t.Fatalf("expected /v1/chat/completions, got %s", r.URL.Path)
}
if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
t.Fatalf("unexpected auth header %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil {
t.Fatalf("decode request: %v", err)
}
return &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"application/json"}},
Body: io.NopCloser(bytes.NewBufferString(`{"choices":[{"message":{"content":"summary ok"}}]}`)),
}, nil
})
client, err := NewVisionClient(domain.LLMProviderQwen, domain.LLMProviderConfig{
BaseURL: "https://example.test/v1",
APIKey: "test-key",
Model: "qwen-vl-plus",
MaxTokens: 321,
Temperature: 0.2,
TimeoutSecs: 5,
})
if err != nil {
t.Fatalf("new client: %v", err)
}
client.httpClient.Transport = transport
text, err := client.SummarizeImage(context.Background(), "describe", "https://cdn.example.com/a.png")
if err != nil {
t.Fatalf("summarize image: %v", err)
}
if text != "summary ok" {
t.Fatalf("expected response text, got %q", text)
}
if requestBody["model"] != "qwen-vl-plus" {
t.Fatalf("expected model forwarded, got %#v", requestBody["model"])
}
if requestBody["max_tokens"] != float64(321) {
t.Fatalf("expected max_tokens forwarded, got %#v", requestBody["max_tokens"])
}
messages := requestBody["messages"].([]any)
content := messages[0].(map[string]any)["content"].([]any)
if content[0].(map[string]any)["text"] != "describe" {
t.Fatalf("expected prompt content, got %#v", content[0])
}
image := content[1].(map[string]any)
if image["type"] != "image_url" {
t.Fatalf("expected image_url item, got %#v", image)
}
url := image["image_url"].(map[string]any)["url"]
if url != "https://cdn.example.com/a.png" {
t.Fatalf("expected image url forwarded, got %#v", url)
}
}
func TestChatCompletionsURLAcceptsFullEndpoint(t *testing.T) {
full := "https://example.com/v1/chat/completions"
if got := chatCompletionsURL(full); got != full {
t.Fatalf("expected full endpoint unchanged, got %q", got)
}
}
+12
View File
@@ -74,6 +74,18 @@ func nativeOverlaySaveRemote(x, y, w, h C.int, annotationsJSON *C.char) {
}()
}
//export nativeOverlaySummarize
func nativeOverlaySummarize(x, y, w, h C.int, annotationsJSON *C.char) {
app := consumeNativeOverlayApp()
if app == nil {
return
}
result := nativeCaptureResult(x, y, w, h, annotationsJSON)
go func() {
_ = app.SummarizeNativeRegion(result)
}()
}
//export nativeOverlayCancel
func nativeOverlayCancel() {
app := consumeNativeOverlayApp()
+29 -8
View File
@@ -15,6 +15,7 @@ extern void nativeOverlayConfirm(int x, int y, int w, int h, const char *annotat
extern void nativeOverlayCopy(int x, int y, int w, int h, const char *annotationsJSON);
extern void nativeOverlaySave(int x, int y, int w, int h, const char *annotationsJSON, const char *dir);
extern void nativeOverlaySaveRemote(int x, int y, int w, int h, const char *annotationsJSON);
extern void nativeOverlaySummarize(int x, int y, int w, int h, const char *annotationsJSON);
extern void nativeOverlayCancel(void);
static NSWindow *nativeOverlayWindow = nil;
@@ -101,9 +102,8 @@ static id nativeOverlayKeyMonitor = nil;
// as returning a +1 retained object.
@property(strong) NSButton *clipboardButton;
@property(strong) NSButton *saveButton;
// `saveRemoteButton` triggers a placeholder action ("not yet supported")
// per product spec; the icon comes from save-remote.svg.
@property(strong) NSButton *saveRemoteButton;
@property(strong) NSButton *summaryButton;
@property(strong) NSButton *uploadButton;
@property(strong) NSButton *penButton;
@property(strong) NSButton *rectButton;
@@ -120,6 +120,7 @@ static id nativeOverlayKeyMonitor = nil;
- (void)copySelection;
- (void)saveSelection;
- (void)saveRemoteSelection;
- (void)summarizeSelection;
- (void)cancelSelection;
@end
@@ -144,6 +145,7 @@ static id nativeOverlayKeyMonitor = nil;
_clipboardButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(copySelection)];
_saveButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveSelection)];
_saveRemoteButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(saveRemoteSelection)];
_summaryButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(summarizeSelection)];
_uploadButton = [SnipHoverButton buttonWithTitle:@"" target:self action:@selector(confirmSelection)];
// Tooltip strings shown on hover for each action button.
// Localized in Chinese to match the rest of the action UI surface.
@@ -151,6 +153,7 @@ static id nativeOverlayKeyMonitor = nil;
[_clipboardButton setToolTip:@"复制图片"];
[_saveButton setToolTip:@"保存本地"];
[_saveRemoteButton setToolTip:@"保存远端"];
[_summaryButton setToolTip:@"复制总结"];
[_uploadButton setToolTip:@"上传云端"];
_penButton = [NSButton buttonWithTitle:@"" target:self action:@selector(selectPen)];
_rectButton = [NSButton buttonWithTitle:@"" target:self action:@selector(selectRect)];
@@ -171,10 +174,10 @@ static id nativeOverlayKeyMonitor = nil;
// Add the action toolbar background BEFORE the buttons so it sits
// behind them in the view hierarchy (AppKit z-order = subview order).
[self addSubview:_actionToolbarBg];
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) {
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _colorButton, _undoButton, _paletteView, _sizeLabel, _hintLabel]) {
[self addSubview:view];
}
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) {
for (NSView *view in @[_cancelButton, _clipboardButton, _saveButton, _saveRemoteButton, _summaryButton, _uploadButton, _penButton, _rectButton, _ellipseButton, _colorButton, _undoButton, _paletteView, _actionToolbarBg]) {
[view setHidden:YES];
}
[_sizeLabel setHidden:YES];
@@ -248,6 +251,7 @@ static id nativeOverlayKeyMonitor = nil;
// save-remote.svg — provided by the user; its content is the same arrow
// icon as the previous upload, so we reuse it verbatim here.
NSImage *saveRemoteIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M481.749333 353.792c16.682667-16.64 43.690667-16.64 60.373334 0l120.917333 120.832 3.541333 4.010667a42.666667 42.666667 0 0 1-63.872 56.32L554.666667 486.954667V896a42.666667 42.666667 0 0 1-85.333334 0v-409.173333l-48.170666 48.128a42.666667 42.666667 0 0 1-60.330667 0l-3.541333-4.010667a42.666667 42.666667 0 0 1 3.541333-56.32zM512 85.333333c122.538667 0 227.84 73.813333 273.92 179.370667A213.333333 213.333333 0 0 1 725.333333 682.666667a42.666667 42.666667 0 0 1-4.992-85.034667L725.333333 597.333333a128 128 0 0 0 36.394667-250.794666l-38.144-11.264-15.914667-36.437334a213.376 213.376 0 0 0-407.296 58.026667l-7.381333 58.368-57.173333 13.824A85.418667 85.418667 0 0 0 256 597.333333h42.666667a42.666667 42.666667 0 0 1 0 85.333334H256a170.666667 170.666667 0 0 1-40.277333-336.554667A298.709333 298.709333 0 0 1 512 85.333333z' fill='white'/></svg>"];
NSImage *summaryIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M256 128h448l192 192v576H256V128z m64 64v640h512V352H672V192H320z m416 45.248V288h50.752L736 237.248zM384 448h384v64H384v-64z m0 128h384v64H384v-64z m0 128h256v64H384v-64zM192 256v704h512v64H128V256h64z' fill='white'/></svg>"];
// upload.svg — newly replaced "send/cloud" icon supplied by the user.
NSImage *uploadIcon = [self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M550.528 544.128L640 539.776V678.4l178.005333-178.005333L640 322.389333v129.024l-72.618667 10.922667c-155.52 23.424-251.733333 73.557333-312.874666 164.437333 84.906667-50.602667 178.261333-76.928 296.021333-82.645333zM554.666667 629.376c-27.392 1.322667-53.034667 3.968-77.141334 7.808l-8.192 1.365333c-136.704 23.637333-224.810667 87.168-310.954666 176.128-26.154667 27.008-51.882667 59.648-51.882667 59.648-14.848 18.176-24.021333 14.037333-20.352-9.173333 0 0 4.394667-31.402667 11.690667-65.792C144.938667 577.066667 250.581333 423.68 554.666667 377.941333V209.066667a38.4 38.4 0 0 1 65.536-27.136l291.328 291.285333a38.4 38.4 0 0 1 0 54.314667l-291.328 291.285333A38.4 38.4 0 0 1 554.666667 791.637333v-162.261333z' fill='white'/></svg>"];
@@ -255,6 +259,7 @@ static id nativeOverlayKeyMonitor = nil;
[self styleIconButton:_clipboardButton image:copyIcon];
[self styleIconButton:_saveButton image:saveIcon];
[self styleIconButton:_saveRemoteButton image:saveRemoteIcon];
[self styleIconButton:_summaryButton image:summaryIcon];
[self styleIconButton:_uploadButton image:uploadIcon];
// Configure hover colors for the 5 action buttons. White is the shared
@@ -267,11 +272,13 @@ static id nativeOverlayKeyMonitor = nil;
SnipHoverButton *clipboardHB = (SnipHoverButton *)_clipboardButton;
SnipHoverButton *saveHB = (SnipHoverButton *)_saveButton;
SnipHoverButton *saveRemoteHB = (SnipHoverButton *)_saveRemoteButton;
SnipHoverButton *summaryHB = (SnipHoverButton *)_summaryButton;
SnipHoverButton *uploadHB = (SnipHoverButton *)_uploadButton;
cancelHB.baseColor = baseWhite; cancelHB.hoverColor = hoverRed;
clipboardHB.baseColor = baseWhite; clipboardHB.hoverColor = hoverBlue;
saveHB.baseColor = baseWhite; saveHB.hoverColor = hoverBlue;
saveRemoteHB.baseColor = baseWhite; saveRemoteHB.hoverColor = hoverBlue;
summaryHB.baseColor = baseWhite; summaryHB.hoverColor = hoverBlue;
uploadHB.baseColor = baseWhite; uploadHB.hoverColor = hoverBlue;
[self styleIconButton:_penButton image:[self iconFromSVG:@"<svg viewBox='0 0 1024 1024' xmlns='http://www.w3.org/2000/svg'><path d='M742.72 752.064c-29.696 28.576-42.976 48.96-42.976 77.12 0 98.4 143.776 142.464 229.728 65.536l-21.344-23.84c-66.976 59.936-176.384 26.4-176.384-41.696 0-16.832 9.28-31.104 33.184-54.08l13.44-12.736c61.024-58.016 75.328-102.72 29.312-179.84-43.84-73.44-96.8-88.288-162.784-50.56-47.232 27.04-68.64 47.456-208.32 190.4-70.624 72.288-153.92 77.088-217.344 26.784-59.712-47.328-79.872-127.36-42.176-188.64 24.512-39.84 62.656-72.896 136.64-124.48 5.952-4.192 27.04-18.816 31.104-21.664 12.16-8.448 21.536-15.104 30.4-21.536 107.552-78.272 140.8-139.136 86.048-219.904-76.992-113.472-202.304-90.016-367.744 61.472l21.6 23.616c153.088-140.224 256.896-159.616 319.68-67.104 41.632 61.408 16.896 106.688-78.4 176.032-8.64 6.304-17.92 12.832-29.888 21.184l-31.136 21.632c-77.504 54.08-117.984 89.184-145.536 133.984-46.784 76.032-22.176 173.664 49.536 230.496 76.224 60.416 177.952 54.56 260.096-29.504 136.16-139.36 158.08-160.224 201.312-184.96 50.656-28.96 84.416-19.52 119.424 39.168 36.896 61.824 27.52 91.392-23.808 140.16 0.096-0.064-10.976 10.368-13.632 12.96z' fill='white'/></svg>"]];
@@ -591,6 +598,7 @@ static id nativeOverlayKeyMonitor = nil;
[_clipboardButton setHidden:!visible];
[_saveButton setHidden:!visible];
[_saveRemoteButton setHidden:!visible];
[_summaryButton setHidden:!visible];
[_uploadButton setHidden:!visible];
[_actionToolbarBg setHidden:!visible];
[_penButton setHidden:!visible];
@@ -608,12 +616,12 @@ static id nativeOverlayKeyMonitor = nil;
[_sizeLabel setStringValue:[NSString stringWithFormat:@"%.0f × %.0f", _selection.size.width, _selection.size.height]];
[_sizeLabel setFrame:NSMakeRect(_selection.origin.x, MAX(0, _selection.origin.y - 26), 110, 22)];
// Action toolbar layout — 5 icon buttons, each 28x28, separated by 8px,
// with 4px outer padding. Total = 5*28 + 4*8 + 2*4 = 180px wide.
// Action toolbar layout — 6 icon buttons, each 28x28, separated by 8px,
// with 4px outer padding. Total = 6*28 + 5*8 + 2*4 = 216px wide.
CGFloat actionBtnSize = 28;
CGFloat actionGap = 8;
CGFloat actionPad = 4;
NSInteger actionCount = 5;
NSInteger actionCount = 6;
CGFloat toolbarW = actionPad * 2 + actionBtnSize * actionCount + actionGap * (actionCount - 1);
CGFloat toolbarH = 40;
CGFloat x = _selection.origin.x + _selection.size.width - toolbarW;
@@ -631,7 +639,8 @@ static id nativeOverlayKeyMonitor = nil;
[_clipboardButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 1, btnY, actionBtnSize, actionBtnSize)];
[_saveButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 2, btnY, actionBtnSize, actionBtnSize)];
[_saveRemoteButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 3, btnY, actionBtnSize, actionBtnSize)];
[_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)];
[_summaryButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 4, btnY, actionBtnSize, actionBtnSize)];
[_uploadButton setFrame:NSMakeRect(baseX + (actionBtnSize + actionGap) * 5, btnY, actionBtnSize, actionBtnSize)];
// Mark toolbar sits to the LEFT of the action toolbar (same row) with an
// 8px gap between the two groups, so they never overlap on tiny selections.
@@ -799,6 +808,18 @@ static id nativeOverlayKeyMonitor = nil;
NSString *json = [self annotationsJSON];
nativeOverlaySaveRemote((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]);
}
// `summarizeSelection` uploads the screenshot to S3, sends it to the
// configured multimodal model, and copies the generated summary.
- (void)summarizeSelection {
if (!_hasSelection) {
return;
}
NSRect r = [self globalRectForSelection:_selection];
[self closeOverlayWindow];
NSString *json = [self annotationsJSON];
nativeOverlaySummarize((int)llround(r.origin.x), (int)llround(r.origin.y), (int)llround(r.size.width), (int)llround(r.size.height), [json UTF8String]);
}
@end
// screenContainingCursor returns the NSScreen the cursor is currently on,
+163
View File
@@ -0,0 +1,163 @@
//go:build darwin
package main
/*
#cgo CFLAGS: -x objective-c -fobjc-arc -fblocks
#cgo LDFLAGS: -framework AppKit -framework QuartzCore
#include <stdlib.h>
#include <dispatch/dispatch.h>
#import <AppKit/AppKit.h>
#import <QuartzCore/QuartzCore.h>
static NSPanel *operationPanel = nil;
static NSTextField *operationTitleLabel = nil;
static NSTextField *operationDetailLabel = nil;
static NSTextField *operationIconLabel = nil;
static NSProgressIndicator *operationSpinner = nil;
static NSString *statusString(const char *value) {
if (value == NULL) {
return @"";
}
NSString *text = [NSString stringWithUTF8String:value];
return text ?: @"";
}
static void ensureOperationPanel(void) {
if (operationPanel != nil) {
return;
}
NSRect frame = NSMakeRect(0, 0, 360, 118);
operationPanel = [[NSPanel alloc]
initWithContentRect:frame
styleMask:NSWindowStyleMaskBorderless
backing:NSBackingStoreBuffered
defer:NO];
[operationPanel setOpaque:NO];
[operationPanel setBackgroundColor:[NSColor clearColor]];
[operationPanel setLevel:NSFloatingWindowLevel];
[operationPanel setHidesOnDeactivate:NO];
[operationPanel setCollectionBehavior:
NSWindowCollectionBehaviorCanJoinAllSpaces |
NSWindowCollectionBehaviorFullScreenAuxiliary |
NSWindowCollectionBehaviorStationary];
NSView *root = [[NSView alloc] initWithFrame:frame];
[root setWantsLayer:YES];
[[root layer] setCornerRadius:14];
[[root layer] setBackgroundColor:[[NSColor colorWithCalibratedWhite:0.08 alpha:0.92] CGColor]];
[[root layer] setShadowColor:[[NSColor blackColor] CGColor]];
[[root layer] setShadowOpacity:0.28];
[[root layer] setShadowRadius:18];
[[root layer] setShadowOffset:CGSizeMake(0, -8)];
operationSpinner = [[NSProgressIndicator alloc] initWithFrame:NSMakeRect(24, 42, 34, 34)];
[operationSpinner setStyle:NSProgressIndicatorStyleSpinning];
[operationSpinner setIndeterminate:YES];
operationIconLabel = [NSTextField labelWithString:@"OK"];
[operationIconLabel setFrame:NSMakeRect(21, 43, 42, 30)];
[operationIconLabel setAlignment:NSTextAlignmentCenter];
[operationIconLabel setFont:[NSFont systemFontOfSize:17 weight:NSFontWeightSemibold]];
[operationIconLabel setTextColor:[NSColor colorWithCalibratedRed:74.0/255.0 green:222.0/255.0 blue:128.0/255.0 alpha:1.0]];
[operationIconLabel setHidden:YES];
operationTitleLabel = [NSTextField labelWithString:@""];
[operationTitleLabel setFrame:NSMakeRect(76, 61, 260, 24)];
[operationTitleLabel setFont:[NSFont systemFontOfSize:15 weight:NSFontWeightSemibold]];
[operationTitleLabel setTextColor:[NSColor whiteColor]];
operationDetailLabel = [NSTextField labelWithString:@""];
[operationDetailLabel setFrame:NSMakeRect(76, 34, 260, 22)];
[operationDetailLabel setFont:[NSFont systemFontOfSize:12 weight:NSFontWeightRegular]];
[operationDetailLabel setTextColor:[NSColor colorWithCalibratedWhite:0.82 alpha:1.0]];
[operationDetailLabel setLineBreakMode:NSLineBreakByTruncatingTail];
[root addSubview:operationSpinner];
[root addSubview:operationIconLabel];
[root addSubview:operationTitleLabel];
[root addSubview:operationDetailLabel];
[operationPanel setContentView:root];
}
static void snipShowOperationStatus(const char *titleC, const char *detailC, int state) {
NSString *title = statusString(titleC);
NSString *detail = statusString(detailC);
dispatch_async(dispatch_get_main_queue(), ^{
ensureOperationPanel();
[operationTitleLabel setStringValue:title];
[operationDetailLabel setStringValue:detail];
if (state == 0) {
[operationIconLabel setHidden:YES];
[operationSpinner setHidden:NO];
[operationSpinner startAnimation:nil];
} else {
[operationSpinner stopAnimation:nil];
[operationSpinner setHidden:YES];
[operationIconLabel setHidden:NO];
if (state == 2) {
[operationIconLabel setStringValue:@"!"];
[operationIconLabel setTextColor:[NSColor colorWithCalibratedRed:248.0/255.0 green:113.0/255.0 blue:113.0/255.0 alpha:1.0]];
} else {
[operationIconLabel setStringValue:@"OK"];
[operationIconLabel setTextColor:[NSColor colorWithCalibratedRed:74.0/255.0 green:222.0/255.0 blue:128.0/255.0 alpha:1.0]];
}
}
NSScreen *screen = [NSScreen mainScreen];
if (screen != nil) {
NSRect visible = [screen visibleFrame];
NSRect panelFrame = [operationPanel frame];
panelFrame.origin.x = NSMidX(visible) - panelFrame.size.width / 2;
panelFrame.origin.y = NSMaxY(visible) - panelFrame.size.height - 48;
[operationPanel setFrame:panelFrame display:YES];
}
[operationPanel orderFrontRegardless];
});
}
static void snipHideOperationStatus(void) {
dispatch_async(dispatch_get_main_queue(), ^{
if (operationPanel != nil) {
[operationSpinner stopAnimation:nil];
[operationPanel orderOut:nil];
}
});
}
*/
import "C"
import (
"sync/atomic"
"time"
"unsafe"
)
const (
operationStatusRunning = iota
operationStatusSuccess
operationStatusError
)
var operationStatusSeq atomic.Uint64
func showOperationStatus(title, detail string, state int) {
operationStatusSeq.Add(1)
ctitle := C.CString(title)
cdetail := C.CString(detail)
defer C.free(unsafe.Pointer(ctitle))
defer C.free(unsafe.Pointer(cdetail))
C.snipShowOperationStatus(ctitle, cdetail, C.int(state))
}
func hideOperationStatusAfter(delay time.Duration) {
seq := operationStatusSeq.Load()
time.AfterFunc(delay, func() {
if operationStatusSeq.Load() == seq {
C.snipHideOperationStatus()
}
})
}
+15
View File
@@ -0,0 +1,15 @@
//go:build !darwin
package main
import "time"
const (
operationStatusRunning = iota
operationStatusSuccess
operationStatusError
)
func showOperationStatus(_ string, _ string, _ int) {}
func hideOperationStatusAfter(_ time.Duration) {}