diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..96879df --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + name: ${{ matrix.os }}-${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: x86_64 + runner: ubuntu-24.04 + - os: linux + arch: arm64 + runner: ubuntu-24.04-arm + - os: macos + arch: x86_64 + runner: macos-13 + - os: macos + arch: arm64 + runner: macos-14 + steps: + - uses: actions/checkout@v4 + + - name: Install Nim + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: stable + + - name: Install system deps (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Build + run: nimble release + + - name: Smoke test + run: | + ./build/lazybookmarks --help + ./build/lazybookmarks status + + - uses: actions/upload-artifact@v4 + with: + name: lazybookmarks-${{ matrix.os }}-${{ matrix.arch }} + path: build/lazybookmarks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..88fbc81 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,73 @@ +name: Release + +on: + push: + tags: ["v*"] + +permissions: + contents: write + +jobs: + build: + name: ${{ matrix.os }}-${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: x86_64 + runner: ubuntu-24.04 + - os: linux + arch: arm64 + runner: ubuntu-24.04-arm + - os: macos + arch: x86_64 + runner: macos-13 + - os: macos + arch: arm64 + runner: macos-14 + steps: + - uses: actions/checkout@v4 + + - name: Install Nim + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: stable + + - name: Install system deps (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Build + run: nimble release + + - name: Package + run: | + mkdir -p dist + tar -czf "dist/lazybookmarks-${{ matrix.os }}-${{ matrix.arch }}.tar.gz" -C build lazybookmarks + ( cd dist && shasum -a 256 "lazybookmarks-${{ matrix.os }}-${{ matrix.arch }}.tar.gz" > "lazybookmarks-${{ matrix.os }}-${{ matrix.arch }}.tar.gz.sha256" ) + + - uses: actions/upload-artifact@v4 + with: + name: lazybookmarks-${{ matrix.os }}-${{ matrix.arch }} + path: dist/* + + release: + name: Publish + needs: build + runs-on: ubuntu-24.04 + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Generate SHA256SUMS + run: ( cd dist && cat *.sha256 > SHA256SUMS ) + + - name: Create release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: dist/* diff --git a/.gitignore b/.gitignore index 991ade9..fade97d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ *.gguf /*.html +/build/ +/dist/ +/lazybookmarks/ +/.opencode/plans diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8df15ae --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,141 @@ +# AGENTS.md — Lazybookmarks Development Guide + +## Project Overview +- Chrome extension (AI-powered bookmark organizer using Gemini Nano) ported to standalone Nim CLI +- Target: Linux arm64, dev on macOS arm64 +- Dependencies: cligen, db_connector, jsony (only 3) +- Build: `nimble release` (NOT `nimble build`) + +## Build & Toolchain +- Nim 2.2.8 via Homebrew +- `nimble release` outputs to `build/lazybookmarks` +- Do NOT use `nimble build` — it ignores custom tasks +- `nim.cfg`: --opt:size, --mm:orc, NO -d:ssl +- `nimble release` auto-resolves dependencies + +## LLM Backend +- Default: Ollama native `/api/chat` (constrained decoding via `format` param) +- OpenAI-compatible `/v1/chat/completions` fallback when `runtimeManaged=false` (LLM_URL set) +- `format` param = grammar-based constrained decoding (model physically cannot generate invalid tokens) +- `"options": {"think": false}` suppresses qwen3.5 thinking mode +- Model lineup: qwen3.5:0.8b, qwen3.5:2b (default), qwen3.5:4b, gemma4:e2b +- Model managed via `ollama pull`, not custom download code + +## Architecture +- Config priority: CLI > env vars > config.toml > defaults +- `runtimeManaged` flag: true = Ollama (native endpoints), false = custom LLM_URL (OpenAI endpoints) +- Link checking: `curl` + `xargs -P` (Nim SSL broken with OpenSSL 3.6+) +- `__skip__` handling: bookmarks classified as `__skip__` remain `organised_at = NULL` + +## 3-Phase Pipeline Internals + +The organize command (`organizer.nim`) runs a 3-phase pipeline to classify bookmarks into folders. + +### Phase 1: Taxonomy Analysis (`runTaxonomyPhase`) +- **Input:** All folders with their bookmarks, enriched with TF-IDF keywords, domain patterns, and exemplar bookmarks +- **Process:** Single LLM call asking the model to describe each folder and provide keywords +- **Schema:** `TaxonomySchemaJson` — array of `{folderId, folderPath, description, keywords[]}` +- **Caching:** Results keyed by a fingerprint of folder UUIDs + bookmark counts (`buildFingerprint`). Cache stored in `taxonomy_cache` table. Survives across runs unless folder structure changes. +- **Key helpers:** `computeTFIDF` (term frequency-inverse document frequency per folder), `extractDomainPatterns` (top domains per folder above 20% threshold), `sampleExemplars` (2 most recent bookmark titles/urls per folder) + +### Phase 1.5: Cluster/Theme Grouping (`runClusterPhase`) +- **Input:** All unorganized bookmarks, existing taxonomy categories, root-level folders +- **Process:** Single LLM call to identify 2-6 thematic groups among unorganized bookmarks that deserve a new folder +- **Schema:** `buildClusterSchemaJson(rootFolderIds)` — array of `{name, description, keywords[], parentFolderId}`. The `parentFolderId` is constrained to root folder UUIDs via JSON enum. +- **Output:** `seq[ClusterSuggestion]` — these become synthetic folders prefixed with `__new_` (e.g., `__new_Hardware`) in the taxonomy for Phase 2 + +### Phase 2: Per-Bookmark Classification (`runClassificationPhase`) +- **Input:** Unorganized bookmarks (chunked into batches), full taxonomy (original + new cluster folders) +- **Process:** For each batch, calls `pruneTaxonomy` to reduce the folder list to the most relevant ~15 folders (based on keyword overlap with the batch's titles via TF-IDF), then asks the LLM to classify each bookmark +- **Schema:** `buildClassificationSchemaJson(folderIds, bookmarkIds)` — array of `{bookmarkId, targetFolderId, confidence, reason}`. Both `bookmarkId` and `targetFolderId` are constrained to exact IDs via JSON enum, plus `"__skip__"` as a valid target. +- **Concurrency:** Uses `classifyBatchAsync` with sliding window — up to `concurrency` (default 4) batches in flight simultaneously via `AsyncHttpClient`. Batch size auto-set by model size (5 for small, 10 for normal). +- **`pruneTaxonomy`:** Scores each taxonomy category by how many of its TF-IDF keywords appear in the batch's token set. Keeps top N (max 15, min 5). This prevents overwhelming small models with 30+ folder options. +- **Bookmarks classified as `__skip__`** are silently dropped (not applied). Low-confidence matches can be reviewed interactively or auto-skipped. + +### Data Flow +1. `organizeBookmarks` loads all bookmarks, builds `folderBookmarks` table mapping folder UUID → bookmark entries +2. Phase 1 enriches folders with TF-IDF/domain/exemplar data, runs LLM, caches result +3. Phase 1.5 takes unorganized bookmarks + taxonomy + root folders, suggests new folders +4. Phase 2 merges new folders into taxonomy, chunks unorganized bookmarks, classifies each batch with pruned taxonomy +5. Results become `seq[Suggestion]` with `bookmarkId`, `targetFolderId`, `targetFolderPath`, `confidence`, `reason`, `isNewFolder` +6. Suggestions are applied via `applyClassification` (sets `category`, `confidence`, `reason`, `organised_at` on the bookmark row) + +### Key Types +- `TaxonomyCategory`: folderId, folderPath, description, keywords +- `ClusterSuggestion`: name, description, keywords, parentFolderId +- `Classification`: bookmarkId, targetFolderId, confidence, reason +- `Suggestion`: bookmarkId, bookmarkTitle, bookmarkUrl, targetFolderId, targetFolderPath, confidence, reason, isNewFolder + +## Nim Language Pitfalls (The Hard-Won Lessons) + +### Syntax +- `.[^1]` is Python, not Nim — use `seq[seq.len - 1]` +- `mapIt` cannot have multi-line blocks — use explicit `for` loops +- `=>` lambda syntax doesn't exist in Nim +- Anonymous tuple fields can't be accessed by name (`.score`) — use `[0]`, `[1]` +- `findIt` on seqs returns `int` (index), not the element — use `seq[index]` +- `{}` set literals only support values 0..255 — HTTP codes 302/404/410 must use `==` chains +- `re.match` requires full-string match — use `re.find` for substring matching +- `&"..."` format strings require `strformat` import +- Variable names can't conflict with keywords (e.g., `file` parameter) + +### Standard Library +- `std/terminal` has `hideCursor`/`showCursor` templates that conflict with custom procs +- `postContent` doesn't take a `headers` param — set `client.headers` before calling +- `HttpClient` has no `onProgress` field +- `filterIt`/`mapIt` are in `std/sequtils`, NOT `std/sugar` in Nim 2.2.8 +- `split()` requires `strutils` import +- `parseFloat` requires `strutils` import +- `sort()` requires `std/algorithm` import +- `sum()` doesn't exist as a standalone proc — use manual loop with `.inc` +- `sleep` → `os.sleep` or `execShellCmd` +- `execShellCmd` is in `std/os`, not `std/osproc` +- `/` operator is for filesystem paths, not URL concatenation — use `&` +- `reversed()` doesn't exist — use manual reverse loop with index +- `rfind` doesn't exist — use manual loop or reverse approach +- `chunk` doesn't exist — implement manually +- `formatFloat` uses `precision` not `ffDecimal` named param + +### Async +- `AsyncHttpClient` is inside `std/httpclient`, no built-in timeout +- `withTimeout(fut, ms)` returns `Future[bool]` — true if completed +- `one()` doesn't exist — use polling with `sleepAsync` + `.finished` +- Async macro can't capture `var` parameters — use return values +- `pump()` closures capturing locals from enclosing procs violate borrow checker — inline +- `std/channels` doesn't exist in 2.2.8; `threadpool` is deprecated +- `waitFor()` requires `std/asyncdispatch` + +### JSON & Data +- `HttpHeaders` doesn't have 3-arg `get(key, default)` — use `hasKey` + `[]` +- `HttpCode` is `range[0..255]` — can't use in `{}` +- `toHex` from `nimcrypto/utils` returns uppercase — `toLowerAscii()` for comparisons + +### Build System +- `nimble build` always runs its own default build via `bin` field — custom `build` tasks are ignored +- `self.exec` is old nimble syntax — use just `exec` +- `--out:path` doesn't work in `nim.cfg` — only `--outdir:dir` +- Circular imports cause "undeclared identifier" — restructure to avoid cycles +- Forward declarations needed for procs called before their definition +- Inline `if` in string concatenation within proc call arguments doesn't work — extract to `let` binding + +### SSL +- Nim 2.2.8 SSL bindings are incompatible with OpenSSL 3.6+ +- Do NOT add `-d:ssl` — use `curl` via `execShellCmd` for HTTPS + +## Code Conventions +- No comments unless asked +- Imports grouped: std/*, then local ./* modules +- Procs use `*` export marker for public API +- CLI subcommands use `cmdXxx` naming, flat names in dispatchMulti (e.g., `model-list`) + +## Testing +- Manual e2e testing with real bookmark data (692 bookmarks) +- `./build/lazybookmarks --verbose` for debug output +- Smoke test: `./build/lazybookmarks status` / `./build/lazybookmarks --help` + +## Ollama API Reference (Native Endpoints) +- Chat: `POST /api/chat` with `format` for structured output, `stream: false` +- Models: `GET /api/tags` +- Health: `GET /api/tags` (200 = running) +- Pull: `ollama pull :` (CLI, not API) +- Response: `{"message": {"content": "..."}}` (not OpenAI's `choices[0].message.content`) diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000..78132d9 --- /dev/null +++ b/BUILD.md @@ -0,0 +1,43 @@ +# Building lazybookmarks + +## Prerequisites + +- **Nim** >= 2.0.0 — https://nim-lang.org/install.html +- **Ollama** — https://ollama.com/download (runtime dependency, not build-time) +- **curl** — required at runtime for `check-links` (link health checking) + +### macOS + +```sh +brew install nim ollama curl +``` + +### Ubuntu/Debian + +```sh +sudo apt install nim curl +curl -fsSL https://ollama.com/install.sh | sh +``` + +### Arch Linux + +```sh +sudo pacman -S nim curl +yay -S ollama-cuda # or ollama-rocm for AMD +``` + +## Build + +```sh +nimble release +``` + +The binary will be at `build/lazybookmarks`. + +## Cross-compiling for Linux (from macOS) + +Install a Linux cross-compiler, then: + +```sh +nim c -d:release --os:linux --cpu:arm64 -o:build/lazybookmarks-linux-arm64 src/lazybookmarks/main.nim +``` diff --git a/assets/models.json b/assets/models.json new file mode 100644 index 0000000..31ac120 --- /dev/null +++ b/assets/models.json @@ -0,0 +1,24 @@ +{ + "entries": [ + { + "name": "qwen3.5-0.8b", + "ollamaModel": "qwen3.5", + "ollamaTag": "0.8b" + }, + { + "name": "qwen3.5-2b", + "ollamaModel": "qwen3.5", + "ollamaTag": "2b" + }, + { + "name": "qwen3.5-4b", + "ollamaModel": "qwen3.5", + "ollamaTag": "4b" + }, + { + "name": "gemma4-e2b", + "ollamaModel": "gemma4", + "ollamaTag": "e2b" + } + ] +} diff --git a/lazybookmarks.nimble b/lazybookmarks.nimble new file mode 100644 index 0000000..ae90c35 --- /dev/null +++ b/lazybookmarks.nimble @@ -0,0 +1,23 @@ +# Package + +version = "0.1.0" +author = "corv89" +description = "CLI bookmark organizer powered by local LLM" +license = "MIT" +srcDir = "src" +bin = @["lazybookmarks/main"] + +# Dependencies + +requires "nim >= 2.0.0" +requires "cligen >= 1.6" +requires "db_connector >= 0.1" +requires "jsony >= 1.1" + +# Tasks + +task release, "Build release binary to build/": + exec "nim c -d:release -o:build/lazybookmarks src/lazybookmarks/main.nim" + +task debug, "Build debug binary to build/": + exec "nim c -o:build/lazybookmarks src/lazybookmarks/main.nim" diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..a57e87d --- /dev/null +++ b/nim.cfg @@ -0,0 +1,2 @@ +--opt:size +--mm:orc diff --git a/src/lazybookmarks/bootstrap.nim b/src/lazybookmarks/bootstrap.nim new file mode 100644 index 0000000..f16cb91 --- /dev/null +++ b/src/lazybookmarks/bootstrap.nim @@ -0,0 +1,18 @@ +import ./config +import ./model +import ./runtime +import ./ui + +proc ensureReady*(cfg: var Config, registry: ModelRegistry) = + if not cfg.runtimeManaged: + return + + requireRuntime(cfg) + + let entry = findModel(registry, cfg.modelVariant) + cfg.modelName = ollamaRef(entry) + + if not isEntryReady(entry, cfg): + pullModel(entry) + else: + infoMsg "Model ready: " & ollamaRef(entry) diff --git a/src/lazybookmarks/client.nim b/src/lazybookmarks/client.nim new file mode 100644 index 0000000..95d1f55 --- /dev/null +++ b/src/lazybookmarks/client.nim @@ -0,0 +1,215 @@ +import std/[httpclient, json, os, re, strutils, asyncdispatch] +import ./config + +type + Message* = object + role*: string + content*: string + +proc stripThinkTags*(s: string): string = + result = s + result = result.replace(re"💭[\s\S]*?💭", "") + result = result.replace(re"]*>", "") + result = result.replace(re"[\s\S]*?", "") + result = result.replace(re"```json\s*", "") + result = result.replace(re"```\s*$", "") + result = result.strip() + +proc closeJson*(s: string): string = + var opens: seq[char] = @[] + var inStr = false + var j = 0 + while j < s.len: + if not inStr: + case s[j] + of '{', '[': opens.add(s[j]) + of '}': + if opens.len > 0 and opens[opens.high] == '{': discard opens.pop() + of ']': + if opens.len > 0 and opens[opens.high] == '[': discard opens.pop() + of '"': inStr = true + else: discard + else: + if s[j] == '"' and (j == 0 or s[j - 1] != '\\'): + inStr = false + inc j + result = s + var k = opens.high + while k >= 0: + let closing = if opens[k] == '{': '}' else: ']' + result.add(closing) + dec k + +proc extractJson*(s: string): string = + var cleaned = stripThinkTags(s) + let start = cleaned.find('{') + if start < 0: + return "" + var depth = 0 + var inStr = false + var endPos = -1 + for i in start .. cleaned.high: + let c = cleaned[i] + if inStr: + if c == '"' and (i == 0 or cleaned[i - 1] != '\\'): + inStr = false + else: + case c + of '"': inStr = true + of '{': inc depth + of '}': + dec depth + if depth == 0: + endPos = i + break + else: discard + if endPos < 0: + return closeJson(cleaned[start .. cleaned.high]) + return cleaned[start .. endPos] + +proc buildRequestBody(cfg: Config, messages: seq[Message], jsonSchema: string): JsonNode = + let native = cfg.runtimeManaged + if native: + result = %*{ + "model": cfg.modelName, + "messages": messages, + "stream": false, + "temperature": 0.1, + "options": { + "think": false, + }, + } + if jsonSchema.len > 0: + result["format"] = parseJson(jsonSchema) + else: + result = %*{ + "model": cfg.modelName, + "messages": messages, + "temperature": 0.1, + "max_tokens": 2048, + "options": { + "think": false, + }, + } + if jsonSchema.len > 0: + result["response_format"] = %*{ + "type": "json_schema", + "json_schema": { + "strict": true, + "schema": parseJson(jsonSchema), + } + } + +proc chatUrl(cfg: Config): string = + if cfg.runtimeManaged: + cfg.ollamaApiUrl() & "/api/chat" + else: + cfg.llmUrl & "/chat/completions" + +proc extractContent(parsed: JsonNode, native: bool): string = + if native: + parsed["message"]["content"].getStr() + else: + parsed["choices"][0]["message"]["content"].getStr() + +proc parseChatResponse(response: string, native: bool, verbose: bool, prefix: string): JsonNode = + let parsed = parseJson(response) + let rawContent = extractContent(parsed, native) + let content = extractJson(rawContent) + if content.len > 0: + if verbose: + stderr.writeLine("[" & prefix & "] response: " & content[0..min(200, content.high)]) + return parseJson(content) + else: + raise newException(CatchableError, "No JSON found in response: " & rawContent[0..min(200, rawContent.high)]) + +proc chatCompletion*(cfg: Config, messages: seq[Message], + jsonSchema: string = "", + maxRetries: int = 3): JsonNode = + let body = buildRequestBody(cfg, messages, jsonSchema) + let native = cfg.runtimeManaged + + let client = newHttpClient(timeout = 120000) + client.headers = newHttpHeaders([("Content-Type", "application/json")]) + defer: client.close() + + let url = chatUrl(cfg) + if cfg.verbose: + stderr.writeLine("[chat] POST " & url & " model=" & cfg.modelName) + for m in messages: + stderr.writeLine("[chat] " & m.role & ": " & m.content[0..min(300, m.content.high)]) + + var lastError = "" + for attempt in 1..maxRetries: + try: + let response = client.postContent(url, body = $body) + + if cfg.verbose: + stderr.writeLine("[attempt " & $attempt & "] -> " & $response.len & " bytes") + + return parseChatResponse(response, native, cfg.verbose, "chat") + except CatchableError as e: + lastError = e.msg + if cfg.verbose: + stderr.writeLine("[attempt " & $attempt & "] Error: " & e.msg) + if attempt < maxRetries: + let delay = 1000 * (1 shl (attempt - 1)) + discard execShellCmd("sleep " & $(delay div 1000)) + + raise newException(CatchableError, "chatCompletion failed after " & $maxRetries & " attempts: " & lastError) + +proc chatCompletionSimple*(cfg: Config, systemPrompt: string, userMessage: string, + jsonSchema: string = ""): JsonNode = + let messages = @[ + Message(role: "system", content: systemPrompt), + Message(role: "user", content: userMessage), + ] + return chatCompletion(cfg, messages, jsonSchema) + +proc chatCompletionAsync*(cfg: Config, messages: seq[Message], + jsonSchema: string = "", + maxRetries: int = 3): Future[JsonNode] {.async.} = + let body = buildRequestBody(cfg, messages, jsonSchema) + let native = cfg.runtimeManaged + let url = chatUrl(cfg) + + if cfg.verbose: + stderr.writeLine("[chat-async] POST " & url & " model=" & cfg.modelName) + + var lastError = "" + for attempt in 1..maxRetries: + try: + let client = newAsyncHttpClient() + client.headers = newHttpHeaders([("Content-Type", "application/json")]) + + let postFut = client.postContent(url, body = $body) + let timedOut = not await withTimeout(postFut, 120_000) + client.close() + + if timedOut: + raise newException(CatchableError, "Request timed out (120s)") + + let response = postFut.read() + + if cfg.verbose: + stderr.writeLine("[attempt " & $attempt & "] -> " & $response.len & " bytes") + + return parseChatResponse(response, native, cfg.verbose, "chat-async") + except CatchableError as e: + lastError = e.msg + if cfg.verbose: + stderr.writeLine("[attempt " & $attempt & "] Error: " & e.msg) + + if attempt < maxRetries: + let delay = 1000 * (1 shl (attempt - 1)) + await sleepAsync(delay) + + raise newException(CatchableError, "chatCompletionAsync failed after " & $maxRetries & " attempts: " & lastError) + +proc chatCompletionSimpleAsync*(cfg: Config, systemPrompt: string, userMessage: string, + jsonSchema: string = ""): Future[JsonNode] {.async.} = + let messages = @[ + Message(role: "system", content: systemPrompt), + Message(role: "user", content: userMessage), + ] + return await chatCompletionAsync(cfg, messages, jsonSchema) diff --git a/src/lazybookmarks/config.nim b/src/lazybookmarks/config.nim new file mode 100644 index 0000000..390cbc0 --- /dev/null +++ b/src/lazybookmarks/config.nim @@ -0,0 +1,137 @@ +import std/[os, re, strutils] + +type ParamSize* = enum psSmall, psNormal + +type Config* = object + llmUrl*: string + modelVariant*: string + modelName*: string + dataDir*: string + runtimeManaged*: bool + autoAcceptHigh*: bool + batchSize*: int + concurrency*: int + verbose*: bool + paramSize*: ParamSize + +proc parseParamSize*(variant: string): ParamSize = + for m in variant.findAll(re"[\d.]+[bB]"): + let numPart = m[0 ..< m.len - 1] + try: + if parseFloat(numPart) < 1.5: return psSmall + except: + discard + return psNormal + +proc isSmallModel*(cfg: Config): bool = + cfg.paramSize == psSmall + +const DefaultLlmUrl* = "http://127.0.0.1:11434" +const DefaultModelVariant* = "qwen3.5-2b" +const DefaultConcurrency* = 4 + +proc ollamaApiUrl*(cfg: Config): string = + if cfg.llmUrl.endsWith("/v1"): + cfg.llmUrl[0 ..< cfg.llmUrl.len - 3] + elif cfg.llmUrl.endsWith("/"): + cfg.llmUrl[0 ..< cfg.llmUrl.len - 1] + else: + cfg.llmUrl + +proc xdgDataHome*: string = + result = getEnv("XDG_DATA_HOME") + if result.len == 0: + result = getHomeDir() / ".local" / "share" + +proc xdgConfigHome*: string = + result = getEnv("XDG_CONFIG_HOME") + if result.len == 0: + result = getHomeDir() / ".config" + +proc defaultDataDir*: string = + xdgDataHome() / "lazybookmarks" + +proc defaultConfigDir*: string = + xdgConfigHome() / "lazybookmarks" + +proc ensureDir*(dir: string) = + createDir(dir) + +proc readTomlString(content: string, key: string): string = + for line in content.splitLines(): + let stripped = line.strip() + if stripped.startsWith(key & " = "): + var val = stripped[key.len + 3 .. stripped.high].strip() + if val.startsWith("\"") and val.endsWith("\""): + val = val[1 ..< val.high] + return val + return "" + +proc readTomlBool(content: string, key: string): bool = + let val = readTomlString(content, key) + return val == "true" + +proc loadConfig*(overrides: Config = Config()): Config = + let configPath = defaultConfigDir() / "config.toml" + var fileModel = "" + var fileAutoAccept = false + if fileExists(configPath): + let content = readFile(configPath) + fileModel = readTomlString(content, "modelVariant") + fileAutoAccept = readTomlBool(content, "autoAcceptHigh") + + let variant = if overrides.modelVariant.len > 0: overrides.modelVariant + elif getEnv("LB_MODEL").len > 0: getEnv("LB_MODEL") + elif fileModel.len > 0: fileModel + else: DefaultModelVariant + let ps = parseParamSize(variant) + let defaultBatch = if ps == psSmall: 5 else: 10 + result = Config( + llmUrl: DefaultLlmUrl, + modelVariant: variant, + dataDir: defaultDataDir(), + runtimeManaged: true, + autoAcceptHigh: fileAutoAccept, + batchSize: defaultBatch, + concurrency: DefaultConcurrency, + verbose: false, + paramSize: ps, + ) + + let envLlmUrl = getEnv("LLM_URL") + if envLlmUrl.len > 0: + result.llmUrl = envLlmUrl + result.runtimeManaged = false + + let envModel = getEnv("LB_MODEL") + if envModel.len > 0: + result.modelVariant = envModel + result.paramSize = parseParamSize(envModel) + + let envDataDir = getEnv("LB_DATA_DIR") + if envDataDir.len > 0: + result.dataDir = envDataDir + + let envAutoAccept = getEnv("LB_AUTO_ACCEPT") + if envAutoAccept.len > 0: + result.autoAcceptHigh = true + + if overrides.llmUrl.len > 0 and overrides.llmUrl != DefaultLlmUrl: + result.llmUrl = overrides.llmUrl + result.runtimeManaged = false + if overrides.modelVariant.len > 0: + result.modelVariant = overrides.modelVariant + result.paramSize = parseParamSize(overrides.modelVariant) + if overrides.dataDir.len > 0: + result.dataDir = overrides.dataDir + if overrides.batchSize > 0: + result.batchSize = overrides.batchSize + if overrides.concurrency > 0: + result.concurrency = overrides.concurrency + if overrides.verbose: + result.verbose = true + if overrides.autoAcceptHigh: + result.autoAcceptHigh = true + +proc dbPath*(cfg: Config): string = + cfg.dataDir / "bookmarks.db" diff --git a/src/lazybookmarks/linkchecker.nim b/src/lazybookmarks/linkchecker.nim new file mode 100644 index 0000000..a48e3d0 --- /dev/null +++ b/src/lazybookmarks/linkchecker.nim @@ -0,0 +1,119 @@ +import std/[strutils, os, tables] +import ./config +import ./storage + +type LinkStatus* = enum + lsAlive, lsDead, lsUnknown, lsRedirected + +type LinkResult* = object + bookmark*: BookmarkEntry + status*: LinkStatus + statusCode*: int + redirectUrl*: string + +const TrackingParams = [ + "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", + "fbclid", "gclid", "msclkid", +] + +proc checkBatch(urls: seq[string]): string = + if urls.len == 0: + return "" + let tmpFile = getTempDir() / "lbcheck_urls.txt" + let outFile = getTempDir() / "lbcheck_results.txt" + try: + var f: File + if not open(f, tmpFile, fmWrite): + return "" + for u in urls: + f.writeLine(u) + f.close() + + let cmd = "cat '" & tmpFile & "' | xargs -P " & $urls.len & + " -I{} curl -sI -o /dev/null -w '%{http_code}\\t%{redirect_url}\\t{}\\n' " & + "--max-time 10 -L --max-redirs 5 -A 'Mozilla/5.0 (compatible; lazybookmarks/0.1)' '{}' > '" & outFile & "' 2>/dev/null" + discard execShellCmd(cmd) + + if not fileExists(outFile): + return "" + result = readFile(outFile) + except: + discard + finally: + try: removeFile(tmpFile) + except: discard + try: removeFile(outFile) + except: discard + +proc classifyCode(code: int, redirectUrl: string): LinkStatus = + if code >= 200 and code < 400: + if redirectUrl.len > 0: + var isTracking = false + for tp in TrackingParams: + if redirectUrl.contains("?" & tp & "=") or redirectUrl.contains("&" & tp & "="): + isTracking = true + break + if isTracking: + return lsAlive + return lsRedirected + return lsAlive + if code == 404 or code == 410: + return lsDead + return lsUnknown + +proc checkAllLinks*(cfg: Config, bookmarks: seq[BookmarkEntry], + concurrency: int = 8, + onProgress: proc(current, total: int) = nil): seq[LinkResult] = + result = newSeq[LinkResult](bookmarks.len) + let total = bookmarks.len + if total == 0: + return + + let batchSize = min(concurrency * 4, total) + var offset = 0 + var done = 0 + + while offset < total: + let endIdx = min(offset + batchSize, total) + var urls: seq[string] = @[] + for i in offset ..< endIdx: + urls.add(bookmarks[i].url) + + let content = checkBatch(urls) + var resultMap: Table[string, tuple[code: int, redirectUrl: string]] = initTable[string, tuple[code: int, redirectUrl: string]]() + + for line in content.splitLines(): + if line.len == 0: + continue + let parts = line.split("\t", maxsplit = 2) + if parts.len < 3: + continue + var code = 0 + try: code = parseInt(parts[0]) + except: continue + let redirect = parts[1] + let url = parts[2] + if url.len > 0: + resultMap[url] = (code, redirect) + + for i in offset ..< endIdx: + let url = bookmarks[i].url + if url in resultMap: + let (code, redirect) = resultMap[url] + result[i] = LinkResult( + bookmark: bookmarks[i], + status: classifyCode(code, redirect), + statusCode: code, + redirectUrl: redirect, + ) + else: + result[i] = LinkResult( + bookmark: bookmarks[i], + status: lsUnknown, + statusCode: 0, + ) + + done = endIdx + offset = endIdx + if onProgress != nil: + onProgress(done, total) diff --git a/src/lazybookmarks/main.nim b/src/lazybookmarks/main.nim new file mode 100644 index 0000000..668a7a6 --- /dev/null +++ b/src/lazybookmarks/main.nim @@ -0,0 +1,327 @@ +import std/[os, strutils, terminal, strformat, sequtils, algorithm] +import cligen +import ./config +import ./storage +import ./model +import ./runtime +import ./bootstrap +import ./organizer +import ./linkchecker +import ./ui + +proc cmdImport(file: string, format = "auto", dryRun = false) = + if not fileExists(file): + errorMsg &"File not found: {file}" + quit(1) + + let cfg = loadConfig() + let content = readFile(file) + + let detectedFormat = if format == "auto": detectFormat(content, file) else: format + + if dryRun: + let parsed = parseImport(content, detectedFormat) + infoMsg &"Would import {parsed.len} bookmarks (format: {detectedFormat})" + for (url, title, folder) in parsed[0 .. min(9, parsed.high)]: + echo &" [{folder}] {title} - {url[0..min(79, url.high)]}" + if parsed.len > 10: + dimMsg &"... and {parsed.len - 10} more" + return + + let count = importBookmarks(cfg, content, detectedFormat, extractFilename(file)) + infoMsg &"Imported {count} bookmarks from {file} (format: {detectedFormat})" + +proc cmdOrganise(model = "", autoAcceptHigh = false, autoAcceptAll = false, + limit = 0, batchSize = 0, concurrency = 0, verbose = false) = + let overrides = Config(modelVariant: model, batchSize: batchSize, + concurrency: concurrency, verbose: verbose, + autoAcceptHigh: autoAcceptHigh) + var cfg = loadConfig(overrides) + let registry = loadModelRegistry() + + ensureReady(cfg, registry) + try: + discard cfg.organizeBookmarks(autoAcceptAll = autoAcceptAll, limit = limit) + except CatchableError as e: + if e.name == "EKeyboardInterrupt": + if cfg.runtimeManaged and cfg.modelName.len > 0: + let bin = findOllamaBin() + if bin.len > 0: + discard execShellCmd(bin & " stop " & cfg.modelName & " 2>/dev/null") + raise + +proc cmdList(category = "", unorganised = false) = + let cfg = loadConfig() + var bookmarks: seq[BookmarkEntry] + + if unorganised: + bookmarks = getUnorganisedBookmarks(cfg) + else: + bookmarks = listBookmarks(cfg, category) + + if bookmarks.len == 0: + dimMsg "No bookmarks found." + return + + for b in bookmarks: + let title = if b.title.len > 0: b.title else: "(untitled)" + var cat = "-" + if b.category.len > 0: cat = b.category + elif b.rawFolder.len > 0: cat = b.rawFolder + echo &" {title:<50} {cat}" + echo &"\n {bookmarks.len} bookmarks" + +proc cmdSearch(query: string) = + let cfg = loadConfig() + let bookmarks = searchBookmarks(cfg, query) + + if bookmarks.len == 0: + dimMsg &"No results for \"{query}\"" + return + + for b in bookmarks: + let title = if b.title.len > 0: b.title else: "(untitled)" + echo &" {title:<50} {b.url[0..min(79, b.url.high)]}" + echo &"\n {bookmarks.len} results for \"{query}\"" + +proc cmdUndo = + let cfg = loadConfig() + let count = cfg.undoLastBatch() + if count > 0: + infoMsg &"Undid {count} bookmark classifications" + else: + dimMsg "Nothing to undo" + +proc cmdModelList = + let cfg = loadConfig() + let registry = loadModelRegistry() + cfg.listModels(registry) + +proc cmdModelSet(variant: string) = + let cfgDir = defaultConfigDir() + createDir(cfgDir) + let configPath = cfgDir / "config.toml" + var content = "" + if fileExists(configPath): + content = readFile(configPath) + var lines = content.splitLines() + var replaced = false + var newLines: seq[string] = @[] + for line in lines: + if line.strip().startsWith("modelVariant"): + newLines.add(&"modelVariant = \"{variant}\"") + replaced = true + else: + newLines.add(line) + if not replaced: + newLines.add(&"modelVariant = \"{variant}\"") + writeFile(configPath, newLines.join("\n") & "\n") + infoMsg &"Default model set to {variant}" + +proc cmdModelDownload = + let cfg = loadConfig() + let registry = loadModelRegistry() + cfg.ensureModel(registry) + +proc cmdStatus = + let cfg = loadConfig() + let registry = loadModelRegistry() + + echo "" + styledWriteLine(stdout, styleBright, " Endpoint: ", resetStyle, cfg.llmUrl) + styledWriteLine(stdout, styleBright, " Model: ", resetStyle, cfg.modelVariant) + + let modelReady = isModelReady(cfg, registry) + styledWriteLine(stdout, styleBright, " Model: ", resetStyle, if modelReady: "[ready]" else: "[not pulled]") + + if cfg.runtimeManaged: + let running = isRuntimeRunning(cfg) + styledWriteLine(stdout, styleBright, " Ollama: ", resetStyle, if running: "[running]" else: "[not running]") + + styledWriteLine(stdout, styleBright, " Data dir: ", resetStyle, cfg.dataDir) + echo "" + +proc cmdDoctor = + echo "" + let cfg = loadConfig() + var issues = 0 + + let dbPath = cfg.dbPath() + if fileExists(dbPath): + infoMsg &"Database: {dbPath}" + else: + warnMsg "Database not found (will be created on first import)" + issues.inc + + let ollamaBin = findOllamaBin() + if ollamaBin.len > 0: + infoMsg &"Ollama: {ollamaBin}" + elif not cfg.runtimeManaged: + dimMsg "Runtime: using external endpoint" + else: + warnMsg "Ollama not found in PATH" + when defined(macosx): + stdout.styledWriteLine(styleDim, " Install: brew install ollama", resetStyle) + elif defined(linux): + stdout.styledWriteLine(styleDim, " Install: curl -fsSL https://ollama.com/install.sh | sh", resetStyle) + issues.inc + + let registry = loadModelRegistry() + if isModelReady(cfg, registry): + infoMsg &"Model: {cfg.modelVariant} ready" + elif not cfg.runtimeManaged: + dimMsg "Model: using external endpoint" + else: + warnMsg &"Model not pulled: {cfg.modelVariant}" + + let running = isRuntimeRunning(cfg) + if running: + infoMsg "Ollama: running" + elif not cfg.runtimeManaged: + dimMsg "Runtime: using external endpoint" + else: + warnMsg "Ollama not running" + when defined(macosx): + stdout.styledWriteLine(styleDim, " Start: open -a Ollama", resetStyle) + elif defined(linux): + stdout.styledWriteLine(styleDim, " Start: ollama serve &", resetStyle) + issues.inc + + echo "" + if issues == 0: + infoMsg "All checks passed" + else: + warnMsg &"{issues} issue(s) found" + +proc cmdDedup(interactive = true, autoRemove = false) = + let cfg = loadConfig() + let groups = findDuplicates(cfg) + + if groups.len == 0: + infoMsg "No duplicate bookmarks found." + return + + var totalDupes = 0 + for g in groups: + totalDupes.inc g.dupes.len + dimMsg &"Found {groups.len} duplicate group(s) ({totalDupes} total duplicates)" + echo "" + + var totalRemoved = 0 + + if interactive: + for i, g in groups: + let remove = reviewDuplicateGroup(i + 1, groups.len, g) + if remove: + let ids = g.dupes.mapIt(it.id) + let removed = cfg.removeDuplicates(ids) + totalRemoved.inc removed + if removed > 0: + infoMsg &"Removed {removed} duplicate(s)" + else: + dimMsg "Skipped" + else: + if autoRemove: + for g in groups: + let ids = g.dupes.mapIt(it.id) + let removed = cfg.removeDuplicates(ids) + totalRemoved.inc removed + infoMsg &"Removed {totalRemoved} duplicate(s) across {groups.len} group(s)" + else: + for i, g in groups: + stdout.styledWriteLine(styleBright, &" Group {i+1}/{groups.len} ", resetStyle, styleDim, &"({g.reason})", resetStyle) + let title = if g.keep.title.len > 0: g.keep.title else: "(untitled)" + stdout.styledWriteLine(" Keep: ", fgGreen, title, resetStyle, styleDim, &" [{g.keep.url[0..min(79, g.keep.url.high)]}]", resetStyle) + for d in g.dupes: + let dt = if d.title.len > 0: d.title else: "(untitled)" + stdout.styledWriteLine(styleDim, " - ", resetStyle, dt, styleDim, &" [{d.url[0..min(79, d.url.high)]}]", resetStyle) + echo "" + dimMsg &"Run with --auto-remove to delete duplicates, or --interactive to review each group" + +proc cmdCheckLinks(concurrency = 8, deadOnly = false, deleteDead = false, + unorganised = false) = + let cfg = loadConfig() + + var bookmarks: seq[BookmarkEntry] + if unorganised: + bookmarks = getUnorganisedBookmarks(cfg) + else: + bookmarks = getAllBookmarks(cfg) + + if bookmarks.len == 0: + dimMsg "No bookmarks to check." + return + + infoMsg &"Checking {bookmarks.len} bookmark(s) (concurrency: {concurrency})..." + echo "" + + var results: seq[LinkResult] + + proc onProgress(current, total: int) = + showProgressBar(current, total, " Checking") + + results = checkAllLinks(cfg, bookmarks, concurrency, onProgress) + + stdout.write "\n" + + var filtered: seq[LinkResult] + if deadOnly: + filtered = results.filterIt(it.status == lsDead) + else: + filtered = results + + if filtered.len > 0 and not deadOnly: + filtered.sort(proc(a, b: LinkResult): int = + result = ord(a.status) - ord(b.status)) + + for r in filtered: + showLinkResult(r) + + showLinkSummary(results) + + if deleteDead: + let deadIds = results.filterIt(it.status == lsDead).mapIt(it.bookmark.id) + if deadIds.len > 0: + let removed = cfg.deleteBookmarks(deadIds) + infoMsg &"Deleted {removed} dead bookmark(s)" + +proc cmdExport(output = "", category = "") = + let cfg = loadConfig() + let html = exportBookmarksHtml(cfg, category) + if html.len == 0: + dimMsg "No bookmarks to export." + return + if output.len > 0: + writeFile(output, html) + infoMsg &"Exported {getBookmarksForExport(cfg, category).len} bookmarks to {output}" + else: + stdout.write(html) + +when isMainModule: + dispatchMulti( + [cmdImport, cmdName = "import", doc = "Import bookmarks from a file", + help = {"file": "Path to bookmark file", "format": "Format: auto|html|json|urllist", "dry-run": "Parse only, no database write"}], + [cmdOrganise, cmdName = "organise", doc = "AI-organize unorganized bookmarks", + help = {"model": "Override model variant", "auto-accept-high": "Skip review for high confidence", + "auto-accept-all": "Accept all suggestions", "limit": "Max bookmarks to process", + "batch-size": "Bookmarks per LLM request (0=auto)", "concurrency": "Parallel LLM requests (0=auto)", + "verbose": "Show debug output"}], + [cmdList, cmdName = "list", doc = "List bookmarks", + help = {"category": "Filter by folder path", "unorganised": "Show only unorganized"}], + [cmdSearch, cmdName = "search", doc = "Search bookmarks", + help = {"query": "Search term"}], + [cmdExport, cmdName = "export", doc = "Export bookmarks to Netscape HTML", + help = {"output": "Write to file instead of stdout", "category": "Filter by category"}], + [cmdUndo, cmdName = "undo", doc = "Undo last batch of classifications"], + [cmdModelList, cmdName = "model-list", doc = "List available models"], + [cmdModelSet, cmdName = "model-set", doc = "Set default model variant", + help = {"variant": "Model variant name"}], + [cmdModelDownload, cmdName = "model-download", doc = "Download model without running organise"], + [cmdStatus, cmdName = "status", doc = "Show runtime and model status"], + [cmdDoctor, cmdName = "doctor", doc = "Run self-diagnostic checks"], + [cmdDedup, cmdName = "dedup", doc = "Find and remove duplicate bookmarks", + help = {"interactive": "Review each duplicate group", "auto-remove": "Remove all duplicates without prompting"}], + [cmdCheckLinks, cmdName = "check-links", doc = "Check bookmarks for dead links", + help = {"concurrency": "Parallel requests", "dead-only": "Show only dead links", + "delete-dead": "Delete dead bookmarks", "unorganised": "Only check unorganized bookmarks"}], + ) diff --git a/src/lazybookmarks/model.nim b/src/lazybookmarks/model.nim new file mode 100644 index 0000000..9b57049 --- /dev/null +++ b/src/lazybookmarks/model.nim @@ -0,0 +1,87 @@ +import std/[os, strformat, httpclient, json, terminal] +import jsony +import ./config + +type + ModelEntry* = object + name*: string + ollamaModel*: string + ollamaTag*: string + + ModelRegistry* = object + entries*: seq[ModelEntry] + +const modelRegistryJson = staticRead("../../assets/models.json") + +proc loadModelRegistry*: ModelRegistry = + return modelRegistryJson.fromJson(ModelRegistry) + +proc findModel*(registry: ModelRegistry, variant: string): ModelEntry = + for e in registry.entries: + if e.name == variant: + return e + raise newException(ValueError, &"Unknown model variant: {variant}") + +proc ollamaRef*(entry: ModelEntry): string = + entry.ollamaModel & ":" & entry.ollamaTag + +proc pullModel*(entry: ModelEntry) = + let refStr = ollamaRef(entry) + stdout.styledWriteLine(styleBright, fgGreen, " ✓ ", fgDefault, resetStyle, + "Pulling " & refStr & "...") + + let exitCode = execShellCmd("ollama pull " & quoteShell(refStr) & " 2>&1") + if exitCode != 0: + stdout.styledWriteLine(styleBright, fgRed, " ✗ ", fgDefault, resetStyle, + "Failed to pull " & refStr) + quit(1) + +proc listLocalModels*(cfg: Config): seq[string] = + result = @[] + try: + let client = newHttpClient(timeout = 5000) + defer: client.close() + let body = client.getContent(cfg.ollamaApiUrl() & "/api/tags") + let jsn = parseJson(body) + for m in jsn["models"]: + result.add(m["name"].getStr()) + except: + discard + +proc isEntryReady*(entry: ModelEntry, cfg: Config): bool = + let refStr = ollamaRef(entry) + for localName in listLocalModels(cfg): + if localName == refStr: + return true + return false + +proc isModelReady*(cfg: Config, registry: ModelRegistry): bool = + try: + let entry = findModel(registry, cfg.modelVariant) + return isEntryReady(entry, cfg) + except: + return false + +proc ensureModel*(cfg: Config, registry: ModelRegistry) = + let entry = findModel(registry, cfg.modelVariant) + if not isEntryReady(entry, cfg): + pullModel(entry) + else: + stdout.styledWriteLine(styleBright, fgGreen, " ✓ ", fgDefault, resetStyle, + "Model ready: " & ollamaRef(entry)) + +proc listModels*(cfg: Config, registry: ModelRegistry) = + echo "" + for entry in registry.entries: + let isCurrent = entry.name == cfg.modelVariant + let isReady = isEntryReady(entry, cfg) + let marker = if isCurrent: " *" else: "" + let status = if isReady: "[installed]" else: "[not installed]" + let name = if isCurrent: entry.name & marker else: entry.name + if isCurrent: + stdout.styledWrite(styleBright) + stdout.write " " & name + stdout.styledWrite(resetStyle) + stdout.styledWriteLine(" " & status & " " & ollamaRef(entry)) + echo "" + stdout.styledWriteLine(styleDim, " * = current selection", resetStyle) diff --git a/src/lazybookmarks/organizer.nim b/src/lazybookmarks/organizer.nim new file mode 100644 index 0000000..9f2431c --- /dev/null +++ b/src/lazybookmarks/organizer.nim @@ -0,0 +1,597 @@ +import std/[strutils, strformat, json, re, math, tables, algorithm, sequtils, sets, asyncdispatch, os, times] +import db_connector/db_sqlite +import ./config +import ./storage +import ./client +import ./prompts +import ./ui +import ./runtime + +type + TaxonomyCategory* = object + folderId*: string + folderPath*: string + description*: string + keywords*: seq[string] + + Taxonomy* = object + categories*: seq[TaxonomyCategory] + + ClusterSuggestion* = object + name*: string + description*: string + keywords*: seq[string] + parentFolderId*: string + + Classification* = object + bookmarkId*: string + targetFolderId*: string + confidence*: string + reason*: string + + Suggestion* = object + bookmarkId*: int64 + bookmarkTitle*: string + bookmarkUrl*: string + targetFolderId*: string + targetFolderPath*: string + confidence*: string + reason*: string + isNewFolder*: bool + + BatchResult* = object + suggestions*: seq[Suggestion] + skipped*: int + lowConf*: int + +const StopWords = ["the","a","an","and","or","of","to","in","for","is","on","with","at","by","from","this","that","it","as","are","was","be","has","have"] + +proc tokenizeText*(text: string): seq[string] = + result = @[] + for word in text.toLowerAscii().replace(re"[^a-z0-9\säöüß]", " ").splitWhitespace(): + if word.len > 2 and word notin StopWords: + result.add(word) + +proc extractDomainPatterns*(bookmarks: seq[BookmarkEntry], threshold = 0.2): seq[string] = + var counts: Table[string, int] + for b in bookmarks: + let domain = extractDomain(b.url).replace(re"^www\.", "") + if domain.len > 0: + counts[domain] = counts.getOrDefault(domain, 0) + 1 + let total = max(1, bookmarks.len) + result = @[] + for domain, count in counts: + if count.float / total.float >= threshold: + result.add(domain) + sort(result, proc(a, b: string): int = cmp(counts[b], counts[a])) + if result.len > 5: + result.setLen(5) + +proc computeTFIDF*(folderBookmarks: Table[string, seq[BookmarkEntry]], allBookmarks: seq[BookmarkEntry]): Table[string, seq[string]] = + var df: Table[string, int] + let N = max(1, allBookmarks.len) + + for b in allBookmarks: + for tok in tokenizeText(b.title): + df[tok] = df.getOrDefault(tok, 0) + 1 + + result = initTable[string, seq[string]]() + for folderId, bookmarks in folderBookmarks: + if bookmarks.len == 0: + result[folderId] = @[] + continue + var tf: Table[string, int] + for b in bookmarks: + for tok in tokenizeText(b.title): + tf[tok] = tf.getOrDefault(tok, 0) + 1 + + proc idf(term: string): float = + let d = df.getOrDefault(term, 0) + return ln(N.float / (1.0 + d.float)) + + var scored: seq[tuple[word: string, score: float]] = @[] + for word, freq in tf: + let score = (freq.float / bookmarks.len.float) * idf(word) + scored.add((word, score)) + + scored.sort(proc(a, b: (string, float)): int = cmp(b[1], a[1])) + var keywords: seq[string] = @[] + for s in scored[0 .. min(5, scored.high)]: + keywords.add(s[0]) + result[folderId] = keywords + +proc sampleExemplars*(bookmarks: seq[BookmarkEntry], count = 2): string = + var sorted = bookmarks + sorted.sort(proc(a, b: BookmarkEntry): int = cmp(b.addedAt, a.addedAt)) + var parts: seq[string] = @[] + for i in 0 .. min(count - 1, sorted.high): + let host = extractDomain(sorted[i].url) + let title = if sorted[i].title.len > 40: sorted[i].title[0 .. 39] else: sorted[i].title + parts.add("\"" & title & "\" " & host) + return parts.join(" | ") + +proc buildFingerprint*(folders: seq[FolderEntry]): string = + var parts: seq[string] = @[] + for f in folders: + parts.add(&"{f.uuid}:{f.bookmarkCount}") + parts.sort() + return parts.join(",") + +proc loadCachedTaxonomy*(db: DbConn, fingerprint: string): (bool, Taxonomy) = + try: + let row = db.getRow(sql("SELECT taxonomy FROM taxonomy_cache WHERE fingerprint = ?"), fingerprint) + if row[0].len == 0: + return (false, Taxonomy()) + let json = parseJson(row[0]) + var cats: seq[TaxonomyCategory] = @[] + for elem in json["categories"].getElems(): + var kws: seq[string] = @[] + for kw in elem["keywords"].getElems(): + kws.add(kw.getStr()) + cats.add(TaxonomyCategory( + folderId: elem["folderId"].getStr(), + folderPath: elem["folderPath"].getStr(), + description: elem["description"].getStr(), + keywords: kws, + )) + let tax = Taxonomy(categories: cats) + return (true, tax) + except: + return (false, Taxonomy()) + +proc saveTaxonomy*(db: DbConn, fingerprint: string, taxonomy: Taxonomy) = + try: + db.exec(sql("INSERT OR REPLACE INTO taxonomy_cache (fingerprint, taxonomy, created_at) VALUES (?, ?, strftime('%s','now'))"), + fingerprint, $(%*taxonomy)) + except: + discard + +proc loadCachedClusters*(db: DbConn, fingerprint: string): (bool, seq[ClusterSuggestion]) = + try: + let row = db.getRow(sql("SELECT clusters FROM cluster_cache WHERE fingerprint = ?"), fingerprint) + if row[0].len == 0: + return (false, @[]) + let json = parseJson(row[0]) + var clusters: seq[ClusterSuggestion] = @[] + for elem in json.getElems(): + var kws: seq[string] = @[] + for kw in elem["keywords"].getElems(): + kws.add(kw.getStr()) + clusters.add(ClusterSuggestion( + name: elem["name"].getStr(), + description: elem["description"].getStr(), + keywords: kws, + parentFolderId: elem["parentFolderId"].getStr(), + )) + return (true, clusters) + except: + return (false, @[]) + +proc saveClusters*(db: DbConn, fingerprint: string, clusters: seq[ClusterSuggestion]) = + try: + db.exec(sql("INSERT OR REPLACE INTO cluster_cache (fingerprint, clusters, created_at) VALUES (?, ?, strftime('%s','now'))"), + fingerprint, $(%*clusters)) + except: + discard + +proc pruneTaxonomy*(taxonomy: Taxonomy, batch: seq[BookmarkEntry], + tfidfMap: Table[string, seq[string]], + topN = 15, minN = 5): Taxonomy = + var batchTokens = initHashSet[string]() + for b in batch: + for tok in tokenizeText(b.title): + batchTokens.incl(tok) + + var scored: seq[tuple[cat: TaxonomyCategory, overlap: int]] = @[] + for cat in taxonomy.categories: + let keywords = tfidfMap.getOrDefault(cat.folderId, @[]) + let overlap = keywords.filterIt(it in batchTokens).len + scored.add((cat, overlap)) + + sort(scored, proc(a, b: (TaxonomyCategory, int)): int = + result = cmp(b[1], a[1]) + if result == 0: result = cmp(a[0].folderId, b[0].folderId) + ) + + let count = min(max(topN, minN), scored.len) + var pruned: seq[TaxonomyCategory] = @[] + for s in scored[0 .. count - 1]: + pruned.add(s[0]) + return Taxonomy(categories: pruned) + +proc runTaxonomyPhase*(cfg: Config, folders: seq[FolderEntry], + folderBookmarks: Table[string, seq[BookmarkEntry]], + allBookmarks: seq[BookmarkEntry], + db: DbConn): Taxonomy = + let fingerprint = buildFingerprint(folders) + let (cached, taxonomy) = loadCachedTaxonomy(db, fingerprint) + if cached: + if cfg.verbose: + dimMsg &"Taxonomy cache hit ({taxonomy.categories.len} folders)" + return taxonomy + + if cfg.verbose: + dimMsg "Taxonomy cache miss, running Phase 1..." + + let tfidfMap = computeTFIDF(folderBookmarks, allBookmarks) + + var enriched: seq[tuple[id, path, count: string, domains, keywords, exemplars: string]] = @[] + for folder in folders: + let bookmarks = folderBookmarks.getOrDefault(folder.uuid, @[]) + let domains = extractDomainPatterns(bookmarks) + let keywords = tfidfMap.getOrDefault(folder.uuid, @[]) + let exemplars = sampleExemplars(bookmarks) + + enriched.add(( + id: folder.uuid, + path: folder.path, + count: $folder.bookmarkCount, + domains: domains.join(", "), + keywords: keywords.join(", "), + exemplars: exemplars, + )) + + let prompt = buildTaxonomyPrompt(enriched) + let response = chatCompletionSimple(cfg, SystemPrompt, prompt, TaxonomySchemaJson) + + result = Taxonomy(categories: @[]) + for elem in response["categories"].getElems(): + var kws: seq[string] = @[] + for kw in elem["keywords"].getElems(): + kws.add(kw.getStr()) + result.categories.add(TaxonomyCategory( + folderId: elem["folderId"].getStr(), + folderPath: elem["folderPath"].getStr(), + description: elem["description"].getStr(), + keywords: kws, + )) + + saveTaxonomy(db, fingerprint, result) + if cfg.verbose: + dimMsg &"Taxonomy cached ({result.categories.len} folders)" + +proc runClusterPhase*(cfg: Config, uncategorized: seq[BookmarkEntry], + taxonomy: Taxonomy, folders: seq[FolderEntry]): seq[ClusterSuggestion] = + var rootFolders: seq[tuple[id, title: string]] = @[] + for f in folders: + if f.parentId == 0: + rootFolders.add((id: f.uuid, title: f.path)) + + let batchTuples = uncategorized.mapIt((id: $it.id, title: it.title, url: it.url)) + let taxCats = taxonomy.categories.mapIt((id: it.folderId, path: it.folderPath)) + let rootIds = rootFolders.mapIt(it.id) + + let prompt = buildClusterPrompt(batchTuples, taxCats, rootFolders) + let schema = buildClusterSchemaJson(rootIds) + + let response = chatCompletionSimple(cfg, SystemPrompt, prompt, schema) + + if not response.hasKey("clusters"): + return @[] + + result = @[] + for elem in response["clusters"].getElems(): + var kws: seq[string] = @[] + for kw in elem["keywords"].getElems(): + kws.add(kw.getStr()) + result.add(ClusterSuggestion( + name: elem["name"].getStr(), + description: elem["description"].getStr(), + keywords: kws, + parentFolderId: elem["parentFolderId"].getStr(), + )) + +proc chunk*[T](s: seq[T], size: int): seq[seq[T]] = + if size <= 0 or s.len == 0: return @[] + result = @[] + var i = 0 + while i < s.len: + var batch: seq[T] = @[] + for j in 0 ..< min(size, s.len - i): + batch.add(s[i + j]) + result.add(batch) + i += size + +proc classifyBatchAsync(cfg: Config, batch: seq[BookmarkEntry], + fullTaxonomy: Taxonomy, + tfidfMap: Table[string, seq[string]], + batchIndex: int): Future[BatchResult] {.async.} = + let pruned = pruneTaxonomy(fullTaxonomy, batch, tfidfMap) + let folderIds = pruned.categories.mapIt(it.folderId) + let bookmarkIds = batch.mapIt($it.id) + let schema = buildClassificationSchemaJson(folderIds, bookmarkIds) + + let taxCats = pruned.categories.mapIt( + (id: it.folderId, path: it.folderPath, description: it.description, keywords: it.keywords.join(", ")) + ) + let batchTuples = batch.mapIt((id: $it.id, title: it.title, url: it.url)) + let prompt = buildClassificationPrompt(taxCats, batchTuples) + + try: + let response = await chatCompletionSimpleAsync(cfg, SystemPrompt, prompt, schema) + var suggestions: seq[Suggestion] = @[] + var skipCount = 0 + var lowCount = 0 + + if response.hasKey("moves"): + for move in response["moves"]: + let moveObj = move + let bmId = parseBiggestInt(moveObj["bookmarkId"].getStr()) + let targetId = moveObj["targetFolderId"].getStr() + let conf = moveObj["confidence"].getStr() + let reason = moveObj["reason"].getStr() + + if targetId == "__skip__": + inc skipCount + continue + + if conf == "low": + inc lowCount + + let bmIdx = batch.findIt(it.id == bmId) + var bmTitle = "" + var bmUrl = "" + if bmIdx >= 0: + bmTitle = batch[bmIdx].title + bmUrl = batch[bmIdx].url + let targetIdx = pruned.categories.findIt(it.folderId == targetId) + var targetPath = targetId + if targetIdx >= 0: + targetPath = pruned.categories[targetIdx].folderPath + let isNew = targetId.startsWith("__new_") + + suggestions.add(Suggestion( + bookmarkId: bmId, + bookmarkTitle: bmTitle, + bookmarkUrl: bmUrl, + targetFolderId: targetId, + targetFolderPath: targetPath, + confidence: conf, + reason: reason, + isNewFolder: isNew, + )) + + return BatchResult(suggestions: suggestions, skipped: skipCount, lowConf: lowCount) + except CatchableError as e: + if e.name == "EKeyboardInterrupt": + raise + if cfg.verbose: + errorMsg &"Batch {batchIndex + 1} failed: {e.msg}" + return BatchResult(suggestions: @[], skipped: 0, lowConf: 0) + +proc runClassificationPhase*(cfg: Config, uncategorized: seq[BookmarkEntry], + taxonomy: Taxonomy, + folderBookmarks: Table[string, seq[BookmarkEntry]], + allBookmarks: seq[BookmarkEntry], + clusters: seq[ClusterSuggestion], + autoApply: bool = false): (int, seq[Suggestion]) = + var fullTaxonomy = taxonomy + + let newFolders = clusters.mapIt(TaxonomyCategory( + folderId: &"__new_{it.name}", + folderPath: it.name, + description: it.description, + keywords: it.keywords, + )) + fullTaxonomy.categories.add(newFolders) + + let tfidfMap = computeTFIDF(folderBookmarks, allBookmarks) + + let batches = uncategorized.chunk(cfg.batchSize) + let conc = cfg.concurrency + let totalBookmarks = uncategorized.len + + if batches.len == 0: + return (0, @[]) + + var completedCount = 0 + var appliedCount = 0 + var totalSkipped = 0 + var totalLowConf = 0 + var totalFailed = 0 + var allSuggestions: seq[Suggestion] = @[] + var completedBookmarks = 0 + + proc commitBatch(suggestions: seq[Suggestion]) = + for s in suggestions: + if s.confidence != "low": + applyClassification(cfg, s.bookmarkId, s.targetFolderPath, s.confidence, s.reason) + inc appliedCount + + if conc <= 1: + let startTime = epochTime() + for i, batch in batches: + let elapsed = int(epochTime() - startTime) + let prefix = &"Classifying {completedBookmarks}/{totalBookmarks} bookmarks" + showProgressBar(i + 1, batches.len, prefix, elapsed) + let br = classifyBatchAsync(cfg, batch, fullTaxonomy, tfidfMap, i).waitFor() + allSuggestions.add(br.suggestions) + completedBookmarks += batch.len + totalSkipped += br.skipped + totalLowConf += br.lowConf + if autoApply: + commitBatch(br.suggestions) + dimMsg &"Batch {i + 1}/{batches.len}: {br.suggestions.len} classified, {br.skipped} skipped, {br.lowConf} low-confidence" + echo "" + dimMsg &"Phase 2 complete: {allSuggestions.len} classified, {totalSkipped} skipped, {totalLowConf} low-confidence, {totalFailed} failed" + echo "" + return (appliedCount, allSuggestions) + + var pending: seq[Future[BatchResult]] = @[] + var batchIdx = 0 + let startTime = epochTime() + var lastHeartbeat = 0.0 + + proc hasUnfinished(): bool = + for f in pending: + if not f.finished: return true + return false + + proc drainPending() = + var i = 0 + while i < pending.len: + if pending[i].finished: + let br = pending[i].read() + pending.delete(i) + inc completedCount + let bmCount = if completedCount <= batches.len: min(cfg.batchSize, totalBookmarks - completedBookmarks + cfg.batchSize) else: cfg.batchSize + completedBookmarks += bmCount + allSuggestions.add(br.suggestions) + totalSkipped += br.skipped + totalLowConf += br.lowConf + if autoApply: + commitBatch(br.suggestions) + let elapsed = int(epochTime() - startTime) + let prefix = &"Classifying {completedBookmarks}/{totalBookmarks} bookmarks" + showProgressBar(completedCount, batches.len, prefix, elapsed) + dimMsg &"Batch {completedCount}/{batches.len}: {br.suggestions.len} classified, {br.skipped} skipped, {br.lowConf} low-confidence" + inc i + else: + inc i + + while batchIdx < batches.len or pending.len > 0: + while pending.len < conc and batchIdx < batches.len: + pending.add(classifyBatchAsync(cfg, batches[batchIdx], fullTaxonomy, tfidfMap, batchIdx)) + inc batchIdx + + if hasUnfinished(): + try: + poll() + except CatchableError as e: + if e.name == "EKeyboardInterrupt": + raise + sleep(50) + + drainPending() + + if hasUnfinished(): + let now = epochTime() + if now - lastHeartbeat >= 1.0: + lastHeartbeat = now + let elapsed = int(epochTime() - startTime) + let prefix = &"Classifying {completedBookmarks}/{totalBookmarks} bookmarks" + showProgressBar(completedCount, batches.len, prefix, elapsed) + + for f in pending: + if f.finished: + try: + let br = f.read() + allSuggestions.add(br.suggestions) + totalSkipped += br.skipped + totalLowConf += br.lowConf + if autoApply: + commitBatch(br.suggestions) + except CatchableError: + inc totalFailed + + echo "" + dimMsg &"Phase 2 complete: {allSuggestions.len} classified, {totalSkipped} skipped, {totalLowConf} low-confidence, {totalFailed} failed" + echo "" + return (appliedCount, allSuggestions) + +proc organizeBookmarks*(cfg: Config, autoAcceptAll: bool = false, limit: int = 0): int = + let db = cfg.initDb() + defer: db.close() + + defer: + if cfg.runtimeManaged and cfg.modelName.len > 0: + let bin = findOllamaBin() + if bin.len > 0: + try: + discard execShellCmd(bin & " stop " & cfg.modelName & " 2>/dev/null") + except: + discard + + let uncategorized = getUnorganisedBookmarks(cfg, limit) + let webUncategorized = uncategorized.filterIt(it.url.startsWith("http://") or it.url.startsWith("https://")) + + if webUncategorized.len == 0: + dimMsg "No unorganized bookmarks found." + return 0 + + infoMsg &"Found {webUncategorized.len} unorganized bookmarks" + + let folders = getAllFolders(cfg) + + var folderBookmarks = initTable[string, seq[BookmarkEntry]]() + var allBookmarks: seq[BookmarkEntry] = @[] + + for row in db.fastRows(sql("SELECT id, url, title, raw_folder FROM bookmarks")): + let bm = BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + ) + allBookmarks.add(bm) + if bm.rawFolder.len > 0: + let folderIdx = folders.findIt(it.path == bm.rawFolder) + if folderIdx >= 0: + let folder = folders[folderIdx] + if folder.uuid notin folderBookmarks: + folderBookmarks[folder.uuid] = @[] + folderBookmarks[folder.uuid].add(bm) + + headerMsg "Phase 1: Analyzing folder structure..." + let taxonomy = runTaxonomyPhase(cfg, folders, folderBookmarks, allBookmarks, db) + + headerMsg "Phase 1.5: Identifying new folder opportunities..." + let fingerprint = buildFingerprint(folders) + var clusters: seq[ClusterSuggestion] = @[] + let (clusterCached, cachedClusters) = loadCachedClusters(db, fingerprint) + if clusterCached: + clusters = cachedClusters + if clusters.len > 0 and cfg.verbose: + dimMsg &"Cluster cache hit ({clusters.len} folders)" + else: + try: + clusters = runClusterPhase(cfg, webUncategorized, taxonomy, folders) + if clusters.len > 0: + saveClusters(db, fingerprint, clusters) + if cfg.verbose: + dimMsg &"Found {clusters.len} potential new folders" + except CatchableError as e: + if cfg.verbose: + warnMsg &"Cluster phase skipped: {e.msg}" + + headerMsg "Phase 2: Classifying bookmarks..." + let shouldAutoApply = autoAcceptAll or cfg.autoAcceptHigh + let (appliedCount, suggestions) = runClassificationPhase(cfg, webUncategorized, taxonomy, folderBookmarks, allBookmarks, clusters, autoApply = shouldAutoApply) + + if suggestions.len == 0: + dimMsg "No suggestions generated." + return 0 + + if shouldAutoApply: + infoMsg &"Applied {appliedCount} suggestions automatically" + return appliedCount + + var accepted = 0 + var skipped = 0 + var edited = 0 + + for s in suggestions: + let displayPath = s.targetFolderPath & (if s.isNewFolder: " (new)" else: "") + let action = reviewSuggestion(s.bookmarkUrl, s.bookmarkTitle, + displayPath, s.confidence, s.reason) + + case action + of ReviewAction.accept: + applyClassification(cfg, s.bookmarkId, s.targetFolderPath, s.confidence, s.reason) + accepted.inc + of ReviewAction.skip: + skipped.inc + of ReviewAction.edit: + stdout.write " New folder path: " + stdout.flushFile() + let newPath = stdin.readLine().strip() + if newPath.len > 0: + applyClassification(cfg, s.bookmarkId, newPath, s.confidence, s.reason) + edited.inc + else: + skipped.inc + of ReviewAction.quitReview: + break + + infoMsg &"Done: {accepted} accepted, {skipped} skipped, {edited} edited" + return accepted diff --git a/src/lazybookmarks/prompts.nim b/src/lazybookmarks/prompts.nim new file mode 100644 index 0000000..0f47ce6 --- /dev/null +++ b/src/lazybookmarks/prompts.nim @@ -0,0 +1,114 @@ +import std/strutils +import ./storage + +const SystemPrompt* = "You are a bookmark classifier. Given a user's folder structure and uncategorized bookmarks, assign each to the most appropriate existing folder. If no folder fits well, set targetFolderId to \"__skip__\" instead of forcing a poor match. Respond with ONLY valid JSON matching the required structure. No explanation, no markdown, no other text. Prefer the user's existing folder names. Only suggest new folders when necessary." + +const TaxonomySchemaJson* = """{ + "type": "object", + "properties": { + "categories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "folderId": { "type": "string" }, + "folderPath": { "type": "string" }, + "description": { "type": "string" }, + "keywords": { "type": "array", "items": { "type": "string" }, "maxItems": 10 } + }, + "required": ["folderId", "folderPath", "description", "keywords"], + "additionalProperties": false + } + } + }, + "required": ["categories"], + "additionalProperties": false +}""" + +proc buildClusterSchemaJson*(rootFolderIds: seq[string]): string = + var enumParts: seq[string] = @[] + for id in rootFolderIds: + enumParts.add("\"" & id & "\"") + let enumValues = enumParts.join(", ") + return "{\"type\":\"object\",\"properties\":{\"clusters\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"},\"description\":{\"type\":\"string\"},\"keywords\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"maxItems\":8},\"parentFolderId\":{\"type\":\"string\",\"enum\":[" & enumValues & "]}}}}}}" + +proc buildClassificationSchemaJson*(folderIds: seq[string], bookmarkIds: seq[string]): string = + var folderParts: seq[string] = @[] + for id in folderIds: + folderParts.add("\"" & id & "\"") + let folderEnum = folderParts.join(", ") & ", \"__skip__\"" + var bookmarkParts: seq[string] = @[] + for id in bookmarkIds: + bookmarkParts.add("\"" & id & "\"") + let bookmarkEnum = bookmarkParts.join(", ") + return "{\"type\":\"object\",\"properties\":{\"moves\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"bookmarkId\":{\"type\":\"string\",\"enum\":[" & bookmarkEnum & "]},\"targetFolderId\":{\"type\":\"string\",\"enum\":[" & folderEnum & "]},\"confidence\":{\"type\":\"string\",\"enum\":[\"high\",\"medium\",\"low\"]},\"reason\":{\"type\":\"string\"}},\"required\":[\"bookmarkId\",\"targetFolderId\",\"confidence\",\"reason\"],\"additionalProperties\":false}}},\"required\":[\"moves\"]}" + +proc buildTaxonomyPrompt*(enrichedFolders: seq[tuple[id, path, count: string, domains, keywords, exemplars: string]]): string = + var lines: seq[string] = @[] + for f in enrichedFolders: + var parts: seq[string] = @[] + parts.add("[" & f.id & "] " & f.path & " (" & f.count & ")") + if f.domains.len > 0: parts.add("domains: " & f.domains) + if f.keywords.len > 0: parts.add("keywords: " & f.keywords) + if f.exemplars.len > 0: parts.add("examples: " & f.exemplars) + lines.add(parts.join(" | ")) + return "Analyze these bookmark folders. For each, describe what it contains and provide keywords.\n\n" & + lines.join("\n") & "\n\n" & + "Respond with a JSON object: {\"categories\": [{\"folderId\": \"\", \"folderPath\": \"\", \"description\": \"\", \"keywords\": [\"word1\", \"word2\"]}]}\n\n" & + "Example:\n{\"categories\": [{\"folderId\": \"a1b2c3\", \"folderPath\": \"Tech/Blogs\", \"description\": \"Programming and software development blogs\", \"keywords\": [\"programming\", \"software\", \"code\"]}]}" + +proc formatBookmarkBatch*(bookmarks: seq[tuple[id, title, url: string]]): string = + var lines: seq[string] = @[] + for b in bookmarks: + var shortUrl = extractDomain(b.url) + if shortUrl.len > 60: + shortUrl = shortUrl[0 .. 56] & "..." + let title = if b.title.len > 0: b.title else: "(untitled)" + lines.add("[" & b.id & "] \"" & title & "\" " & shortUrl) + return lines.join("\n") + +proc buildClusterPrompt*(uncategorizedBookmarks: seq[tuple[id, title, url: string]], + taxonomyCategories: seq[tuple[id, path: string]], + rootFolders: seq[tuple[id, title: string]]): string = + var existingParts: seq[string] = @[] + for c in taxonomyCategories: + existingParts.add("[" & c.id & "] " & c.path) + let existingList = existingParts.join("\n") + + var rootParts: seq[string] = @[] + for f in rootFolders: + rootParts.add("[" & f.id & "] " & f.title) + let rootList = rootParts.join("\n") + + let bookmarkList = formatBookmarkBatch(uncategorizedBookmarks) + + return "Analyze these uncategorized bookmarks and identify 2-6 thematic groups that would benefit from a new folder.\n\n" & + "Existing folders (for reference -- do NOT use these as parentFolderId):\n" & + existingList & "\n\n" & + "Valid locations for new folders (use one of these IDs as parentFolderId):\n" & + rootList & "\n\n" & + "Uncategorized bookmarks:\n" & + bookmarkList & "\n\n" & + "For each cluster, suggest a short folder name, a description, keywords, and which root location to create it in (parentFolderId).\n" & + "Only suggest clusters when a meaningful group of 2+ bookmarks shares a clear theme. Do not suggest clusters that duplicate an existing folder's purpose.\n\n" & + "Respond with a JSON object: {\"clusters\": [{\"name\": \"\", \"description\": \"\", \"keywords\": [\"word1\", \"word2\"], \"parentFolderId\": \"\"}]}" + +proc buildClassificationPrompt*(taxonomyCategories: seq[tuple[id, path, description, keywords: string]], + bookmarkBatch: seq[tuple[id, title, url: string]]): string = + var folderParts: seq[string] = @[] + for c in taxonomyCategories: + folderParts.add("[" & c.id & "] " & c.path & ": " & c.description & " (" & c.keywords & ")") + let folderList = folderParts.join("\n") + let bookmarkList = formatBookmarkBatch(bookmarkBatch) + + return "Classify these bookmarks into the most appropriate folders.\n\n" & + "Available folders:\n" & + folderList & "\n\n" & + "Bookmarks to classify (format: [id] \"title\" url):\n" & + bookmarkList & "\n\n" & + "For each bookmark:\n" & + "- Choose the best existing folder (targetFolderId)\n" & + "- Set confidence: \"high\" (obvious match), \"medium\" (reasonable), \"low\" (uncertain)\n" & + "- Give a brief reason\n\n" & + "Use targetFolderId=\"__skip__\" if no folder is a good match.\n\n" & + "Respond with a JSON object: {\"moves\": [{\"bookmarkId\": \"\", \"targetFolderId\": \"\", \"confidence\": \"high|medium|low\", \"reason\": \"\"}]}" diff --git a/src/lazybookmarks/runtime.nim b/src/lazybookmarks/runtime.nim new file mode 100644 index 0000000..2d64cda --- /dev/null +++ b/src/lazybookmarks/runtime.nim @@ -0,0 +1,36 @@ +import std/[os, strutils, httpclient, terminal] +import ./config + +proc findOllamaBin*(): string = + for dir in getEnv("PATH").split(PathSep): + let path = dir / "ollama" + if fileExists(path): + return path + return "" + +proc isRuntimeRunning*(cfg: Config): bool = + try: + let client = newHttpClient(timeout = 2000) + defer: client.close() + discard client.getContent(cfg.ollamaApiUrl() & "/api/tags") + return true + except: + return false + +proc requireRuntime*(cfg: Config) = + if isRuntimeRunning(cfg): + return + if not cfg.runtimeManaged: + stdout.styledWriteLine(styleBright, fgRed, " ✗ ", fgDefault, resetStyle, + "Endpoint not reachable: " & cfg.llmUrl) + quit(1) + stdout.styledWriteLine(styleBright, fgRed, " ✗ ", fgDefault, resetStyle, + "Ollama is not running.") + when defined(macosx): + stdout.styledWriteLine(styleDim, " Start it with: ollama serve &", resetStyle) + stdout.styledWriteLine(styleDim, " Or install: brew install ollama", resetStyle) + elif defined(linux): + stdout.styledWriteLine(styleDim, " Start it with: ollama serve &", resetStyle) + stdout.styledWriteLine(styleDim, " Or install: curl -fsSL https://ollama.com/install.sh | sh", resetStyle) + stdout.styledWriteLine(styleDim, " Manual: https://ollama.com/download", resetStyle) + quit(1) diff --git a/src/lazybookmarks/storage.nim b/src/lazybookmarks/storage.nim new file mode 100644 index 0000000..c215391 --- /dev/null +++ b/src/lazybookmarks/storage.nim @@ -0,0 +1,519 @@ +import std/[re, strutils, strformat, random, times, json, uri, sequtils] +import db_connector/db_sqlite +import ./config + +randomize() + +type + BookmarkEntry* = object + id*: int64 + url*: string + title*: string + rawFolder*: string + category*: string + confidence*: string + reason*: string + importId*: int64 + organisedAt*: int64 + addedAt*: int64 + + FolderEntry* = object + id*: int64 + uuid*: string + path*: string + parentId*: int64 + bookmarkCount*: int + +const Schema = """ +CREATE TABLE IF NOT EXISTS bookmarks ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL UNIQUE, + title TEXT, + raw_folder TEXT, + category TEXT, + confidence TEXT CHECK(confidence IN ('high','medium','low',NULL)), + reason TEXT, + import_id INTEGER REFERENCES imports(id), + organised_at INTEGER, + added_at INTEGER +); + +CREATE TABLE IF NOT EXISTS imports ( + id INTEGER PRIMARY KEY, + filename TEXT, + format TEXT CHECK(format IN ('netscape','json','urllist')), + imported_at INTEGER, + bookmark_count INTEGER +); + +CREATE TABLE IF NOT EXISTS taxonomy_cache ( + fingerprint TEXT PRIMARY KEY, + taxonomy TEXT NOT NULL, + created_at INTEGER +); + +CREATE TABLE IF NOT EXISTS cluster_cache ( + fingerprint TEXT PRIMARY KEY, + clusters TEXT NOT NULL, + created_at INTEGER +); + +CREATE TABLE IF NOT EXISTS folders ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL UNIQUE, + path TEXT NOT NULL UNIQUE, + parent_path TEXT, + bookmark_count INTEGER DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_bookmarks_organised ON bookmarks(organised_at); +CREATE INDEX IF NOT EXISTS idx_bookmarks_import ON bookmarks(import_id); +CREATE INDEX IF NOT EXISTS idx_bookmarks_folder ON bookmarks(raw_folder); +""" + +proc initDb*(cfg: Config): DbConn + +proc genUuid*: string = + const hexChars = "0123456789abcdef" + var s = "" + for i in 0..31: + if i == 8 or i == 12 or i == 16 or i == 20: + s.add '-' + s.add hexChars[rand(15)] + return s + +const TrackingParams = [ + "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", + "fbclid", "gclid", "msclkid", "ref", "source", "mc_cid", "mc_eid", +] + +proc normalizeUrl*(url: string): string = + try: + var u = parseUri(url) + var host = u.hostname.toLowerAscii() + var path = u.path + if path.len > 1 and path.endsWith("/"): + path = path[0 ..< path.len - 1] + var query = u.query + if query.len > 0: + var pairs: seq[string] = @[] + for part in query.split('&'): + let eqIdx = part.find('=') + let key = if eqIdx >= 0: part[0 ..< eqIdx].toLowerAscii() else: part.toLowerAscii() + var isTracking = false + for tp in TrackingParams: + if key == tp: + isTracking = true + break + if not isTracking: + pairs.add(part) + if pairs.len > 0: + query = pairs.join("&") + result = host & path & "?" & query + else: + result = host & path + else: + result = host & path + except: + result = url.toLowerAscii() + +proc extractDomain*(url: string): string = + try: + let u = parseUri(url) + result = u.hostname.toLowerAscii() + except: + let idx = url.find("://") + if idx >= 0: + let rest = url[idx + 3 .. url.high] + let slashIdx = rest.find('/') + result = if slashIdx >= 0: rest[0 ..< slashIdx] else: rest + else: + result = url + +type + DuplicateGroup* = object + keep*: BookmarkEntry + dupes*: seq[BookmarkEntry] + reason*: string + +proc findDuplicates*(cfg: Config): seq[DuplicateGroup] = + let db = cfg.initDb() + defer: db.close() + + for row in db.fastRows(sql("SELECT id, url, title, raw_folder, category, confidence FROM bookmarks ORDER BY added_at DESC")): + let b = BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + ) + + let normUrl = normalizeUrl(b.url) + var matched = false + + for i in 0 ..< result.len: + let g = result[i] + let keepNorm = normalizeUrl(g.keep.url) + if keepNorm == normUrl and normUrl.len > 0: + result[i].dupes.add(b) + matched = true + break + + if not matched: + for i in 0 ..< result.len: + let g = result[i] + let keepDomain = extractDomain(g.keep.url) + let bDomain = extractDomain(b.url) + if keepDomain == bDomain and keepDomain.len > 0: + let keepTitle = g.keep.title.strip().toLowerAscii() + let bTitle = b.title.strip().toLowerAscii() + if keepTitle.len > 3 and keepTitle == bTitle: + result[i].dupes.add(b) + matched = true + break + + if not matched: + result.add(DuplicateGroup(keep: b, dupes: @[], reason: "")) + + for i in 0 ..< result.len: + if result[i].dupes.len > 0: + let keepNorm = normalizeUrl(result[i].keep.url) + var allNormMatch = true + for d in result[i].dupes: + if normalizeUrl(d.url) != keepNorm: + allNormMatch = false + break + result[i].reason = if allNormMatch: "normalized URL match" else: "same domain + title" + + result = result.filterIt(it.dupes.len > 0) + + for i in 0 ..< result.len: + var g = result[i] + if g.dupes.len > 0: + var bestIdx = 0 + for j in 0 ..< g.dupes.len: + if g.dupes[j].category.len > 0 and g.keep.category.len == 0: + bestIdx = j + 1 + break + if bestIdx > 0: + let oldKeep = g.keep + g.keep = g.dupes[bestIdx - 1] + g.dupes[bestIdx - 1] = oldKeep + result[i] = g + +proc removeDuplicates*(cfg: Config, ids: seq[int64]): int = + if ids.len == 0: + return 0 + let db = cfg.initDb() + defer: db.close() + let placeholders = repeat("?", ids.len).join(",") + result = db.execAffectedRows( + sql(&"DELETE FROM bookmarks WHERE id IN ({placeholders})"), ids) + +proc deleteBookmarks*(cfg: Config, ids: seq[int64]): int = + if ids.len == 0: + return 0 + let db = cfg.initDb() + defer: db.close() + let placeholders = repeat("?", ids.len).join(",") + result = db.execAffectedRows( + sql(&"DELETE FROM bookmarks WHERE id IN ({placeholders})"), ids) + +proc initDb*(cfg: Config): DbConn = + ensureDir(cfg.dataDir) + result = open(cfg.dbPath(), "", "", "") + for stmt in Schema.split(';'): + let trimmed = stmt.strip() + if trimmed.len > 0: + result.exec(sql(trimmed)) + +proc getOrCreateFolder(db: DbConn, path: string, parentPath: string = ""): FolderEntry = + let row = db.getRow(sql("SELECT id, uuid, path, parent_path, bookmark_count FROM folders WHERE path = ?"), path) + if row[0].len > 0: + return FolderEntry( + id: parseBiggestInt(row[0]), + uuid: row[1], + path: row[2], + parentId: if row[3].len > 0: parseBiggestInt(row[3]) else: 0, + bookmarkCount: if row[4].len > 0: parseBiggestInt(row[4]) else: 0, + ) + let uuid = genUuid() + result = FolderEntry( + uuid: uuid, + path: path, + parentId: 0, + bookmarkCount: 0, + ) + result.id = db.insertId(sql("INSERT INTO folders (uuid, path, parent_path, bookmark_count) VALUES (?, ?, ?, 0)"), uuid, path, parentPath) + return result + +proc importNetscapeHtml*(content: string): seq[tuple[url, title, folder: string]] = + result = @[] + var folderStack: seq[string] = @[""] + + for line in content.splitLines(): + let trimmed = line.strip() + if "
" in trimmed: + if "= 0: + let url = matches[0] + let titleStart = trimmed.find(">") + let titleEnd = trimmed.find("") + var title = "" + if titleStart >= 0 and titleEnd > titleStart: + title = trimmed[titleStart + 1 .. titleEnd - 1] + title = title.replace(re"<[^>]+>", "") + result.add((url, title, folderStack.join(" / "))) + elif "" in trimmed: + var matches: array[1, string] + if trimmed.find(re"""]*>(.*?)""", matches) >= 0: + var folderName = matches[0].replace(re"<[^>]+>", "") + folderStack.add(folderName) + if "" in trimmed and folderStack.len > 1: + discard folderStack.pop() + +proc importJson*(content: string): seq[tuple[url, title, folder: string]] = + result = @[] + try: + let parsed = parseJson(content) + for item in parsed.getElems(): + let url = item{"url"}.getStr("") + let title = item{"title"}.getStr("") + let folder = item{"folder"}.getStr("") + if url.len > 0: + result.add((url, title, folder)) + except: + discard + +proc importUrlList*(content: string): seq[tuple[url, title, folder: string]] = + result = @[] + for line in content.splitLines(): + let trimmed = line.strip() + if trimmed.len == 0 or trimmed.startsWith("#"): + continue + if trimmed.startsWith("http://") or trimmed.startsWith("https://"): + result.add((trimmed, "", "")) + +proc detectFormat*(content: string, filename: string): string = + let parts = filename.split('.') + let ext = if parts.len > 1: parts[parts.len - 1].toLowerAscii() else: "" + if ext == "json": + return "json" + if ext == "txt" or ext == "url" or ext == "urls": + return "urllist" + if ext == "html" or ext == "htm": + return "netscape" + if "
" in content and " 0: + continue + if folder.len > 0: + discard db.getOrCreateFolder(folder) + try: + db.exec( + sql("INSERT INTO bookmarks (url, title, raw_folder, import_id, added_at) VALUES (?, ?, ?, ?, ?)"), + url, title, folder, importId, now + ) + count.inc + except DbError: + continue + + return count + +proc getUnorganisedBookmarks*(cfg: Config, limit: int = 0): seq[BookmarkEntry] = + let db = cfg.initDb() + defer: db.close() + + var query = "SELECT id, url, title, raw_folder, category, confidence FROM bookmarks WHERE organised_at IS NULL" + if limit > 0: + query.add &" LIMIT {limit}" + query.add " ORDER BY added_at DESC" + + for row in db.fastRows(sql(query)): + result.add(BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + )) + +proc getAllBookmarks*(cfg: Config): seq[BookmarkEntry] = + let db = cfg.initDb() + defer: db.close() + + for row in db.fastRows(sql( + "SELECT id, url, title, raw_folder, category, confidence FROM bookmarks ORDER BY added_at DESC")): + result.add(BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + )) + +proc listBookmarks*(cfg: Config, category: string = ""): seq[BookmarkEntry] = + let db = cfg.initDb() + defer: db.close() + + if category.len > 0: + for row in db.fastRows(sql( + "SELECT id, url, title, raw_folder, category, confidence FROM bookmarks WHERE raw_folder = ? ORDER BY added_at DESC LIMIT 50"), + category): + result.add(BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + )) + else: + for row in db.fastRows(sql( + "SELECT id, url, title, raw_folder, category, confidence FROM bookmarks ORDER BY added_at DESC LIMIT 50")): + result.add(BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + )) + +proc searchBookmarks*(cfg: Config, query: string): seq[BookmarkEntry] = + let db = cfg.initDb() + defer: db.close() + + let pattern = "%" & query & "%" + for row in db.fastRows(sql( + "SELECT id, url, title, raw_folder, category, confidence FROM bookmarks WHERE title LIKE ? OR url LIKE ? OR category LIKE ? LIMIT 20"), + pattern, pattern, pattern): + result.add(BookmarkEntry( + id: parseBiggestInt(row[0]), + url: row[1], + title: row[2], + rawFolder: row[3], + category: row[4], + confidence: row[5], + )) + +proc getAllFolders*(cfg: Config): seq[FolderEntry] = + let db = cfg.initDb() + defer: db.close() + + for row in db.fastRows(sql("SELECT id, uuid, path, parent_path, bookmark_count FROM folders")): + result.add(FolderEntry( + id: parseBiggestInt(row[0]), + uuid: row[1], + path: row[2], + parentId: if row[3].len > 0: parseBiggestInt(row[3]) else: 0, + bookmarkCount: if row[4].len > 0: parseBiggestInt(row[4]) else: 0, + )) + +proc applyClassification*(cfg: Config, bookmarkId: int64, category: string, confidence: string, reason: string) = + let db = cfg.initDb() + defer: db.close() + + let now = getTime().toUnix() + db.exec(sql( + "UPDATE bookmarks SET category = ?, confidence = ?, reason = ?, organised_at = ? WHERE id = ?" + ), category, confidence, reason, now, bookmarkId) + +proc undoLastBatch*(cfg: Config): int = + let db = cfg.initDb() + defer: db.close() + + let row = db.getRow(sql( + "SELECT organised_at FROM bookmarks WHERE organised_at IS NOT NULL ORDER BY organised_at DESC LIMIT 1" + )) + if row.len > 0 and row[0].len > 0: + let batchTime = parseBiggestInt(row[0]) + result = db.execAffectedRows(sql( + "UPDATE bookmarks SET category = NULL, confidence = NULL, reason = NULL, organised_at = NULL WHERE organised_at >= ?" + ), batchTime) + +proc htmlEscape*(s: string): string = + result = s + result = result.replace("&", "&") + result = result.replace("<", "<") + result = result.replace(">", ">") + result = result.replace("\"", """) + +proc getBookmarksForExport*(cfg: Config, categoryFilter = ""): seq[tuple[url, title, category: string]] = + let db = cfg.initDb() + defer: db.close() + + let query = if categoryFilter.len > 0: + sql("SELECT url, title, category FROM bookmarks WHERE category = ? ORDER BY category, title") + else: + sql("SELECT url, title, category FROM bookmarks ORDER BY category, title") + + for row in db.fastRows(query, categoryFilter): + result.add(( + url: row[0], + title: row[1], + category: if row[2].len > 0: row[2] else: "Unorganized", + )) + +proc exportBookmarksHtml*(cfg: Config, categoryFilter = ""): string = + let bookmarks = getBookmarksForExport(cfg, categoryFilter) + if bookmarks.len == 0: + return "" + + var lines: seq[string] = @[] + lines.add("""""") + lines.add("""""") + lines.add("Bookmarks") + lines.add("

Bookmarks

") + lines.add("

") + + var currentCat = "" + for bm in bookmarks: + if bm.category != currentCat: + if currentCat.len > 0: + lines.add("

") + currentCat = bm.category + lines.add("

" & htmlEscape(currentCat) & "

") + lines.add("

") + let title = if bm.title.len > 0: bm.title else: bm.url + lines.add("

" & htmlEscape(title) & "") + + lines.add("

") + lines.add("

") + return lines.join("\n") diff --git a/src/lazybookmarks/ui.nim b/src/lazybookmarks/ui.nim new file mode 100644 index 0000000..6be3656 --- /dev/null +++ b/src/lazybookmarks/ui.nim @@ -0,0 +1,132 @@ +import std/[terminal, strutils, strformat] +import ./storage +import ./linkchecker + +proc infoMsg*(msg: string) = + stdout.styledWriteLine(styleBright, fgGreen, " ✓ ", fgDefault, resetStyle, msg) + +proc warnMsg*(msg: string) = + stdout.styledWriteLine(styleBright, fgYellow, " ! ", fgDefault, resetStyle, msg) + +proc errorMsg*(msg: string) = + stdout.styledWriteLine(styleBright, fgRed, " ✗ ", fgDefault, resetStyle, msg) + +proc dimMsg*(msg: string) = + stdout.styledWrite(styleDim, " " & msg, resetStyle, "\n") + +proc headerMsg*(msg: string) = + stdout.styledWriteLine(styleBright, fgCyan, "\n " & msg, resetStyle, "\n") + +proc formatElapsed*(seconds: int): string = + if seconds < 60: + return &"{seconds}s" + let m = seconds div 60 + let s = seconds mod 60 + return &"{m}m{s:02d}s" + +proc showProgressBar*(current: int, total: int, prefix: string = "", elapsed: int = -1) = + stdout.write "\r\e[2K" + if total == 0: + stdout.write prefix & " 0/0" + stdout.flushFile() + return + let pct = (current * 100) div total + let width = 30 + let filled = (current * width) div total + let bar = repeat("#", filled) & repeat("-", width - filled) + var suffix = " (" & $current & "/" & $total & ")" + if elapsed >= 0: + suffix.add(" " & formatElapsed(elapsed)) + stdout.write prefix & " [" & bar & "] " & $pct & "%" & suffix + stdout.flushFile() + +type ReviewAction* = enum + accept, skip, edit, quitReview + +proc reviewSuggestion*(url: string, title: string, targetFolder: string, confidence: string, reason: string): ReviewAction = + echo "" + stdout.styledWrite(styleBright, " ┌─ ", resetStyle, url, "\n") + stdout.styledWrite(styleBright, " │ ", resetStyle) + stdout.write "\"" & title & "\"\n" + let confColor = case confidence + of "high": fgGreen + of "medium": fgYellow + else: fgRed + stdout.styledWrite(styleBright, " │ ", resetStyle, "→ ") + stdout.styledWrite(confColor, targetFolder, resetStyle) + stdout.write " [" & confidence.toUpperAscii() & "]\n" + stdout.styledWriteLine(styleBright, " │ ", resetStyle, styleDim, reason, resetStyle) + stdout.styledWriteLine(styleBright, " └─ ", resetStyle, styleDim, "[A]ccept [S]kip [e]dit [q]uit", resetStyle) + stdout.write " > " + stdout.flushFile() + + while true: + let input = stdin.readLine().strip().toLowerAscii() + case input + of "a", "accept": return ReviewAction.accept + of "s", "skip": return ReviewAction.skip + of "e", "edit": return ReviewAction.edit + of "q", "quit": return ReviewAction.quitReview + else: + stdout.write "\r\e[2K" + stdout.write " > " + stdout.flushFile() + +proc reviewDuplicateGroup*(idx: int, total: int, group: DuplicateGroup): bool = + echo "" + stdout.styledWriteLine(styleBright, fgCyan, &" Duplicate group {idx}/{total} ", resetStyle, styleDim, &"({group.reason})", resetStyle) + stdout.styledWriteLine(styleBright, " Keep: ", fgGreen, group.keep.title, resetStyle, styleDim, &" [{group.keep.url[0..min(79, group.keep.url.high)]}]", resetStyle) + for i, d in group.dupes: + let title = if d.title.len > 0: d.title else: "(untitled)" + stdout.styledWriteLine(styleDim, " - ", resetStyle, title, styleDim, &" [{d.url[0..min(79, d.url.high)]}]", resetStyle) + stdout.styledWriteLine(styleBright, " └─ ", resetStyle, styleDim, "[R]emove dupes [S]kip [q]uit", resetStyle) + stdout.write " > " + stdout.flushFile() + + while true: + let input = stdin.readLine().strip().toLowerAscii() + case input + of "r", "remove": return true + of "s", "skip": return false + of "q", "quit": return false + else: + stdout.write "\r\e[2K" + stdout.write " > " + stdout.flushFile() + +proc showLinkResult*(r: LinkResult) = + let title = if r.bookmark.title.len > 0: r.bookmark.title else: "(untitled)" + let (label, color) = case r.status + of lsAlive: ("OK", fgGreen) + of lsDead: ("DEAD", fgRed) + of lsUnknown: ("???", fgYellow) + of lsRedirected: ("REDIR", fgCyan) + var extra = "" + if r.status == lsRedirected and r.redirectUrl.len > 0: + extra = &" -> {r.redirectUrl[0..min(60, r.redirectUrl.high)]}" + if r.statusCode > 0: + extra = &" [{r.statusCode}]{extra}" + stdout.styledWrite(" ", color, &"{label:<5}", resetStyle, &" {title:<50}", styleDim, &" {r.bookmark.url[0..min(60, r.bookmark.url.high)]}{extra}", resetStyle, "\n") + +proc showLinkSummary*(results: seq[LinkResult]) = + var alive = 0 + var dead = 0 + var unknown = 0 + var redirected = 0 + for r in results: + case r.status + of lsAlive: inc alive + of lsDead: inc dead + of lsUnknown: inc unknown + of lsRedirected: inc redirected + echo "" + stdout.styledWriteLine(styleBright, " Summary:", resetStyle) + if alive > 0: + stdout.styledWriteLine(" ", fgGreen, &"{alive} alive", resetStyle) + if dead > 0: + stdout.styledWriteLine(" ", fgRed, &"{dead} dead", resetStyle) + if redirected > 0: + stdout.styledWriteLine(" ", fgCyan, &"{redirected} redirected", resetStyle) + if unknown > 0: + stdout.styledWriteLine(" ", fgYellow, &"{unknown} unknown", resetStyle) + echo ""