From 84e4b4c539f732ef064603a7d771147064efe7e7 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 12:48:35 -0400 Subject: [PATCH 01/36] feat(server): ownership-partition debug overlay per provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/debug/partition bakes an ownership-partition PMTiles for every installed provider (which cell renders which ground per band, no portrayed content) via the new tile57.BakePartitionDebug binding, and registers each as the "{provider}-partition" tile set — served at /tiles/{provider}-partition/{z}/{x}/{y}.mvt (layers: "partition" polygons by cell colour, "labels" points with the owning cell name). GET returns each provider's readiness + tile URL, so a debug UI can trigger generation and overlay the result. Reuses encRootDir + setDir + the tileSets registry; the partition math is all in the engine. Verified live against the NOAA provider (3528 cells, z0-12): bake ~12s, tiles serve 200 with both layers. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/debug_partition.go | 90 +++++++++++++++++++++++ internal/engine/server/http.go | 2 + 2 files changed, 92 insertions(+) create mode 100644 internal/engine/server/debug_partition.go diff --git a/internal/engine/server/debug_partition.go b/internal/engine/server/debug_partition.go new file mode 100644 index 0000000..d6a6916 --- /dev/null +++ b/internal/engine/server/debug_partition.go @@ -0,0 +1,90 @@ +package server + +import ( + "fmt" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// partitionSuffix names a provider's ownership-partition debug tile set: it is served +// at /tiles/{provider}-partition/{z}/{x}/{y}.mvt, separate from the real chart set. +const partitionSuffix = "-partition" + +// Zoom range for the debug partition. z0..12 covers overview→approach across a whole +// provider quickly; harbor-level detail (z13+) multiplies the tile count ~4×/zoom, so +// the overlay overzooms past 12 rather than baking it. +const ( + partitionMinZoom = 0 + partitionMaxZoom = 12 +) + +// handleDebugPartition drives the ownership-partition DEBUG overlay. +// +// POST /api/debug/partition → (re)generate the partition for every provider, +// in the background; returns the provider list. +// GET /api/debug/partition → per provider: whether its partition tile set is +// ready + the tile URL template to overlay. +func (s *Server) handleDebugPartition(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + providers := s.installedProviders() + // Bake off the request goroutine — a whole provider is seconds, several add up. + go func() { + for _, p := range providers { + if err := s.bakeProviderPartition(p); err != nil { + log.Printf("partition-debug %s: %v", p, err) + } + } + }() + writeJSON(w, map[string]any{"ok": true, "providers": providers}) + case http.MethodGet: + type item struct { + Provider string `json:"provider"` + Ready bool `json:"ready"` + Tiles string `json:"tiles"` + } + items := []item{} + for _, p := range s.installedProviders() { + _, ready := s.sets.get(p + partitionSuffix) + items = append(items, item{ + Provider: p, + Ready: ready, + Tiles: "tiles/" + p + partitionSuffix + "/{z}/{x}/{y}.mvt", + }) + } + writeJSON(w, map[string]any{"partitions": items}) + default: + apiErr(w, http.StatusMethodNotAllowed, "POST or GET") + } +} + +// bakeProviderPartition bakes ONE provider's ENC_ROOT into an ownership-partition debug +// PMTiles (which cell renders which ground per band; NO portrayed content) and registers +// it as the "{provider}-partition" tile set. Reuses the same ENC_ROOT + output dir as the +// real bake; the partition math is entirely in the engine (tile57.BakePartitionDebug). +func (s *Server) bakeProviderPartition(provider string) error { + if len(s.providerDistricts(provider)) == 0 { + return fmt.Errorf("provider %q has no districts", provider) + } + encRoot := s.encRootDir(provider) + outPath := filepath.Join(s.setDir(provider), "tiles", provider+partitionSuffix+".pmtiles") + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return err + } + n, err := tile57.BakePartitionDebug(encRoot, outPath, partitionMinZoom, partitionMaxZoom, tile57.BandGoverning) + if err != nil { + return fmt.Errorf("bake partition: %w", err) + } + src, err := tilesource.Open(outPath) + if err != nil { + return fmt.Errorf("open partition pmtiles: %w", err) + } + s.sets.register(provider+partitionSuffix, src) + log.Printf("partition-debug: %q (%d cell(s)) → %s", provider, n, outPath) + return nil +} diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 75fa4f3..1cb9555 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -325,6 +325,8 @@ func (s *Server) handleAPI(w http.ResponseWriter, r *http.Request) { s.handleDeleteDistrict(w, r) // DELETE: remove one district + re-bake the provider case r.URL.Path == "/api/proxy": s.serveProxy(w, r) // dumb CORS/Range passthrough for a NOAA URL (e.g. All_ENCs.zip) + case strings.HasPrefix(r.URL.Path, "/api/debug/partition"): + s.handleDebugPartition(w, r) // POST: bake ownership-partition overlays; GET: their status default: apiErr(w, http.StatusNotFound, "unknown endpoint") } From f5c1494165c4bbd9f63a1da297ef1171e71612ac Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 12:54:56 -0400 Subject: [PATCH 02/36] feat(web): partition-debug overlay controls in dev-tools (Advanced) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Advanced gains an "Ownership partition (debug)" section: - "Generate partition overlays" POSTs /api/debug/partition (server bakes one partition PMTiles per installed provider), then polls until each is ready. - Per provider, a "Show/Hide overlay" toggle adds/removes a MapLibre vector source over the chart from /tiles/{provider}-partition: a fill layer coloured by the face "color" property + a symbol layer labelling each face with its owning "cell". Re-applied on style rebuild (mariner change); removed on teardown. Lets you see the composite ownership quilt directly on the chart to debug which cell renders which ground per band. JS syntax-checked; server side verified live. Co-Authored-By: Claude Opus 4.8 (1M context) --- web/src/plugins/dev-tools.mjs | 91 +++++++++++++++++++++++++++++- web/src/plugins/dev-tools.view.mjs | 32 ++++++++++- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/web/src/plugins/dev-tools.mjs b/web/src/plugins/dev-tools.mjs index 66d0aad..bbcb01a 100644 --- a/web/src/plugins/dev-tools.mjs +++ b/web/src/plugins/dev-tools.mjs @@ -91,6 +91,10 @@ export class DevTools { this._inspectLastKey = ""; this._areaCleanup = null; // mirrors the shell's box-download guard (kept null here) this._busy = false; // a rebake is running (in addition to the injected isBusy) + // Ownership-partition debug overlay state. + this._partitions = null; // [{provider, ready, tiles}] from GET /api/debug/partition (null = not loaded) + this._partOn = new Set(); // providers whose partition overlay is currently on the map + this._partGen = false; // a generate run is in flight this._registerSelf(deps.registry); this._wireMap(); @@ -128,12 +132,16 @@ export class DevTools { const busy = this._isBusy(); // The dialog's shadow has its own sheet; inject our chrome once per render so // the dev-tools + inspector classes resolve inside it. - host.innerHTML = `${devToolsPanel(busy, this._inspectMode, this._selectingArea)}`; + const part = { list: this._partitions, on: this._partOn, gen: this._partGen }; + host.innerHTML = `${devToolsPanel(busy, this._inspectMode, this._selectingArea, part)}`; const q = (id) => host.querySelector("#" + id); const rebuild = q("dev-rebuild"); if (rebuild && !rebuild.disabled) rebuild.onclick = (e) => this._rebuildAllPerBand(e.currentTarget); const inspect = q("dev-inspect"); if (inspect) inspect.onclick = () => this.setInspectMode(!this._inspectMode); const area = q("dev-area"); if (area && !area.disabled) area.onclick = () => this._toggleSelectArea(); const feat = q("dev-feat"); if (feat && !feat.disabled) feat.onclick = (e) => this._copyInspectDebug(e.currentTarget); + const pgen = q("dev-part-gen"); if (pgen && !pgen.disabled) pgen.onclick = () => this._generatePartitions(); + host.querySelectorAll("[data-part-toggle]").forEach((el) => (el.onclick = () => this._togglePartitionOverlay(el.dataset.partToggle, !this._partOn.has(el.dataset.partToggle)))); + if (this._partitions == null && !this._partLoading) this._loadPartitions(); // lazy: fetch status once // If inspect is on with a live selection, repaint the result panel. if (this._inspectMode && this._inspectFeats.length) this._renderInspect(); else if (this._inspectMode) this._inspectHint(INSPECT_HINT); @@ -195,6 +203,82 @@ export class DevTools { if (btn) flashBtn(btn, `✓ rebuilt ${todo.length}`); } + // --- ownership-partition debug overlay ----------------------------------- + // GET the per-provider partition status (is each provider's partition tile set baked + // and ready to overlay?) and repaint the panel. + async _loadPartitions() { + this._partLoading = true; + const assets = this._d.assets || ""; + try { + const j = await fetch(`${assets}api/debug/partition?t=${Date.now()}`).then((r) => (r.ok ? r.json() : null)); + this._partitions = (j && j.partitions) || []; + } catch (e) { this._partitions = []; } + this._partLoading = false; + this._refreshPanel(); + } + + // POST to (re)generate every provider's partition PMTiles on the server, then poll the + // status until they're all ready (or give up), repainting as each lands. + async _generatePartitions() { + if (this._partGen) return; + const assets = this._d.assets || ""; + this._partGen = true; + this._refreshPanel(); + try { await fetch(`${assets}api/debug/partition`, { method: "POST" }); } catch (e) { console.warn("[partition] generate", e); } + for (let i = 0; i < 90; i++) { + await new Promise((r) => setTimeout(r, 2000)); + await this._loadPartitions(); + const ps = this._partitions || []; + if (ps.length && ps.every((p) => p.ready)) break; + } + this._partGen = false; + this._refreshPanel(); + } + + // Show/hide a provider's partition overlay on the chart map. + _togglePartitionOverlay(provider, on) { + if (!provider) return; + if (on) { this._partOn.add(provider); this._addPartitionLayers(provider); } + else { this._partOn.delete(provider); this._removePartitionLayers(provider); } + this._refreshPanel(); + } + + _partSourceId(provider) { return `dbg-part-${provider}`; } + + // Add the vector source + fill (by face colour) + symbol (cell name) layers for a + // provider's partition, pointing at the server's /tiles/{provider}-partition set. + _addPartitionLayers(provider) { + const map = this._map; + if (!map || !map.isStyleLoaded || !map.isStyleLoaded()) return; + this._removePartitionLayers(provider); // idempotent re-add + const sid = this._partSourceId(provider); + const base = new URL(`${this._d.assets || ""}tiles/${provider}-partition/`, document.baseURI).href; + try { + map.addSource(sid, { type: "vector", tiles: [base + "{z}/{x}/{y}.mvt"], minzoom: 0, maxzoom: 12 }); + map.addLayer({ + id: sid + "-fill", type: "fill", source: sid, "source-layer": "partition", + paint: { "fill-color": ["get", "color"], "fill-opacity": 0.35, "fill-outline-color": "#000000" }, + }); + map.addLayer({ + id: sid + "-label", type: "symbol", source: sid, "source-layer": "labels", + layout: { "text-field": ["get", "cell"], "text-font": ["Noto Sans Regular"], "text-size": 11, "text-allow-overlap": false }, + paint: { "text-color": "#ffffff", "text-halo-color": "#000000", "text-halo-width": 1.4 }, + }); + } catch (e) { console.warn("[partition] add layers", provider, e); } + } + + _removePartitionLayers(provider) { + const map = this._map; + if (!map) return; + const sid = this._partSourceId(provider); + for (const id of [sid + "-fill", sid + "-label"]) { if (map.getLayer && map.getLayer(id)) map.removeLayer(id); } + if (map.getSource && map.getSource(sid)) map.removeSource(sid); + } + + // The chart style is rebuilt on mariner changes (setStyle drops every source); re-add + // any active partition overlays once the new style has loaded. + _reapplyPartitions() { for (const p of this._partOn) this._addPartitionLayers(p); } + // --- feature inspector --------------------------------------------------- // Add the inspector's map listeners ONCE. They all no-op unless inspect mode is // on (so the shell's own click handler need only defer to `inspecting`). The @@ -273,6 +357,9 @@ export class DevTools { c.addEventListener("pointerup", this._onPointerUp); c.addEventListener("pointercancel", this._onPointerUp); map.on("click", this._onClick); + // A mariner change rebuilds the style (setStyle), dropping our overlay sources — + // re-add any that are toggled on once the new style loads. + map.on("style.load", this._onStyleLoad = () => this._reapplyPartitions()); } _unwireMap() { @@ -285,6 +372,8 @@ export class DevTools { if (this._onPointerUp) { c.removeEventListener("pointerup", this._onPointerUp); c.removeEventListener("pointercancel", this._onPointerUp); } } if (this._onClick) map.off("click", this._onClick); + if (this._onStyleLoad) { map.off("style.load", this._onStyleLoad); this._onStyleLoad = null; } + try { this._partOn.forEach((p) => this._removePartitionLayers(p)); } catch (e) { /* map gone */ } if (this._hoverRaf) { cancelAnimationFrame(this._hoverRaf); this._hoverRaf = 0; } } diff --git a/web/src/plugins/dev-tools.view.mjs b/web/src/plugins/dev-tools.view.mjs index 207b762..412dc5a 100644 --- a/web/src/plugins/dev-tools.view.mjs +++ b/web/src/plugins/dev-tools.view.mjs @@ -91,10 +91,36 @@ export function inspectorSection(inspecting, selectingArea) { `; } -// The whole dev-tools panel skeleton: rebake + inspector sections + the +// The ownership-partition debug section: a "Generate" button (the server bakes one +// partition PMTiles per provider) + a per-provider "Show/Hide overlay" toggle. `part` +// = { list, on, gen }: list = [{provider, ready, tiles}] (null while loading), on = the +// Set of providers whose overlay is shown, gen = a generate run is in flight. +export function partitionSection(part, busy) { + const list = part && part.list; + const on = (part && part.on) || new Set(); + const gen = part && part.gen; + let rows; + if (list == null) rows = `

Loading…

`; + else if (!list.length) rows = `

No providers installed.

`; + else rows = list.map((p) => { + const shown = on.has(p.provider); + const ctrl = p.ready + ? `` + : `generating…`; + return `
${esc(p.provider)}${ctrl}
`; + }).join(""); + return `
+
Ownership partition (debug)
+ + ${rows} +

A PMTiles per provider showing which cell owns each region per band (finer cells win; coarser fill the gaps). Enable a provider's overlay to see the composite quilt on the chart — each face is coloured and labelled with its cell.

+
`; +} + +// The whole dev-tools panel skeleton: rebake + partition + inspector sections + the // inspect-result container (filled separately by the logic on hover/click). -export function devToolsPanel(busy, inspecting, selectingArea) { - return `
${rebakeSection(busy)}${inspectorSection(inspecting, selectingArea)}
`; +export function devToolsPanel(busy, inspecting, selectingArea, part) { + return `
${rebakeSection(busy)}${partitionSection(part, busy)}${inspectorSection(inspecting, selectingArea)}
`; } // One inspected feature card. `label`/`acr`/`named` come from the logic's injected From dded2b1c9d4257955cb146af6bfd673aafed45b8 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 17:26:23 -0400 Subject: [PATCH 03/36] feat(bake): per-cell composite bake replaces the in-bake combiner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bakeProvider now bakes each cell to its own native-scale PMTiles (coverage embedded) and streams them through the engine's ownership partition into tiles/chart.pmtiles (tile57.BakeCell -> ComposeFiles), replacing tile57.BakeBundle's in-bake cross-cell combiner. Only tiles/chart.pmtiles is needed by the serving path — the MapLibre style is built dynamically (tile57.Style) and the sprite/glyphs/colortables are global server assets — so the composite bake skips the bundle's (unused) assets/style/ manifest emission. Progress is now per-cell (bake i/N) then a compose phase, driven from Go. Escape hatch: TILE57_LEGACY_BAKE=1 restores the BakeBundle path while the composite model beds in. KNOWN GAPS to flush out: per-cell bake is serial (a warmup + worker pool is the next lever for whole-district speed); the compose phase has no fine-grained progress. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/tile57_bake.go | 116 ++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 7 deletions(-) diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 37b2069..08304cf 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -2,9 +2,11 @@ package server import ( "fmt" + "io/fs" "log" "os" "path/filepath" + "strings" "time" "github.com/beetlebugorg/chartplotter/internal/engine/auxfiles" @@ -122,17 +124,34 @@ func (s *Server) bakeProvider(jobID, provider string) bool { j.Phase, j.Band, j.Unit, j.Note, j.Done, j.Total = "bake", "", "cells", "Preparing charts", 0, 0 }) created := time.Now().UTC().Format(time.RFC3339) - // MaxZoom 24 = the ABI's "no clamp" (each cell's full native band); MaxZoom 0 would - // clamp every band down to z0 — an EMPTY archive. - n, bbox, err := tile57.BakeBundle(encRoot, outDir, tile57.BakeOpts{Created: created, MaxZoom: 24}, s.bakeProgress(jobID)) + + // The per-cell COMPOSITE model (default): bake each cell to its own native-scale PMTiles, + // then combine them via the engine's ownership partition into tiles/chart.pmtiles. This + // replaces the in-bake cross-cell combiner (BakeBundle) — kept behind TILE57_LEGACY_BAKE=1 + // as an escape hatch while the composite model beds in. The served bundle only needs + // tiles/chart.pmtiles: the MapLibre style is built dynamically (tile57.Style) and the + // sprite/glyphs/colortables are global server assets, so the composite path skips the + // bundle's (unused) assets/style/manifest emission. + var n int + var err error + if os.Getenv("TILE57_LEGACY_BAKE") != "" { + // MaxZoom 24 = the ABI's "no clamp" (each cell's full native band). + var bbox [4]float64 + n, bbox, err = tile57.BakeBundle(encRoot, outDir, tile57.BakeOpts{Created: created, MaxZoom: 24}, s.bakeProgress(jobID)) + if err == nil && (n == 0 || bbox[2] <= bbox[0] || bbox[3] <= bbox[1]) { + os.RemoveAll(outDir) + return fail(fmt.Errorf("import produced no coverage (%d cell(s), no valid S-57 data)", n)) + } + } else { + n, err = s.composeProvider(jobID, encRoot, outDir) + } if err != nil { return fail(err) } - // An inverted/empty bbox (or zero cells) means nothing valid parsed. Treat it as a - // failed import (don't register an empty pack) and drop the stub bundle it wrote. - if n == 0 || bbox[2] <= bbox[0] || bbox[3] <= bbox[1] { + // Zero cells means nothing valid parsed → a failed import (don't register an empty pack). + if n == 0 { os.RemoveAll(outDir) - return fail(fmt.Errorf("import produced no coverage (%d cell(s), no valid S-57 data)", n)) + return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) } s.imports.update(jobID, func(j *importJob) { j.Phase, j.Note = "meta", "Reading chart metadata" }) @@ -147,6 +166,89 @@ func (s *Server) bakeProvider(jobID, provider string) bool { return true } +// composeProvider bakes each cell under encRoot to its own native-scale PMTiles (coverage +// embedded in the metadata) and streams them through the engine's ownership partition into +// /tiles/chart.pmtiles. Per-cell archives go to a temp dir (mmap'd by the compositor, +// then discarded), so the whole cell set is never resident. Returns the count of cells that +// contributed to the composite, or 0 if none produced coverage. +func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { + cells, err := listCells(encRoot) + if err != nil { + return 0, err + } + if len(cells) == 0 { + return 0, nil + } + + // Per-cell PMTiles cache (temp; discarded after the compose reads them). + cellsDir, err := os.MkdirTemp("", "tile57-cells-*") + if err != nil { + return 0, err + } + defer os.RemoveAll(cellsDir) + + // 1. Bake each cell to its own PMTiles (one cell resident at a time — the bytes are freed + // as soon as they are written). Serial for now; a warmup + worker pool is the next lever. + perCell := make([]string, 0, len(cells)) + for i, cp := range cells { + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Unit, j.Note = "bake", "", "cells", "Baking charts" + j.Done, j.Total = i, len(cells) + }) + b, err := tile57.BakeCell(cp) + if err != nil { + log.Printf("import %s: bake cell %s: %v (skipping)", jobID, filepath.Base(cp), err) + continue + } + if len(b) == 0 { + continue + } + pc := filepath.Join(cellsDir, filepath.Base(cp)+".pmtiles") + if err := os.WriteFile(pc, b, 0o644); err != nil { + log.Printf("import %s: write per-cell %s: %v (skipping)", jobID, filepath.Base(cp), err) + continue + } + perCell = append(perCell, pc) + } + if len(perCell) == 0 { + return 0, nil + } + + // 2. Stream-compose the per-cell archives into tiles/chart.pmtiles via the partition. + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Unit, j.Note, j.Done, j.Total = "bake", "", "cells", "Composing tiles", len(cells), len(cells) + }) + tilesDir := filepath.Join(outDir, "tiles") + if err := os.MkdirAll(tilesDir, 0o755); err != nil { + return 0, err + } + n, err := tile57.ComposeFiles(perCell, filepath.Join(tilesDir, "chart.pmtiles")) + if err != nil { + return 0, err + } + return n, nil +} + +// listCells returns every base cell (.000) path under encRoot, deduped by stem (a boundary +// cell shared by two districts bakes once). +func listCells(encRoot string) ([]string, error) { + var out []string + seen := map[string]bool{} + err := filepath.WalkDir(encRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".000") { + return nil + } + stem := strings.TrimSuffix(filepath.Base(path), ".000") + if seen[stem] { + return nil + } + seen[stem] = true + out = append(out, path) + return nil + }) + return out, err +} + // dropProviderSet unregisters a provider whose ENC_ROOT is now empty and removes its // (regenerable) baked bundle + its (now-empty) provider data tree. The cell index is // rebuilt so its cells stop counting as installed. From 9c28bf6a8fba2bd21fb4dc362cd552bffca83a2b Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 17:52:38 -0400 Subject: [PATCH 04/36] fix(bake): composeProvider errors when all cells fail to bake (don't drop the tree) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit composeProvider swallowed per-cell BakeCell errors and returned (0, nil) when every cell failed, which bakeProvider treats as "no coverage" → os.RemoveAll(outDir). In the colocated cache+data test layout that deletes the provider's ENC_ROOT source. BakeBundle errors in this case (no removeAll), so match it: return an error when cells were present but none baked. Fixes TestImportFetchDownloadOnly on the compose path. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/tile57_bake.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 08304cf..62a2666 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -211,6 +211,12 @@ func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { perCell = append(perCell, pc) } if len(perCell) == 0 { + // Cells were present but none baked (e.g. all unparseable) → a bake ERROR, not empty + // coverage. Returning an error (like BakeBundle) lets the caller fail WITHOUT dropping + // the provider's stub/source; returning 0,nil would trip the "no coverage" RemoveAll. + if len(cells) > 0 { + return 0, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) + } return 0, nil } From 45e296d50762d72c1ff15063e8346c29691c26f6 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 17:56:38 -0400 Subject: [PATCH 05/36] feat(bake): CLI `chartplotter bake` uses the per-cell composite too (shared helper) Extract the compose flow (walk cells -> BakeCell each -> ComposeFiles) into baker.ComposeENCRoot, shared by the server bakeProvider and the CLI runTile57Archive. Both now default to the ownership-partition composite; TILE57_LEGACY_BAKE=1 restores the in-bake BakeBundle combiner. The CLI reads the manifest bbox back from the composed archive (OpenPMTiles.Info). Server composeProvider is now a thin wrapper that maps the onProgress/onSkip callbacks to the import job. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/tile57_bake.go | 61 ++++++++++++---- internal/engine/baker/compose.go | 101 ++++++++++++++++++++++++++ internal/engine/server/tile57_bake.go | 96 ++++-------------------- 3 files changed, 163 insertions(+), 95 deletions(-) create mode 100644 internal/engine/baker/compose.go diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go index 902acb9..3a59713 100644 --- a/cmd/chartplotter/tile57_bake.go +++ b/cmd/chartplotter/tile57_bake.go @@ -7,6 +7,7 @@ import ( "sort" "strings" + "github.com/beetlebugorg/chartplotter/internal/engine/baker" tile57 "github.com/beetlebugorg/tile57/bindings/go" ) @@ -57,21 +58,55 @@ func (c bakeCmd) runTile57Archive() error { if err != nil { return err } - tmp, err := os.MkdirTemp(filepath.Dir(outAbs), ".bake-*") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - n, bbox, err := bakeTile57Bundle(input, tmp, c.MaxZoom, c.Format, nil) - if err != nil { - return err - } - if err := os.Rename(filepath.Join(tmp, "tiles", "chart.pmtiles"), outAbs); err != nil { - return err + var n int + var bbox [4]float64 + if os.Getenv("TILE57_LEGACY_BAKE") != "" { + // Legacy in-bake combiner: BakeBundle into a temp dir beside -o, then rename the + // finished tiles/chart.pmtiles onto -o and discard the bundle scaffolding. + tmp, terr := os.MkdirTemp(filepath.Dir(outAbs), ".bake-*") + if terr != nil { + return terr + } + defer os.RemoveAll(tmp) + n, bbox, err = bakeTile57Bundle(input, tmp, c.MaxZoom, c.Format, nil) + if err != nil { + return err + } + if err := os.Rename(filepath.Join(tmp, "tiles", "chart.pmtiles"), outAbs); err != nil { + return err + } + st, _ := os.Stat(outAbs) + fmt.Printf("baked %d cell(s) → %s (%.1f MB) via libtile57 (streamed)\n", n, c.Out, float64(st.Size())/(1<<20)) + } else { + // Per-cell COMPOSITE (default): bake each cell at its native scale, then combine them + // via the engine's ownership partition straight into -o. --max-zoom/--format do not + // apply — each cell bakes at its native band and the compositor expands zoom. + n, err = baker.ComposeENCRoot(input, outAbs, + func(done, total int) { + if done < total { + fmt.Printf("\rbaking cells %d/%d… ", done, total) + } else { + fmt.Printf("\rcomposing %d cells… ", total) + } + }, + func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) + fmt.Println() + if err != nil { + return err + } + if n == 0 { + return fmt.Errorf("no coverage: no valid S-57 cells under %s", input) + } + st, _ := os.Stat(outAbs) + fmt.Printf("composed %d cell(s) → %s (%.1f MB) via the per-cell ownership partition\n", n, c.Out, float64(st.Size())/(1<<20)) + // bbox for the manifest, read back from the composed archive. + if src, e := tile57.OpenPMTiles(outAbs); e == nil { + info := src.Info() + bbox = [4]float64{info.West, info.South, info.East, info.North} + src.Close() + } } - st, _ := os.Stat(outAbs) - fmt.Printf("baked %d cell(s) → %s (%.1f MB) via libtile57 (streamed)\n", n, c.Out, float64(st.Size())/(1<<20)) // Aux content walks the INPUT tree (for a lone .000, its directory). auxRoot := input diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go new file mode 100644 index 0000000..0e7da29 --- /dev/null +++ b/internal/engine/baker/compose.go @@ -0,0 +1,101 @@ +package baker + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// ComposeENCRoot is the per-cell COMPOSITE bake: it bakes each cell under `input` (a single +// .000 or an ENC_ROOT directory) to its own native-scale PMTiles (coverage embedded in the +// metadata), then streams them through the engine's ownership partition into `outPath`. Per-cell +// archives go to a temp dir (mmap'd by the compositor, then discarded), so the whole cell set is +// never resident. This replaces the in-bake cross-cell combiner (tile57.BakeBundle) — the tiles +// half only; a host builds the style dynamically and serves global assets. +// +// onProgress(done, total) is called before each cell bake (done 0..total-1) and once more with +// done==total when the bakes are finished and the partition compose begins; nil to skip. onSkip +// (nil to skip) reports a cell that failed to bake. Returns the count of cells that contributed; +// an error (not 0) is returned when cells were present but none baked. +func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onSkip func(cell string, err error)) (int, error) { + cells, err := ListCells(input) + if err != nil { + return 0, err + } + if len(cells) == 0 { + return 0, nil + } + + cellsDir, err := os.MkdirTemp("", "tile57-cells-*") + if err != nil { + return 0, err + } + defer os.RemoveAll(cellsDir) + + // 1. Bake each cell to its own PMTiles (one cell resident at a time — the bytes are freed as + // soon as they are written). + perCell := make([]string, 0, len(cells)) + for i, cp := range cells { + if onProgress != nil { + onProgress(i, len(cells)) + } + b, err := tile57.BakeCell(cp) + if err != nil { + if onSkip != nil { + onSkip(filepath.Base(cp), err) + } + continue + } + if len(b) == 0 { + continue + } + pc := filepath.Join(cellsDir, filepath.Base(cp)+".pmtiles") + if err := os.WriteFile(pc, b, 0o644); err != nil { + if onSkip != nil { + onSkip(filepath.Base(cp), err) + } + continue + } + perCell = append(perCell, pc) + } + if len(perCell) == 0 { + // Cells were present but none baked → a bake ERROR, not empty coverage (so the caller + // fails without dropping the provider/source as "no coverage"). + return 0, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) + } + + // 2. Stream-compose the per-cell archives into outPath via the ownership partition. + if onProgress != nil { + onProgress(len(cells), len(cells)) + } + if dir := filepath.Dir(outPath); dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + return 0, err + } + } + return tile57.ComposeFiles(perCell, outPath) +} + +// ListCells returns every base cell (.000) path under `root` (a single file or a directory), +// deduped by stem (a boundary cell shared by two districts bakes once). +func ListCells(root string) ([]string, error) { + var out []string + seen := map[string]bool{} + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".000") { + return nil + } + stem := strings.TrimSuffix(filepath.Base(path), ".000") + if seen[stem] { + return nil + } + seen[stem] = true + out = append(out, path) + return nil + }) + return out, err +} diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 62a2666..f2741bf 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -2,11 +2,9 @@ package server import ( "fmt" - "io/fs" "log" "os" "path/filepath" - "strings" "time" "github.com/beetlebugorg/chartplotter/internal/engine/auxfiles" @@ -172,87 +170,21 @@ func (s *Server) bakeProvider(jobID, provider string) bool { // then discarded), so the whole cell set is never resident. Returns the count of cells that // contributed to the composite, or 0 if none produced coverage. func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { - cells, err := listCells(encRoot) - if err != nil { - return 0, err - } - if len(cells) == 0 { - return 0, nil - } - - // Per-cell PMTiles cache (temp; discarded after the compose reads them). - cellsDir, err := os.MkdirTemp("", "tile57-cells-*") - if err != nil { - return 0, err - } - defer os.RemoveAll(cellsDir) - - // 1. Bake each cell to its own PMTiles (one cell resident at a time — the bytes are freed - // as soon as they are written). Serial for now; a warmup + worker pool is the next lever. - perCell := make([]string, 0, len(cells)) - for i, cp := range cells { - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Unit, j.Note = "bake", "", "cells", "Baking charts" - j.Done, j.Total = i, len(cells) + tilesPath := filepath.Join(outDir, "tiles", "chart.pmtiles") + return baker.ComposeENCRoot(encRoot, tilesPath, + func(done, total int) { + note := "Baking charts" + if done >= total { // bakes finished → the partition compose runs + note = "Composing tiles" + } + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Unit, j.Note = "bake", "", "cells", note + j.Done, j.Total = done, total + }) + }, + func(cell string, err error) { + log.Printf("import %s: bake cell %s: %v (skipping)", jobID, cell, err) }) - b, err := tile57.BakeCell(cp) - if err != nil { - log.Printf("import %s: bake cell %s: %v (skipping)", jobID, filepath.Base(cp), err) - continue - } - if len(b) == 0 { - continue - } - pc := filepath.Join(cellsDir, filepath.Base(cp)+".pmtiles") - if err := os.WriteFile(pc, b, 0o644); err != nil { - log.Printf("import %s: write per-cell %s: %v (skipping)", jobID, filepath.Base(cp), err) - continue - } - perCell = append(perCell, pc) - } - if len(perCell) == 0 { - // Cells were present but none baked (e.g. all unparseable) → a bake ERROR, not empty - // coverage. Returning an error (like BakeBundle) lets the caller fail WITHOUT dropping - // the provider's stub/source; returning 0,nil would trip the "no coverage" RemoveAll. - if len(cells) > 0 { - return 0, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) - } - return 0, nil - } - - // 2. Stream-compose the per-cell archives into tiles/chart.pmtiles via the partition. - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Unit, j.Note, j.Done, j.Total = "bake", "", "cells", "Composing tiles", len(cells), len(cells) - }) - tilesDir := filepath.Join(outDir, "tiles") - if err := os.MkdirAll(tilesDir, 0o755); err != nil { - return 0, err - } - n, err := tile57.ComposeFiles(perCell, filepath.Join(tilesDir, "chart.pmtiles")) - if err != nil { - return 0, err - } - return n, nil -} - -// listCells returns every base cell (.000) path under encRoot, deduped by stem (a boundary -// cell shared by two districts bakes once). -func listCells(encRoot string) ([]string, error) { - var out []string - seen := map[string]bool{} - err := filepath.WalkDir(encRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() || !strings.HasSuffix(path, ".000") { - return nil - } - stem := strings.TrimSuffix(filepath.Base(path), ".000") - if seen[stem] { - return nil - } - seen[stem] = true - out = append(out, path) - return nil - }) - return out, err } // dropProviderSet unregisters a provider whose ENC_ROOT is now empty and removes its From d02368a8aa93c37730f153b2016ea85512c3b42d Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Wed, 8 Jul 2026 20:16:18 -0400 Subject: [PATCH 06/36] feat(import): show an ETA on the composite bake progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composite provider bake now reports seconds-remaining (mean per-cell rate over cells baked so far) on the import job, and the CLI prints it too. The web progress UI ignored the job's free-text `note`, so the earlier note-only ETA never showed; add a dedicated `eta` field to the import job and render it in chart-service._formatStatus via a compact fmtEta → "1,234 / 4,567 charts · ~2m left". Rebuild + hard-refresh to pick up the embedded web asset. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/tile57_bake.go | 11 ++++++++--- internal/engine/server/import.go | 1 + internal/engine/server/tile57_bake.go | 7 ++++++- web/src/data/chart-service.mjs | 11 +++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go index 3a59713..da45831 100644 --- a/cmd/chartplotter/tile57_bake.go +++ b/cmd/chartplotter/tile57_bake.go @@ -6,6 +6,7 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/beetlebugorg/chartplotter/internal/engine/baker" tile57 "github.com/beetlebugorg/tile57/bindings/go" @@ -82,12 +83,16 @@ func (c bakeCmd) runTile57Archive() error { // Per-cell COMPOSITE (default): bake each cell at its native scale, then combine them // via the engine's ownership partition straight into -o. --max-zoom/--format do not // apply — each cell bakes at its native band and the compositor expands zoom. + start := time.Now() n, err = baker.ComposeENCRoot(input, outAbs, func(done, total int) { - if done < total { - fmt.Printf("\rbaking cells %d/%d… ", done, total) + if done >= total { + fmt.Printf("\rcomposing %d cells… ", total) + } else if done > 0 { + per := time.Since(start) / time.Duration(done) + fmt.Printf("\rbaking cells %d/%d · ~%s left ", done, total, (per * time.Duration(total-done)).Round(time.Second)) } else { - fmt.Printf("\rcomposing %d cells… ", total) + fmt.Printf("\rbaking cells %d/%d… ", done, total) } }, func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 33d5259..84fe262 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -42,6 +42,7 @@ type importJob struct { Note string `json:"note"` // human-readable current step (e.g. "downloading US5MD1MC") Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) Total int `json:"total"` // phase total (0 until known) + ETA int `json:"eta,omitempty"` // seconds remaining in this phase (0 = unknown/none) Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" Cells int `json:"cells"` // cells successfully parsed Err string `json:"error,omitempty"` diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index f2741bf..08ab740 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -171,15 +171,20 @@ func (s *Server) bakeProvider(jobID, provider string) bool { // contributed to the composite, or 0 if none produced coverage. func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { tilesPath := filepath.Join(outDir, "tiles", "chart.pmtiles") + start := time.Now() return baker.ComposeENCRoot(encRoot, tilesPath, func(done, total int) { note := "Baking charts" + eta := 0 if done >= total { // bakes finished → the partition compose runs note = "Composing tiles" + } else if done > 0 { // seconds remaining from the mean per-cell rate so far + per := time.Since(start) / time.Duration(done) + eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) } s.imports.update(jobID, func(j *importJob) { j.Phase, j.Band, j.Unit, j.Note = "bake", "", "cells", note - j.Done, j.Total = done, total + j.Done, j.Total, j.ETA = done, total, eta }) }, func(cell string, err error) { diff --git a/web/src/data/chart-service.mjs b/web/src/data/chart-service.mjs index 87cc28a..3f19c29 100644 --- a/web/src/data/chart-service.mjs +++ b/web/src/data/chart-service.mjs @@ -36,6 +36,14 @@ function fmtBytes(n) { return `${n.toFixed(n < 10 && i > 0 ? 1 : 0)} ${u[i]}`; } +// Seconds remaining → compact "~2m 30s" / "~45s" / "~1h" for a progress ETA. +function fmtEta(sec) { + if (!sec || sec < 0) return ""; + if (sec >= 3600) return `~${Math.round(sec / 3600)}h`; + if (sec >= 60) { const m = Math.floor(sec / 60), r = sec % 60; return r ? `~${m}m ${r}s` : `~${m}m`; } + return `~${sec}s`; +} + export class ChartService { constructor({ assets = "" } = {}) { this._assets = assets; @@ -170,6 +178,9 @@ export class ChartService { const u = s.unit === "cells" ? "charts" : (s.unit || ""); detail = `${s.done.toLocaleString()} / ${s.total.toLocaleString()} ${u}`.trim(); } + // Time remaining, when the server reports one (the composite bake): "1,234 / 4,567 charts · ~2m left". + const eta = fmtEta(s.eta); + if (eta) detail = detail ? `${detail} · ${eta} left` : `${eta} left`; const frac = s.total ? s.done / s.total : (s.percent ? s.percent / 100 : null); // pack/packNum/packTotal identify WHICH pack a multi-pack batch is on right now (the // caller resolves pack — a set key — to a friendly name), so progress reads From 77dff00c51013a4792f845b8507ea8a1661b8df2 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 04:51:39 -0400 Subject: [PATCH 07/36] feat(import): live compose progress + server-owned status wording The composite bake's 3-5 min partition compose was one opaque step: the UI stuck on "Preparing" (the client re-derived the verb from phase+unit and pinned bake+cells -> "Preparing") behind a frozen 100% bar. Now: - composeProvider feeds tile57's new ComposeFiles progress callback into the job: "Composing tiles" with a real bar (engine zoom-weight fraction), a "zoom N of M" detail, and an ETA extrapolated from elapsed compose time. The per-cell stage reports "Baking charts". - The server owns the status wording: statusJSON emits display-ready action + detail + frac (bar fill; null = indeterminate) built server-side; the client renders them verbatim instead of interpreting phase/unit/note. The dead client PHASE map / fmtBytes / fmtEta and the "Preparing" heuristic are removed; the raw fields still ride along for pack/region context. - The earlier note-only ETA now actually reaches the client (statusJSON had dropped the eta field, so the wired-up fmtEta path was dead). Rebuild with `make TILE57=../tile57 build` + hard-refresh to pick up the embedded web asset. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/tile57_bake.go | 1 + internal/engine/baker/compose.go | 12 +- internal/engine/server/import.go | 159 ++++++++++++++++++++++++-- internal/engine/server/tile57_bake.go | 44 +++++-- web/src/chartplotter.mjs | 2 +- web/src/data/chart-service.mjs | 64 +++-------- 6 files changed, 209 insertions(+), 73 deletions(-) diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go index da45831..6ff9362 100644 --- a/cmd/chartplotter/tile57_bake.go +++ b/cmd/chartplotter/tile57_bake.go @@ -95,6 +95,7 @@ func (c bakeCmd) runTile57Archive() error { fmt.Printf("\rbaking cells %d/%d… ", done, total) } }, + nil, // CLI shows no separate compose bar; the per-cell line above suffices func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) fmt.Println() if err != nil { diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 0e7da29..1ec682c 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -18,10 +18,12 @@ import ( // half only; a host builds the style dynamically and serves global assets. // // onProgress(done, total) is called before each cell bake (done 0..total-1) and once more with -// done==total when the bakes are finished and the partition compose begins; nil to skip. onSkip -// (nil to skip) reports a cell that failed to bake. Returns the count of cells that contributed; -// an error (not 0) is returned when cells were present but none baked. -func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onSkip func(cell string, err error)) (int, error) { +// done==total when the bakes are finished and the partition compose begins; nil to skip. +// onCompose (nil to skip) then reports live progress THROUGH that partition compose as it walks +// the zoom ladder (a smooth Done/Total fraction). onSkip (nil to skip) reports a cell that failed +// to bake. Returns the count of cells that contributed; an error (not 0) is returned when cells +// were present but none baked. +func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onCompose func(tile57.ComposeProgress), onSkip func(cell string, err error)) (int, error) { cells, err := ListCells(input) if err != nil { return 0, err @@ -77,7 +79,7 @@ func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onS return 0, err } } - return tile57.ComposeFiles(perCell, outPath) + return tile57.ComposeFiles(perCell, outPath, onCompose) } // ListCells returns every base cell (.000) path under `root` (a single file or a directory), diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 84fe262..6e90afc 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -43,8 +44,13 @@ type importJob struct { Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) Total int `json:"total"` // phase total (0 until known) ETA int `json:"eta,omitempty"` // seconds remaining in this phase (0 = unknown/none) - Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" + Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" | "" (opaque) Cells int `json:"cells"` // cells successfully parsed + // Compose-step zoom progress: the ownership-partition compose walks the zoom ladder, so it + // reports "zoom N of M" instead of a chart count. Zoom == 0 means "not composing". + Zoom int `json:"-"` + ZoomMin int `json:"-"` + ZoomMax int `json:"-"` Err string `json:"error,omitempty"` Started string `json:"started"` } @@ -586,16 +592,153 @@ func (s *Server) setCells(set string) ([]string, bool) { return stems, true } -// statusJSON renders a job snapshot as the status JSON line (shared by the polling -// endpoint and the SSE stream). +// statusJSON renders a job snapshot as the status JSON line (shared by the polling endpoint and +// the SSE stream). The server owns the human wording: `action` (the live step) and `detail` (the +// count/ETA beside it) are display-ready strings the client renders verbatim, and `frac` is the +// bar fill (0..1, or null for an indeterminate/opaque-step bar). The raw fields ride along for the +// client's own use (pack context, region title), but it must not re-derive the status from them. func (j importJob) statusJSON() string { - pct := 0 + action, detail, frac := j.statusDisplay() + return fmt.Sprintf( + `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"eta":%d,"cells":%d,"action":%q,"detail":%q,"frac":%s,"error":%q}`, + j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, j.ETA, j.Cells, action, detail, fracJSON(frac), j.Err) +} + +// statusDisplay composes the display-ready progress strings the client renders verbatim: +// +// action — the live step ("Baking charts", "Composing tiles", "Downloading charts") +// detail — the count beside it ("1,234 / 4,567 charts · ~2m left", "zoom 12 / 16 · ~1m left") +// frac — bar fill 0..1, or a negative value for an indeterminate (opaque-step) bar. +func (j importJob) statusDisplay() (action, detail string, frac float64) { + action = j.actionText() + + switch { + case j.Zoom > 0: // the compose step reports its zoom position, not a chart count + detail = fmt.Sprintf("zoom %d / %d", j.Zoom-j.ZoomMin+1, j.ZoomMax-j.ZoomMin+1) + case j.Unit == "bytes": + if j.Total > 0 { + detail = fmtBytes(j.Done) + " / " + fmtBytes(j.Total) + } else { + detail = fmtBytes(j.Done) + } + case j.Total > 0: + unit := j.Unit + if unit == "cells" { + unit = "charts" // "charts" reads friendlier than the internal "cells" + } + detail = commaInt(j.Done) + " / " + commaInt(j.Total) + " " + unit + } + if eta := fmtEtaSecs(j.ETA); eta != "" { + if detail != "" { + detail += " · " + eta + " left" + } else { + detail = eta + " left" + } + } + + // A known total → a determinate bar (the compose weights or the chart count both work). An + // opaque step with no total (initial "Preparing", the compose warm-up) sweeps. if j.Total > 0 { - pct = j.Done * 100 / j.Total + frac = float64(j.Done) / float64(j.Total) + } else { + frac = -1 + } + return +} + +// actionText is the live step verb. The bake/meta phases carry a clean, self-contained note that +// names the step ("Baking charts", "Composing tiles", "Reading chart metadata"); use it verbatim. +// The download/extract notes echo a source filename ("Downloading US5MD1MC"), so those get a +// friendly generic verb instead. +func (j importJob) actionText() string { + switch j.Phase { + case "bake", "meta": + if j.Note != "" { + if j.Band != "" { + return j.Note + " " + j.Band // legacy per-band bake: "Generating tiles coastal" + } + return j.Note + } + case "download": + return "Downloading charts" + case "extract": + return "Extracting charts" + } + if j.Note != "" { + return j.Note + } + if j.Phase != "" { + return strings.ToUpper(j.Phase[:1]) + j.Phase[1:] + } + return "Working" +} + +// fracJSON renders a bar fill as a JSON number, or null for an indeterminate bar. +func fracJSON(frac float64) string { + if frac < 0 { + return "null" + } + if frac > 1 { + frac = 1 + } + return strconv.FormatFloat(frac, 'f', 4, 64) +} + +// fmtBytes renders a byte count as a compact "12 MB" / "1.4 KB". +func fmtBytes(n int) string { + if n <= 0 { + return "0 B" + } + units := []string{"B", "KB", "MB", "GB"} + f, i := float64(n), 0 + for f >= 1024 && i < len(units)-1 { + f /= 1024 + i++ + } + if f < 10 && i > 0 { + return fmt.Sprintf("%.1f %s", f, units[i]) + } + return fmt.Sprintf("%.0f %s", f, units[i]) +} + +// commaInt renders an int with thousands separators ("1234567" → "1,234,567"). +func commaInt(n int) string { + s := strconv.Itoa(n) + neg := strings.HasPrefix(s, "-") + if neg { + s = s[1:] + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if i > 0 && (len(s)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteByte(s[i]) + } + if neg { + return "-" + b.String() + } + return b.String() +} + +// fmtEtaSecs renders seconds-remaining as a compact "~2m 30s" / "~45s" / "~1h" (matches the +// client's old fmtEta); "" when unknown. +func fmtEtaSecs(sec int) string { + if sec <= 0 { + return "" + } + switch { + case sec >= 3600: + return fmt.Sprintf("~%dh", (sec+1800)/3600) + case sec >= 60: + m, r := sec/60, sec%60 + if r != 0 { + return fmt.Sprintf("~%dm %ds", m, r) + } + return fmt.Sprintf("~%dm", m) + default: + return fmt.Sprintf("~%ds", sec) } - return fmt.Sprintf( - `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"percent":%d,"cells":%d,"error":%q}`, - j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, pct, j.Cells, j.Err) } // importStatus returns a job's state as JSON (one-shot poll). diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 08ab740..ae07b7c 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -152,7 +152,9 @@ func (s *Server) bakeProvider(jobID, provider string) bool { return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) } - s.imports.update(jobID, func(j *importJob) { j.Phase, j.Note = "meta", "Reading chart metadata" }) + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Note, j.Zoom, j.Unit, j.Done, j.Total, j.ETA = "meta", "Reading chart metadata", 0, "", 0, 0, 0 + }) cells := s.providerCellData(provider) aux := s.providerAux(provider) cat := s.providerCatalog(provider) @@ -172,19 +174,43 @@ func (s *Server) bakeProvider(jobID, provider string) bool { func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { tilesPath := filepath.Join(outDir, "tiles", "chart.pmtiles") start := time.Now() + var composeStart time.Time // set when the per-cell bakes finish and the compose begins return baker.ComposeENCRoot(encRoot, tilesPath, func(done, total int) { - note := "Baking charts" + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Zoom = "bake", "", 0 + if done >= total { + // Per-cell bakes done → the ownership-partition compose runs. Until its first + // zoom-progress arrives, sweep (total 0) under "Composing tiles". + composeStart = time.Now() + j.Unit, j.Note, j.Done, j.Total, j.ETA = "", "Composing tiles", 0, 0, 0 + return + } + // Per-cell portrayal: a determinate bar plus an ETA from the mean per-cell rate so far. + eta := 0 + if done > 0 { + per := time.Since(start) / time.Duration(done) + eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) + } + j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", "Baking charts", done, total, eta + }) + }, + func(p tile57.ComposeProgress) { + // Live compose progress: the engine weights zooms by tile count, so p.Done/p.Total is + // a smooth fraction. ETA extrapolates from elapsed compose time and that fraction. + if composeStart.IsZero() { + composeStart = time.Now() + } eta := 0 - if done >= total { // bakes finished → the partition compose runs - note = "Composing tiles" - } else if done > 0 { // seconds remaining from the mean per-cell rate so far - per := time.Since(start) / time.Duration(done) - eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) + if p.Total > 0 && p.Done > 0 && p.Done < p.Total { + frac := float64(p.Done) / float64(p.Total) + remain := time.Duration(float64(time.Since(composeStart)) * (1 - frac) / frac) + eta = int(remain.Round(time.Second).Seconds()) } s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Unit, j.Note = "bake", "", "cells", note - j.Done, j.Total, j.ETA = done, total, eta + j.Phase, j.Band, j.Unit, j.Note = "bake", "", "", "Composing tiles" + j.Done, j.Total, j.ETA = int(p.Done), int(p.Total), eta + j.Zoom, j.ZoomMin, j.ZoomMax = p.Zoom, p.MinZoom, p.MaxZoom }) }, func(cell string, err error) { diff --git a/web/src/chartplotter.mjs b/web/src/chartplotter.mjs index 0421001..c6e9884 100644 --- a/web/src/chartplotter.mjs +++ b/web/src/chartplotter.mjs @@ -1010,7 +1010,7 @@ export class ChartPlotter extends HTMLElement { if (!j || j.state !== "running" || !j.id) return; this._task = { kind: "download", status: "running" }; const name = this._reattachName(j.set); - this._setProgress({ label: "Working", pill: `Working on ${name}`, sub: "", frac: j.percent ? j.percent / 100 : null }); + this._setProgress({ label: "Working", pill: `Working on ${name}`, sub: j.action ? `${j.action}…` : "", frac: (j.frac === null || j.frac === undefined) ? null : j.frac }); this._pollImport(j.id, (p) => this._setProgress(p), name).catch(() => {}).then(async () => { this._task = null; this._setProgress(null); diff --git a/web/src/data/chart-service.mjs b/web/src/data/chart-service.mjs index 3f19c29..9e6ca9b 100644 --- a/web/src/data/chart-service.mjs +++ b/web/src/data/chart-service.mjs @@ -16,34 +16,6 @@ // filenames. Methods throw on hard failure; pollJob resolves with the final // status object or rejects on error/timeout. -// Job phase → [verb, noun] for the progress label, e.g. bake → "Generating … tiles". -// The noun is what's being acted on at that phase (source charts while fetching / -// reading; tiles while baking; nothing while finishing). -const PHASE = { - download: ["Downloading", "charts"], fetch: ["Downloading", "charts"], dl: ["Downloading", "charts"], - upload: ["Uploading", "charts"], - extract: ["Extracting", "charts"], unzip: ["Extracting", "charts"], expand: ["Extracting", "charts"], - parse: ["Reading", "charts"], read: ["Reading", "charts"], import: ["Reading", "charts"], - bake: ["Generating", "tiles"], tiles: ["Generating", "tiles"], render: ["Generating", "tiles"], - register: ["Finishing", "up"], finalize: ["Finishing", "up"], index: ["Finishing", "up"], -}; - -// Bytes → compact "12 MB" / "1.4 KB" (local copy so this module is self-contained). -function fmtBytes(n) { - if (!n) return "0 B"; - const u = ["B", "KB", "MB", "GB"]; let i = 0; - while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; } - return `${n.toFixed(n < 10 && i > 0 ? 1 : 0)} ${u[i]}`; -} - -// Seconds remaining → compact "~2m 30s" / "~45s" / "~1h" for a progress ETA. -function fmtEta(sec) { - if (!sec || sec < 0) return ""; - if (sec >= 3600) return `~${Math.round(sec / 3600)}h`; - if (sec >= 60) { const m = Math.floor(sec / 60), r = sec % 60; return r ? `~${m}m ${r}s` : `~${m}m`; } - return `~${sec}s`; -} - export class ChartService { constructor({ assets = "" } = {}) { this._assets = assets; @@ -164,28 +136,20 @@ export class ChartService { // tiles", "12 MB / 45 MB"). The action carries the band but not the noun, so the // unit isn't repeated — "Generating coastal…" + "1,234 / 4,567 tiles". _formatStatus(s, name) { - const m = s.phase ? PHASE[s.phase] : null; - let verb = m ? m[0] : (s.phase ? s.phase[0].toUpperCase() + s.phase.slice(1) : "Working"); - // The bake phase has two visible stages the server distinguishes by unit: - // "cells" while a band's charts are parsed + portrayed (the long gap before - // any tile emits), then "tiles" while that band's tiles are generated. - if (m && m[1] === "tiles" && s.unit === "cells") verb = "Preparing"; - const sub = [verb, s.band].filter(Boolean).join(" ") + "…"; // "Preparing coastal…" - // Count, with the unit named in every stage so a bare number is never ambiguous. - let detail = ""; - if (s.unit === "bytes") detail = s.total ? `${fmtBytes(s.done)} / ${fmtBytes(s.total)}` : fmtBytes(s.done); - else if (s.total) { - const u = s.unit === "cells" ? "charts" : (s.unit || ""); - detail = `${s.done.toLocaleString()} / ${s.total.toLocaleString()} ${u}`.trim(); - } - // Time remaining, when the server reports one (the composite bake): "1,234 / 4,567 charts · ~2m left". - const eta = fmtEta(s.eta); - if (eta) detail = detail ? `${detail} · ${eta} left` : `${eta} left`; - const frac = s.total ? s.done / s.total : (s.percent ? s.percent / 100 : null); - // pack/packNum/packTotal identify WHICH pack a multi-pack batch is on right now (the - // caller resolves pack — a set key — to a friendly name), so progress reads - // "Mid-Atlantic (2 of 4) · Generating coastal…" instead of looping on one line. - return { label: name || "", sub, detail, frac, pack: s.pack || "", packNum: s.packNum || 0, packTotal: s.packTotal || 0 }; + // The SERVER owns the wording: `action` is the live step and `detail` the count/ETA beside + // it, both display-ready — the client renders them verbatim rather than re-deriving the + // status from the raw phase/unit/note fields. `frac` is the bar fill (0..1, or null → the + // shell sweeps an indeterminate bar). The client only supplies the region title (label) and + // the pack context (pack/packNum/packTotal, resolved to a friendly name by the caller), so + // progress reads "Mid-Atlantic (2 of 4) · Composing tiles…". + const frac = (s.frac === null || s.frac === undefined) ? null : s.frac; + return { + label: name || "", + sub: s.action ? `${s.action}…` : "", + detail: s.detail || "", + frac, + pack: s.pack || "", packNum: s.packNum || 0, packTotal: s.packTotal || 0, + }; } // GET /api/packs — the installed-pack registry (single source of truth, incl. From dd9d189ac2fbc8e4c9d2c94110d2393bc154eaaf Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 05:03:23 -0400 Subject: [PATCH 08/36] feat(import): name the chart being baked + pin the count/ETA so they don't jitter - The per-cell bake stage now names the chart in flight ("Baking US5MD1MD" instead of a generic "Baking charts"): ComposeENCRoot's progress callback passes the cell stem, and composeProvider puts it in the action. The CLI prints it too. - The server reports the ETA as its OWN field, separate from the count `detail`, so the client renders each in a fixed, right-aligned tabular slot (the ETA reserves a min-width). Glued together they changed width every tick and the whole line shifted around; split, the count and ETA stay put. The chart-library batch banner re-joins them for its one-line layout. Rebuild with `make TILE57=../tile57 build` + hard-refresh. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/tile57_bake.go | 6 +++--- internal/engine/baker/compose.go | 18 +++++++++--------- internal/engine/server/import.go | 25 ++++++++++++------------- internal/engine/server/tile57_bake.go | 11 ++++++++--- web/src/chartplotter.mjs | 2 ++ web/src/chartplotter.view.mjs | 7 ++++++- web/src/data/chart-service.mjs | 1 + web/src/plugins/chart-library.mjs | 10 ++++++++-- 8 files changed, 49 insertions(+), 31 deletions(-) diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go index 6ff9362..003cd07 100644 --- a/cmd/chartplotter/tile57_bake.go +++ b/cmd/chartplotter/tile57_bake.go @@ -85,14 +85,14 @@ func (c bakeCmd) runTile57Archive() error { // apply — each cell bakes at its native band and the compositor expands zoom. start := time.Now() n, err = baker.ComposeENCRoot(input, outAbs, - func(done, total int) { + func(done, total int, cell string) { if done >= total { fmt.Printf("\rcomposing %d cells… ", total) } else if done > 0 { per := time.Since(start) / time.Duration(done) - fmt.Printf("\rbaking cells %d/%d · ~%s left ", done, total, (per * time.Duration(total-done)).Round(time.Second)) + fmt.Printf("\rbaking %s (%d/%d) · ~%s left ", cell, done, total, (per * time.Duration(total-done)).Round(time.Second)) } else { - fmt.Printf("\rbaking cells %d/%d… ", done, total) + fmt.Printf("\rbaking %s (%d/%d)… ", cell, done, total) } }, nil, // CLI shows no separate compose bar; the per-cell line above suffices diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 1ec682c..6e827e6 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -17,13 +17,13 @@ import ( // never resident. This replaces the in-bake cross-cell combiner (tile57.BakeBundle) — the tiles // half only; a host builds the style dynamically and serves global assets. // -// onProgress(done, total) is called before each cell bake (done 0..total-1) and once more with -// done==total when the bakes are finished and the partition compose begins; nil to skip. -// onCompose (nil to skip) then reports live progress THROUGH that partition compose as it walks -// the zoom ladder (a smooth Done/Total fraction). onSkip (nil to skip) reports a cell that failed -// to bake. Returns the count of cells that contributed; an error (not 0) is returned when cells -// were present but none baked. -func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onCompose func(tile57.ComposeProgress), onSkip func(cell string, err error)) (int, error) { +// onProgress(done, total, cell) is called before each cell bake (done 0..total-1, cell = the +// stem about to bake, e.g. "US5MD1MD") and once more with done==total (cell "") when the bakes +// are finished and the partition compose begins; nil to skip. onCompose (nil to skip) then +// reports live progress THROUGH that partition compose as it walks the zoom ladder (a smooth +// Done/Total fraction). onSkip (nil to skip) reports a cell that failed to bake. Returns the count +// of cells that contributed; an error (not 0) is returned when cells were present but none baked. +func ComposeENCRoot(input, outPath string, onProgress func(done, total int, cell string), onCompose func(tile57.ComposeProgress), onSkip func(cell string, err error)) (int, error) { cells, err := ListCells(input) if err != nil { return 0, err @@ -43,7 +43,7 @@ func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onC perCell := make([]string, 0, len(cells)) for i, cp := range cells { if onProgress != nil { - onProgress(i, len(cells)) + onProgress(i, len(cells), strings.TrimSuffix(filepath.Base(cp), ".000")) } b, err := tile57.BakeCell(cp) if err != nil { @@ -72,7 +72,7 @@ func ComposeENCRoot(input, outPath string, onProgress func(done, total int), onC // 2. Stream-compose the per-cell archives into outPath via the ownership partition. if onProgress != nil { - onProgress(len(cells), len(cells)) + onProgress(len(cells), len(cells), "") } if dir := filepath.Dir(outPath); dir != "" { if err := os.MkdirAll(dir, 0o755); err != nil { diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 6e90afc..36538aa 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -598,18 +598,21 @@ func (s *Server) setCells(set string) ([]string, bool) { // bar fill (0..1, or null for an indeterminate/opaque-step bar). The raw fields ride along for the // client's own use (pack context, region title), but it must not re-derive the status from them. func (j importJob) statusJSON() string { - action, detail, frac := j.statusDisplay() + action, detail, eta, frac := j.statusDisplay() return fmt.Sprintf( - `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"eta":%d,"cells":%d,"action":%q,"detail":%q,"frac":%s,"error":%q}`, - j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, j.ETA, j.Cells, action, detail, fracJSON(frac), j.Err) + `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"cells":%d,"action":%q,"detail":%q,"eta":%q,"frac":%s,"error":%q}`, + j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, j.Cells, action, detail, eta, fracJSON(frac), j.Err) } -// statusDisplay composes the display-ready progress strings the client renders verbatim: +// statusDisplay composes the display-ready progress strings the client renders verbatim. `detail` +// (the count) and `eta` are kept SEPARATE so the client can pin each in its own fixed, right- +// aligned slot — glued together they change width and the whole line jitters as the numbers move: // -// action — the live step ("Baking charts", "Composing tiles", "Downloading charts") -// detail — the count beside it ("1,234 / 4,567 charts · ~2m left", "zoom 12 / 16 · ~1m left") +// action — the live step ("Baking US5MD1MD", "Composing tiles", "Downloading charts") +// detail — the count beside it ("1,234 / 4,567 charts", "zoom 12 / 16") +// eta — the time remaining ("~2m 30s left"), or "" when unknown // frac — bar fill 0..1, or a negative value for an indeterminate (opaque-step) bar. -func (j importJob) statusDisplay() (action, detail string, frac float64) { +func (j importJob) statusDisplay() (action, detail, eta string, frac float64) { action = j.actionText() switch { @@ -628,12 +631,8 @@ func (j importJob) statusDisplay() (action, detail string, frac float64) { } detail = commaInt(j.Done) + " / " + commaInt(j.Total) + " " + unit } - if eta := fmtEtaSecs(j.ETA); eta != "" { - if detail != "" { - detail += " · " + eta + " left" - } else { - detail = eta + " left" - } + if e := fmtEtaSecs(j.ETA); e != "" { + eta = e + " left" } // A known total → a determinate bar (the compose weights or the chart count both work). An diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index ae07b7c..5320c3a 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -176,7 +176,7 @@ func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { start := time.Now() var composeStart time.Time // set when the per-cell bakes finish and the compose begins return baker.ComposeENCRoot(encRoot, tilesPath, - func(done, total int) { + func(done, total int, cell string) { s.imports.update(jobID, func(j *importJob) { j.Phase, j.Band, j.Zoom = "bake", "", 0 if done >= total { @@ -186,13 +186,18 @@ func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { j.Unit, j.Note, j.Done, j.Total, j.ETA = "", "Composing tiles", 0, 0, 0 return } - // Per-cell portrayal: a determinate bar plus an ETA from the mean per-cell rate so far. + // Per-cell portrayal: name the chart being baked, plus a determinate bar and an ETA + // from the mean per-cell rate so far. + note := "Baking charts" + if cell != "" { + note = "Baking " + cell + } eta := 0 if done > 0 { per := time.Since(start) / time.Duration(done) eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) } - j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", "Baking charts", done, total, eta + j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", note, done, total, eta }) }, func(p tile57.ComposeProgress) { diff --git a/web/src/chartplotter.mjs b/web/src/chartplotter.mjs index c6e9884..8779424 100644 --- a/web/src/chartplotter.mjs +++ b/web/src/chartplotter.mjs @@ -1875,9 +1875,11 @@ export class ChartPlotter extends HTMLElement { const title = p.label || p.pill || ""; const action = p.error ? String(p.error) : (p.sub || ""); const count = p.error ? "" : (p.detail || ""); + const eta = p.error ? "" : (p.eta || ""); r.getElementById("db-prog-title").textContent = title; r.getElementById("db-prog-action").textContent = action; r.getElementById("db-prog-count").textContent = count; + r.getElementById("db-prog-eta").textContent = eta; const fill = r.getElementById("db-prog-fill"); fill.style.width = p.frac != null ? `${Math.round(p.frac * 100)}%` : "100%"; fill.classList.toggle("indet", p.frac == null && !done); // slow sweep when no fraction diff --git a/web/src/chartplotter.view.mjs b/web/src/chartplotter.view.mjs index daa1cd1..e11458d 100644 --- a/web/src/chartplotter.view.mjs +++ b/web/src/chartplotter.view.mjs @@ -462,7 +462,11 @@ export const STYLE = ` .db-prog-status { display:flex; align-items:baseline; gap:10px; font:500 11.5px/1.3 system-ui,sans-serif; color:var(--ui-text-dim); } .db-prog-action { flex:1; min-width:0; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } - .db-prog-count { flex:none; font-variant-numeric:tabular-nums; } + /* Count + ETA are fixed, right-aligned tabular slots so the text stays put as the numbers + change width (the ETA reserves a min-width; both right-align so their right edge is pinned). */ + .db-prog-count { flex:none; text-align:right; font-variant-numeric:tabular-nums; } + .db-prog-eta { flex:none; min-width:7em; text-align:right; font-variant-numeric:tabular-nums; } + .db-prog-eta:empty { display:none; } .db-prog-status:empty { display:none; } .db-prog.error .db-prog-title, .db-prog.error .db-prog-action { color:#c0392b; } .db-prog-track { position:relative; width:100%; height:6px; border-radius:3px; overflow:hidden; background:var(--ui-surface-2); } @@ -611,6 +615,7 @@ export const CHROME = `
+
diff --git a/web/src/data/chart-service.mjs b/web/src/data/chart-service.mjs index 9e6ca9b..7aaca40 100644 --- a/web/src/data/chart-service.mjs +++ b/web/src/data/chart-service.mjs @@ -147,6 +147,7 @@ export class ChartService { label: name || "", sub: s.action ? `${s.action}…` : "", detail: s.detail || "", + eta: s.eta || "", // kept separate from detail so each pins in its own fixed slot frac, pack: s.pack || "", packNum: s.packNum || 0, packTotal: s.packTotal || 0, }; diff --git a/web/src/plugins/chart-library.mjs b/web/src/plugins/chart-library.mjs index 02e33a5..d88d048 100644 --- a/web/src/plugins/chart-library.mjs +++ b/web/src/plugins/chart-library.mjs @@ -906,10 +906,16 @@ export class ChartLibrary extends HTMLElement { return (pack ? `${pack}${count} · ` : "") + ((p && p.sub) || "Downloading…"); } + // "1,234 / 4,567 charts · ~2m left" — the count and (when present) the ETA, which the server + // reports separately so the card can pin each; the banner is one line, so re-join them here. + _batchDetail(p) { + return (p && p.detail ? p.detail : "") + (p && p.eta ? ` · ${p.eta}` : ""); + } + // The in-dialog banner descriptor for the packs header (bar + text). _batchBanner() { const p = this._batchStatus; - return { busy: true, sub: this._batchSub(p), detail: (p && p.detail) || "", frac: (p && p.frac) || 0 }; + return { busy: true, sub: this._batchSub(p), detail: this._batchDetail(p), frac: (p && p.frac) || 0 }; } // Update the banner in place from a formatted status (no re-render → the bar animates). @@ -921,7 +927,7 @@ export class ChartLibrary extends HTMLElement { const sub = r.querySelector(".dl-banner .dl-sub"); if (sub) sub.textContent = this._batchSub(p); const detail = r.querySelector(".dl-banner .dl-detail"); - if (detail) detail.textContent = p.detail || ""; + if (detail) detail.textContent = this._batchDetail(p); } // Find-a-chart search box. From 8ccb524255cf23a28f94e3566dc8df39901ba9cc Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 07:38:40 -0400 Subject: [PATCH 09/36] feat(tiles): live runtime-compositor tile source (tile57 ComposeSource) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Composer TileSource backed by tile57's on-demand compositor: per-cell PMTiles held mmap'd + the ownership partition resident, composing each tile as the camera asks instead of serving a prebaked archive. It returns decompressed MLT (the HTTP layer gzips on the wire) with TileType=mlt. A dev/test hook wires it in: set TILE57_LIVE_COMPOSE to a dir of per-cell archives (from `tile57 compose --keep-cells`), optionally TILE57_LIVE_PARTITION to a sidecar (`--save-partition`) and TILE57_LIVE_NAME (default "live"), and the set is served at /tiles/{name}/{z}/{x}/{y}. No-op when unset. Verified against NOAA d05 (848 cells): the server opens the compositor at boot (loads the partition sidecar), and GET /tiles/live/14/4706/6244 returns the 3480-byte MLT byte-matching the batch reference — 0.61 ms/tile end-to-end over HTTP, with gzip-on-the-wire and 204 for out-of-coverage tiles. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/http.go | 1 + internal/engine/server/live_compose.go | 59 ++++++++++++++++++++++++++ internal/engine/tilesource/composer.go | 51 ++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 internal/engine/server/live_compose.go create mode 100644 internal/engine/tilesource/composer.go diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 1cb9555..33b9ac8 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -92,6 +92,7 @@ func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { if len(s.packs) > 0 { log.Printf("tilesets: %d pack(s) on disk, %d enabled (from %s)", len(s.packs), n, cacheDir) } + s.registerLiveCompose() // dev/test: on-demand runtime compositor from TILE57_LIVE_COMPOSE (no-op if unset) s.rebakeMissingProviders() // self-heal: bake any provider with an ENC_ROOT but no bundle (post-migration) return s } diff --git a/internal/engine/server/live_compose.go b/internal/engine/server/live_compose.go new file mode 100644 index 0000000..6b2f34e --- /dev/null +++ b/internal/engine/server/live_compose.go @@ -0,0 +1,59 @@ +package server + +import ( + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// registerLiveCompose registers a live runtime-composited tile set when TILE57_LIVE_COMPOSE names +// a directory of per-cell PMTiles (each from `tile57 compose --keep-cells`). It serves tiles by +// on-demand composition (the tile57 ComposeSource) instead of a prebaked archive. Optional env: +// +// TILE57_LIVE_PARTITION a partition sidecar (`tile57 compose --save-partition`) to load +// instead of building the ownership partition at open; +// TILE57_LIVE_NAME the set name (default "live"), served at /tiles//{z}/{x}/{y}. +// +// A dev/test hook for the on-demand compositor — a no-op when TILE57_LIVE_COMPOSE is unset. +func (s *Server) registerLiveCompose() { + dir := os.Getenv("TILE57_LIVE_COMPOSE") + if dir == "" { + return + } + entries, err := os.ReadDir(dir) + if err != nil { + log.Printf("live-compose: cannot read %s: %v", dir, err) + return + } + var paths []string + for _, e := range entries { + if e.IsDir() { + continue + } + if n := e.Name(); strings.HasSuffix(n, ".cell.tmp") || strings.HasSuffix(n, ".pmtiles") { + paths = append(paths, filepath.Join(dir, n)) + } + } + sort.Strings(paths) + if len(paths) == 0 { + log.Printf("live-compose: no per-cell archives (*.cell.tmp / *.pmtiles) in %s", dir) + return + } + name := os.Getenv("TILE57_LIVE_NAME") + if name == "" { + name = "live" + } + src, err := tilesource.NewComposer(paths, os.Getenv("TILE57_LIVE_PARTITION")) + if err != nil { + log.Printf("live-compose: open %s failed: %v", dir, err) + return + } + s.sets.register(name, src) + m := src.Meta() + log.Printf("live-compose: registered %q — %d archive(s), z%d..%d, bounds [%.3f, %.3f, %.3f, %.3f] (GET /tiles/%s/{z}/{x}/{y})", + name, len(paths), m.MinZoom, m.MaxZoom, m.W, m.S, m.E, m.N, name) +} diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go new file mode 100644 index 0000000..e95b4bc --- /dev/null +++ b/internal/engine/tilesource/composer.go @@ -0,0 +1,51 @@ +package tilesource + +import ( + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// Composer is a TileSource backed by the tile57 runtime compositor: the per-cell PMTiles +// held mmap'd and the ownership partition resident, composing each tile on demand. It is the +// live counterpart of a prebaked .pmtiles archive — no district bake; tiles are built as the +// camera asks (a classify + one decode/clip or a decompress per tile). Serve is serialised +// inside tile57.ComposeSource, so this satisfies TileSource with no extra locking. +type Composer struct { + src *tile57.ComposeSource + meta TileMeta +} + +// NewComposer opens a runtime compositor over the per-cell PMTiles at paths (each from +// `tile57 compose --keep-cells` / tile57.BakeCell). partitionPath (or "") names a partition +// sidecar (`tile57 compose --save-partition`) to load and skip the owned-face build. Close it +// when done — callers must not Close while any request can still call Tile. +func NewComposer(paths []string, partitionPath string) (*Composer, error) { + src, err := tile57.OpenCompose(paths, partitionPath) + if err != nil { + return nil, err + } + m := src.Meta() + return &Composer{ + src: src, + meta: TileMeta{ + MinZoom: m.MinZoom, + MaxZoom: m.MaxZoom, + W: m.West, + S: m.South, + E: m.East, + N: m.North, + Gzipped: false, // Serve returns decompressed MLT; the HTTP layer gzips on the wire + TileType: "mlt", + }, + }, nil +} + +// Tile composes (z,x,y) on demand, returning decompressed MLT, or (nil, nil) for a blank tile. +func (c *Composer) Tile(z uint8, x, y uint32) ([]byte, error) { + return c.src.Serve(z, x, y) +} + +// Meta returns the compositor's display metadata (zoom range + coverage bounds). +func (c *Composer) Meta() TileMeta { return c.meta } + +// Close releases the compositor (io.Closer, so tilesource.Close finds it). +func (c *Composer) Close() error { return c.src.Close() } From 4f68330855cb7bc1ff8c33f48bf158ba5fe6d45b Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 07:49:16 -0400 Subject: [PATCH 10/36] feat(tiles): surface registry-only sets in /api/packs so they render as map layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handlePacks built the provider list from disk packs + ENC_ROOT dirs only, so a set that lives only in the s.sets registry (the live runtime compositor) never reached the client and never became a map layer. Include s.sets.names() in the union. Everything downstream already works for a registry set: the TileJSON (/tiles/{name}.json) derives bounds/zoom/encoding from src.Meta(), and the engine style defaults to all registered sets — so the client creates the chart- source and the style supplies its layers. Verified: with TILE57_LIVE_COMPOSE set, /api/packs lists "live", and the web map renders the Chesapeake from on-demand composed tiles (full S-52 symbology). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/tilesets.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/engine/server/tilesets.go b/internal/engine/server/tilesets.go index 0f49c23..8ffe102 100644 --- a/internal/engine/server/tilesets.go +++ b/internal/engine/server/tilesets.go @@ -176,6 +176,12 @@ func (s *Server) handlePacks(w http.ResponseWriter, r *http.Request) { for _, prov := range s.installedProviders() { seen[prov] = true } + // Runtime-registered sets with no disk pack (e.g. the live runtime compositor) — so a set + // that only lives in the registry still surfaces as a first-class, toggleable map layer. + // Its bounds/metadata come from the TileJSON (src.Meta()), not a disk pack. + for _, name := range s.sets.names() { + seen[providerOf(name)] = true + } providers := make([]string, 0, len(seen)) for p := range seen { providers = append(providers, p) From f064a60865bc3e70c71bbd28255094f734751ede Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 08:17:13 -0400 Subject: [PATCH 11/36] feat(tiles): default imports to the live runtime compositor (no district compose) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing a provider now builds the live compose structure itself — no CLI, no env vars. bakeProvider bakes each cell to a KEPT per-provider dir (/cells-pm, incremental: adding a district re-bakes only its new cells), opens a runtime Composer over them, and saves the ownership-partition sidecar (/partition. tpart) — skipping the whole-district compose pass (tiles compose on demand). The provider registers + renders exactly like a baked pack. On boot, registerLive Providers re-opens a Composer for every provider with kept archives (loading the sidecar → ~25ms), so live layers survive a restart; rebakeMissingProviders skips providers already served. TILE57_BATCH_COMPOSE=1 keeps the portable chart.pmtiles path; a tile57-engine-commit stamp invalidates stale per-cell archives on upgrade. Refactors: registerBakedSet → registerProviderSet(src, packPath) so a live source (no disk pmtiles) registers with the same tail (cell manifest, aux, meta); baker. PrepareLive is the input-prep half; tilesource.Composer.SavePartition persists the sidecar. Fix: scanPacks now skips the cells-pm dir — its per-cell PMTiles are compositor INPUTS, not packs, so they no longer each surface as a provider (which made the client probe /tiles/.json → 404 spam). Verified on NOAA d05 (848 cells): boot registers one "noaa" live provider (z0..16), /api/packs lists it with district d05, and the web map renders the Chesapeake from on-demand composed tiles. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/baker/compose.go | 63 ++++++++++ internal/engine/server/http.go | 6 +- internal/engine/server/live_provider.go | 150 ++++++++++++++++++++++++ internal/engine/server/prefs.go | 10 +- internal/engine/server/tile57_bake.go | 55 ++++++--- internal/engine/tilesource/composer.go | 4 + 6 files changed, 271 insertions(+), 17 deletions(-) create mode 100644 internal/engine/server/live_provider.go diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 6e827e6..703c8ab 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -82,6 +82,69 @@ func ComposeENCRoot(input, outPath string, onProgress func(done, total int, cell return tile57.ComposeFiles(perCell, outPath, onCompose) } +// PrepareLive bakes each cell under `input` (a .000 or an ENC_ROOT dir) to its own native-scale +// PMTiles in `cellsDir` (KEPT — the runtime compositor mmaps these directly), and returns the +// per-cell archive paths. There is NO district compose pass: tiles compose on demand at serve +// time (see tile57.OpenCompose). It is the input-prep half of the live compositor. +// +// Incremental: a cell whose per-cell archive already exists, is non-empty, and is at least as new +// as its source .000 is reused — so adding a district re-bakes only its new cells (not the whole +// provider). onProgress(done, total, cell) fires before each cell actually baked (nil to skip); +// onSkip reports a cell that failed (nil to skip). Returns an error (not an empty slice) only when +// cells were present but none are available. +func PrepareLive(input, cellsDir string, onProgress func(done, total int, cell string), onSkip func(cell string, err error)) ([]string, error) { + cells, err := ListCells(input) + if err != nil { + return nil, err + } + if len(cells) == 0 { + return nil, nil + } + if err := os.MkdirAll(cellsDir, 0o755); err != nil { + return nil, err + } + perCell := make([]string, 0, len(cells)) + for i, cp := range cells { + stem := strings.TrimSuffix(filepath.Base(cp), ".000") + pc := filepath.Join(cellsDir, stem+".pmtiles") + // Reuse an up-to-date archive: present, non-empty, and not older than its source cell (a + // re-downloaded district rewrites the .000, so its mtime advances → we re-bake it). + if fi, err := os.Stat(pc); err == nil && fi.Size() > 0 { + if si, err := os.Stat(cp); err == nil && !si.ModTime().After(fi.ModTime()) { + perCell = append(perCell, pc) + continue + } + } + if onProgress != nil { + onProgress(i, len(cells), stem) + } + b, err := tile57.BakeCell(cp) + if err != nil { + if onSkip != nil { + onSkip(filepath.Base(cp), err) + } + continue + } + if len(b) == 0 { + continue + } + if err := os.WriteFile(pc, b, 0o644); err != nil { + if onSkip != nil { + onSkip(filepath.Base(cp), err) + } + continue + } + perCell = append(perCell, pc) + } + if len(perCell) == 0 { + return nil, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) + } + if onProgress != nil { + onProgress(len(cells), len(cells), "") + } + return perCell, nil +} + // ListCells returns every base cell (.000) path under `root` (a single file or a directory), // deduped by stem (a boundary cell shared by two districts bakes once). func ListCells(root string) ([]string, error) { diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 33b9ac8..8766f4a 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -92,8 +92,9 @@ func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { if len(s.packs) > 0 { log.Printf("tilesets: %d pack(s) on disk, %d enabled (from %s)", len(s.packs), n, cacheDir) } + s.registerLiveProviders() // re-register live runtime compositors from kept per-cell archives (survives restart) s.registerLiveCompose() // dev/test: on-demand runtime compositor from TILE57_LIVE_COMPOSE (no-op if unset) - s.rebakeMissingProviders() // self-heal: bake any provider with an ENC_ROOT but no bundle (post-migration) + s.rebakeMissingProviders() // self-heal: bake any provider with an ENC_ROOT but no set (→ live prep) return s } @@ -105,6 +106,9 @@ func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { func (s *Server) rebakeMissingProviders() { var missing []string for _, prov := range s.installedProviders() { + if _, live := s.sets.get(prov); live { + continue // already serving (a live compositor from kept archives, or a registered pack) + } if _, ok := s.packPath(prov); !ok { missing = append(missing, prov) } diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go new file mode 100644 index 0000000..f12da66 --- /dev/null +++ b/internal/engine/server/live_provider.go @@ -0,0 +1,150 @@ +package server + +import ( + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/beetlebugorg/chartplotter/internal/engine/baker" + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// liveCellsDir is /cells-pm — the KEPT per-cell PMTiles the runtime compositor mmaps. +func (s *Server) liveCellsDir(provider string) string { + return filepath.Join(s.setDir(provider), "cells-pm") +} + +// livePartitionPath is /partition.tpart — the saved ownership-partition sidecar. +func (s *Server) livePartitionPath(provider string) string { + return filepath.Join(s.setDir(provider), "partition.tpart") +} + +// liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). +func (s *Server) liveCellArchives(provider string) []string { + dir := s.liveCellsDir(provider) + ents, err := os.ReadDir(dir) + if err != nil { + return nil + } + var paths []string + for _, e := range ents { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".pmtiles") { + paths = append(paths, filepath.Join(dir, e.Name())) + } + } + sort.Strings(paths) + return paths +} + +// openLiveComposer opens a runtime compositor over a provider's kept per-cell archives, loading the +// partition sidecar when present (else building it and saving one for next time). Returns nil when +// the provider has no live-cells dir yet. +func (s *Server) openLiveComposer(provider string) (*tilesource.Composer, error) { + paths := s.liveCellArchives(provider) + if len(paths) == 0 { + return nil, nil + } + sidecar := s.livePartitionPath(provider) + load := "" + if fi, err := os.Stat(sidecar); err == nil && fi.Size() > 0 { + load = sidecar + } + c, err := tilesource.NewComposer(paths, load) + if err != nil { + return nil, err + } + if load == "" { // freshly built the partition — persist it so the next open is a fast load + if err := c.SavePartition(sidecar); err != nil { + log.Printf("live %s: save partition sidecar: %v", provider, err) + } + } + return c, nil +} + +// prepareLiveProvider bakes the provider's cells to its kept live-cells dir (incremental) and opens +// a runtime compositor over them — the live counterpart of composeProvider, with NO district +// compose pass (tiles compose on demand). Returns the contributing cell count + the Composer. +func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tilesource.TileSource, error) { + cellsDir := s.liveCellsDir(provider) + s.invalidateLiveOnEngineChange(cellsDir) + start := time.Now() + paths, err := baker.PrepareLive(encRoot, cellsDir, + func(done, total int, cell string) { + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Zoom = "bake", "", 0 + if done >= total { + j.Unit, j.Note, j.Done, j.Total, j.ETA = "", "Preparing live tiles", 0, 0, 0 + return + } + note := "Baking charts" + if cell != "" { + note = "Baking " + cell + } + eta := 0 + if done > 0 { + per := time.Since(start) / time.Duration(done) + eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) + } + j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", note, done, total, eta + }) + }, + func(cell string, err error) { log.Printf("live %s: bake cell %s: %v (skipping)", provider, cell, err) }) + if err != nil { + return 0, nil, err + } + if len(paths) == 0 { + return 0, nil, nil + } + c, err := s.openLiveComposer(provider) + if err != nil { + return 0, nil, err + } + if c == nil { + return 0, nil, nil + } + return len(paths), c, nil +} + +// invalidateLiveOnEngineChange drops the kept per-cell archives when the tile57 engine commit +// changed since they were baked (a new engine portrays different tiles), so a binary upgrade +// re-bakes them. The partition sidecar is coverage-derived (engine-independent) and self-validates +// via its input key, so it is left in place. First bake just records the stamp. +func (s *Server) invalidateLiveOnEngineChange(cellsDir string) { + if s.EngineCommit == "" { + return + } + stamp := filepath.Join(cellsDir, ".enginever") + if b, err := os.ReadFile(stamp); err == nil && string(b) == s.EngineCommit { + return + } + if _, err := os.Stat(cellsDir); err == nil { + _ = os.RemoveAll(cellsDir) + } + _ = os.MkdirAll(cellsDir, 0o755) + _ = os.WriteFile(stamp, []byte(s.EngineCommit), 0o644) +} + +// registerLiveProviders re-registers, at boot, a runtime compositor for every installed provider +// that has kept per-cell archives (a live provider from a previous run) and isn't disabled — so +// live layers survive a restart without re-baking. A batch pack registered for the same provider is +// replaced (live wins). Runs before rebakeMissingProviders, which then skips the ones registered here. +func (s *Server) registerLiveProviders() { + for _, prov := range s.installedProviders() { + if s.prefs.isDisabled(prov) || len(s.liveCellArchives(prov)) == 0 { + continue + } + c, err := s.openLiveComposer(prov) + if err != nil || c == nil { + if err != nil { + log.Printf("live %s: reopen at boot: %v", prov, err) + } + continue + } + s.sets.register(prov, c) + m := c.Meta() + log.Printf("live: registered %q from kept per-cell archives (z%d..%d)", prov, m.MinZoom, m.MaxZoom) + } +} diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 4cc63ab..805cddc 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -103,7 +103,15 @@ func (p *prefs) setDisabled(set string, off bool) { func scanPacks(cacheDir string) map[string]string { out := map[string]string{} _ = filepath.WalkDir(cacheDir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() { + if err != nil { + return nil + } + if d.IsDir() { + // The live compositor's per-cell PMTiles are INPUTS, not packs — skip them, else every + // cell would surface as its own provider (and the client would probe /tiles/.json). + if d.Name() == "cells-pm" { + return filepath.SkipDir + } return nil } if !strings.HasSuffix(path, ".pmtiles") && !strings.HasSuffix(path, ".mbtiles") { diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 5320c3a..96ab1da 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -45,24 +45,36 @@ func (s *Server) bakeProgress(jobID string) func(tile57.BakeProgress) { // Returns false (recording a job error) if the bundle can't be opened. `set` is the // provider name (one archive per provider). func (s *Server) registerBakedSet(jobID, set string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { - outDir := s.setDir(set) - chart := filepath.Join(outDir, "tiles", "chart.pmtiles") + chart := filepath.Join(s.setDir(set), "tiles", "chart.pmtiles") src, err := tilesource.Open(chart) if err != nil { log.Printf("import %s (%s): open baked bundle: %v", jobID, set, err) return false } + return s.registerProviderSet(jobID, set, src, chart, cells, aux, cat, created) +} + +// registerProviderSet registers `src` as provider `set`'s live tile set and writes its tail (bake +// stamps, cell manifest, companion aux, metadata sidecar) so the provider is as complete in the +// chart library as any pack. `packPath` is the on-disk chart.pmtiles of a BATCH-baked provider +// (recorded for the pack list + stamped with the bake/engine version), or "" for a LIVE +// runtime-compositor provider (no disk archive — its bounds come from the TileSource Meta/TileJSON, +// and it is surfaced in /api/packs straight from the registry). +func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSource, packPath string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { + outDir := s.setDir(set) s.sets.register(set, src) - s.packAdd(set, chart) s.prefs.setDisabled(set, false) - if s.Version != "" { - _ = os.WriteFile(chart+bakeVerExt, []byte(s.Version), 0o644) - } - // Bake-time engine stamp (.enginever): the tile57 commit THIS binary links, - // recorded beside the pack so the set's TileJSON can report which engine baked - // these tiles even after the binary is upgraded. Best-effort, like .bakever. - if s.EngineCommit != "" { - _ = os.WriteFile(chart+engineVerExt, []byte(s.EngineCommit), 0o644) + if packPath != "" { + s.packAdd(set, packPath) + if s.Version != "" { + _ = os.WriteFile(packPath+bakeVerExt, []byte(s.Version), 0o644) + } + // Bake-time engine stamp (.enginever): the tile57 commit THIS binary links, + // recorded beside the pack so the set's TileJSON can report which engine baked + // these tiles even after the binary is upgraded. Best-effort, like .bakever. + if s.EngineCommit != "" { + _ = os.WriteFile(packPath+engineVerExt, []byte(s.EngineCommit), 0o644) + } } if err := s.writeSetCells(set, cells); err != nil { log.Printf("import %s: cell manifest %q: %v", jobID, set, err) @@ -131,8 +143,10 @@ func (s *Server) bakeProvider(jobID, provider string) bool { // sprite/glyphs/colortables are global server assets, so the composite path skips the // bundle's (unused) assets/style/manifest emission. var n int + var liveSrc tilesource.TileSource // non-nil → register the live compositor directly; nil → batch chart.pmtiles var err error - if os.Getenv("TILE57_LEGACY_BAKE") != "" { + switch { + case os.Getenv("TILE57_LEGACY_BAKE") != "": // MaxZoom 24 = the ABI's "no clamp" (each cell's full native band). var bbox [4]float64 n, bbox, err = tile57.BakeBundle(encRoot, outDir, tile57.BakeOpts{Created: created, MaxZoom: 24}, s.bakeProgress(jobID)) @@ -140,8 +154,13 @@ func (s *Server) bakeProvider(jobID, provider string) bool { os.RemoveAll(outDir) return fail(fmt.Errorf("import produced no coverage (%d cell(s), no valid S-57 data)", n)) } - } else { + case os.Getenv("TILE57_BATCH_COMPOSE") != "": + // Escape hatch: produce a portable district chart.pmtiles (the per-cell composite batch). n, err = s.composeProvider(jobID, encRoot, outDir) + default: + // DEFAULT: live runtime compositor — the per-cell bakes are KEPT and tiles compose on + // demand. No district compose pass, and adding a district re-bakes only its new cells. + n, liveSrc, err = s.prepareLiveProvider(jobID, encRoot, provider) } if err != nil { return fail(err) @@ -158,8 +177,14 @@ func (s *Server) bakeProvider(jobID, provider string) bool { cells := s.providerCellData(provider) aux := s.providerAux(provider) cat := s.providerCatalog(provider) - if !s.registerBakedSet(jobID, provider, cells, aux, cat, created) { - return fail(fmt.Errorf("could not register baked bundle for %q", provider)) + registered := false + if liveSrc != nil { + registered = s.registerProviderSet(jobID, provider, liveSrc, "", cells, aux, cat, created) + } else { + registered = s.registerBakedSet(jobID, provider, cells, aux, cat, created) + } + if !registered { + return fail(fmt.Errorf("could not register set for %q", provider)) } s.imports.update(jobID, func(j *importJob) { j.Cells = n }) log.Printf("import %s: baked provider %q (%d cell(s)) → %s", jobID, provider, n, outDir) diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go index e95b4bc..cb562fa 100644 --- a/internal/engine/tilesource/composer.go +++ b/internal/engine/tilesource/composer.go @@ -47,5 +47,9 @@ func (c *Composer) Tile(z uint8, x, y uint32) ([]byte, error) { // Meta returns the compositor's display metadata (zoom range + coverage bounds). func (c *Composer) Meta() TileMeta { return c.meta } +// SavePartition persists the resident ownership partition to path, so a later NewComposer can load +// it (as partitionPath) and skip the owned-face build. +func (c *Composer) SavePartition(path string) error { return c.src.SavePartition(path) } + // Close releases the compositor (io.Closer, so tilesource.Close finds it). func (c *Composer) Close() error { return c.src.Close() } From ebce1ccbc2e477f9800e0706a02bcb520bd26279 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 08:44:46 -0400 Subject: [PATCH 12/36] feat(tiles): ownership-aware tile caching for the live compositor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the ownership partition to decide caching, so a live provider never caches a blank that will fill in. The compositor reports `owned` per tile (a cell's face covers it → SHOULD render). tileserve then: - content, or unowned blank (true empty ocean) → immutable per ?g; - owned-but-blank (a cell owns it but produced nothing), or the set is still baking (imports.runningFor) → no-cache, so it re-fetches once the cell lands; - owned-but-blank AFTER bakes are done → logged as a likely missing/failed cell. Generation token: a live provider now carries a ?g (live.gen, bumped on each completed import via bumpLiveGen; packGen falls back to it), so the client re-fetches with a fresh URL when the cell set changes — invalidating tiles it cached against a previous, less-complete set. registerLiveProviders only re-serves a provider whose import completed (live.gen present); a set left partial by an interrupted bake is finished by rebakeMissingProviders instead of served partial. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/import.go | 14 +++++++++ internal/engine/server/live_provider.go | 23 ++++++++++++++ internal/engine/server/prefs.go | 5 +++ internal/engine/server/tile57_bake.go | 4 +++ internal/engine/server/tileserve.go | 42 +++++++++++++++++-------- internal/engine/tilesource/composer.go | 15 +++++++++ 6 files changed, 90 insertions(+), 13 deletions(-) diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 36538aa..cb161d1 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -121,6 +121,20 @@ func (j *importJobs) running() (importJob, bool) { return *best, true } +// runningFor reports whether a bake/import is in flight for `set` — its own job, or the pack being +// processed in a multi-pack batch. The tile server keeps caching OFF for a set while its content is +// in flux (a blank tile now may fill in when a cell finishes baking). +func (j *importJobs) runningFor(set string) bool { + j.mu.Lock() + defer j.mu.Unlock() + for _, job := range j.m { + if job.State == "running" && (job.Set == set || job.Pack == set) { + return true + } + } + return false +} + // handleImport routes the import endpoints (already past the /api host check). func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/import/status" { diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index f12da66..3301a4d 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -22,6 +22,23 @@ func (s *Server) livePartitionPath(provider string) string { return filepath.Join(s.setDir(provider), "partition.tpart") } +// liveGenPath is /live.gen — its mtime is the provider's tile GENERATION token (packGen +// reads it), bumped on each completed import so the client's tile URLs change and it re-fetches, +// invalidating tiles it cached against a previous, less-complete cell set. Its existence also marks +// "an import completed" — registerLiveProviders only re-serves a provider that has one, so a set +// left partial by an interrupted bake is completed by rebakeMissingProviders instead of served. +func (s *Server) liveGenPath(provider string) string { + return filepath.Join(s.setDir(provider), "live.gen") +} + +// bumpLiveGen advances the live provider's generation token (writes live.gen; the MTIME is the +// token). Called when an import registration completes. +func (s *Server) bumpLiveGen(provider string) { + p := s.liveGenPath(provider) + _ = os.MkdirAll(filepath.Dir(p), 0o755) + _ = os.WriteFile(p, []byte(time.Now().UTC().Format(time.RFC3339Nano)), 0o644) +} + // liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). func (s *Server) liveCellArchives(provider string) []string { dir := s.liveCellsDir(provider) @@ -136,6 +153,12 @@ func (s *Server) registerLiveProviders() { if s.prefs.isDisabled(prov) || len(s.liveCellArchives(prov)) == 0 { continue } + // Only re-serve a provider whose import COMPLETED (live.gen written at registration). A set + // left partial by an interrupted bake has cells-pm but no live.gen → skip it here, and + // rebakeMissingProviders finishes it (incremental) instead of serving a partial map. + if _, err := os.Stat(s.liveGenPath(prov)); err != nil { + continue + } c, err := s.openLiveComposer(prov) if err != nil || c == nil { if err != nil { diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 805cddc..7394489 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -165,6 +165,11 @@ func (s *Server) packGen(set string) int64 { return fi.ModTime().UnixNano() } } + // Live runtime-compositor provider (no disk pack): the live.gen file's mtime, bumped on each + // completed import — so its tiles are content-addressed by ?g exactly like a baked pack's. + if fi, err := os.Stat(s.liveGenPath(set)); err == nil { + return fi.ModTime().UnixNano() + } return 0 } diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 96ab1da..be9a921 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -75,6 +75,10 @@ func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSourc if s.EngineCommit != "" { _ = os.WriteFile(packPath+engineVerExt, []byte(s.EngineCommit), 0o644) } + } else { + // Live compositor: advance the generation token so the client re-fetches with a fresh ?g, + // invalidating any tiles (incl. blank 204s) it cached against a previous, less-complete set. + s.bumpLiveGen(set) } if err := s.writeSetCells(set, cells); err != nil { log.Printf("import %s: cell manifest %q: %v", jobID, set, err) diff --git a/internal/engine/server/tileserve.go b/internal/engine/server/tileserve.go index 4b0c171..f4c973a 100644 --- a/internal/engine/server/tileserve.go +++ b/internal/engine/server/tileserve.go @@ -3,6 +3,7 @@ package server import ( "compress/gzip" "fmt" + "log" "net/http" "os" "strconv" @@ -60,27 +61,42 @@ func (s *Server) serveTileSet(w http.ResponseWriter, r *http.Request) { apiErr(w, http.StatusNotFound, "unknown tile set") return } - body, err := src.Tile(uint8(z), x, y) + // Fetch the tile, plus — where the backend can tell us (the runtime compositor) — whether the + // ownership partition says a cell SHOULD render here. `owned` lets us tell a transient/erroneous + // empty (a cell owns this ground but produced nothing) from true empty ocean. + var body []byte + var owned bool + var err error + if ot, ok := src.(tilesource.OwnershipTiler); ok { + body, owned, err = ot.TileOwned(uint8(z), x, y) + } else { + body, err = src.Tile(uint8(z), x, y) + } if err != nil { apiErr(w, http.StatusInternalServerError, err.Error()) return } - // Cache policy first, so it rides the 204 too: an EMPTY tile at a given ?g is - // as content-addressed as a full one (emptiness is baked into that generation), - // and empty ocean tiles are the MAJORITY of a viewport's grid — caching those - // 204s saves the most round-trips. Tiles are immutable per bake generation: the - // ?g token in the URL (the pack archive's mtime) changes on every re-bake, so a - // given tile URL always maps to identical bytes/emptiness and can cache forever. - // The live/dynamic set carries no generation (?g absent or 0) and regenerates on - // demand, so it stays no-cache. Keying off the token — not pack-vs-live plumbing - // — ties the policy exactly to the content-addressing guarantee. - if g := r.URL.Query().Get("g"); g != "" && g != "0" { + + // Cache policy (rides the 204 too). A tile is immutable-per-generation ONLY when it is stable: + // it has content, or it is truly empty (no cell owns this ground — open ocean, the majority of a + // viewport). Two cases must NEVER cache, so they re-fetch once content lands (the ?g bumps on + // bake completion): the set is still baking (its content is in flux), or the partition says a + // cell SHOULD render here but it came back blank (its per-cell bake hasn't produced the tile + // yet). Emptiness at a given ?g is otherwise as content-addressed as bytes are. + blank := len(body) == 0 + generating := s.imports.runningFor(set) + shouldFill := blank && owned // expected content, not here yet + if g := r.URL.Query().Get("g"); g != "" && g != "0" && !generating && !shouldFill { w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") } else { w.Header().Set("Cache-Control", "no-cache") } - if len(body) == 0 { - w.WriteHeader(http.StatusNoContent) // blank/missing tile (still cacheable per ?g) + if shouldFill && !generating { + // Bakes are done, yet a tile a cell owns rendered nothing — a missing or failed cell. + log.Printf("tiles %s: z%d/%d/%d owned but empty (missing or failed cell?)", set, z, x, y) + } + if blank { + w.WriteHeader(http.StatusNoContent) // blank/missing tile return } // Tiles serve BYTES-VERBATIM in the set's stored encoding (no transcode). diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go index cb562fa..a95c544 100644 --- a/internal/engine/tilesource/composer.go +++ b/internal/engine/tilesource/composer.go @@ -39,8 +39,23 @@ func NewComposer(paths []string, partitionPath string) (*Composer, error) { }, nil } +// OwnershipTiler is a TileSource that also reports tile OWNERSHIP: whether its data model says a +// cell SHOULD render at (z,x,y). A blank (nil body) from an OWNED tile is transient (a cell's bake +// is still running) or an error (bake done) — the HTTP layer must not cache it, so it re-fetches +// once content lands; a blank from an UNOWNED tile is true empty ocean (safe to cache). The runtime +// Composer implements this; prebaked archives do not (their blanks are always true empty). +type OwnershipTiler interface { + TileOwned(z uint8, x, y uint32) (body []byte, owned bool, err error) +} + // Tile composes (z,x,y) on demand, returning decompressed MLT, or (nil, nil) for a blank tile. func (c *Composer) Tile(z uint8, x, y uint32) ([]byte, error) { + body, _, err := c.src.Serve(z, x, y) + return body, err +} + +// TileOwned is Tile plus the ownership flag (implements OwnershipTiler). +func (c *Composer) TileOwned(z uint8, x, y uint32) (body []byte, owned bool, err error) { return c.src.Serve(z, x, y) } From b370696b5184e7088fdde63258dacd4beb743eed Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 09:14:45 -0400 Subject: [PATCH 13/36] refactor(bake): make live cell-bake + compose the only tile-production path (host) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step A of the legacy cleanup — remove every host caller of the batch/bundle tile-production paths so cell-bake + live-compose is the only route: - bakeProvider collapses to prepareLiveProvider → registerProviderSet; drop the TILE57_LEGACY_BAKE (BakeBundle) and TILE57_BATCH_COMPOSE (composeProvider) branches. - delete composeProvider, registerBakedSet (the chart.pmtiles opener), bakeProgress (BakeBundle's per-band bar), and baker.ComposeENCRoot (BakeCell → ComposeFiles batch). - CLI `bake` now writes the LIVE STRUCTURE: /tiles/*.pmtiles (per cell, via baker.PrepareLive, incremental) + /partition.tpart (NewComposer + SavePartition). No merged chart.pmtiles, no per-scheme style/assets/manifest emission — style + assets are served at runtime. Deletes runTile57Archive/runTile57Bundle/bakeTile57Bundle + writeManifestJSON. Nothing here calls tile57.BakeBundle/BakePmtiles/Compose/ComposeFiles anymore — the Go bindings + C ABI for those come out next (Steps B–D). Builds + vets clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/bake.go | 85 +++++++------- cmd/chartplotter/tile57_bake.go | 159 -------------------------- internal/engine/baker/compose.go | 72 ------------ internal/engine/server/tile57_bake.go | 141 ++--------------------- 4 files changed, 53 insertions(+), 404 deletions(-) delete mode 100644 cmd/chartplotter/tile57_bake.go diff --git a/cmd/chartplotter/bake.go b/cmd/chartplotter/bake.go index 87986a6..ddf646e 100644 --- a/cmd/chartplotter/bake.go +++ b/cmd/chartplotter/bake.go @@ -2,15 +2,16 @@ package main import ( "archive/zip" - "encoding/json" "fmt" "io" "io/fs" "os" "path/filepath" "strings" + "time" "github.com/beetlebugorg/chartplotter/internal/engine/baker" + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" ) // bakeCmd bakes S-57 ENC base cells into a PMTiles archive of MVT tiles, for @@ -28,42 +29,59 @@ type bakeCmd struct { } func (c bakeCmd) Run() error { - // libtile57 is the sole bake engine. A *.pmtiles/-mbtiles -o writes ONE flat - // merged archive (+ optional --manifest / aux.zip) — the coverage-clipped - // composite resolves best-available inside the single archive, so there are - // no per-band archives any more (the retired --bands). Any other -o is a - // self-contained bundle directory (tiles/chart.pmtiles + per-scheme style + - // assets + manifest.json). - switch strings.ToLower(filepath.Ext(c.Out)) { - case ".pmtiles", ".mbtiles": - return c.runTile57Archive() - } - return c.runTile57Bundle() -} - -// runTile57Bundle bakes the ENC inputs with the native libtile57 engine into a -// self-contained chart bundle (tiles/chart.pmtiles + per-scheme SCAMIN-bucketed -// style-*.json + assets + manifest.json) under the output directory. The engine -// reads the ENC from disk, so a lone directory or .000 is handed over directly and -// only zips / multiple inputs are staged into a temp directory of cells first. -func (c bakeCmd) runTile57Bundle() error { + // libtile57 makes tiles ONE way: bake each cell to its own native-scale PMTiles + build the + // ownership partition. This CLI writes that LIVE STRUCTURE — the exact inputs the runtime + // compositor consumes: /tiles/.pmtiles + /partition.tpart. There is no + // merged chart.pmtiles bundle; the style + assets are produced separately (the asset commands / + // the server's runtime style), not per bake. input, cleanup, err := c.tile57Input() if err != nil { return err } defer cleanup() - outDir := bundleOutDir(c.Out) - if err := os.MkdirAll(outDir, 0o755); err != nil { + outDir := bundleOutDir(c.Out) // a *.pmtiles path → its stem dir; any other -o used verbatim + tilesDir := filepath.Join(outDir, "tiles") + if err := os.MkdirAll(tilesDir, 0o755); err != nil { + return err + } + + // Bake each cell to /tiles/*.pmtiles (incremental — reruns re-bake only changed cells). + start := time.Now() + paths, err := baker.PrepareLive(input, tilesDir, + func(done, total int, cell string) { + switch { + case done >= total: + fmt.Printf("\rbuilding partition over %d cell(s)… ", total) + case done > 0: + per := time.Since(start) / time.Duration(done) + fmt.Printf("\rbaking %s (%d/%d) · ~%s left ", cell, done, total, (per * time.Duration(total-done)).Round(time.Second)) + default: + fmt.Printf("\rbaking %s (%d/%d)… ", cell, done, total) + } + }, + func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) + fmt.Println() + if err != nil { return err } - // nil progress → the lib's built-in per-band console progress (good CLI output). - n, bbox, err := bakeTile57Bundle(input, outDir, c.MaxZoom, c.Format, nil) + if len(paths) == 0 { + return fmt.Errorf("no coverage: no valid S-57 cells under %s", input) + } + + // Build + save the ownership-partition sidecar next to the tiles (the compositor loads it on + // open to skip the build). + comp, err := tilesource.NewComposer(paths, "") if err != nil { return err } - fmt.Printf("baked %d cell(s) → %s/ via libtile57 — bundle: tiles/chart.pmtiles + assets/style-{day,dusk,night}.json + manifest.json (bbox %.4f,%.4f,%.4f,%.4f)\n", - n, outDir, bbox[0], bbox[1], bbox[2], bbox[3]) + defer comp.Close() + if err := comp.SavePartition(filepath.Join(outDir, "partition.tpart")); err != nil { + return err + } + m := comp.Meta() + fmt.Printf("baked %d cell(s) → %s/tiles/*.pmtiles + partition.tpart (z%d..%d, bounds %.4f,%.4f,%.4f,%.4f)\n", + len(paths), outDir, m.MinZoom, m.MaxZoom, m.W, m.S, m.E, m.N) return nil } @@ -120,21 +138,6 @@ func (c bakeCmd) tile57Input() (path string, cleanup func(), err error) { return dir, func() { os.RemoveAll(dir) }, nil } -// writeManifestJSON writes a charts-index.json manifest (indented) to path. -func writeManifestJSON(path string, man map[string]any) error { - mf, err := os.Create(path) - if err != nil { - return err - } - enc := json.NewEncoder(mf) - enc.SetIndent("", " ") - if err := enc.Encode(man); err != nil { - mf.Close() - return err - } - return mf.Close() -} - // bundleOutDir derives the bundle output DIRECTORY from the -o value: a *.pmtiles / // *.mbtiles path becomes its stem (charts.pmtiles → charts/), otherwise -o is used // as the directory verbatim. diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go deleted file mode 100644 index 003cd07..0000000 --- a/cmd/chartplotter/tile57_bake.go +++ /dev/null @@ -1,159 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/beetlebugorg/chartplotter/internal/engine/baker" - tile57 "github.com/beetlebugorg/tile57/bindings/go" -) - -// bakeTile57Bundle bakes an on-disk ENC input (a .000 cell or a directory of cells) -// into a self-contained chart bundle under outDir via the native libtile57 engine. -// maxZoom caps the highest baked zoom (0 = no cap). format selects the tile -// encoding ("mlt"/"mvt"; "" = the engine default, MLT). progress nil uses the -// lib's built-in console progress. Returns the cell count + bbox (w,s,e,n). -func bakeTile57Bundle(input, outDir string, maxZoom int, format string, progress func(tile57.BakeProgress)) (int, [4]float64, error) { - // MaxZoom 24 = the ABI's "no clamp" (bake each cell's full native band); MaxZoom 0 - // would clamp every band down to z0 — an EMPTY archive. Only narrow on --max-zoom. - opts := tile57.BakeOpts{MaxZoom: 24, Format: bakeFormat(format)} - if maxZoom > 0 && maxZoom < 24 { - opts.MaxZoom = uint8(maxZoom) - } - return tile57.BakeBundle(input, outDir, opts, progress) -} - -// bakeFormat maps the --format flag to the engine's bake format. "" = the engine -// default (MLT); "mvt" keeps the legacy Mapbox Vector Tile output. -func bakeFormat(format string) tile57.TileFormat { - switch format { - case "mvt": - return tile57.FormatMVT - case "mlt": - return tile57.FormatMLT - } - return tile57.FormatDefault -} - -// runTile57Archive bakes the ENC inputs into ONE flat merged archive at -o via -// the STREAMING bundle driver — super-tile locality, geometry LRU, coarse -// riders, capped overscale fill-up, TILE57_SUPER_DZ / TILE57_LRU_BUDGET -// tuning, tiles streamed to disk instead of an in-memory whole-archive buffer. -// BakeBundle bakes into a temp dir beside -o; the finished tiles/chart.pmtiles -// then renames onto -o (same filesystem) and the bundle scaffolding (assets/ -// styles/manifest.json) is discarded. Honors --max-zoom; writes --manifest -// (one band-less entry) + aux.zip (aux content collected from the input tree — -// zip inputs stage theirs beside the cells). -func (c bakeCmd) runTile57Archive() error { - input, cleanup, err := c.tile57Input() - if err != nil { - return err - } - defer cleanup() - - outAbs, err := filepath.Abs(c.Out) - if err != nil { - return err - } - - var n int - var bbox [4]float64 - if os.Getenv("TILE57_LEGACY_BAKE") != "" { - // Legacy in-bake combiner: BakeBundle into a temp dir beside -o, then rename the - // finished tiles/chart.pmtiles onto -o and discard the bundle scaffolding. - tmp, terr := os.MkdirTemp(filepath.Dir(outAbs), ".bake-*") - if terr != nil { - return terr - } - defer os.RemoveAll(tmp) - n, bbox, err = bakeTile57Bundle(input, tmp, c.MaxZoom, c.Format, nil) - if err != nil { - return err - } - if err := os.Rename(filepath.Join(tmp, "tiles", "chart.pmtiles"), outAbs); err != nil { - return err - } - st, _ := os.Stat(outAbs) - fmt.Printf("baked %d cell(s) → %s (%.1f MB) via libtile57 (streamed)\n", n, c.Out, float64(st.Size())/(1<<20)) - } else { - // Per-cell COMPOSITE (default): bake each cell at its native scale, then combine them - // via the engine's ownership partition straight into -o. --max-zoom/--format do not - // apply — each cell bakes at its native band and the compositor expands zoom. - start := time.Now() - n, err = baker.ComposeENCRoot(input, outAbs, - func(done, total int, cell string) { - if done >= total { - fmt.Printf("\rcomposing %d cells… ", total) - } else if done > 0 { - per := time.Since(start) / time.Duration(done) - fmt.Printf("\rbaking %s (%d/%d) · ~%s left ", cell, done, total, (per * time.Duration(total-done)).Round(time.Second)) - } else { - fmt.Printf("\rbaking %s (%d/%d)… ", cell, done, total) - } - }, - nil, // CLI shows no separate compose bar; the per-cell line above suffices - func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) - fmt.Println() - if err != nil { - return err - } - if n == 0 { - return fmt.Errorf("no coverage: no valid S-57 cells under %s", input) - } - st, _ := os.Stat(outAbs) - fmt.Printf("composed %d cell(s) → %s (%.1f MB) via the per-cell ownership partition\n", n, c.Out, float64(st.Size())/(1<<20)) - // bbox for the manifest, read back from the composed archive. - if src, e := tile57.OpenPMTiles(outAbs); e == nil { - info := src.Info() - bbox = [4]float64{info.West, info.South, info.East, info.North} - src.Close() - } - } - - // Aux content walks the INPUT tree (for a lone .000, its directory). - auxRoot := input - if fi, e := os.Stat(input); e == nil && !fi.IsDir() { - auxRoot = filepath.Dir(input) - } - ext := filepath.Ext(c.Out) - stem := strings.TrimSuffix(c.Out, ext) - auxManifest, err := writeAuxDir(stem, collectAuxDir(auxRoot)) - if err != nil { - return err - } - - if c.Manifest != "" { - entry := map[string]any{"file": filepath.Base(c.Out), "bounds": bbox[:]} - man := map[string]any{"districts": []map[string]any{entry}} - if auxManifest != "" { - man["aux"] = auxManifest - } - if err := writeManifestJSON(c.Manifest, man); err != nil { - return err - } - fmt.Printf("wrote manifest %s\n", c.Manifest) - } - return nil -} - -// orderedUpdates returns a cell's update bodies sorted by filename so libtile57 -// applies them in sequence (.001, .002, …). -func orderedUpdates(m map[string][]byte) [][]byte { - if len(m) == 0 { - return nil - } - names := make([]string, 0, len(m)) - for n := range m { - names = append(names, n) - } - sort.Strings(names) - out := make([][]byte, len(names)) - for i, n := range names { - out[i] = m[n] - } - return out -} diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 703c8ab..1d27ab5 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -10,78 +10,6 @@ import ( tile57 "github.com/beetlebugorg/tile57/bindings/go" ) -// ComposeENCRoot is the per-cell COMPOSITE bake: it bakes each cell under `input` (a single -// .000 or an ENC_ROOT directory) to its own native-scale PMTiles (coverage embedded in the -// metadata), then streams them through the engine's ownership partition into `outPath`. Per-cell -// archives go to a temp dir (mmap'd by the compositor, then discarded), so the whole cell set is -// never resident. This replaces the in-bake cross-cell combiner (tile57.BakeBundle) — the tiles -// half only; a host builds the style dynamically and serves global assets. -// -// onProgress(done, total, cell) is called before each cell bake (done 0..total-1, cell = the -// stem about to bake, e.g. "US5MD1MD") and once more with done==total (cell "") when the bakes -// are finished and the partition compose begins; nil to skip. onCompose (nil to skip) then -// reports live progress THROUGH that partition compose as it walks the zoom ladder (a smooth -// Done/Total fraction). onSkip (nil to skip) reports a cell that failed to bake. Returns the count -// of cells that contributed; an error (not 0) is returned when cells were present but none baked. -func ComposeENCRoot(input, outPath string, onProgress func(done, total int, cell string), onCompose func(tile57.ComposeProgress), onSkip func(cell string, err error)) (int, error) { - cells, err := ListCells(input) - if err != nil { - return 0, err - } - if len(cells) == 0 { - return 0, nil - } - - cellsDir, err := os.MkdirTemp("", "tile57-cells-*") - if err != nil { - return 0, err - } - defer os.RemoveAll(cellsDir) - - // 1. Bake each cell to its own PMTiles (one cell resident at a time — the bytes are freed as - // soon as they are written). - perCell := make([]string, 0, len(cells)) - for i, cp := range cells { - if onProgress != nil { - onProgress(i, len(cells), strings.TrimSuffix(filepath.Base(cp), ".000")) - } - b, err := tile57.BakeCell(cp) - if err != nil { - if onSkip != nil { - onSkip(filepath.Base(cp), err) - } - continue - } - if len(b) == 0 { - continue - } - pc := filepath.Join(cellsDir, filepath.Base(cp)+".pmtiles") - if err := os.WriteFile(pc, b, 0o644); err != nil { - if onSkip != nil { - onSkip(filepath.Base(cp), err) - } - continue - } - perCell = append(perCell, pc) - } - if len(perCell) == 0 { - // Cells were present but none baked → a bake ERROR, not empty coverage (so the caller - // fails without dropping the provider/source as "no coverage"). - return 0, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) - } - - // 2. Stream-compose the per-cell archives into outPath via the ownership partition. - if onProgress != nil { - onProgress(len(cells), len(cells), "") - } - if dir := filepath.Dir(outPath); dir != "" { - if err := os.MkdirAll(dir, 0o755); err != nil { - return 0, err - } - } - return tile57.ComposeFiles(perCell, outPath, onCompose) -} - // PrepareLive bakes each cell under `input` (a .000 or an ENC_ROOT dir) to its own native-scale // PMTiles in `cellsDir` (KEPT — the runtime compositor mmaps these directly), and returns the // per-cell archive paths. There is NO district compose pass: tiles compose on demand at serve diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index be9a921..3243e4c 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -13,46 +13,6 @@ import ( tile57 "github.com/beetlebugorg/tile57/bindings/go" ) -// bakeProgress returns a per-band progress callback for a provider bake. BakeBundle -// fires stage-0 (portraying a band's cells) then stage-1 (writing that band's tiles) -// events per navigational-purpose band. A calm per-band bar: it fills 0→100% per band -// and resets at each band — clamped monotonic WITHIN a band (the engine double-counts a -// band's tiles across its parallel-gen + serial-write phases, which would otherwise -// rewind). The multi-district DOWNLOAD phase sets pack "N of M" on the job separately. -func (s *Server) bakeProgress(jobID string) func(tile57.BakeProgress) { - curBand, bandDoneMax := -1, 0 - return func(p tile57.BakeProgress) { - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band = "bake", p.BandName - if p.Stage == 0 { // portraying this band's cells - j.Unit, j.Note, j.Done, j.Total = "cells", "Preparing charts", p.Done, p.Total - return - } - if p.BandIndex != curBand { // new band → reset the within-band floor - curBand, bandDoneMax = p.BandIndex, 0 - } - if p.Done > bandDoneMax { - bandDoneMax = p.Done - } - j.Unit, j.Note, j.Done, j.Total = "tiles", "Generating tiles", bandDoneMax, p.Total - }) - } -} - -// registerBakedSet registers a freshly-baked PROVIDER's chart.pmtiles as the live tile -// set and writes its tail — bake stamps, cell manifest, companion aux.zip, metadata -// sidecar — so a tile57 provider is as complete in the chart library as any pack. -// Returns false (recording a job error) if the bundle can't be opened. `set` is the -// provider name (one archive per provider). -func (s *Server) registerBakedSet(jobID, set string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { - chart := filepath.Join(s.setDir(set), "tiles", "chart.pmtiles") - src, err := tilesource.Open(chart) - if err != nil { - log.Printf("import %s (%s): open baked bundle: %v", jobID, set, err) - return false - } - return s.registerProviderSet(jobID, set, src, chart, cells, aux, cat, created) -} // registerProviderSet registers `src` as provider `set`'s live tile set and writes its tail (bake // stamps, cell manifest, companion aux, metadata sidecar) so the provider is as complete in the @@ -139,37 +99,17 @@ func (s *Server) bakeProvider(jobID, provider string) bool { }) created := time.Now().UTC().Format(time.RFC3339) - // The per-cell COMPOSITE model (default): bake each cell to its own native-scale PMTiles, - // then combine them via the engine's ownership partition into tiles/chart.pmtiles. This - // replaces the in-bake cross-cell combiner (BakeBundle) — kept behind TILE57_LEGACY_BAKE=1 - // as an escape hatch while the composite model beds in. The served bundle only needs - // tiles/chart.pmtiles: the MapLibre style is built dynamically (tile57.Style) and the - // sprite/glyphs/colortables are global server assets, so the composite path skips the - // bundle's (unused) assets/style/manifest emission. - var n int - var liveSrc tilesource.TileSource // non-nil → register the live compositor directly; nil → batch chart.pmtiles - var err error - switch { - case os.Getenv("TILE57_LEGACY_BAKE") != "": - // MaxZoom 24 = the ABI's "no clamp" (each cell's full native band). - var bbox [4]float64 - n, bbox, err = tile57.BakeBundle(encRoot, outDir, tile57.BakeOpts{Created: created, MaxZoom: 24}, s.bakeProgress(jobID)) - if err == nil && (n == 0 || bbox[2] <= bbox[0] || bbox[3] <= bbox[1]) { - os.RemoveAll(outDir) - return fail(fmt.Errorf("import produced no coverage (%d cell(s), no valid S-57 data)", n)) - } - case os.Getenv("TILE57_BATCH_COMPOSE") != "": - // Escape hatch: produce a portable district chart.pmtiles (the per-cell composite batch). - n, err = s.composeProvider(jobID, encRoot, outDir) - default: - // DEFAULT: live runtime compositor — the per-cell bakes are KEPT and tiles compose on - // demand. No district compose pass, and adding a district re-bakes only its new cells. - n, liveSrc, err = s.prepareLiveProvider(jobID, encRoot, provider) - } + // Live runtime compositor — the ONLY tile-production path. Bake each cell to its own + // native-scale PMTiles (KEPT under /tiles), build + save the ownership-partition + // sidecar, and register a Composer that composes tiles on demand. Adding a district re-bakes + // only its new cells. There is no district compose pass and no chart.pmtiles bundle; the + // MapLibre style + sprite/glyph/colortable assets are served at runtime (tile57.Style + the + // asset endpoints), not emitted per provider. + n, liveSrc, err := s.prepareLiveProvider(jobID, encRoot, provider) if err != nil { return fail(err) } - // Zero cells means nothing valid parsed → a failed import (don't register an empty pack). + // Zero cells means nothing valid parsed → a failed import (don't register an empty set). if n == 0 { os.RemoveAll(outDir) return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) @@ -181,13 +121,7 @@ func (s *Server) bakeProvider(jobID, provider string) bool { cells := s.providerCellData(provider) aux := s.providerAux(provider) cat := s.providerCatalog(provider) - registered := false - if liveSrc != nil { - registered = s.registerProviderSet(jobID, provider, liveSrc, "", cells, aux, cat, created) - } else { - registered = s.registerBakedSet(jobID, provider, cells, aux, cat, created) - } - if !registered { + if !s.registerProviderSet(jobID, provider, liveSrc, "", cells, aux, cat, created) { return fail(fmt.Errorf("could not register set for %q", provider)) } s.imports.update(jobID, func(j *importJob) { j.Cells = n }) @@ -195,63 +129,6 @@ func (s *Server) bakeProvider(jobID, provider string) bool { return true } -// composeProvider bakes each cell under encRoot to its own native-scale PMTiles (coverage -// embedded in the metadata) and streams them through the engine's ownership partition into -// /tiles/chart.pmtiles. Per-cell archives go to a temp dir (mmap'd by the compositor, -// then discarded), so the whole cell set is never resident. Returns the count of cells that -// contributed to the composite, or 0 if none produced coverage. -func (s *Server) composeProvider(jobID, encRoot, outDir string) (int, error) { - tilesPath := filepath.Join(outDir, "tiles", "chart.pmtiles") - start := time.Now() - var composeStart time.Time // set when the per-cell bakes finish and the compose begins - return baker.ComposeENCRoot(encRoot, tilesPath, - func(done, total int, cell string) { - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Zoom = "bake", "", 0 - if done >= total { - // Per-cell bakes done → the ownership-partition compose runs. Until its first - // zoom-progress arrives, sweep (total 0) under "Composing tiles". - composeStart = time.Now() - j.Unit, j.Note, j.Done, j.Total, j.ETA = "", "Composing tiles", 0, 0, 0 - return - } - // Per-cell portrayal: name the chart being baked, plus a determinate bar and an ETA - // from the mean per-cell rate so far. - note := "Baking charts" - if cell != "" { - note = "Baking " + cell - } - eta := 0 - if done > 0 { - per := time.Since(start) / time.Duration(done) - eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) - } - j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", note, done, total, eta - }) - }, - func(p tile57.ComposeProgress) { - // Live compose progress: the engine weights zooms by tile count, so p.Done/p.Total is - // a smooth fraction. ETA extrapolates from elapsed compose time and that fraction. - if composeStart.IsZero() { - composeStart = time.Now() - } - eta := 0 - if p.Total > 0 && p.Done > 0 && p.Done < p.Total { - frac := float64(p.Done) / float64(p.Total) - remain := time.Duration(float64(time.Since(composeStart)) * (1 - frac) / frac) - eta = int(remain.Round(time.Second).Seconds()) - } - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Unit, j.Note = "bake", "", "", "Composing tiles" - j.Done, j.Total, j.ETA = int(p.Done), int(p.Total), eta - j.Zoom, j.ZoomMin, j.ZoomMax = p.Zoom, p.MinZoom, p.MaxZoom - }) - }, - func(cell string, err error) { - log.Printf("import %s: bake cell %s: %v (skipping)", jobID, cell, err) - }) -} - // dropProviderSet unregisters a provider whose ENC_ROOT is now empty and removes its // (regenerable) baked bundle + its (now-empty) provider data tree. The cell index is // rebuilt so its cells stop counting as installed. From fb508947d4299a08b820e7423eb672850acb5f99 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 11:32:09 -0400 Subject: [PATCH 14/36] refactor(live): rename the server's live per-cell dir cells-pm -> tiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the runtime compositor's kept per-cell archives with the layout the `tile57 bake -o ` CLI writes (/tiles/.pmtiles + partition.tpart), so a CLI-baked structure drops straight into a provider's set dir. liveCellsDir now returns /tiles. scanPacks recognizes a live provider by its partition.tpart sidecar and skips the whole tiles/ dir (its per-cell archives are compositor INPUTS, not packs) — else every cell would surface as a phantom provider. A legacy batch bundle's tiles/chart.pmtiles (no sidecar) still registers by provider name. Tests: new TestScanPacksSkipsLiveProvider locks the skip/keep behavior; the stale TestImportPacks (asserted the removed batch tiles/chart.pmtiles) now asserts the live structure — per-cell tiles/.pmtiles + partition.tpart. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/import_packs_test.go | 14 +++--- internal/engine/server/live_provider.go | 8 ++-- internal/engine/server/prefs.go | 14 ++++-- internal/engine/server/prefs_test.go | 47 +++++++++++++++++++++ 4 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 internal/engine/server/prefs_test.go diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index 87c6e0c..036f818 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -79,14 +79,18 @@ func TestImportPacks(t *testing.T) { t.Fatalf("job state=%q error=%q", st.State, st.Error) } - // ONE provider set ("noaa") registered, with a baked chart.pmtiles under the cache - // dir, and both districts' source cells kept under the data dir. + // ONE provider set ("noaa") registered, serving from the live-composite structure + // under the cache dir (per-cell tiles/.pmtiles + partition.tpart), and both + // districts' source cells kept under the data dir. if _, ok := s.sets.get("noaa"); !ok { t.Errorf("provider set %q not registered", "noaa") } - chart := filepath.Join(s.setDir("noaa"), "tiles", "chart.pmtiles") - if fi, err := os.Stat(chart); err != nil || fi.Size() == 0 { - t.Errorf("provider %q: no baked pmtiles (%v)", "noaa", err) + cellArc := filepath.Join(s.setDir("noaa"), "tiles", "US5MD1MC.pmtiles") + if fi, err := os.Stat(cellArc); err != nil || fi.Size() == 0 { + t.Errorf("provider %q: no baked per-cell tiles (%v)", "noaa", err) + } + if _, err := os.Stat(filepath.Join(s.setDir("noaa"), "partition.tpart")); err != nil { + t.Errorf("provider %q: no partition sidecar (%v)", "noaa", err) } for _, dist := range []string{"d5", "d7"} { if _, err := os.Stat(filepath.Join(s.districtDir("noaa", dist), "US5MD1MC.000")); err != nil { diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index 3301a4d..e878310 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -12,9 +12,11 @@ import ( "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" ) -// liveCellsDir is /cells-pm — the KEPT per-cell PMTiles the runtime compositor mmaps. +// liveCellsDir is /tiles — the KEPT per-cell PMTiles the runtime compositor mmaps. +// This is the same layout `tile57 bake -o ` writes (/tiles/.pmtiles next to +// /partition.tpart), so a CLI-baked structure drops straight into a provider's set dir. func (s *Server) liveCellsDir(provider string) string { - return filepath.Join(s.setDir(provider), "cells-pm") + return filepath.Join(s.setDir(provider), "tiles") } // livePartitionPath is /partition.tpart — the saved ownership-partition sidecar. @@ -154,7 +156,7 @@ func (s *Server) registerLiveProviders() { continue } // Only re-serve a provider whose import COMPLETED (live.gen written at registration). A set - // left partial by an interrupted bake has cells-pm but no live.gen → skip it here, and + // left partial by an interrupted bake has tiles/ but no live.gen → skip it here, and // rebakeMissingProviders finishes it (incremental) instead of serving a partial map. if _, err := os.Stat(s.liveGenPath(prov)); err != nil { continue diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 7394489..1107ea1 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -107,10 +107,16 @@ func scanPacks(cacheDir string) map[string]string { return nil } if d.IsDir() { - // The live compositor's per-cell PMTiles are INPUTS, not packs — skip them, else every - // cell would surface as its own provider (and the client would probe /tiles/.json). - if d.Name() == "cells-pm" { - return filepath.SkipDir + // A live runtime-compositor provider keeps its per-cell PMTiles under + // /tiles next to a partition.tpart sidecar. Those archives are INPUTS, + // not packs — skip the whole dir, else every cell would surface as its own + // provider (and the client would probe /tiles/.json). Detect the live + // provider by the sidecar, so a legacy batch bundle's tiles/chart.pmtiles still + // registers below. + if d.Name() == "tiles" { + if _, err := os.Stat(filepath.Join(filepath.Dir(path), "partition.tpart")); err == nil { + return filepath.SkipDir + } } return nil } diff --git a/internal/engine/server/prefs_test.go b/internal/engine/server/prefs_test.go new file mode 100644 index 0000000..e8fbafa --- /dev/null +++ b/internal/engine/server/prefs_test.go @@ -0,0 +1,47 @@ +package server + +import ( + "os" + "path/filepath" + "testing" +) + +// TestScanPacksSkipsLiveProvider: a live runtime-compositor provider keeps its per-cell +// PMTiles under /tiles next to a partition.tpart sidecar. scanPacks must skip that +// whole dir — those archives are compositor INPUTS — so no cell surfaces as a phantom provider, +// while a legacy batch bundle's tiles/chart.pmtiles (no sidecar) still registers by provider name. +func TestScanPacksSkipsLiveProvider(t *testing.T) { + cache := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(cache, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + // A live provider: per-cell archives under tiles/ + the partition sidecar alongside. + write("NOAA/tiles/US5MD1MC.pmtiles", "pm") + write("NOAA/tiles/US4MD81M.pmtiles", "pm") + write("NOAA/partition.tpart", "part") + + // A legacy batch bundle: tiles/chart.pmtiles, no sidecar. + write("LEGACY/tiles/chart.pmtiles", "pm") + + got := scanPacks(cache) + + if _, ok := got["legacy"]; !ok { + t.Errorf("legacy batch bundle not registered: %v", got) + } + for _, phantom := range []string{"noaa", "us5md1mc", "us4md81m", "chart", "partition"} { + if p, ok := got[phantom]; ok { + t.Errorf("live-provider input mis-registered as pack %q -> %q", phantom, p) + } + } + if len(got) != 1 { + t.Errorf("scanPacks = %v, want only {legacy}", got) + } +} From f84fcea6c3ab33ee192235cf7ebb02d2d27fe671 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 11:48:21 -0400 Subject: [PATCH 15/36] feat(live): make live-compositor providers first-class in the pack handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live runtime-compositor provider has no disk pack (it lives only in the set registry), so several handlers that assumed a packPath left it half-supported. Surface it provider-centrically instead: - /api/packs: emit coverage bounds for a live provider from the registry when enabled, else from its meta sidecar (BBox) — not only for disk packs. - /api/set/enable: re-register a disabled live provider by re-opening its runtime compositor from the kept per-cell archives, not just re-opening a disk pack (previously disable->enable dropped a live set until restart). - enabledPackCells (?active): include live providers' cells from the same per-provider cell manifest, so their charts count as "on the map". TestImportPacks now also asserts the live provider's bounds appear in /api/packs and that disable->enable round-trips through the registry. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/import_packs_test.go | 37 ++++++++++++++++++ internal/engine/server/tilesets.go | 42 ++++++++++++++++++--- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index 036f818..741cd23 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -101,4 +101,41 @@ func TestImportPacks(t *testing.T) { if stems, ok := s.setCells("noaa"); !ok || len(stems) != 1 || stems[0] != "US5MD1MC" { t.Errorf("provider cell manifest = %v (ok=%t), want [US5MD1MC]", stems, ok) } + + // A live provider is a FIRST-CLASS set, surfaced provider-centrically (no disk pack): + // /api/packs reports its coverage bounds from the registry. + pr, _ := http.Get(ts.URL + "/api/packs") + pb, _ := io.ReadAll(pr.Body) + pr.Body.Close() + var packs struct { + Packs []struct { + Name string `json:"name"` + Bounds []float64 `json:"bounds"` + } `json:"packs"` + } + json.Unmarshal(pb, &packs) + var noaaBounds []float64 + for _, p := range packs.Packs { + if p.Name == "noaa" { + noaaBounds = p.Bounds + } + } + if len(noaaBounds) != 4 { + t.Errorf("/api/packs: live provider noaa has no bounds: %v", noaaBounds) + } + + // disable → enable round-trips through the registry (re-opening the runtime + // compositor from kept per-cell archives), not just disk packs. + if resp, err := http.Post(ts.URL+"/api/set/disable?set=noaa", "", nil); err == nil { + resp.Body.Close() + } + if _, live := s.sets.get("noaa"); live { + t.Error("disabled live provider still registered") + } + if resp, err := http.Post(ts.URL+"/api/set/enable?set=noaa", "", nil); err == nil { + resp.Body.Close() + } + if _, live := s.sets.get("noaa"); !live { + t.Error("re-enabled live provider not re-registered from kept archives") + } } diff --git a/internal/engine/server/tilesets.go b/internal/engine/server/tilesets.go index 8ffe102..9b9ff5f 100644 --- a/internal/engine/server/tilesets.go +++ b/internal/engine/server/tilesets.go @@ -195,18 +195,26 @@ func (s *Server) handlePacks(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, ",") } fmt.Fprintf(w, `{"name":%q,"enabled":%t`, prov, !s.prefs.isDisabled(prov)) - // Bounds straight from the baked archive on disk — works even while DISABLED (its - // tiles aren't served, but the boundary still marks "you have this chart here"). + meta, hasMeta := s.readSetMeta(prov) + // Coverage bounds, so the client marks "you have this chart here" even while a set is + // DISABLED. A standalone archive reads them off the file; a live runtime-compositor + // provider (no disk archive) takes them from the registry when enabled, else from its + // meta sidecar — surfaced first-class from provider structure, not a pack path. if path, ok := s.packPath(prov); ok { if src, err := tilesource.Open(path); err == nil { m := src.Meta() _ = tilesource.Close(src) fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, m.W, m.S, m.E, m.N) } + } else if src, live := s.sets.get(prov); live { + m := src.Meta() + fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, m.W, m.S, m.E, m.N) + } else if hasMeta && len(meta.BBox) == 4 { + fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, meta.BBox[0], meta.BBox[1], meta.BBox[2], meta.BBox[3]) } // Extracted metadata (title/agency/scale range/counts/imported date), from the // .meta.json sidecar. Cells omitted here — GET /api/pack/. - if m, ok := s.readSetMeta(prov); ok { + if m := meta; hasMeta { if m.Title != "" { fmt.Fprintf(w, `,"title":%q`, m.Title) } @@ -273,14 +281,22 @@ func (s *Server) handleSetEnabled(w http.ResponseWriter, r *http.Request) { enable := strings.HasSuffix(r.URL.Path, "/enable") s.prefs.setDisabled(provider, !enable) if enable { - if path, ok := s.packPath(provider); ok { - if _, live := s.sets.get(provider); !live { + if _, live := s.sets.get(provider); !live { + if path, ok := s.packPath(provider); ok { + // Standalone/overlay archive: re-open the disk pack. if src, err := tilesource.Open(path); err == nil { s.sets.register(provider, src) } else { apiErr(w, http.StatusInternalServerError, err.Error()) return } + } else if c, err := s.openLiveComposer(provider); err != nil { + apiErr(w, http.StatusInternalServerError, err.Error()) + return + } else if c != nil { + // Live runtime-compositor provider (no disk pack): re-open the compositor + // from its kept per-cell archives + partition sidecar. + s.sets.register(provider, c) } } } else { @@ -358,6 +374,22 @@ func (s *Server) enabledPackCells() (map[string]bool, [][4]float64) { legacy = append(legacy, [4]float64{m.W, m.S, m.E, m.N}) } } + // Live runtime-compositor providers (no disk pack): their cells come from the SAME + // per-provider cell manifest (writeSetCells at import), keyed provider-centrically — + // so a live provider's charts count as "on the map" for ?active just like a pack's. + for _, prov := range s.installedProviders() { + if s.prefs.isDisabled(prov) { + continue + } + if _, isPack := s.packPath(prov); isPack { + continue // already covered above + } + if stems, ok := s.setCells(prov); ok { + for _, st := range stems { + cells[st] = true + } + } + } return cells, legacy } From 79a2a6d7714bb7053c94b1e1fc39fbe51cfd2778 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 11:51:25 -0400 Subject: [PATCH 16/36] fix(ui): brand a bare provider set as its display name (NOAA, not noaa) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _setLabel branded district / river sets (noaa-d5 -> "NOAA · Mid-Atlantic", ienc-* -> "IENC · …") but a BARE provider set — which the live-composite model registers, one set per provider ("noaa") — fell through to the raw lowercase routing key. Map it to the branded display name from the existing _providers() list ("noaa" -> "NOAA", "ienc" -> "Inland ENC", "user" -> "User Charts"). The routing key stays lowercase; this is display only. Co-Authored-By: Claude Opus 4.8 (1M context) --- web/src/plugins/chart-library.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/src/plugins/chart-library.mjs b/web/src/plugins/chart-library.mjs index d88d048..7576b69 100644 --- a/web/src/plugins/chart-library.mjs +++ b/web/src/plugins/chart-library.mjs @@ -738,7 +738,10 @@ export class ChartLibrary extends HTMLElement { if (name === "import") return "Imported charts"; const ie = /^ienc-(.+)$/.exec(name); if (ie) return `IENC · ${ie[1]}`; - return name; + // A bare provider set (the live-composite model registers ONE set per provider, + // e.g. "noaa") → its branded display name ("NOAA"), not the lowercase routing key. + const prov = this._providers().find((p) => p.id === this._providerKey(name)); + return prov ? prov.name : name; } _wirePacks() { this._wireMillerRows(); this._wireDetailButtons(); } From 38568c3ae31ccfe802678362279d597929ed896b Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 11:53:13 -0400 Subject: [PATCH 17/36] refactor(discovery): scanPacks reads only the flat standalone-archive dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make provider discovery fully provider-centric. scanPacks now reads ONLY the flat /tiles dir for hand-dropped standalone .pmtiles/.mbtiles archives, instead of walking the whole cache. Live runtime-compositor providers own //tiles/*.pmtiles + partition.tpart and are discovered by registerLiveProviders (iterating installed providers) — never scavenged here. This supersedes the earlier partition.tpart-guarded skip with a stronger invariant: an interrupted import (per-cell archives present, partition not saved yet) can no longer leak its cells as phantom sets, because provider dirs aren't scanned at all. TestScanPacksStandaloneArchivesOnly covers flat discovery + the provider-input skip, both with and without a saved partition sidecar. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/prefs.go | 60 +++++++++++----------------- internal/engine/server/prefs_test.go | 37 +++++++++-------- 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 1107ea1..9b6335f 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -95,47 +95,35 @@ func (p *prefs) setDisabled(set string, off bool) { } } -// scanPacks walks the cache and returns every baked provider bundle keyed by set name -// (= provider) → path. The tile57 bundle writes //tiles/chart.pmtiles -// (one archive per provider, provider-enc-root), so the set name is the provider dir — -// otherwise every bundle collapses to "chart" and the chart library can't list them -// after a restart. +// scanPacks discovers STANDALONE tile archives — prebaked .pmtiles/.mbtiles hand-dropped +// into the flat /tiles dir — keyed set name (the file basename) → path. This is the +// narrow overlay path for hand-placed archives. +// +// It scans ONLY the flat tiles dir, never the whole cache: a live runtime-compositor +// provider owns //tiles/*.pmtiles + partition.tpart, and those per-cell +// archives are compositor INPUTS discovered provider-centrically by registerLiveProviders — +// never scavenged here (else every cell would surface as its own phantom set, and an +// interrupted import that hasn't saved its partition yet would leak cells as packs). func scanPacks(cacheDir string) map[string]string { out := map[string]string{} - _ = filepath.WalkDir(cacheDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - if d.IsDir() { - // A live runtime-compositor provider keeps its per-cell PMTiles under - // /tiles next to a partition.tpart sidecar. Those archives are INPUTS, - // not packs — skip the whole dir, else every cell would surface as its own - // provider (and the client would probe /tiles/.json). Detect the live - // provider by the sidecar, so a legacy batch bundle's tiles/chart.pmtiles still - // registers below. - if d.Name() == "tiles" { - if _, err := os.Stat(filepath.Join(filepath.Dir(path), "partition.tpart")); err == nil { - return filepath.SkipDir - } - } - return nil - } - if !strings.HasSuffix(path, ".pmtiles") && !strings.HasSuffix(path, ".mbtiles") { - return nil + dir := tilesDir(cacheDir) + entries, err := os.ReadDir(dir) + if err != nil { + return out + } + for _, e := range entries { + if e.IsDir() { + continue } - name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) - // tile57 bundle: //tiles/chart.pmtiles → the provider name. - if name == "chart" && filepath.Base(filepath.Dir(path)) == "tiles" { - provider := filepath.Base(filepath.Dir(filepath.Dir(path))) - if provider != "" && provider != "." { - name = strings.ToLower(provider) - } + name := e.Name() + if !strings.HasSuffix(name, ".pmtiles") && !strings.HasSuffix(name, ".mbtiles") { + continue } - if isSetName(name) { - out[name] = path + set := strings.TrimSuffix(name, filepath.Ext(name)) + if isSetName(set) { + out[set] = filepath.Join(dir, name) } - return nil - }) + } return out } diff --git a/internal/engine/server/prefs_test.go b/internal/engine/server/prefs_test.go index e8fbafa..d7c0037 100644 --- a/internal/engine/server/prefs_test.go +++ b/internal/engine/server/prefs_test.go @@ -6,11 +6,12 @@ import ( "testing" ) -// TestScanPacksSkipsLiveProvider: a live runtime-compositor provider keeps its per-cell -// PMTiles under /tiles next to a partition.tpart sidecar. scanPacks must skip that -// whole dir — those archives are compositor INPUTS — so no cell surfaces as a phantom provider, -// while a legacy batch bundle's tiles/chart.pmtiles (no sidecar) still registers by provider name. -func TestScanPacksSkipsLiveProvider(t *testing.T) { +// TestScanPacksStandaloneArchivesOnly: scanPacks discovers only STANDALONE archives dropped +// into the flat /tiles dir (keyed by basename). A live runtime-compositor provider owns +// /tiles/*.pmtiles + partition.tpart — compositor INPUTS discovered +// provider-centrically by registerLiveProviders — and must NEVER be scavenged here (no phantom +// per-cell sets), even when its partition sidecar isn't saved yet. +func TestScanPacksStandaloneArchivesOnly(t *testing.T) { cache := t.TempDir() write := func(rel, body string) { t.Helper() @@ -23,25 +24,29 @@ func TestScanPacksSkipsLiveProvider(t *testing.T) { } } - // A live provider: per-cell archives under tiles/ + the partition sidecar alongside. + // Standalone archives in the flat /tiles dir → registered by basename. + write("tiles/charts.pmtiles", "pm") + write("tiles/overlay.mbtiles", "mb") + + // A live provider's per-cell inputs under /tiles — WITH a partition sidecar... write("NOAA/tiles/US5MD1MC.pmtiles", "pm") - write("NOAA/tiles/US4MD81M.pmtiles", "pm") write("NOAA/partition.tpart", "part") - - // A legacy batch bundle: tiles/chart.pmtiles, no sidecar. - write("LEGACY/tiles/chart.pmtiles", "pm") + // ...and one mid-import (archives present, partition not saved yet). + write("IENC/tiles/US4MD81M.pmtiles", "pm") got := scanPacks(cache) - if _, ok := got["legacy"]; !ok { - t.Errorf("legacy batch bundle not registered: %v", got) + for _, want := range []string{"charts", "overlay"} { + if _, ok := got[want]; !ok { + t.Errorf("standalone archive %q not discovered: %v", want, got) + } } - for _, phantom := range []string{"noaa", "us5md1mc", "us4md81m", "chart", "partition"} { + for _, phantom := range []string{"noaa", "ienc", "us5md1mc", "us4md81m", "US5MD1MC", "US4MD81M", "partition"} { if p, ok := got[phantom]; ok { - t.Errorf("live-provider input mis-registered as pack %q -> %q", phantom, p) + t.Errorf("live-provider input mis-registered as set %q -> %q", phantom, p) } } - if len(got) != 1 { - t.Errorf("scanPacks = %v, want only {legacy}", got) + if len(got) != 2 { + t.Errorf("scanPacks = %v, want {charts, overlay}", got) } } From e5779fb47a8d093bb333af6e21cacfe4fc8c12f1 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 12:03:16 -0400 Subject: [PATCH 18/36] feat(cache): content-addressed ?g token for live providers (sha-of-shas) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the live provider's live.gen MTIME cache-bust token with a CONTENT token: a sha-of-shas over its per-cell archives. PrepareLive writes an .sha sidecar (the archive's content sha) at bake; liveGenToken sorts the per-cell "stem:sha" lines and hashes them into a positive int63, stored in live.gen and read by packGen. So a no-op re-bake keeps the token (the client's cached tiles stay valid), and the token advances exactly when the cell set or a cell's content changes — the foundation for progressive per-batch re-keying during import. TestLiveGenTokenContentAddressed covers determinism, content-sensitivity, and add/remove. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/baker/compose.go | 7 ++++ internal/engine/server/live_provider.go | 45 +++++++++++++++++---- internal/engine/server/prefs.go | 12 ++++-- internal/engine/server/prefs_test.go | 54 +++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 1d27ab5..92f9426 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -1,6 +1,8 @@ package baker import ( + "crypto/sha256" + "encoding/hex" "fmt" "io/fs" "os" @@ -62,6 +64,11 @@ func PrepareLive(input, cellsDir string, onProgress func(done, total int, cell s } continue } + // Content sha of the archive, written beside it (.sha), for the provider's + // content-addressed cache-bust token (a sha-of-shas). A reused cell keeps its sidecar, + // so a re-key reads N tiny sidecars instead of re-hashing the whole archive set. + sum := sha256.Sum256(b) + _ = os.WriteFile(pc+".sha", []byte(hex.EncodeToString(sum[:])), 0o644) perCell = append(perCell, pc) } if len(perCell) == 0 { diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index e878310..e2f8550 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -1,10 +1,14 @@ package server import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" "log" "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -24,21 +28,48 @@ func (s *Server) livePartitionPath(provider string) string { return filepath.Join(s.setDir(provider), "partition.tpart") } -// liveGenPath is /live.gen — its mtime is the provider's tile GENERATION token (packGen -// reads it), bumped on each completed import so the client's tile URLs change and it re-fetches, -// invalidating tiles it cached against a previous, less-complete cell set. Its existence also marks -// "an import completed" — registerLiveProviders only re-serves a provider that has one, so a set +// liveGenPath is /live.gen — it holds the provider's CONTENT cache-bust token (a +// decimal of the sha-of-shas over its per-cell archives; see liveGenToken), which packGen reads +// and stamps into tile URLs as ?g. It changes exactly when the cell set or any cell's content +// changes, so a no-op re-bake keeps the client's cached tiles. Its existence also marks "an +// import registered" — registerLiveProviders only re-serves a provider that has one, so a set // left partial by an interrupted bake is completed by rebakeMissingProviders instead of served. func (s *Server) liveGenPath(provider string) string { return filepath.Join(s.setDir(provider), "live.gen") } -// bumpLiveGen advances the live provider's generation token (writes live.gen; the MTIME is the -// token). Called when an import registration completes. +// liveGenToken is the provider's CONTENT cache-bust token: a sha-of-shas over its per-cell +// archives (each archive's content sha, from the .sha sidecar written at bake). The lines are +// sorted so the token is order-independent, and the low 63 bits become the positive ?g int. It +// changes exactly when the set of cells or any cell's content changes. +func (s *Server) liveGenToken(provider string) int64 { + paths := s.liveCellArchives(provider) + if len(paths) == 0 { + return 0 + } + lines := make([]string, 0, len(paths)) + for _, p := range paths { + stem := strings.TrimSuffix(filepath.Base(p), ".pmtiles") + sha, err := os.ReadFile(p + ".sha") + if err != nil { // no sidecar (shouldn't happen post-bake) — hash the archive itself + if b, e := os.ReadFile(p); e == nil { + sum := sha256.Sum256(b) + sha = []byte(hex.EncodeToString(sum[:])) + } + } + lines = append(lines, stem+":"+strings.TrimSpace(string(sha))) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return int64(binary.BigEndian.Uint64(sum[:8]) & 0x7fffffffffffffff) // 63 bits → non-negative +} + +// bumpLiveGen recomputes and persists the live provider's content token (writes live.gen). +// Called when an import registration completes, and per batch during a progressive bake. func (s *Server) bumpLiveGen(provider string) { p := s.liveGenPath(provider) _ = os.MkdirAll(filepath.Dir(p), 0o755) - _ = os.WriteFile(p, []byte(time.Now().UTC().Format(time.RFC3339Nano)), 0o644) + _ = os.WriteFile(p, []byte(strconv.FormatInt(s.liveGenToken(provider), 10)), 0o644) } // liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 9b6335f..13e7d83 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" ) @@ -159,10 +160,13 @@ func (s *Server) packGen(set string) int64 { return fi.ModTime().UnixNano() } } - // Live runtime-compositor provider (no disk pack): the live.gen file's mtime, bumped on each - // completed import — so its tiles are content-addressed by ?g exactly like a baked pack's. - if fi, err := os.Stat(s.liveGenPath(set)); err == nil { - return fi.ModTime().UnixNano() + // Live runtime-compositor provider (no disk pack): the CONTENT token stored in live.gen (a + // sha-of-shas over its per-cell archives), so its tiles are content-addressed by ?g — a no-op + // re-bake keeps the token, and each progressive re-key advances it as cells fill in. + if b, err := os.ReadFile(s.liveGenPath(set)); err == nil { + if n, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64); err == nil { + return n + } } return 0 } diff --git a/internal/engine/server/prefs_test.go b/internal/engine/server/prefs_test.go index d7c0037..459b36d 100644 --- a/internal/engine/server/prefs_test.go +++ b/internal/engine/server/prefs_test.go @@ -50,3 +50,57 @@ func TestScanPacksStandaloneArchivesOnly(t *testing.T) { t.Errorf("scanPacks = %v, want {charts, overlay}", got) } } + +// TestLiveGenTokenContentAddressed: a live provider's ?g token is a content sha-of-shas over its +// per-cell archives — deterministic (same content → same token), order-independent, and it moves +// only when a cell's content changes or the set gains/loses a cell. A no-op re-bake keeps the +// token so the client's cached tiles stay valid. +func TestLiveGenTokenContentAddressed(t *testing.T) { + s := &Server{cacheDir: t.TempDir()} + dir := s.liveCellsDir("noaa") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + cell := func(stem, sha string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, stem+".pmtiles"), []byte("pm"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, stem+".pmtiles.sha"), []byte(sha), 0o644); err != nil { + t.Fatal(err) + } + } + + cell("US5MD1MC", "aaaa") + cell("US4MD81M", "bbbb") + base := s.liveGenToken("noaa") + if base <= 0 { + t.Fatalf("token = %d, want positive", base) + } + if got := s.liveGenToken("noaa"); got != base { // deterministic + t.Errorf("token not stable: %d vs %d", got, base) + } + + cell("US4MD81M", "cccc") // a cell's content changed → token changes + if got := s.liveGenToken("noaa"); got == base { + t.Errorf("token unchanged after a cell's sha changed") + } + cell("US4MD81M", "bbbb") // restore → original token (content-addressed, history-independent) + if got := s.liveGenToken("noaa"); got != base { + t.Errorf("token not restored: %d vs %d", got, base) + } + + cell("US3MD01M", "dddd") // adding a cell changes it; removing restores + if got := s.liveGenToken("noaa"); got == base { + t.Errorf("token unchanged after adding a cell") + } + _ = os.Remove(filepath.Join(dir, "US3MD01M.pmtiles")) + _ = os.Remove(filepath.Join(dir, "US3MD01M.pmtiles.sha")) + if got := s.liveGenToken("noaa"); got != base { + t.Errorf("token not restored after removing the cell: %d vs %d", got, base) + } + + if got := (&Server{cacheDir: t.TempDir()}).liveGenToken("noaa"); got != 0 { // empty set → 0 + t.Errorf("empty token = %d, want 0", got) + } +} From 762fbf083ccb84d590a22584cf49f9da199fbbbf Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 12:08:40 -0400 Subject: [PATCH 19/36] feat(live): progressive per-batch re-key so an import fills in the map live Register + serve a live provider DURING its import instead of only when the whole provider finishes: every liveReKeyBatch (48) freshly-baked cells, re-open the compositor over what's baked so far, register + enable it, and advance the content token (progressiveReKey, driven from the bake progress callback). The client re-fetches on each new ?g and the map fills in batch by batch. The per-re-key partition rebuild is negligible against the per-cell (seconds-each) bake, and it runs synchronously in the paused bake loop so it never races the writer. openLiveComposer now ALWAYS persists the (possibly rebuilt) partition sidecar, so a re-key over a changed cell set leaves the on-disk sidecar current for a fast boot instead of stale. TestImportPacks exercises progressiveReKey over the real baked archives. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/server/import_packs_test.go | 12 +++++++ internal/engine/server/live_provider.go | 39 ++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index 741cd23..1073e83 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -138,4 +138,16 @@ func TestImportPacks(t *testing.T) { if _, live := s.sets.get("noaa"); !live { t.Error("re-enabled live provider not re-registered from kept archives") } + + // progressiveReKey re-opens + re-registers the set over the cells baked so far — the + // mechanism that fills the map in batch-by-batch during a long import — and stamps the + // content token so the client re-fetches. + s.sets.remove("noaa") + s.progressiveReKey("noaa") + if _, live := s.sets.get("noaa"); !live { + t.Error("progressiveReKey did not register the live set") + } + if s.packGen("noaa") <= 0 { + t.Error("progressiveReKey did not persist the content token") + } } diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index e2f8550..7ff65e6 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -106,14 +106,39 @@ func (s *Server) openLiveComposer(provider string) (*tilesource.Composer, error) if err != nil { return nil, err } - if load == "" { // freshly built the partition — persist it so the next open is a fast load - if err := c.SavePartition(sidecar); err != nil { - log.Printf("live %s: save partition sidecar: %v", provider, err) - } + // Persist the (possibly rebuilt) partition so the on-disk sidecar always matches the current + // cell set: a progressive re-key or an added/removed district changes the inputs, the + // compositor rebuilds from a stale sidecar, and saving keeps the sidecar current for a fast + // boot. (When the sidecar was loaded intact this re-writes identical bytes.) + if err := c.SavePartition(sidecar); err != nil { + log.Printf("live %s: save partition sidecar: %v", provider, err) } return c, nil } +// liveReKeyBatch is how many freshly-baked cells trigger a progressive re-key during an import. +// Small enough that coverage appears promptly on the map; large enough that the per-re-key +// partition rebuild is a negligible fraction of the (per-cell, seconds-each) bake. +const liveReKeyBatch = 48 + +// progressiveReKey re-opens the live compositor over the cells baked SO FAR, registers it as the +// provider's (enabled) set, and advances the content token — so a long import fills in on the map +// batch by batch instead of appearing only when the whole provider finishes. Called synchronously +// from the bake loop (which is paused, so it never races the writer). Best-effort: a transient +// open failure just skips this batch; the next one (or the final register) catches up. +func (s *Server) progressiveReKey(provider string) { + c, err := s.openLiveComposer(provider) + if err != nil || c == nil { + if err != nil { + log.Printf("live %s: progressive re-key: %v", provider, err) + } + return + } + s.sets.register(provider, c) + s.prefs.setDisabled(provider, false) + s.bumpLiveGen(provider) +} + // prepareLiveProvider bakes the provider's cells to its kept live-cells dir (incremental) and opens // a runtime compositor over them — the live counterpart of composeProvider, with NO district // compose pass (tiles compose on demand). Returns the contributing cell count + the Composer. @@ -140,6 +165,12 @@ func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tile } j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", note, done, total, eta }) + // Progressive: every liveReKeyBatch freshly-baked cells, re-open + re-key the set so + // the map fills in live during a long import. The final register (registerProviderSet) + // handles done == total, so skip it here. + if done > 0 && done < total && done%liveReKeyBatch == 0 { + s.progressiveReKey(provider) + } }, func(cell string, err error) { log.Printf("live %s: bake cell %s: %v (skipping)", provider, cell, err) }) if err != nil { From 7fe8d1ee5c145d07cc057a5e71c215fc7fe5c5b3 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 16:08:31 -0400 Subject: [PATCH 20/36] =?UTF-8?q?feat(bake):=20parallel=20tree-bake=20?= =?UTF-8?q?=E2=80=94=20the=20host=20bakes=20cells=20concurrently=20via=20t?= =?UTF-8?q?ile57?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewire the live-provider bake onto tile57.BakeTree: one call walks the ENC tree and bakes every cell IN PARALLEL to the mirrored /.pmtiles (+ .sha), the engine writing + freeing each archive (peak memory ~ workers). This restores parallelism after making the cell the sole parallel unit (the host had gone fully serial). - PrepareLive is now a thin BakeTree wrapper (drops the serial per-cell loop, the .sha write, and ListCells-based iteration — the library does all of it). - liveCellArchives walks the mirrored tree (was a flat ReadDir). - liveBakeWorkers bounds concurrency (min(cpus,8), CHARTPLOTTER_BAKE_WORKERS override). - Progress flows through BakeTree's (done,total) callback; drop the mid-bake progressive re-key (each re-key rebuilt the partition → StalePartition churn) — the provider registers once at the end. - bakeProvider's failed-import cleanup removes only the baked tiles dir, not the whole set dir (which holds ENC_ROOT in single-dir mode) — fixes TestImportFetchDownloadOnly. - TestImportPacks asserts the cell's archive in the mirrored tree. Server package green. A cell shared by two districts now bakes once per district dir (the ownership partition tie-breaks). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/engine/baker/compose.go | 82 +++----------------- internal/engine/server/import_packs_test.go | 16 +++- internal/engine/server/live_provider.go | 83 ++++++++++----------- internal/engine/server/tile57_bake.go | 4 +- 4 files changed, 67 insertions(+), 118 deletions(-) diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go index 92f9426..0d537f9 100644 --- a/internal/engine/baker/compose.go +++ b/internal/engine/baker/compose.go @@ -1,87 +1,27 @@ package baker import ( - "crypto/sha256" - "encoding/hex" - "fmt" "io/fs" - "os" "path/filepath" "strings" tile57 "github.com/beetlebugorg/tile57/bindings/go" ) -// PrepareLive bakes each cell under `input` (a .000 or an ENC_ROOT dir) to its own native-scale -// PMTiles in `cellsDir` (KEPT — the runtime compositor mmaps these directly), and returns the -// per-cell archive paths. There is NO district compose pass: tiles compose on demand at serve -// time (see tile57.OpenCompose). It is the input-prep half of the live compositor. -// -// Incremental: a cell whose per-cell archive already exists, is non-empty, and is at least as new -// as its source .000 is reused — so adding a district re-bakes only its new cells (not the whole -// provider). onProgress(done, total, cell) fires before each cell actually baked (nil to skip); -// onSkip reports a cell that failed (nil to skip). Returns an error (not an empty slice) only when -// cells were present but none are available. -func PrepareLive(input, cellsDir string, onProgress func(done, total int, cell string), onSkip func(cell string, err error)) ([]string, error) { - cells, err := ListCells(input) - if err != nil { - return nil, err - } - if len(cells) == 0 { - return nil, nil - } - if err := os.MkdirAll(cellsDir, 0o755); err != nil { - return nil, err - } - perCell := make([]string, 0, len(cells)) - for i, cp := range cells { - stem := strings.TrimSuffix(filepath.Base(cp), ".000") - pc := filepath.Join(cellsDir, stem+".pmtiles") - // Reuse an up-to-date archive: present, non-empty, and not older than its source cell (a - // re-downloaded district rewrites the .000, so its mtime advances → we re-bake it). - if fi, err := os.Stat(pc); err == nil && fi.Size() > 0 { - if si, err := os.Stat(cp); err == nil && !si.ModTime().After(fi.ModTime()) { - perCell = append(perCell, pc) - continue - } - } - if onProgress != nil { - onProgress(i, len(cells), stem) - } - b, err := tile57.BakeCell(cp) - if err != nil { - if onSkip != nil { - onSkip(filepath.Base(cp), err) - } - continue - } - if len(b) == 0 { - continue - } - if err := os.WriteFile(pc, b, 0o644); err != nil { - if onSkip != nil { - onSkip(filepath.Base(cp), err) - } - continue - } - // Content sha of the archive, written beside it (.sha), for the provider's - // content-addressed cache-bust token (a sha-of-shas). A reused cell keeps its sidecar, - // so a re-key reads N tiny sidecars instead of re-hashing the whole archive set. - sum := sha256.Sum256(b) - _ = os.WriteFile(pc+".sha", []byte(hex.EncodeToString(sum[:])), 0o644) - perCell = append(perCell, pc) - } - if len(perCell) == 0 { - return nil, fmt.Errorf("bake failed for all %d cell(s)", len(cells)) - } - if onProgress != nil { - onProgress(len(cells), len(cells), "") - } - return perCell, nil +// PrepareLive walks the ENC data under `input` and bakes each S-57 base cell (*.000) IN PARALLEL to +// the SAME relative path under `cellsDir` with a .pmtiles extension (plus a .sha content sidecar), +// creating subdirs — the mirrored tree the runtime compositor mmaps directly (there is NO district +// compose pass; tiles compose on demand). A cell whose archive is already up to date (newer than its +// .000 and its update chain) is skipped, so re-baking a provider only bakes what changed. The engine +// writes and frees each archive as it goes, so this never holds N archives in memory (peak ~ +// workers). `workers` bounds concurrency (a MEMORY bound). `onProgress(done, total)` fires per baked +// cell for the import UI (may be called concurrently). Returns the number of cells baked this pass. +func PrepareLive(input, cellsDir string, workers int, onProgress func(done, total int)) (int, error) { + return tile57.BakeTree(input, cellsDir, workers, onProgress) } // ListCells returns every base cell (.000) path under `root` (a single file or a directory), -// deduped by stem (a boundary cell shared by two districts bakes once). +// deduped by stem (a boundary cell shared by two districts appears once). func ListCells(root string) ([]string, error) { var out []string seen := map[string]bool{} diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index 1073e83..e81c8a3 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -85,9 +85,19 @@ func TestImportPacks(t *testing.T) { if _, ok := s.sets.get("noaa"); !ok { t.Errorf("provider set %q not registered", "noaa") } - cellArc := filepath.Join(s.setDir("noaa"), "tiles", "US5MD1MC.pmtiles") - if fi, err := os.Stat(cellArc); err != nil || fi.Size() == 0 { - t.Errorf("provider %q: no baked per-cell tiles (%v)", "noaa", err) + // The bake mirrors the ENC tree, so the cell's archive lives under its district subdir + // (tiles/d5/US5MD1MC.pmtiles) — walk for it. + var found bool + for _, p := range s.liveCellArchives("noaa") { + if strings.HasSuffix(p, "US5MD1MC.pmtiles") { + found = true + if fi, err := os.Stat(p); err != nil || fi.Size() == 0 { + t.Errorf("provider %q: empty archive %s (%v)", "noaa", p, err) + } + } + } + if !found { + t.Errorf("provider %q: no baked US5MD1MC archive in the mirrored tree", "noaa") } if _, err := os.Stat(filepath.Join(s.setDir("noaa"), "partition.tpart")); err != nil { t.Errorf("provider %q: no partition sidecar (%v)", "noaa", err) diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index 7ff65e6..882cb0c 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -65,30 +66,46 @@ func (s *Server) liveGenToken(provider string) int64 { } // bumpLiveGen recomputes and persists the live provider's content token (writes live.gen). -// Called when an import registration completes, and per batch during a progressive bake. +// Called when an import registration completes. func (s *Server) bumpLiveGen(provider string) { p := s.liveGenPath(provider) _ = os.MkdirAll(filepath.Dir(p), 0o755) _ = os.WriteFile(p, []byte(strconv.FormatInt(s.liveGenToken(provider), 10)), 0o644) } -// liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). +// liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). The bake mirrors the +// ENC tree, so the archives live in subdirs (/d1/US4CT1AA.pmtiles, …) — walk, don't ReadDir. func (s *Server) liveCellArchives(provider string) []string { - dir := s.liveCellsDir(provider) - ents, err := os.ReadDir(dir) - if err != nil { - return nil - } + root := s.liveCellsDir(provider) var paths []string - for _, e := range ents { - if !e.IsDir() && strings.HasSuffix(e.Name(), ".pmtiles") { - paths = append(paths, filepath.Join(dir, e.Name())) + _ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil } - } + if !d.IsDir() && strings.HasSuffix(p, ".pmtiles") { + paths = append(paths, p) + } + return nil + }) sort.Strings(paths) return paths } +// liveBakeWorkers is how many cells bake in parallel — a MEMORY bound (each concurrent bake holds a +// whole cell's parse+portray+raster working set), so the CPU count capped modestly, overridable via +// CHARTPLOTTER_BAKE_WORKERS. +func liveBakeWorkers() int { + if v := os.Getenv("CHARTPLOTTER_BAKE_WORKERS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + if n := runtime.NumCPU(); n < 8 { + return n + } + return 8 +} + // openLiveComposer opens a runtime compositor over a provider's kept per-cell archives, loading the // partition sidecar when present (else building it and saving one for next time). Returns nil when // the provider has no live-cells dir yet. @@ -116,11 +133,6 @@ func (s *Server) openLiveComposer(provider string) (*tilesource.Composer, error) return c, nil } -// liveReKeyBatch is how many freshly-baked cells trigger a progressive re-key during an import. -// Small enough that coverage appears promptly on the map; large enough that the per-re-key -// partition rebuild is a negligible fraction of the (per-cell, seconds-each) bake. -const liveReKeyBatch = 48 - // progressiveReKey re-opens the live compositor over the cells baked SO FAR, registers it as the // provider's (enabled) set, and advances the content token — so a long import fills in on the map // batch by batch instead of appearing only when the whole provider finishes. Called synchronously @@ -146,36 +158,21 @@ func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tile cellsDir := s.liveCellsDir(provider) s.invalidateLiveOnEngineChange(cellsDir) start := time.Now() - paths, err := baker.PrepareLive(encRoot, cellsDir, - func(done, total int, cell string) { - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band, j.Zoom = "bake", "", 0 - if done >= total { - j.Unit, j.Note, j.Done, j.Total, j.ETA = "", "Preparing live tiles", 0, 0, 0 - return - } - note := "Baking charts" - if cell != "" { - note = "Baking " + cell - } - eta := 0 - if done > 0 { - per := time.Since(start) / time.Duration(done) - eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) - } - j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", note, done, total, eta - }) - // Progressive: every liveReKeyBatch freshly-baked cells, re-open + re-key the set so - // the map fills in live during a long import. The final register (registerProviderSet) - // handles done == total, so skip it here. - if done > 0 && done < total && done%liveReKeyBatch == 0 { - s.progressiveReKey(provider) + if _, err := baker.PrepareLive(encRoot, cellsDir, liveBakeWorkers(), func(done, total int) { + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Zoom = "bake", "", 0 + eta := 0 + if done > 0 && total > done { + per := time.Since(start) / time.Duration(done) + eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) } - }, - func(cell string, err error) { log.Printf("live %s: bake cell %s: %v (skipping)", provider, cell, err) }) - if err != nil { + j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", "Baking charts", done, total, eta + }) + }); err != nil { return 0, nil, err } + // The mirrored tree holds every kept per-cell archive (baked + reused). + paths := s.liveCellArchives(provider) if len(paths) == 0 { return 0, nil, nil } diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 3243e4c..9fc15dc 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -110,8 +110,10 @@ func (s *Server) bakeProvider(jobID, provider string) bool { return fail(err) } // Zero cells means nothing valid parsed → a failed import (don't register an empty set). + // Clean only the (empty) baked tiles dir, NOT the whole set dir — in single-dir mode the set + // dir also holds the source ENC_ROOT, which must survive a failed bake. if n == 0 { - os.RemoveAll(outDir) + os.RemoveAll(s.liveCellsDir(provider)) return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) } From 1faa9add146b85df6554b11f5823accd2d24925e Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Thu, 9 Jul 2026 16:10:33 -0400 Subject: [PATCH 21/36] fix(cli): update `chartplotter bake` to the parallel BakeTree PrepareLive PrepareLive's signature changed (workers + (done,total) progress, returns a count). The CLI now bakes in parallel and walks /tiles for the mirrored per-cell archives to build the partition sidecar. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/chartplotter/bake.go | 42 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/cmd/chartplotter/bake.go b/cmd/chartplotter/bake.go index ddf646e..924377e 100644 --- a/cmd/chartplotter/bake.go +++ b/cmd/chartplotter/bake.go @@ -7,6 +7,8 @@ import ( "io/fs" "os" "path/filepath" + "runtime" + "sort" "strings" "time" @@ -46,25 +48,33 @@ func (c bakeCmd) Run() error { return err } - // Bake each cell to /tiles/*.pmtiles (incremental — reruns re-bake only changed cells). + // Bake each cell IN PARALLEL to /tiles/.pmtiles, mirroring the input tree + // (incremental — reruns re-bake only changed cells). The engine writes + frees each archive. start := time.Now() - paths, err := baker.PrepareLive(input, tilesDir, - func(done, total int, cell string) { - switch { - case done >= total: - fmt.Printf("\rbuilding partition over %d cell(s)… ", total) - case done > 0: - per := time.Since(start) / time.Duration(done) - fmt.Printf("\rbaking %s (%d/%d) · ~%s left ", cell, done, total, (per * time.Duration(total-done)).Round(time.Second)) - default: - fmt.Printf("\rbaking %s (%d/%d)… ", cell, done, total) - } - }, - func(cell string, e error) { fmt.Fprintf(os.Stderr, "\nwarning: bake %s: %v (skipping)\n", cell, e) }) - fmt.Println() - if err != nil { + if _, err := baker.PrepareLive(input, tilesDir, runtime.NumCPU(), func(done, total int) { + if done >= total { + fmt.Printf("\rbaked %d cell(s), building partition… ", total) + } else if done > 0 { + per := time.Since(start) / time.Duration(done) + fmt.Printf("\rbaking %d/%d · ~%s left ", done, total, (per * time.Duration(total-done)).Round(time.Second)) + } else { + fmt.Printf("\rbaking %d/%d… ", done, total) + } + }); err != nil { + fmt.Println() return err } + fmt.Println() + + // Collect the mirrored per-cell archives (baked + reused) for the compositor. + var paths []string + _ = filepath.WalkDir(tilesDir, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, ".pmtiles") { + paths = append(paths, p) + } + return nil + }) + sort.Strings(paths) if len(paths) == 0 { return fmt.Errorf("no coverage: no valid S-57 cells under %s", input) } From e7ac8fad292748a1ad298608bebcf0f34c43f915 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 05:42:12 -0400 Subject: [PATCH 22/36] =?UTF-8?q?feat(engine):=20track=20the=20tile57=20v0?= =?UTF-8?q?.2.0=20ABI=20=E2=80=94=20handle-free=20Cells/FeaturesBytes,=20C?= =?UTF-8?q?omposeSource.Tile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cell-index and bake-metadata scans call tile57.Cells(path) directly (no chart handle over raw S-57 anymore); the simulator water mask uses tile57.FeaturesBytes; the composer serves via ComposeSource.Tile. Submodule at tile57 599f8ad (bake/render/compose sections, PMTiles-only mmap'd chart handle, compose borrows charts, tile57_status everywhere). Co-Authored-By: Claude Fable 5 --- cmd/chartplotter/simulate.go | 7 +------ internal/engine/baker/meta.go | 7 +------ internal/engine/nmea/sim/water_test.go | 5 +---- internal/engine/server/cellindex.go | 7 +------ internal/engine/server/provider.go | 10 +++++----- internal/engine/tilesource/composer.go | 4 ++-- tile57 | 2 +- 7 files changed, 12 insertions(+), 30 deletions(-) diff --git a/cmd/chartplotter/simulate.go b/cmd/chartplotter/simulate.go index 009da96..be49e84 100644 --- a/cmd/chartplotter/simulate.go +++ b/cmd/chartplotter/simulate.go @@ -172,12 +172,7 @@ func loadWaterFeatures(p string) ([]tile57.Feature, error) { if err != nil { return nil, err } - src, err := tile57.OpenChartBytes(data) - if err != nil { - return nil, err - } - defer src.Close() - return src.Features("DEPARE", "DRGARE") + return tile57.FeaturesBytes(data, "DEPARE", "DRGARE") } // readBaseCell returns the raw base-cell bytes from a .000 file or the first diff --git a/internal/engine/baker/meta.go b/internal/engine/baker/meta.go index 6b21906..1cbc96d 100644 --- a/internal/engine/baker/meta.go +++ b/internal/engine/baker/meta.go @@ -83,12 +83,7 @@ func ExtractCellMeta(cells map[string]CellData, onSkip func(name string, err err } } - src, err := tile57.Open(dir) - if err != nil { - return skipAll(err) - } - defer src.Close() - infos, err := src.Cells() + infos, err := tile57.Cells(dir) if err != nil { return skipAll(err) } diff --git a/internal/engine/nmea/sim/water_test.go b/internal/engine/nmea/sim/water_test.go index 9c715d9..9160bb1 100644 --- a/internal/engine/nmea/sim/water_test.go +++ b/internal/engine/nmea/sim/water_test.go @@ -29,10 +29,7 @@ func TestWaterMask_RealCell(t *testing.T) { if err != nil { t.Skip("test cell not available") } - src, err := tile57.OpenChartBytes(data) - require.NoError(t, err) - defer src.Close() - feats, err := src.Features("DEPARE", "DRGARE") + feats, err := tile57.FeaturesBytes(data, "DEPARE", "DRGARE") require.NoError(t, err) m := NewWaterMask(feats, 2) require.NotNil(t, m, "cell should yield navigable depth areas") diff --git a/internal/engine/server/cellindex.go b/internal/engine/server/cellindex.go index 854397e..12ddadb 100644 --- a/internal/engine/server/cellindex.go +++ b/internal/engine/server/cellindex.go @@ -171,12 +171,7 @@ func (ci *cellIndex) scan() { if _, ok := ci.get(name); ok { return nil // already indexed (forget() drops a re-imported cell so it re-parses) } - src, err := tile57.Open(path) - if err != nil { - return nil - } - infos, err := src.Cells() - src.Close() + infos, err := tile57.Cells(path) if err != nil || len(infos) == 0 { return nil } diff --git a/internal/engine/server/provider.go b/internal/engine/server/provider.go index 97259c4..0cc1ff3 100644 --- a/internal/engine/server/provider.go +++ b/internal/engine/server/provider.go @@ -54,7 +54,7 @@ func (s *Server) providerDataDir(provider string) string { } // encRootDir is //ENC_ROOT/ — the bake input, walked recursively by -// tile57.BakeBundle for every *.000 across all installed district subfolders. It +// tile57.BakeTree for every *.000 across all installed district subfolders. It // lives under the DATA dir (persistent, survives a cache wipe: it is the downloads' // home + the bake input), so ClearCache never touches it. func (s *Server) encRootDir(provider string) string { @@ -111,15 +111,15 @@ func (s *Server) installedProviders() []string { } // districtCatFile is the per-district sidecar of parsed CATALOG.031 titles (the raw -// catalogue isn't kept at the ENC_ROOT root — that would flip BakeBundle into -// catalog-only mode — so titles are stashed here and re-gathered per provider bake). +// catalogue isn't kept at the ENC_ROOT root, so titles are stashed here and +// re-gathered per provider bake). const districtCatFile = "_catalog.json" // cacheDistrict writes one district's downloaded exchange-set content into its // ENC_ROOT subfolder: each cell FLAT as .000 (+ .001… updates), aux content // files (TXTDSC/PICREP) flat beside them, and a _catalog.json of parsed titles. The // bake reads the whole provider ENC_ROOT from here (no temp staging); the flat layout -// is transparent to BakeBundle's recursive *.000 walk. Best-effort; errors logged. +// is transparent to BakeTree's recursive *.000 walk. Best-effort; errors logged. func (s *Server) cacheDistrict(provider, district string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry) { dir := s.districtDir(provider, district) if err := os.MkdirAll(dir, 0o755); err != nil { @@ -177,7 +177,7 @@ func (s *Server) cacheDistrict(provider, district string, cells map[string]baker // providerCellData reads every base cell (+ its .001… updates) under a provider's // ENC_ROOT, DE-DUPLICATED by stem (a boundary cell shared by two districts is read // once), for the bake's metadata sidecar + cell manifest. The bake itself reads the -// tree directly (BakeBundle); this is only the in-memory view the register tail needs. +// tree directly (BakeTree); this is only the in-memory view the register tail needs. func (s *Server) providerCellData(provider string) map[string]baker.CellData { root := s.encRootDir(provider) cells := map[string]baker.CellData{} // stem+".000" → base diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go index a95c544..587b461 100644 --- a/internal/engine/tilesource/composer.go +++ b/internal/engine/tilesource/composer.go @@ -50,13 +50,13 @@ type OwnershipTiler interface { // Tile composes (z,x,y) on demand, returning decompressed MLT, or (nil, nil) for a blank tile. func (c *Composer) Tile(z uint8, x, y uint32) ([]byte, error) { - body, _, err := c.src.Serve(z, x, y) + body, _, err := c.src.Tile(z, x, y) return body, err } // TileOwned is Tile plus the ownership flag (implements OwnershipTiler). func (c *Composer) TileOwned(z uint8, x, y uint32) (body []byte, owned bool, err error) { - return c.src.Serve(z, x, y) + return c.src.Tile(z, x, y) } // Meta returns the compositor's display metadata (zoom range + coverage bounds). diff --git a/tile57 b/tile57 index a2022b6..599f8ad 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit a2022b6758dc26258bc83fc9a4b3c22beb5eba75 +Subproject commit 599f8ad28a24c39e8efb457e412360d51bb9cd56 From 04770cb7c7cc94d05227af8bb3ea465420809180 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 07:24:34 -0400 Subject: [PATCH 23/36] =?UTF-8?q?chore(engine):=20tile57=20@=20042dd5e=20?= =?UTF-8?q?=E2=80=94=20complex-linestyle=20style=20layers=20(+SYMBOL=5FSCA?= =?UTF-8?q?LE=20icons),=20sector=20figures=20whole=20across=20seams,=20com?= =?UTF-8?q?pose=20output=20backends,=20tile57=5Ffree(ptr)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- tile57 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile57 b/tile57 index 599f8ad..042dd5e 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit 599f8ad28a24c39e8efb457e412360d51bb9cd56 +Subproject commit 042dd5ef74dc948789b7c51b072a756c62ff42ac From 244fd165eef1fa2f05b295682516afc135a2f3d7 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 07:34:25 -0400 Subject: [PATCH 24/36] =?UTF-8?q?chore(engine):=20tile57=20@=20c64de73=20?= =?UTF-8?q?=E2=80=94=20tessellated=20complex=20linestyles=20(re-bake=20req?= =?UTF-8?q?uired=20for=20decoration)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- tile57 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile57 b/tile57 index 042dd5e..c64de73 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit 042dd5ef74dc948789b7c51b072a756c62ff42ac +Subproject commit c64de7307df74a688f99e5b7601a5194ad6bd4b5 From 223bb938307fd54a7307738c21c421b11f88c6e8 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 07:39:18 -0400 Subject: [PATCH 25/36] =?UTF-8?q?fix(import):=20read=20chart=20metadata=20?= =?UTF-8?q?from=20the=20ENC=20tree=20=E2=80=94=20the=20'Reading=20chart=20?= =?UTF-8?q?metadata'=20phase=20no=20longer=20holds=20the=20catalogue=20in?= =?UTF-8?q?=20RAM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit providerCellData loaded every base cell + update of the whole provider into memory (~8 GB for the full NOAA catalogue), only for ExtractCellMeta to write it all back to a temp dir and re-scan it. registerProviderSet now reads the per-cell metadata straight from the provider's on-disk ENC_ROOT (baker.ExtractCellMetaDir over tile57.Cells — the engine streams cells and never holds the set), and the cell manifest is written from the resulting stems. The zip-upload path keeps its in-memory CellData (district-sized, already in RAM from the upload). Co-Authored-By: Claude Fable 5 --- internal/engine/baker/meta.go | 32 ++++++++++++++ internal/engine/server/cells_active_test.go | 3 +- internal/engine/server/import.go | 6 +-- internal/engine/server/provider.go | 48 --------------------- internal/engine/server/tile57_bake.go | 20 ++++++--- 5 files changed, 47 insertions(+), 62 deletions(-) diff --git a/internal/engine/baker/meta.go b/internal/engine/baker/meta.go index 1cbc96d..f5cfd84 100644 --- a/internal/engine/baker/meta.go +++ b/internal/engine/baker/meta.go @@ -42,6 +42,38 @@ type CellMeta struct { HasBBox bool `json:"-"` } +// ExtractCellMetaDir reads per-cell metadata straight from an on-disk ENC tree: +// the engine walks the .000s (applying each cell's update chain) and reports the +// inventory — no cell bytes are held in memory, so a whole-catalogue import stays +// flat. Duplicate stems (a boundary cell shared by two district subfolders) +// collapse to one entry, matching the first-copy-wins district dedup. +func ExtractCellMetaDir(root string, onSkip func(name string, err error)) map[string]CellMeta { + infos, err := tile57.Cells(root) + if err != nil { + if onSkip != nil { + onSkip(root, err) + } + return map[string]CellMeta{} + } + out := make(map[string]CellMeta, len(infos)) + for _, ci := range infos { + if _, seen := out[ci.Name]; seen { + continue + } + out[ci.Name] = CellMeta{ + Name: ci.Name, + Scale: ci.Scale, + Edition: ci.Edition, + Update: ci.Update, + IssueDate: ci.IssueDate, + Agency: ci.Agency, + BBox: ci.BBox, + HasBBox: ci.HasBBox, + } + } + return out +} + // ExtractCellMeta stages the cells (base + updates) to a temporary ENC dir, // opens it with the native engine, and returns per-cell metadata keyed by cell // stem (the DSNM stem the engine reports). Cells that fail to parse are diff --git a/internal/engine/server/cells_active_test.go b/internal/engine/server/cells_active_test.go index 78de08a..6772d88 100644 --- a/internal/engine/server/cells_active_test.go +++ b/internal/engine/server/cells_active_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "testing" - "github.com/beetlebugorg/chartplotter/internal/engine/baker" ) // activeCells GETs /api/cells?active=1 and returns the cell-name set. @@ -55,7 +54,7 @@ func TestActiveCellsDropOnDelete(t *testing.T) { } // Register the provider set with an exact cell manifest, and add it as an enabled pack. const provider = "noaa" - if err := s.writeSetCells(provider, map[string]baker.CellData{cell + ".000": {}}); err != nil { + if err := s.writeSetCells(provider, []string{cell}); err != nil { t.Fatal(err) } manifest := filepath.Join(s.setDir(provider), provider+".cells.json") diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index cb161d1..27714de 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -574,11 +574,7 @@ func (s *Server) looseCellsDir() string { // (/.cells.json). /api/cells?active reads these to return exactly the // installed cells, instead of every cached cell whose bounds overlap the pack's // (often global, for a worldwide-scattered import) bounding box. -func (s *Server) writeSetCells(set string, cells map[string]baker.CellData) error { - stems := make([]string, 0, len(cells)) - for n := range cells { - stems = append(stems, strings.TrimSuffix(n, ".000")) - } +func (s *Server) writeSetCells(set string, stems []string) error { sort.Strings(stems) dir := s.setDir(set) if err := os.MkdirAll(dir, 0o755); err != nil { diff --git a/internal/engine/server/provider.go b/internal/engine/server/provider.go index 0cc1ff3..5926f31 100644 --- a/internal/engine/server/provider.go +++ b/internal/engine/server/provider.go @@ -174,54 +174,6 @@ func (s *Server) cacheDistrict(provider, district string, cells map[string]baker } } -// providerCellData reads every base cell (+ its .001… updates) under a provider's -// ENC_ROOT, DE-DUPLICATED by stem (a boundary cell shared by two districts is read -// once), for the bake's metadata sidecar + cell manifest. The bake itself reads the -// tree directly (BakeTree); this is only the in-memory view the register tail needs. -func (s *Server) providerCellData(provider string) map[string]baker.CellData { - root := s.encRootDir(provider) - cells := map[string]baker.CellData{} // stem+".000" → base - updates := map[string]map[string][]byte{} // stem → update-name → bytes - _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - ext := encExtServer(d.Name()) - if ext == "" { - return nil - } - stem := strings.TrimSuffix(d.Name(), filepath.Ext(d.Name())) - if !isCellName(stem) { - return nil - } - if ext == ".000" { - if _, seen := cells[stem+".000"]; seen { - return nil // dedup: first district's copy wins - } - if b, e := os.ReadFile(path); e == nil { - cells[stem+".000"] = baker.CellData{Base: b} - } - return nil - } - if updates[stem] == nil { - updates[stem] = map[string][]byte{} - } - if _, seen := updates[stem][d.Name()]; !seen { - if b, e := os.ReadFile(path); e == nil { - updates[stem][d.Name()] = b - } - } - return nil - }) - for name, cd := range cells { - if u := updates[strings.TrimSuffix(name, ".000")]; len(u) > 0 { - cd.Updates = u - cells[name] = cd - } - } - return cells -} - // providerAux gathers the aux content files (TXTDSC/PICREP text + pictures) across a // provider's ENC_ROOT, de-duplicated by upper-cased basename, for the provider's one // companion aux.zip. diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 9fc15dc..fff1158 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -20,7 +20,7 @@ import ( // (recorded for the pack list + stamped with the bake/engine version), or "" for a LIVE // runtime-compositor provider (no disk archive — its bounds come from the TileSource Meta/TileJSON, // and it is surfaced in /api/packs straight from the registry). -func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSource, packPath string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { +func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSource, packPath string, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { outDir := s.setDir(set) s.sets.register(set, src) s.prefs.setDisabled(set, false) @@ -40,7 +40,17 @@ func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSourc // invalidating any tiles (incl. blank 204s) it cached against a previous, less-complete set. s.bumpLiveGen(set) } - if err := s.writeSetCells(set, cells); err != nil { + // Per-cell metadata straight from the provider's on-disk ENC tree — the whole + // point of the path-based read is that a catalogue-sized import never holds + // cell bytes in memory (the previous in-RAM staging peaked in the gigabytes). + cellMeta := baker.ExtractCellMetaDir(s.encRootDir(set), func(name string, e error) { + log.Printf("import %s: meta skip %s: %v", jobID, name, e) + }) + stems := make([]string, 0, len(cellMeta)) + for n := range cellMeta { + stems = append(stems, n) + } + if err := s.writeSetCells(set, stems); err != nil { log.Printf("import %s: cell manifest %q: %v", jobID, set, err) } @@ -57,9 +67,6 @@ func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSourc // Per-pack metadata sidecar for the chart library (per-cell scale/edition/date/ // agency/coverage + catalogue titles). - cellMeta := baker.ExtractCellMeta(cells, func(name string, e error) { - log.Printf("import %s: meta skip %s: %v", jobID, name, e) - }) meta := buildSetMeta(set, cellMeta, cat) meta.Imported = created if err := s.writeSetMeta(set, meta); err != nil { @@ -120,10 +127,9 @@ func (s *Server) bakeProvider(jobID, provider string) bool { s.imports.update(jobID, func(j *importJob) { j.Phase, j.Note, j.Zoom, j.Unit, j.Done, j.Total, j.ETA = "meta", "Reading chart metadata", 0, "", 0, 0, 0 }) - cells := s.providerCellData(provider) aux := s.providerAux(provider) cat := s.providerCatalog(provider) - if !s.registerProviderSet(jobID, provider, liveSrc, "", cells, aux, cat, created) { + if !s.registerProviderSet(jobID, provider, liveSrc, "", aux, cat, created) { return fail(fmt.Errorf("could not register set for %q", provider)) } s.imports.update(jobID, func(j *importJob) { j.Cells = n }) From 10e416908adb034212600c1f47224b8fc59ebbb9 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 07:58:52 -0400 Subject: [PATCH 26/36] =?UTF-8?q?chore(engine):=20tile57=20@=207f2a6c0=20?= =?UTF-8?q?=E2=80=94=20S-52=20edge=20masking=20in=20boundary=20fallbacks?= =?UTF-8?q?=20(kills=20false=20IALA=20boundaries=20at=20cell=20junctions)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- tile57 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile57 b/tile57 index c64de73..7f2a6c0 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit c64de7307df74a688f99e5b7601a5194ad6bd4b5 +Subproject commit 7f2a6c0b944456e434d805c1ea1792939a5e9df5 From d49066a2a29c157b91ddfd758f47965e6b93fead Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 09:45:38 -0400 Subject: [PATCH 27/36] =?UTF-8?q?chore(engine):=20tile57=20@=20046f312=20?= =?UTF-8?q?=E2=80=94=20fill-up=20quilting=20(finer=20cells=20serve=20coars?= =?UTF-8?q?e=20zooms,=20scamin-restricted;=20re-bake=20required)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- tile57 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile57 b/tile57 index 7f2a6c0..046f312 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit 7f2a6c0b944456e434d805c1ea1792939a5e9df5 +Subproject commit 046f3129f88cda85ad196bdd620ea520c431bfdb From e3f94bdb2dc6250258c80ef4b8aafa2dbdc8228e Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 13:40:07 -0400 Subject: [PATCH 28/36] fix(serve): don't re-serve kept live archives baked by another engine build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registerLiveProviders re-registered kept per-cell archives at boot without the .enginever check (that only ran on the import path), so a binary upgrade kept serving stale tiles until a manual re-import. Skip stale sets at boot — rebakeMissingProviders then re-bakes them through prepareLiveProvider, which drops + re-stamps the dir. Also bumps the tile57 engine to 3c46b23 (sector- light fixes: sub-band SCAMIN cull, 512-px figure sizing, compose reach ring). Co-Authored-By: Claude Fable 5 --- internal/engine/server/live_provider.go | 27 +++++++++++++++++++------ tile57 | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index 882cb0c..3344607 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -186,23 +186,31 @@ func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tile return len(paths), c, nil } +// liveEngineCurrent reports whether the kept per-cell archives in cellsDir were baked by the +// RUNNING engine (the .enginever stamp matches the linked tile57 commit). An unstamped dir +// counts as stale — the next prepareLiveProvider re-bakes and stamps it. With no engine +// commit linked in, everything counts as current (nothing to compare against). +func (s *Server) liveEngineCurrent(cellsDir string) bool { + if s.EngineCommit == "" { + return true + } + b, err := os.ReadFile(filepath.Join(cellsDir, ".enginever")) + return err == nil && string(b) == s.EngineCommit +} + // invalidateLiveOnEngineChange drops the kept per-cell archives when the tile57 engine commit // changed since they were baked (a new engine portrays different tiles), so a binary upgrade // re-bakes them. The partition sidecar is coverage-derived (engine-independent) and self-validates // via its input key, so it is left in place. First bake just records the stamp. func (s *Server) invalidateLiveOnEngineChange(cellsDir string) { - if s.EngineCommit == "" { - return - } - stamp := filepath.Join(cellsDir, ".enginever") - if b, err := os.ReadFile(stamp); err == nil && string(b) == s.EngineCommit { + if s.liveEngineCurrent(cellsDir) { return } if _, err := os.Stat(cellsDir); err == nil { _ = os.RemoveAll(cellsDir) } _ = os.MkdirAll(cellsDir, 0o755) - _ = os.WriteFile(stamp, []byte(s.EngineCommit), 0o644) + _ = os.WriteFile(filepath.Join(cellsDir, ".enginever"), []byte(s.EngineCommit), 0o644) } // registerLiveProviders re-registers, at boot, a runtime compositor for every installed provider @@ -220,6 +228,13 @@ func (s *Server) registerLiveProviders() { if _, err := os.Stat(s.liveGenPath(prov)); err != nil { continue } + // Archives baked by an older engine are stale (a new engine portrays different + // tiles) — don't re-serve them; rebakeMissingProviders re-bakes the provider, + // and prepareLiveProvider drops + re-stamps the dir. + if !s.liveEngineCurrent(s.liveCellsDir(prov)) { + log.Printf("live %s: kept archives are from another engine build — re-baking", prov) + continue + } c, err := s.openLiveComposer(prov) if err != nil || c == nil { if err != nil { diff --git a/tile57 b/tile57 index 046f312..3c46b23 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit 046f3129f88cda85ad196bdd620ea520c431bfdb +Subproject commit 3c46b234cc9d6577a9856369e975c8e2face2a2a From a3944fea2b6d1a2e04711ce62fb59cdcfaff62d7 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 19:21:35 -0400 Subject: [PATCH 29/36] fix(server): stamp the engine commit into the live cache-bust token The live compositor generates tiles at SERVE time, so the ?g content token must address the engine build too: a serve-path engine fix used to leave the token unchanged (archives byte-identical), and clients kept their cached broken tiles until a manual hard refresh. liveGenToken now mixes Server.EngineCommit into the sha-of-shas, and registerLiveProviders re-persists live.gen at boot so a restart under a new engine mints the new token without waiting for the next import. Co-Authored-By: Claude Fable 5 --- internal/engine/server/live_provider.go | 17 +++++++++++++---- internal/engine/server/prefs_test.go | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index 3344607..47c7828 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -40,15 +40,21 @@ func (s *Server) liveGenPath(provider string) string { } // liveGenToken is the provider's CONTENT cache-bust token: a sha-of-shas over its per-cell -// archives (each archive's content sha, from the .sha sidecar written at bake). The lines are -// sorted so the token is order-independent, and the low 63 bits become the positive ?g int. It -// changes exactly when the set of cells or any cell's content changes. +// archives (each archive's content sha, from the .sha sidecar written at bake) plus the engine +// commit serving them. The lines are sorted so the token is order-independent, and the low 63 +// bits become the positive ?g int. It changes exactly when the set of cells, any cell's +// content, or the engine build changes — the engine composes live tiles at SERVE time, so its +// identity is part of a tile's content address: a serve-path fix must bust client caches even +// when the baked archives are byte-identical. func (s *Server) liveGenToken(provider string) int64 { paths := s.liveCellArchives(provider) if len(paths) == 0 { return 0 } - lines := make([]string, 0, len(paths)) + lines := make([]string, 0, len(paths)+1) + if s.EngineCommit != "" { + lines = append(lines, "engine:"+s.EngineCommit) + } for _, p := range paths { stem := strings.TrimSuffix(filepath.Base(p), ".pmtiles") sha, err := os.ReadFile(p + ".sha") @@ -243,6 +249,9 @@ func (s *Server) registerLiveProviders() { continue } s.sets.register(prov, c) + // Re-persist the token: the archives are unchanged, but the serving + // engine may not be the one that wrote live.gen, and the token embeds it. + s.bumpLiveGen(prov) m := c.Meta() log.Printf("live: registered %q from kept per-cell archives (z%d..%d)", prov, m.MinZoom, m.MaxZoom) } diff --git a/internal/engine/server/prefs_test.go b/internal/engine/server/prefs_test.go index 459b36d..6a8c2d3 100644 --- a/internal/engine/server/prefs_test.go +++ b/internal/engine/server/prefs_test.go @@ -100,6 +100,23 @@ func TestLiveGenTokenContentAddressed(t *testing.T) { t.Errorf("token not restored after removing the cell: %d vs %d", got, base) } + // The engine composes live tiles at serve time, so its commit is part of the + // content address: a rebuilt engine must move the token even over identical + // archives (serve-path fixes bust client caches), deterministically. + s.EngineCommit = "abc123" + withEngine := s.liveGenToken("noaa") + if withEngine == base { + t.Errorf("token unchanged after engine commit set") + } + if got := s.liveGenToken("noaa"); got != withEngine { + t.Errorf("engine-stamped token not stable: %d vs %d", got, withEngine) + } + s.EngineCommit = "def456" + if got := s.liveGenToken("noaa"); got == withEngine { + t.Errorf("token unchanged after engine commit changed") + } + s.EngineCommit = "" + if got := (&Server{cacheDir: t.TempDir()}).liveGenToken("noaa"); got != 0 { // empty set → 0 t.Errorf("empty token = %d, want 0", got) } From 5c5fac9588114a49ab7ec8f43d541b45068abba6 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 20:14:50 -0400 Subject: [PATCH 30/36] =?UTF-8?q?feat(engine):=20tile57=20v0.3.0=20family?= =?UTF-8?q?=20rename=20=E2=80=94=20Charts/BakeChart=20call=20sites,=20subm?= =?UTF-8?q?odule=20to=20b8e8499?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine renamed its public surface so every symbol leads with its family (tile57_chart_* handle methods, bake_charts/enc_charts, Go Cells -> Charts, BakeCell -> BakeChart). Host call sites follow; host-internal Cells vocabulary (import packs, set meta, ienc) is untouched. Binary rebuilt against the new libtile57.a (engineCommit b8e8499, tile57_chart_* symbols verified in the binary); baker/tilesource/server tests pass. Co-Authored-By: Claude Fable 5 --- internal/engine/baker/meta.go | 4 ++-- internal/engine/server/cellindex.go | 2 +- internal/engine/tilesource/composer.go | 2 +- tile57 | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/engine/baker/meta.go b/internal/engine/baker/meta.go index f5cfd84..817e918 100644 --- a/internal/engine/baker/meta.go +++ b/internal/engine/baker/meta.go @@ -48,7 +48,7 @@ type CellMeta struct { // flat. Duplicate stems (a boundary cell shared by two district subfolders) // collapse to one entry, matching the first-copy-wins district dedup. func ExtractCellMetaDir(root string, onSkip func(name string, err error)) map[string]CellMeta { - infos, err := tile57.Cells(root) + infos, err := tile57.Charts(root) if err != nil { if onSkip != nil { onSkip(root, err) @@ -115,7 +115,7 @@ func ExtractCellMeta(cells map[string]CellData, onSkip func(name string, err err } } - infos, err := tile57.Cells(dir) + infos, err := tile57.Charts(dir) if err != nil { return skipAll(err) } diff --git a/internal/engine/server/cellindex.go b/internal/engine/server/cellindex.go index 12ddadb..0ac86eb 100644 --- a/internal/engine/server/cellindex.go +++ b/internal/engine/server/cellindex.go @@ -171,7 +171,7 @@ func (ci *cellIndex) scan() { if _, ok := ci.get(name); ok { return nil // already indexed (forget() drops a re-imported cell so it re-parses) } - infos, err := tile57.Cells(path) + infos, err := tile57.Charts(path) if err != nil || len(infos) == 0 { return nil } diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go index 587b461..797ad6c 100644 --- a/internal/engine/tilesource/composer.go +++ b/internal/engine/tilesource/composer.go @@ -15,7 +15,7 @@ type Composer struct { } // NewComposer opens a runtime compositor over the per-cell PMTiles at paths (each from -// `tile57 compose --keep-cells` / tile57.BakeCell). partitionPath (or "") names a partition +// `tile57 compose --keep-cells` / tile57.BakeChart). partitionPath (or "") names a partition // sidecar (`tile57 compose --save-partition`) to load and skip the owned-face build. Close it // when done — callers must not Close while any request can still call Tile. func NewComposer(paths []string, partitionPath string) (*Composer, error) { diff --git a/tile57 b/tile57 index 3c46b23..b8e8499 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit 3c46b234cc9d6577a9856369e975c8e2face2a2a +Subproject commit b8e84992ee83eb3bf49b3fc42dbe277ffd9bbb25 From 1798ffd2be8ca13552f31ebea5e276d126f1890c Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 20:34:13 -0400 Subject: [PATCH 31/36] =?UTF-8?q?fix(server):=20engine=20commit=20reaches?= =?UTF-8?q?=20the=20boot=20staleness=20gate=20=E2=80=94=20New=20takes=20it?= =?UTF-8?q?=20as=20an=20arg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registerLiveProviders and rebakeMissingProviders run inside server.New, but EngineCommit was assigned by the caller AFTER New returned — so the .enginever stamp check always ran against "" ("counts as current") and stale kept archives registered on every boot, silencing the self-heal re-bake. This is why an engine upgrade never re-baked the live NOAA set: tiles baked pre- meta-bounds-complement (and interrupted at d1+d5 of five districts) kept serving. The commit is construction-time data, so it is now a New parameter, set before boot registration; serve.go passes the ldflags stamp, tests pass "". Also bumps the engine submodule to aeac905 (tile57_version now reports 0.3.0 — the capi string had missed the version bump). Co-Authored-By: Claude Fable 5 --- cmd/chartplotter/serve.go | 3 +-- internal/engine/server/aux_test.go | 4 ++-- internal/engine/server/cells_active_test.go | 2 +- internal/engine/server/connections_test.go | 10 +++++----- internal/engine/server/http.go | 7 ++++++- internal/engine/server/import_meta_test.go | 4 ++-- internal/engine/server/import_packs_test.go | 2 +- internal/engine/server/import_test.go | 10 +++++----- internal/engine/server/packs_test.go | 2 +- internal/engine/server/security_test.go | 4 ++-- internal/engine/server/server_test.go | 10 +++++----- internal/engine/server/tileserve_test.go | 10 +++++----- tile57 | 2 +- 13 files changed, 37 insertions(+), 33 deletions(-) diff --git a/cmd/chartplotter/serve.go b/cmd/chartplotter/serve.go index 9deff73..ae2afb9 100644 --- a/cmd/chartplotter/serve.go +++ b/cmd/chartplotter/serve.go @@ -67,10 +67,9 @@ func (c serveCmd) Run() error { // Loopback bind → enforce the Host-header DNS-rebind check on /api. Any // other bind means the operator opted into network exposure. allowRemote := !(c.Host == "127.0.0.1" || c.Host == "localhost" || c.Host == "::1") - srv := server.New(c.Assets, cacheDir, dataDir, allowRemote) + srv := server.New(c.Assets, cacheDir, dataDir, allowRemote, engineCommit) srv.SetAssetFallback(s101AssetDir) // emitted S-101 assets, searched after --assets, before embedded srv.Version = version - srv.EngineCommit = engineCommit // stamped onto every bake srv.ReportStaleCache() // loud warning if any served pack predates this binary addr := net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)) diff --git a/internal/engine/server/aux_test.go b/internal/engine/server/aux_test.go index 2033f47..b3905a5 100644 --- a/internal/engine/server/aux_test.go +++ b/internal/engine/server/aux_test.go @@ -68,7 +68,7 @@ func TestServeAux(t *testing.T) { writeLegacyAuxZip(t, filepath.Join(cache, "NOAA", "D17-OVERVIEW"), "noaa-d17-overview", map[string][]byte{ "NOTE17.TXT": []byte("Caution: ice"), }) - s := New("", cache, cache, true) + s := New("", cache, cache, true, "") // Manifest lists every referenced filename → {stored,type}, across both layouts. rec := httptest.NewRecorder() @@ -117,7 +117,7 @@ func TestServeAux(t *testing.T) { func TestAuxIndexInvalidate(t *testing.T) { cache := t.TempDir() - s := New("", cache, cache, true) + s := New("", cache, cache, true, "") if got := s.auxIdx.manifest(cache); len(got) != 0 { t.Fatalf("empty cache manifest = %v", got) } diff --git a/internal/engine/server/cells_active_test.go b/internal/engine/server/cells_active_test.go index 6772d88..f1150ea 100644 --- a/internal/engine/server/cells_active_test.go +++ b/internal/engine/server/cells_active_test.go @@ -40,7 +40,7 @@ func activeCells(t *testing.T, base string) map[string]bool { // delete also reclaims the provider's source ENC_ROOT. func TestActiveCellsDropOnDelete(t *testing.T) { dir := t.TempDir() - s := New(dir, dir, dir, false) + s := New(dir, dir, dir, false, "") const cell = "US5MD11M" // The cell's source .000 must exist in the provider ENC_ROOT (serveCells lists diff --git a/internal/engine/server/connections_test.go b/internal/engine/server/connections_test.go index 37aba6f..4c43954 100644 --- a/internal/engine/server/connections_test.go +++ b/internal/engine/server/connections_test.go @@ -46,7 +46,7 @@ func doReq(t *testing.T, method, url, body string) (int, connResp) { func TestConnections_CRUDAndPersistence(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() @@ -88,7 +88,7 @@ func TestConnections_CRUDAndPersistence(t *testing.T) { func TestConnections_Validation(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() @@ -105,7 +105,7 @@ func TestConnections_Validation(t *testing.T) { func TestConnections_PersistAcrossRestart(t *testing.T) { dir := t.TempDir() - s1 := New("", dir, dir, true) + s1 := New("", dir, dir, true, "") ts1 := httptest.NewServer(s1) code, _ := doReq(t, http.MethodPost, ts1.URL+"/api/connections", `{"name":"Mux","host":"h","port":2000,"enabled":false}`) @@ -114,7 +114,7 @@ func TestConnections_PersistAcrossRestart(t *testing.T) { s1.Close() // A fresh Server over the same dataDir reloads the connection. - s2 := New("", dir, dir, true) + s2 := New("", dir, dir, true, "") defer s2.Close() ts2 := httptest.NewServer(s2) defer ts2.Close() @@ -126,7 +126,7 @@ func TestConnections_PersistAcrossRestart(t *testing.T) { func TestVessel_SnapshotAndStream(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 8766f4a..1c3aae0 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -63,13 +63,18 @@ type Server struct { // that must survive a cache wipe — pass "" to default it to cacheDir (single-dir // mode). allowRemote is true when the bind host is not loopback (the operator // opted into network exposure), which skips the per-request Host-header check. -func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { +func New(assetsDir, cacheDir, dataDir string, allowRemote bool, engineCommit string) *Server { if dataDir == "" { dataDir = cacheDir } migrateLegacyENCRoot(dataDir) // one-time: retired flat ENC_ROOT → loose/cells (before indexing) migrateProviderEncRoot(dataDir, cacheDir) // one-time: per-district-pack layout → per-provider ENC_ROOT s := &Server{assetsDir: assetsDir, cacheDir: cacheDir, dataDir: dataDir, allowRemote: allowRemote, sets: newTileSets(), imports: newImportJobs(), auxIdx: newAuxIndex(), cellIdx: newCellIndex(dataDir)} + // The engine commit must be known BEFORE the boot registration below: + // registerLiveProviders/rebakeMissingProviders gate on the .enginever stamp, + // and an empty commit counts every kept archive as current (stale tiles + // would register and the self-heal re-bake would never fire). + s.EngineCommit = engineCommit s.cellIdx.build() // backfill cell bounds in the background (kick spawns its own goroutine) // Discover every baked pack on disk (provider trees + flat tiles/), then // register the ENABLED ones (disabled packs stay on disk but off the map). State diff --git a/internal/engine/server/import_meta_test.go b/internal/engine/server/import_meta_test.go index 35dceb0..f72f5f7 100644 --- a/internal/engine/server/import_meta_test.go +++ b/internal/engine/server/import_meta_test.go @@ -50,7 +50,7 @@ func buildExchangeZip(t *testing.T) []byte { // title falls back to the cell's dataset name; the client resolves a nicer name // where it can. func TestImport_NoCatalog(t *testing.T) { - s := New(t.TempDir(), t.TempDir(), t.TempDir(), false) + s := New(t.TempDir(), t.TempDir(), t.TempDir(), false, "") cell, err := os.ReadFile("../../../testdata/US5MD1MC.000") if err != nil { t.Fatal(err) @@ -94,7 +94,7 @@ func TestImport_NoCatalog(t *testing.T) { // post-bake metadata tail (ExtractCellMeta → sidecar) independently of a real bake. func TestImport_AutoNameAndMeta(t *testing.T) { cacheDir, dataDir := t.TempDir(), t.TempDir() - s := New(t.TempDir(), cacheDir, dataDir, false) + s := New(t.TempDir(), cacheDir, dataDir, false, "") zipData := buildExchangeZip(t) cells, _, cat, err := extractZipCells(zipData) diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index e81c8a3..c1fa7ed 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -25,7 +25,7 @@ func TestImportPacks(t *testing.T) { t.Skipf("testdata cell absent: %v", err) } cacheDir, dataDir := t.TempDir(), t.TempDir() - s := New(t.TempDir(), cacheDir, dataDir, false) + s := New(t.TempDir(), cacheDir, dataDir, false, "") ts := httptest.NewServer(s) defer ts.Close() diff --git a/internal/engine/server/import_test.go b/internal/engine/server/import_test.go index 24b75ea..e3221eb 100644 --- a/internal/engine/server/import_test.go +++ b/internal/engine/server/import_test.go @@ -76,7 +76,7 @@ func TestExtractZipCells(t *testing.T) { func TestImportValidation(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Bad set name → 400. @@ -121,7 +121,7 @@ func TestImportValidation(t *testing.T) { // real cell. No archive is registered. func TestImportJobErrorPath(t *testing.T) { dir := t.TempDir() - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") ts := httptest.NewServer(srv) defer ts.Close() @@ -175,7 +175,7 @@ func TestImportJobErrorPath(t *testing.T) { func TestImportFetchValidation(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() post := func(body string) int { @@ -210,7 +210,7 @@ func TestImportFetchDownloadOnly(t *testing.T) { if err := os.WriteFile(cp, []byte("cell-bytes"), 0o644); err != nil { t.Fatal(err) } - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") // The seeded USER/ENC_ROOT makes New() kick off a self-heal bake of "user" in the // background; drain it (srv.Close → bakeWG.Wait) before t.TempDir's RemoveAll, else // that bake's USER/assets write races cleanup ("directory not empty" on CI). @@ -266,7 +266,7 @@ func TestServeCells(t *testing.T) { } os.WriteFile(p, []byte("x"), 0o644) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/api/cells") body, _ := io.ReadAll(resp.Body) diff --git a/internal/engine/server/packs_test.go b/internal/engine/server/packs_test.go index 9aa59ac..e308508 100644 --- a/internal/engine/server/packs_test.go +++ b/internal/engine/server/packs_test.go @@ -36,7 +36,7 @@ func TestProviderKeys(t *testing.T) { // installed districts read from the ENC_ROOT folder listing (not per-district sets). func TestHandlePacksProviders(t *testing.T) { dir := t.TempDir() - s := New(dir, dir, dir, false) + s := New(dir, dir, dir, false, "") // Create ENC_ROOT district folders (each with a placeholder .000) for two providers. for _, dd := range []struct{ prov, dist string }{ diff --git a/internal/engine/server/security_test.go b/internal/engine/server/security_test.go index 3b0f6b9..442b5ae 100644 --- a/internal/engine/server/security_test.go +++ b/internal/engine/server/security_test.go @@ -63,7 +63,7 @@ func TestCrossSiteWrite(t *testing.T) { // A cross-site POST to a state-changing endpoint must be rejected before any // handler runs, and security headers must be present on every response. func TestHandlerBlocksCrossSiteAndSetsHeaders(t *testing.T) { - s := New("", t.TempDir(), t.TempDir(), false) + s := New("", t.TempDir(), t.TempDir(), false, "") defer s.Close() r := httptest.NewRequest("POST", "http://127.0.0.1:8080/api/settings", nil) @@ -85,7 +85,7 @@ func TestHandlerBlocksCrossSiteAndSetsHeaders(t *testing.T) { // The proxy must refuse a non-provider URL (SSRF). func TestProxyRejectsNonProviderURL(t *testing.T) { - s := New("", t.TempDir(), t.TempDir(), false) + s := New("", t.TempDir(), t.TempDir(), false, "") defer s.Close() r := httptest.NewRequest("GET", "http://127.0.0.1:8080/api/proxy?url=http://169.254.169.254/", nil) diff --git a/internal/engine/server/server_test.go b/internal/engine/server/server_test.go index b43c4e1..ff1d91b 100644 --- a/internal/engine/server/server_test.go +++ b/internal/engine/server/server_test.go @@ -19,7 +19,7 @@ func TestServeStaticAndRange(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "data.pmtiles"), body, 0o644); err != nil { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Root serves index.html. @@ -62,7 +62,7 @@ func TestServeCell(t *testing.T) { if err := os.WriteFile(cp, cell, 0o644); err != nil { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/api/cell/US5MD1MC") @@ -87,7 +87,7 @@ func TestServeCell(t *testing.T) { func TestAPIHealthAndHostCheck(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) // loopback-only + ts := httptest.NewServer(New(dir, dir, dir, false, "")) // loopback-only defer ts.Close() resp, _ := http.Get(ts.URL + "/api/health") @@ -115,7 +115,7 @@ func TestAPIHealthAndHostCheck(t *testing.T) { // disk (share.json persistence). func TestShareAndUpload(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // No snapshot yet → 404. @@ -166,7 +166,7 @@ func TestShareAndUpload(t *testing.T) { } // A fresh Server over the same cache dir reloads the snapshot from share.json. - ts2 := httptest.NewServer(New(dir, dir, dir, false)) + ts2 := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts2.Close() resp, _ = http.Get(ts2.URL + "/api/share") got, _ = io.ReadAll(resp.Body) diff --git a/internal/engine/server/tileserve_test.go b/internal/engine/server/tileserve_test.go index fe11cb5..937e537 100644 --- a/internal/engine/server/tileserve_test.go +++ b/internal/engine/server/tileserve_test.go @@ -36,7 +36,7 @@ func TestServeTileSet(t *testing.T) { dir := t.TempDir() body := writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // A present tile → 200 with the MVT body and the vector-tile content type. @@ -120,7 +120,7 @@ func TestServeTileSet(t *testing.T) { func TestServeTileGzip(t *testing.T) { dir := t.TempDir() body := writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Go's transport transparently decodes gzip unless we set Accept-Encoding @@ -148,7 +148,7 @@ func TestServeTileGzip(t *testing.T) { func TestServeTileJSONAndList(t *testing.T) { dir := t.TempDir() writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // TileJSON descriptor. @@ -205,7 +205,7 @@ func TestServeTileJSONEngineStamp(t *testing.T) { t.Fatal(err) } - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") srv.EngineCommit = "fff999000" // the RUNNING binary's engine // A dynamic set: registered without a pack path (plugin tiles). live, err := tilesource.Open(filepath.Join(tilesDir(dir), "legacy.pmtiles")) @@ -263,7 +263,7 @@ func TestServeTileSetMLT(t *testing.T) { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/tiles/mltcharts.json") diff --git a/tile57 b/tile57 index b8e8499..aeac905 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit b8e84992ee83eb3bf49b3fc42dbe277ffd9bbb25 +Subproject commit aeac905c84df29828b28a39f5ce844a3a56f0eab From 6c4c26668cbf1a5b2b6be32f697da8d2c0e2f295 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 10 Jul 2026 20:45:52 -0400 Subject: [PATCH 32/36] =?UTF-8?q?feat(server):=20engine=20re-bakes=20stage?= =?UTF-8?q?=20to=20tiles.next=20and=20swap=20LAST=20=E2=80=94=20old=20tile?= =?UTF-8?q?s=20serve=20throughout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An engine upgrade used to RemoveAll the kept per-cell archives up front and re-bake in place: an interrupted or failed re-bake left the provider with no tiles at all (and boot refused to serve the stale survivors, blanking the map for the whole multi-minute bake). Now stale archives keep serving — stale tiles beat no tiles — while the full re-bake lands in a staging tree (/tiles.next, stamped with the baking engine up front so a same-build re-run resumes incrementally and a different build starts over). The swap into place is the LAST step, after the whole bake succeeded; a staged bake that produces nothing is discarded. registerProviderSet's re-register closes the old composer, so the dropped archives' mmaps release on swap. Boot recovers the one crash window (old tree dropped, staged tree not yet renamed) by moving the staged tree into place and routing the set through the self-heal bake. invalidateLiveOnEngineChange is gone; rebakeMissingProviders now also queues sets that are serving from another engine build's archives. Covered by TestEngineRebakeStagesAndSwapsLast (failure path keeps the served tree untouched; success path swaps, restamps, and leaves no staging behind). Co-Authored-By: Claude Fable 5 --- internal/engine/server/http.go | 8 +- internal/engine/server/live_provider.go | 103 +++++++++++++++------- internal/engine/server/live_stage_test.go | 83 +++++++++++++++++ internal/engine/server/tile57_bake.go | 6 +- 4 files changed, 165 insertions(+), 35 deletions(-) create mode 100644 internal/engine/server/live_stage_test.go diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 1c3aae0..013f06f 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -112,7 +112,13 @@ func (s *Server) rebakeMissingProviders() { var missing []string for _, prov := range s.installedProviders() { if _, live := s.sets.get(prov); live { - continue // already serving (a live compositor from kept archives, or a registered pack) + // Serving, but from another engine build's archives: re-bake to a staging + // tree and swap when done (prepareLiveProvider) — the old tiles keep + // serving meanwhile, and registerProviderSet replaces the composer. + if !s.liveEngineCurrent(s.liveCellsDir(prov)) { + missing = append(missing, prov) + } + continue } if _, ok := s.packPath(prov); !ok { missing = append(missing, prov) diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go index 47c7828..aa0ac7a 100644 --- a/internal/engine/server/live_provider.go +++ b/internal/engine/server/live_provider.go @@ -82,7 +82,12 @@ func (s *Server) bumpLiveGen(provider string) { // liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). The bake mirrors the // ENC tree, so the archives live in subdirs (/d1/US4CT1AA.pmtiles, …) — walk, don't ReadDir. func (s *Server) liveCellArchives(provider string) []string { - root := s.liveCellsDir(provider) + return archivesUnder(s.liveCellsDir(provider)) +} + +// archivesUnder lists the per-cell PMTiles under any tree (sorted) — the kept live +// dir or an engine re-bake's staging dir. +func archivesUnder(root string) []string { var paths []string _ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { if err != nil { @@ -157,14 +162,38 @@ func (s *Server) progressiveReKey(provider string) { s.bumpLiveGen(provider) } -// prepareLiveProvider bakes the provider's cells to its kept live-cells dir (incremental) and opens -// a runtime compositor over them — the live counterpart of composeProvider, with NO district -// compose pass (tiles compose on demand). Returns the contributing cell count + the Composer. +// stagingSuffix names the engine re-bake staging tree beside the kept live dir +// (/tiles.next): a full re-bake lands there while the previous archives +// keep serving, and the swap into place is the LAST step. +const stagingSuffix = ".next" + +// prepareLiveProvider bakes the provider's cells to its kept live-cells dir and opens a runtime +// compositor over them — the live counterpart of composeProvider, with NO district compose pass +// (tiles compose on demand). Same-engine bakes are incremental IN PLACE (adding a district bakes +// only its new cells). An ENGINE change re-bakes everything into a staging tree (tiles.next) +// while the previous archives keep serving, and replaces the served tree only as the LAST step — +// a failed or interrupted re-bake leaves the old tiles in place (and the staging tree resumes +// incrementally on the next run under the same build). Returns the contributing cell count + +// the Composer. func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tilesource.TileSource, error) { cellsDir := s.liveCellsDir(provider) - s.invalidateLiveOnEngineChange(cellsDir) + bakeDir := cellsDir + staged := !s.liveEngineCurrent(cellsDir) + if staged { + bakeDir = cellsDir + stagingSuffix + // A staging tree left by a DIFFERENT engine build is itself stale — start over. + if !s.liveEngineCurrent(bakeDir) { + _ = os.RemoveAll(bakeDir) + } + } + if err := os.MkdirAll(bakeDir, 0o755); err != nil { + return 0, nil, err + } + // Stamp the tree with the baking engine up front: an interrupted run resumes + // incrementally under the same build and restarts under a different one. + _ = os.WriteFile(filepath.Join(bakeDir, ".enginever"), []byte(s.EngineCommit), 0o644) start := time.Now() - if _, err := baker.PrepareLive(encRoot, cellsDir, liveBakeWorkers(), func(done, total int) { + if _, err := baker.PrepareLive(encRoot, bakeDir, liveBakeWorkers(), func(done, total int) { s.imports.update(jobID, func(j *importJob) { j.Phase, j.Band, j.Zoom = "bake", "", 0 eta := 0 @@ -175,11 +204,26 @@ func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tile j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", "Baking charts", done, total, eta }) }); err != nil { - return 0, nil, err + return 0, nil, err // old tiles untouched; a staging tree stays for resume + } + if staged { + // Everything baked — swap the staged tree into place. This is the ONLY point + // the previous archives are dropped. + if len(archivesUnder(bakeDir)) == 0 { + _ = os.RemoveAll(bakeDir) // staged bake produced nothing — keep serving the old tiles + return 0, nil, nil + } + if err := os.RemoveAll(cellsDir); err != nil { + return 0, nil, err + } + if err := os.Rename(bakeDir, cellsDir); err != nil { + return 0, nil, err + } } // The mirrored tree holds every kept per-cell archive (baked + reused). paths := s.liveCellArchives(provider) if len(paths) == 0 { + _ = os.RemoveAll(cellsDir) // nothing valid parsed — clean the empty tree return 0, nil, nil } c, err := s.openLiveComposer(provider) @@ -192,10 +236,10 @@ func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tile return len(paths), c, nil } -// liveEngineCurrent reports whether the kept per-cell archives in cellsDir were baked by the -// RUNNING engine (the .enginever stamp matches the linked tile57 commit). An unstamped dir -// counts as stale — the next prepareLiveProvider re-bakes and stamps it. With no engine -// commit linked in, everything counts as current (nothing to compare against). +// liveEngineCurrent reports whether the per-cell archives in a tree were baked by the +// RUNNING engine (the .enginever stamp matches the linked tile57 commit). An unstamped +// tree counts as stale — prepareLiveProvider re-bakes it to staging and swaps. With no +// engine commit linked in, everything counts as current (nothing to compare against). func (s *Server) liveEngineCurrent(cellsDir string) bool { if s.EngineCommit == "" { return true @@ -204,27 +248,25 @@ func (s *Server) liveEngineCurrent(cellsDir string) bool { return err == nil && string(b) == s.EngineCommit } -// invalidateLiveOnEngineChange drops the kept per-cell archives when the tile57 engine commit -// changed since they were baked (a new engine portrays different tiles), so a binary upgrade -// re-bakes them. The partition sidecar is coverage-derived (engine-independent) and self-validates -// via its input key, so it is left in place. First bake just records the stamp. -func (s *Server) invalidateLiveOnEngineChange(cellsDir string) { - if s.liveEngineCurrent(cellsDir) { - return - } - if _, err := os.Stat(cellsDir); err == nil { - _ = os.RemoveAll(cellsDir) - } - _ = os.MkdirAll(cellsDir, 0o755) - _ = os.WriteFile(filepath.Join(cellsDir, ".enginever"), []byte(s.EngineCommit), 0o644) -} - // registerLiveProviders re-registers, at boot, a runtime compositor for every installed provider // that has kept per-cell archives (a live provider from a previous run) and isn't disabled — so // live layers survive a restart without re-baking. A batch pack registered for the same provider is // replaced (live wins). Runs before rebakeMissingProviders, which then skips the ones registered here. func (s *Server) registerLiveProviders() { for _, prov := range s.installedProviders() { + // Finish an interrupted engine re-bake swap: a crash between dropping the old + // tree and renaming the staged one leaves tiles/ missing with tiles.next + // present (the staged tree was complete — the swap only runs after a full + // bake). Move it into place; dropping live.gen routes the set through the + // self-heal bake (a fast incremental no-op) before it re-registers. + cellsDir := s.liveCellsDir(prov) + if _, err := os.Stat(cellsDir); os.IsNotExist(err) { + if _, e := os.Stat(cellsDir + stagingSuffix); e == nil { + log.Printf("live %s: recovering an interrupted engine re-bake swap", prov) + _ = os.Rename(cellsDir+stagingSuffix, cellsDir) + _ = os.Remove(s.liveGenPath(prov)) + } + } if s.prefs.isDisabled(prov) || len(s.liveCellArchives(prov)) == 0 { continue } @@ -234,12 +276,11 @@ func (s *Server) registerLiveProviders() { if _, err := os.Stat(s.liveGenPath(prov)); err != nil { continue } - // Archives baked by an older engine are stale (a new engine portrays different - // tiles) — don't re-serve them; rebakeMissingProviders re-bakes the provider, - // and prepareLiveProvider drops + re-stamps the dir. + // Archives baked by an older engine are stale (a new engine portrays + // different tiles) — but stale tiles beat NO tiles: keep serving them while + // rebakeMissingProviders re-bakes to staging and swaps when done. if !s.liveEngineCurrent(s.liveCellsDir(prov)) { - log.Printf("live %s: kept archives are from another engine build — re-baking", prov) - continue + log.Printf("live %s: kept archives are from another engine build — serving them while the re-bake runs", prov) } c, err := s.openLiveComposer(prov) if err != nil || c == nil { diff --git a/internal/engine/server/live_stage_test.go b/internal/engine/server/live_stage_test.go new file mode 100644 index 0000000..9bebd69 --- /dev/null +++ b/internal/engine/server/live_stage_test.go @@ -0,0 +1,83 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// TestEngineRebakeStagesAndSwapsLast: an engine change re-bakes into the staging tree +// (tiles.next) while the previously baked archives stay in place, and replaces them only +// as the LAST step, after the whole bake succeeded — a staged bake that produces nothing +// leaves the served tiles (and their stamp) untouched. +func TestEngineRebakeStagesAndSwapsLast(t *testing.T) { + cell, err := os.ReadFile("../../../testdata/US5MD1MC.000") + if err != nil { + t.Skipf("testdata cell absent: %v", err) + } + cacheDir, dataDir := t.TempDir(), t.TempDir() + s := New(t.TempDir(), cacheDir, dataDir, false, "engine-B") + + // The provider's source ENC_ROOT holds the real cell. + src := filepath.Join(s.districtDir("noaa", "d5"), "US5MD1MC.000") + if err := os.MkdirAll(filepath.Dir(src), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(src, cell, 0o644); err != nil { + t.Fatal(err) + } + + // A previously served tree, baked by another engine build. + tiles := s.liveCellsDir("noaa") + old := filepath.Join(tiles, "d5", "OLD.pmtiles") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(old, []byte("previous archive"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tiles, ".enginever"), []byte("engine-A"), 0o644); err != nil { + t.Fatal(err) + } + + // A staged bake over an ENC root with no cells produces nothing: the served + // tree must survive, stamp and all, and the staging tree must be cleaned up. + n, ts, err := s.prepareLiveProvider("job-empty", t.TempDir(), "noaa") + if err != nil { + t.Fatalf("empty staged bake errored: %v", err) + } + if n != 0 || ts != nil { + t.Fatalf("empty staged bake: n=%d src=%v, want 0/nil", n, ts) + } + if _, err := os.Stat(old); err != nil { + t.Error("failed staged bake dropped the served tiles") + } + if b, _ := os.ReadFile(filepath.Join(tiles, ".enginever")); string(b) != "engine-A" { + t.Errorf("served tree re-stamped by a failed staged bake: %q", b) + } + if _, err := os.Stat(tiles + stagingSuffix); !os.IsNotExist(err) { + t.Error("empty staging tree not cleaned up") + } + + // The real staged bake fills tiles.next and swaps it into place — only now do + // the old archives disappear, replaced by the fresh tree with the new stamp. + n, ts, err = s.prepareLiveProvider("job-real", s.encRootDir("noaa"), "noaa") + if err != nil || n == 0 || ts == nil { + t.Fatalf("staged re-bake: n=%d err=%v", n, err) + } + defer func() { _ = tilesource.Close(ts) }() + if _, err := os.Stat(old); !os.IsNotExist(err) { + t.Error("old archive survived the swap") + } + if _, err := os.Stat(filepath.Join(tiles, "d5", "US5MD1MC.pmtiles")); err != nil { + t.Errorf("swapped tree missing the fresh archive: %v", err) + } + if b, _ := os.ReadFile(filepath.Join(tiles, ".enginever")); string(b) != "engine-B" { + t.Errorf("swapped tree stamp = %q, want engine-B", b) + } + if _, err := os.Stat(tiles + stagingSuffix); !os.IsNotExist(err) { + t.Error("staging tree left behind after the swap") + } +} diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index fff1158..9d9f8c1 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -117,10 +117,10 @@ func (s *Server) bakeProvider(jobID, provider string) bool { return fail(err) } // Zero cells means nothing valid parsed → a failed import (don't register an empty set). - // Clean only the (empty) baked tiles dir, NOT the whole set dir — in single-dir mode the set - // dir also holds the source ENC_ROOT, which must survive a failed bake. + // prepareLiveProvider already cleaned whichever tree it baked (a fresh dir, or a staged + // engine re-bake that never replaced the served tiles); the set dir itself survives — in + // single-dir mode it also holds the source ENC_ROOT. if n == 0 { - os.RemoveAll(s.liveCellsDir(provider)) return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) } From 43895cd6973367cf1445f3513fab4d311712efa6 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Sat, 11 Jul 2026 11:22:50 -0400 Subject: [PATCH 33/36] style: gofmt the tree + scope fmt to this repo (not the tile57 submodule) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four files weren't gofmt-clean (Go 1.26 toolchain gofmt). Also scope `make fmt` /`fmt-check` to `git ls-files '*.go'` instead of `.` — the latter walked into the ./tile57 engine submodule and failed the gate on the engine's own Go bindings, which chartplotter-go doesn't own. git ls-files lists only this repo's tracked files, so the submodule is naturally excluded. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 10 +++---- cmd/chartplotter/serve.go | 2 +- internal/engine/server/cells_active_test.go | 1 - internal/engine/server/import.go | 32 ++++++++++----------- internal/engine/server/tile57_bake.go | 1 - 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 6ea229f..9411d08 100644 --- a/Makefile +++ b/Makefile @@ -305,16 +305,16 @@ vet: # Format with the gofmt of the toolchain go.mod pins (Go 1.26), NOT whatever # gofmt happens to be on PATH — gofmt's rules change between Go minor releases, -# so a stray 1.25 gofmt reintroduces drift that the 1.26 CI check rejects. Invoke -# gofmt over `.` (not `go fmt ./...`, which can skip build-tagged files) so the -# file set matches the CI `gofmt -l .` gate exactly. +# so a stray 1.25 gofmt reintroduces drift that the 1.26 CI check rejects. Scope +# to THIS repo's tracked *.go via `git ls-files` (not `.`, which walks into the +# ./tile57 engine submodule — chartplotter-go doesn't own its formatting). fmt: - @"$$(go env GOROOT)/bin/gofmt" -w . + @"$$(go env GOROOT)/bin/gofmt" -w $$(git ls-files '*.go') # Mirror the CI gofmt gate exactly, using the same toolchain gofmt as `fmt`. fmt-check: @GOFMT="$$(go env GOROOT)/bin/gofmt"; \ - out="$$($$GOFMT -l .)"; \ + out="$$($$GOFMT -l $$(git ls-files '*.go'))"; \ test -z "$$out" || { echo "needs gofmt:"; echo "$$out"; exit 1; } tidy: diff --git a/cmd/chartplotter/serve.go b/cmd/chartplotter/serve.go index ae2afb9..db228d6 100644 --- a/cmd/chartplotter/serve.go +++ b/cmd/chartplotter/serve.go @@ -70,7 +70,7 @@ func (c serveCmd) Run() error { srv := server.New(c.Assets, cacheDir, dataDir, allowRemote, engineCommit) srv.SetAssetFallback(s101AssetDir) // emitted S-101 assets, searched after --assets, before embedded srv.Version = version - srv.ReportStaleCache() // loud warning if any served pack predates this binary + srv.ReportStaleCache() // loud warning if any served pack predates this binary addr := net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)) remoteNote := "" diff --git a/internal/engine/server/cells_active_test.go b/internal/engine/server/cells_active_test.go index f1150ea..aa25065 100644 --- a/internal/engine/server/cells_active_test.go +++ b/internal/engine/server/cells_active_test.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "testing" - ) // activeCells GETs /api/cells?active=1 and returns the cell-name set. diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 27714de..2d5401a 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -34,25 +34,25 @@ import ( type importJob struct { ID string `json:"id"` Set string `json:"set"` - State string `json:"state"` // "running" | "done" | "error" - Phase string `json:"phase"` // "download" | "extract" | "bake" - Band string `json:"band"` // usage band being baked (e.g. "coastal"); "" outside the bake phase - Pack string `json:"pack"` // set key of the pack being processed now (multi-pack import); "" for a single set - PackNum int `json:"packNum"` // 1-based position of the current pack in the batch (0 = n/a) - PackTotal int `json:"packTotal"` // packs in the batch (0/1 = single, no "N of M" shown) - Note string `json:"note"` // human-readable current step (e.g. "downloading US5MD1MC") - Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) - Total int `json:"total"` // phase total (0 until known) + State string `json:"state"` // "running" | "done" | "error" + Phase string `json:"phase"` // "download" | "extract" | "bake" + Band string `json:"band"` // usage band being baked (e.g. "coastal"); "" outside the bake phase + Pack string `json:"pack"` // set key of the pack being processed now (multi-pack import); "" for a single set + PackNum int `json:"packNum"` // 1-based position of the current pack in the batch (0 = n/a) + PackTotal int `json:"packTotal"` // packs in the batch (0/1 = single, no "N of M" shown) + Note string `json:"note"` // human-readable current step (e.g. "downloading US5MD1MC") + Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) + Total int `json:"total"` // phase total (0 until known) ETA int `json:"eta,omitempty"` // seconds remaining in this phase (0 = unknown/none) - Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" | "" (opaque) - Cells int `json:"cells"` // cells successfully parsed + Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" | "" (opaque) + Cells int `json:"cells"` // cells successfully parsed // Compose-step zoom progress: the ownership-partition compose walks the zoom ladder, so it // reports "zoom N of M" instead of a chart count. Zoom == 0 means "not composing". - Zoom int `json:"-"` - ZoomMin int `json:"-"` - ZoomMax int `json:"-"` - Err string `json:"error,omitempty"` - Started string `json:"started"` + Zoom int `json:"-"` + ZoomMin int `json:"-"` + ZoomMax int `json:"-"` + Err string `json:"error,omitempty"` + Started string `json:"started"` } // importJobs is the (in-memory) registry of import jobs. diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 9d9f8c1..d178c0d 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -13,7 +13,6 @@ import ( tile57 "github.com/beetlebugorg/tile57/bindings/go" ) - // registerProviderSet registers `src` as provider `set`'s live tile set and writes its tail (bake // stamps, cell manifest, companion aux, metadata sidecar) so the provider is as complete in the // chart library as any pack. `packPath` is the on-disk chart.pmtiles of a BATCH-baked provider From b61982c4d4936f0d77209503ed17e41a0e1696cc Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Sat, 11 Jul 2026 11:31:26 -0400 Subject: [PATCH 34/36] security: bump toolchain to go1.26.5 (fixes GO-2026-5856) govulncheck flagged GO-2026-5856 (Encrypted Client Hello privacy leak in crypto/tls), reached from serveOSM's HTTP client. Fixed in the standard library as of go1.26.5. Bump the toolchain directive; govulncheck is clean afterward. Co-Authored-By: Claude Opus 4.8 (1M context) --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index eab5947..86e45f2 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/beetlebugorg/chartplotter go 1.26.0 -toolchain go1.26.4 +toolchain go1.26.5 require ( github.com/BertoldVdb/go-ais v0.4.0 From abbc32f7add9daf7087ad2a1bd543a081817b51d Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Sat, 11 Jul 2026 18:36:31 -0400 Subject: [PATCH 35/36] chore(ci): use Go 1.26.5 so govulncheck picks up the crypto/tls fix (GO-2026-5856) go.mod already pins `toolchain go1.26.5`, but CI ignored it: setup-go `go-version: '1.26'` installed the cached 1.26.4 and GOTOOLCHAIN=local blocks the auto-upgrade to the toolchain directive, so govulncheck ran under 1.26.4 and flagged GO-2026-5856. Pin the exact patch (1.26.5) in every workflow; release.yml especially, so shipped binaries link the patched crypto/tls. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 6 +++--- .github/workflows/docs.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5f0414..c690fea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig @@ -65,7 +65,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig @@ -95,7 +95,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a7c7d52..91192fc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,7 +51,7 @@ jobs: - uses: actions/setup-go@v6 with: - go-version: "1.26" + go-version: "1.26.5" cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89dede8..3c4ebff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: "1.26" + go-version: "1.26.5" cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig From ae9a0ed83625d37ea0b10d36deaf5e4e42d2a177 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Sat, 11 Jul 2026 18:42:31 -0400 Subject: [PATCH 36/36] build(deps): move tile57 submodule pin to the Windows-fix commit (4b6d73c) The old pin (aeac905c8) predates the cross-platform filemap.zig refactor, so a non-floating `submodule update --init` checkout still hit the `std.posix.mmap` void-flag break on the windows cross-compile. Bump the last-known-good pin to 4b6d73c (feat/per-cell-composite tip, now merged to tile57 main), which builds clean for x86_64-windows-gnu. Co-Authored-By: Claude Opus 4.8 (1M context) --- tile57 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tile57 b/tile57 index aeac905..4b6d73c 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit aeac905c84df29828b28a39f5ce844a3a56f0eab +Subproject commit 4b6d73c5ae23955156a9799d62f3ef6abea4ce69