diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5f0414..c690fea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig @@ -65,7 +65,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig @@ -95,7 +95,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: '1.26' + go-version: '1.26.5' cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a7c7d52..91192fc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,7 +51,7 @@ jobs: - uses: actions/setup-go@v6 with: - go-version: "1.26" + go-version: "1.26.5" cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89dede8..3c4ebff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: git -C chartplotter submodule update --init --recursive tile57 - uses: actions/setup-go@v6 with: - go-version: "1.26" + go-version: "1.26.5" cache: true cache-dependency-path: chartplotter/go.sum - name: Install Zig diff --git a/Makefile b/Makefile index 6ea229f..9411d08 100644 --- a/Makefile +++ b/Makefile @@ -305,16 +305,16 @@ vet: # Format with the gofmt of the toolchain go.mod pins (Go 1.26), NOT whatever # gofmt happens to be on PATH — gofmt's rules change between Go minor releases, -# so a stray 1.25 gofmt reintroduces drift that the 1.26 CI check rejects. Invoke -# gofmt over `.` (not `go fmt ./...`, which can skip build-tagged files) so the -# file set matches the CI `gofmt -l .` gate exactly. +# so a stray 1.25 gofmt reintroduces drift that the 1.26 CI check rejects. Scope +# to THIS repo's tracked *.go via `git ls-files` (not `.`, which walks into the +# ./tile57 engine submodule — chartplotter-go doesn't own its formatting). fmt: - @"$$(go env GOROOT)/bin/gofmt" -w . + @"$$(go env GOROOT)/bin/gofmt" -w $$(git ls-files '*.go') # Mirror the CI gofmt gate exactly, using the same toolchain gofmt as `fmt`. fmt-check: @GOFMT="$$(go env GOROOT)/bin/gofmt"; \ - out="$$($$GOFMT -l .)"; \ + out="$$($$GOFMT -l $$(git ls-files '*.go'))"; \ test -z "$$out" || { echo "needs gofmt:"; echo "$$out"; exit 1; } tidy: diff --git a/cmd/chartplotter/bake.go b/cmd/chartplotter/bake.go index 87986a6..924377e 100644 --- a/cmd/chartplotter/bake.go +++ b/cmd/chartplotter/bake.go @@ -2,15 +2,18 @@ package main import ( "archive/zip" - "encoding/json" "fmt" "io" "io/fs" "os" "path/filepath" + "runtime" + "sort" "strings" + "time" "github.com/beetlebugorg/chartplotter/internal/engine/baker" + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" ) // bakeCmd bakes S-57 ENC base cells into a PMTiles archive of MVT tiles, for @@ -28,42 +31,67 @@ type bakeCmd struct { } func (c bakeCmd) Run() error { - // libtile57 is the sole bake engine. A *.pmtiles/-mbtiles -o writes ONE flat - // merged archive (+ optional --manifest / aux.zip) — the coverage-clipped - // composite resolves best-available inside the single archive, so there are - // no per-band archives any more (the retired --bands). Any other -o is a - // self-contained bundle directory (tiles/chart.pmtiles + per-scheme style + - // assets + manifest.json). - switch strings.ToLower(filepath.Ext(c.Out)) { - case ".pmtiles", ".mbtiles": - return c.runTile57Archive() - } - return c.runTile57Bundle() -} - -// runTile57Bundle bakes the ENC inputs with the native libtile57 engine into a -// self-contained chart bundle (tiles/chart.pmtiles + per-scheme SCAMIN-bucketed -// style-*.json + assets + manifest.json) under the output directory. The engine -// reads the ENC from disk, so a lone directory or .000 is handed over directly and -// only zips / multiple inputs are staged into a temp directory of cells first. -func (c bakeCmd) runTile57Bundle() error { + // libtile57 makes tiles ONE way: bake each cell to its own native-scale PMTiles + build the + // ownership partition. This CLI writes that LIVE STRUCTURE — the exact inputs the runtime + // compositor consumes: /tiles/.pmtiles + /partition.tpart. There is no + // merged chart.pmtiles bundle; the style + assets are produced separately (the asset commands / + // the server's runtime style), not per bake. input, cleanup, err := c.tile57Input() if err != nil { return err } defer cleanup() - outDir := bundleOutDir(c.Out) - if err := os.MkdirAll(outDir, 0o755); err != nil { + outDir := bundleOutDir(c.Out) // a *.pmtiles path → its stem dir; any other -o used verbatim + tilesDir := filepath.Join(outDir, "tiles") + if err := os.MkdirAll(tilesDir, 0o755); err != nil { + return err + } + + // Bake each cell IN PARALLEL to /tiles/.pmtiles, mirroring the input tree + // (incremental — reruns re-bake only changed cells). The engine writes + frees each archive. + start := time.Now() + if _, err := baker.PrepareLive(input, tilesDir, runtime.NumCPU(), func(done, total int) { + if done >= total { + fmt.Printf("\rbaked %d cell(s), building partition… ", total) + } else if done > 0 { + per := time.Since(start) / time.Duration(done) + fmt.Printf("\rbaking %d/%d · ~%s left ", done, total, (per * time.Duration(total-done)).Round(time.Second)) + } else { + fmt.Printf("\rbaking %d/%d… ", done, total) + } + }); err != nil { + fmt.Println() return err } - // nil progress → the lib's built-in per-band console progress (good CLI output). - n, bbox, err := bakeTile57Bundle(input, outDir, c.MaxZoom, c.Format, nil) + fmt.Println() + + // Collect the mirrored per-cell archives (baked + reused) for the compositor. + var paths []string + _ = filepath.WalkDir(tilesDir, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, ".pmtiles") { + paths = append(paths, p) + } + return nil + }) + sort.Strings(paths) + if len(paths) == 0 { + return fmt.Errorf("no coverage: no valid S-57 cells under %s", input) + } + + // Build + save the ownership-partition sidecar next to the tiles (the compositor loads it on + // open to skip the build). + comp, err := tilesource.NewComposer(paths, "") if err != nil { return err } - fmt.Printf("baked %d cell(s) → %s/ via libtile57 — bundle: tiles/chart.pmtiles + assets/style-{day,dusk,night}.json + manifest.json (bbox %.4f,%.4f,%.4f,%.4f)\n", - n, outDir, bbox[0], bbox[1], bbox[2], bbox[3]) + defer comp.Close() + if err := comp.SavePartition(filepath.Join(outDir, "partition.tpart")); err != nil { + return err + } + m := comp.Meta() + fmt.Printf("baked %d cell(s) → %s/tiles/*.pmtiles + partition.tpart (z%d..%d, bounds %.4f,%.4f,%.4f,%.4f)\n", + len(paths), outDir, m.MinZoom, m.MaxZoom, m.W, m.S, m.E, m.N) return nil } @@ -120,21 +148,6 @@ func (c bakeCmd) tile57Input() (path string, cleanup func(), err error) { return dir, func() { os.RemoveAll(dir) }, nil } -// writeManifestJSON writes a charts-index.json manifest (indented) to path. -func writeManifestJSON(path string, man map[string]any) error { - mf, err := os.Create(path) - if err != nil { - return err - } - enc := json.NewEncoder(mf) - enc.SetIndent("", " ") - if err := enc.Encode(man); err != nil { - mf.Close() - return err - } - return mf.Close() -} - // bundleOutDir derives the bundle output DIRECTORY from the -o value: a *.pmtiles / // *.mbtiles path becomes its stem (charts.pmtiles → charts/), otherwise -o is used // as the directory verbatim. diff --git a/cmd/chartplotter/serve.go b/cmd/chartplotter/serve.go index 9deff73..db228d6 100644 --- a/cmd/chartplotter/serve.go +++ b/cmd/chartplotter/serve.go @@ -67,11 +67,10 @@ func (c serveCmd) Run() error { // Loopback bind → enforce the Host-header DNS-rebind check on /api. Any // other bind means the operator opted into network exposure. allowRemote := !(c.Host == "127.0.0.1" || c.Host == "localhost" || c.Host == "::1") - srv := server.New(c.Assets, cacheDir, dataDir, allowRemote) + srv := server.New(c.Assets, cacheDir, dataDir, allowRemote, engineCommit) srv.SetAssetFallback(s101AssetDir) // emitted S-101 assets, searched after --assets, before embedded srv.Version = version - srv.EngineCommit = engineCommit // stamped onto every bake - srv.ReportStaleCache() // loud warning if any served pack predates this binary + srv.ReportStaleCache() // loud warning if any served pack predates this binary addr := net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)) remoteNote := "" diff --git a/cmd/chartplotter/simulate.go b/cmd/chartplotter/simulate.go index 009da96..be49e84 100644 --- a/cmd/chartplotter/simulate.go +++ b/cmd/chartplotter/simulate.go @@ -172,12 +172,7 @@ func loadWaterFeatures(p string) ([]tile57.Feature, error) { if err != nil { return nil, err } - src, err := tile57.OpenChartBytes(data) - if err != nil { - return nil, err - } - defer src.Close() - return src.Features("DEPARE", "DRGARE") + return tile57.FeaturesBytes(data, "DEPARE", "DRGARE") } // readBaseCell returns the raw base-cell bytes from a .000 file or the first diff --git a/cmd/chartplotter/tile57_bake.go b/cmd/chartplotter/tile57_bake.go deleted file mode 100644 index 902acb9..0000000 --- a/cmd/chartplotter/tile57_bake.go +++ /dev/null @@ -1,118 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - tile57 "github.com/beetlebugorg/tile57/bindings/go" -) - -// bakeTile57Bundle bakes an on-disk ENC input (a .000 cell or a directory of cells) -// into a self-contained chart bundle under outDir via the native libtile57 engine. -// maxZoom caps the highest baked zoom (0 = no cap). format selects the tile -// encoding ("mlt"/"mvt"; "" = the engine default, MLT). progress nil uses the -// lib's built-in console progress. Returns the cell count + bbox (w,s,e,n). -func bakeTile57Bundle(input, outDir string, maxZoom int, format string, progress func(tile57.BakeProgress)) (int, [4]float64, error) { - // MaxZoom 24 = the ABI's "no clamp" (bake each cell's full native band); MaxZoom 0 - // would clamp every band down to z0 — an EMPTY archive. Only narrow on --max-zoom. - opts := tile57.BakeOpts{MaxZoom: 24, Format: bakeFormat(format)} - if maxZoom > 0 && maxZoom < 24 { - opts.MaxZoom = uint8(maxZoom) - } - return tile57.BakeBundle(input, outDir, opts, progress) -} - -// bakeFormat maps the --format flag to the engine's bake format. "" = the engine -// default (MLT); "mvt" keeps the legacy Mapbox Vector Tile output. -func bakeFormat(format string) tile57.TileFormat { - switch format { - case "mvt": - return tile57.FormatMVT - case "mlt": - return tile57.FormatMLT - } - return tile57.FormatDefault -} - -// runTile57Archive bakes the ENC inputs into ONE flat merged archive at -o via -// the STREAMING bundle driver — super-tile locality, geometry LRU, coarse -// riders, capped overscale fill-up, TILE57_SUPER_DZ / TILE57_LRU_BUDGET -// tuning, tiles streamed to disk instead of an in-memory whole-archive buffer. -// BakeBundle bakes into a temp dir beside -o; the finished tiles/chart.pmtiles -// then renames onto -o (same filesystem) and the bundle scaffolding (assets/ -// styles/manifest.json) is discarded. Honors --max-zoom; writes --manifest -// (one band-less entry) + aux.zip (aux content collected from the input tree — -// zip inputs stage theirs beside the cells). -func (c bakeCmd) runTile57Archive() error { - input, cleanup, err := c.tile57Input() - if err != nil { - return err - } - defer cleanup() - - outAbs, err := filepath.Abs(c.Out) - if err != nil { - return err - } - tmp, err := os.MkdirTemp(filepath.Dir(outAbs), ".bake-*") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - n, bbox, err := bakeTile57Bundle(input, tmp, c.MaxZoom, c.Format, nil) - if err != nil { - return err - } - if err := os.Rename(filepath.Join(tmp, "tiles", "chart.pmtiles"), outAbs); err != nil { - return err - } - st, _ := os.Stat(outAbs) - fmt.Printf("baked %d cell(s) → %s (%.1f MB) via libtile57 (streamed)\n", n, c.Out, float64(st.Size())/(1<<20)) - - // Aux content walks the INPUT tree (for a lone .000, its directory). - auxRoot := input - if fi, e := os.Stat(input); e == nil && !fi.IsDir() { - auxRoot = filepath.Dir(input) - } - ext := filepath.Ext(c.Out) - stem := strings.TrimSuffix(c.Out, ext) - auxManifest, err := writeAuxDir(stem, collectAuxDir(auxRoot)) - if err != nil { - return err - } - - if c.Manifest != "" { - entry := map[string]any{"file": filepath.Base(c.Out), "bounds": bbox[:]} - man := map[string]any{"districts": []map[string]any{entry}} - if auxManifest != "" { - man["aux"] = auxManifest - } - if err := writeManifestJSON(c.Manifest, man); err != nil { - return err - } - fmt.Printf("wrote manifest %s\n", c.Manifest) - } - return nil -} - -// orderedUpdates returns a cell's update bodies sorted by filename so libtile57 -// applies them in sequence (.001, .002, …). -func orderedUpdates(m map[string][]byte) [][]byte { - if len(m) == 0 { - return nil - } - names := make([]string, 0, len(m)) - for n := range m { - names = append(names, n) - } - sort.Strings(names) - out := make([][]byte, len(names)) - for i, n := range names { - out[i] = m[n] - } - return out -} diff --git a/go.mod b/go.mod index eab5947..86e45f2 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/beetlebugorg/chartplotter go 1.26.0 -toolchain go1.26.4 +toolchain go1.26.5 require ( github.com/BertoldVdb/go-ais v0.4.0 diff --git a/internal/engine/baker/compose.go b/internal/engine/baker/compose.go new file mode 100644 index 0000000..0d537f9 --- /dev/null +++ b/internal/engine/baker/compose.go @@ -0,0 +1,41 @@ +package baker + +import ( + "io/fs" + "path/filepath" + "strings" + + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// PrepareLive walks the ENC data under `input` and bakes each S-57 base cell (*.000) IN PARALLEL to +// the SAME relative path under `cellsDir` with a .pmtiles extension (plus a .sha content sidecar), +// creating subdirs — the mirrored tree the runtime compositor mmaps directly (there is NO district +// compose pass; tiles compose on demand). A cell whose archive is already up to date (newer than its +// .000 and its update chain) is skipped, so re-baking a provider only bakes what changed. The engine +// writes and frees each archive as it goes, so this never holds N archives in memory (peak ~ +// workers). `workers` bounds concurrency (a MEMORY bound). `onProgress(done, total)` fires per baked +// cell for the import UI (may be called concurrently). Returns the number of cells baked this pass. +func PrepareLive(input, cellsDir string, workers int, onProgress func(done, total int)) (int, error) { + return tile57.BakeTree(input, cellsDir, workers, onProgress) +} + +// ListCells returns every base cell (.000) path under `root` (a single file or a directory), +// deduped by stem (a boundary cell shared by two districts appears once). +func ListCells(root string) ([]string, error) { + var out []string + seen := map[string]bool{} + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".000") { + return nil + } + stem := strings.TrimSuffix(filepath.Base(path), ".000") + if seen[stem] { + return nil + } + seen[stem] = true + out = append(out, path) + return nil + }) + return out, err +} diff --git a/internal/engine/baker/meta.go b/internal/engine/baker/meta.go index 6b21906..817e918 100644 --- a/internal/engine/baker/meta.go +++ b/internal/engine/baker/meta.go @@ -42,6 +42,38 @@ type CellMeta struct { HasBBox bool `json:"-"` } +// ExtractCellMetaDir reads per-cell metadata straight from an on-disk ENC tree: +// the engine walks the .000s (applying each cell's update chain) and reports the +// inventory — no cell bytes are held in memory, so a whole-catalogue import stays +// flat. Duplicate stems (a boundary cell shared by two district subfolders) +// collapse to one entry, matching the first-copy-wins district dedup. +func ExtractCellMetaDir(root string, onSkip func(name string, err error)) map[string]CellMeta { + infos, err := tile57.Charts(root) + if err != nil { + if onSkip != nil { + onSkip(root, err) + } + return map[string]CellMeta{} + } + out := make(map[string]CellMeta, len(infos)) + for _, ci := range infos { + if _, seen := out[ci.Name]; seen { + continue + } + out[ci.Name] = CellMeta{ + Name: ci.Name, + Scale: ci.Scale, + Edition: ci.Edition, + Update: ci.Update, + IssueDate: ci.IssueDate, + Agency: ci.Agency, + BBox: ci.BBox, + HasBBox: ci.HasBBox, + } + } + return out +} + // ExtractCellMeta stages the cells (base + updates) to a temporary ENC dir, // opens it with the native engine, and returns per-cell metadata keyed by cell // stem (the DSNM stem the engine reports). Cells that fail to parse are @@ -83,12 +115,7 @@ func ExtractCellMeta(cells map[string]CellData, onSkip func(name string, err err } } - src, err := tile57.Open(dir) - if err != nil { - return skipAll(err) - } - defer src.Close() - infos, err := src.Cells() + infos, err := tile57.Charts(dir) if err != nil { return skipAll(err) } diff --git a/internal/engine/nmea/sim/water_test.go b/internal/engine/nmea/sim/water_test.go index 9c715d9..9160bb1 100644 --- a/internal/engine/nmea/sim/water_test.go +++ b/internal/engine/nmea/sim/water_test.go @@ -29,10 +29,7 @@ func TestWaterMask_RealCell(t *testing.T) { if err != nil { t.Skip("test cell not available") } - src, err := tile57.OpenChartBytes(data) - require.NoError(t, err) - defer src.Close() - feats, err := src.Features("DEPARE", "DRGARE") + feats, err := tile57.FeaturesBytes(data, "DEPARE", "DRGARE") require.NoError(t, err) m := NewWaterMask(feats, 2) require.NotNil(t, m, "cell should yield navigable depth areas") diff --git a/internal/engine/server/aux_test.go b/internal/engine/server/aux_test.go index 2033f47..b3905a5 100644 --- a/internal/engine/server/aux_test.go +++ b/internal/engine/server/aux_test.go @@ -68,7 +68,7 @@ func TestServeAux(t *testing.T) { writeLegacyAuxZip(t, filepath.Join(cache, "NOAA", "D17-OVERVIEW"), "noaa-d17-overview", map[string][]byte{ "NOTE17.TXT": []byte("Caution: ice"), }) - s := New("", cache, cache, true) + s := New("", cache, cache, true, "") // Manifest lists every referenced filename → {stored,type}, across both layouts. rec := httptest.NewRecorder() @@ -117,7 +117,7 @@ func TestServeAux(t *testing.T) { func TestAuxIndexInvalidate(t *testing.T) { cache := t.TempDir() - s := New("", cache, cache, true) + s := New("", cache, cache, true, "") if got := s.auxIdx.manifest(cache); len(got) != 0 { t.Fatalf("empty cache manifest = %v", got) } diff --git a/internal/engine/server/cellindex.go b/internal/engine/server/cellindex.go index 854397e..0ac86eb 100644 --- a/internal/engine/server/cellindex.go +++ b/internal/engine/server/cellindex.go @@ -171,12 +171,7 @@ func (ci *cellIndex) scan() { if _, ok := ci.get(name); ok { return nil // already indexed (forget() drops a re-imported cell so it re-parses) } - src, err := tile57.Open(path) - if err != nil { - return nil - } - infos, err := src.Cells() - src.Close() + infos, err := tile57.Charts(path) if err != nil || len(infos) == 0 { return nil } diff --git a/internal/engine/server/cells_active_test.go b/internal/engine/server/cells_active_test.go index 78de08a..aa25065 100644 --- a/internal/engine/server/cells_active_test.go +++ b/internal/engine/server/cells_active_test.go @@ -8,8 +8,6 @@ import ( "os" "path/filepath" "testing" - - "github.com/beetlebugorg/chartplotter/internal/engine/baker" ) // activeCells GETs /api/cells?active=1 and returns the cell-name set. @@ -41,7 +39,7 @@ func activeCells(t *testing.T, base string) map[string]bool { // delete also reclaims the provider's source ENC_ROOT. func TestActiveCellsDropOnDelete(t *testing.T) { dir := t.TempDir() - s := New(dir, dir, dir, false) + s := New(dir, dir, dir, false, "") const cell = "US5MD11M" // The cell's source .000 must exist in the provider ENC_ROOT (serveCells lists @@ -55,7 +53,7 @@ func TestActiveCellsDropOnDelete(t *testing.T) { } // Register the provider set with an exact cell manifest, and add it as an enabled pack. const provider = "noaa" - if err := s.writeSetCells(provider, map[string]baker.CellData{cell + ".000": {}}); err != nil { + if err := s.writeSetCells(provider, []string{cell}); err != nil { t.Fatal(err) } manifest := filepath.Join(s.setDir(provider), provider+".cells.json") diff --git a/internal/engine/server/connections_test.go b/internal/engine/server/connections_test.go index 37aba6f..4c43954 100644 --- a/internal/engine/server/connections_test.go +++ b/internal/engine/server/connections_test.go @@ -46,7 +46,7 @@ func doReq(t *testing.T, method, url, body string) (int, connResp) { func TestConnections_CRUDAndPersistence(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() @@ -88,7 +88,7 @@ func TestConnections_CRUDAndPersistence(t *testing.T) { func TestConnections_Validation(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() @@ -105,7 +105,7 @@ func TestConnections_Validation(t *testing.T) { func TestConnections_PersistAcrossRestart(t *testing.T) { dir := t.TempDir() - s1 := New("", dir, dir, true) + s1 := New("", dir, dir, true, "") ts1 := httptest.NewServer(s1) code, _ := doReq(t, http.MethodPost, ts1.URL+"/api/connections", `{"name":"Mux","host":"h","port":2000,"enabled":false}`) @@ -114,7 +114,7 @@ func TestConnections_PersistAcrossRestart(t *testing.T) { s1.Close() // A fresh Server over the same dataDir reloads the connection. - s2 := New("", dir, dir, true) + s2 := New("", dir, dir, true, "") defer s2.Close() ts2 := httptest.NewServer(s2) defer ts2.Close() @@ -126,7 +126,7 @@ func TestConnections_PersistAcrossRestart(t *testing.T) { func TestVessel_SnapshotAndStream(t *testing.T) { dir := t.TempDir() - s := New("", dir, dir, true) + s := New("", dir, dir, true, "") defer s.Close() ts := httptest.NewServer(s) defer ts.Close() diff --git a/internal/engine/server/debug_partition.go b/internal/engine/server/debug_partition.go new file mode 100644 index 0000000..d6a6916 --- /dev/null +++ b/internal/engine/server/debug_partition.go @@ -0,0 +1,90 @@ +package server + +import ( + "fmt" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// partitionSuffix names a provider's ownership-partition debug tile set: it is served +// at /tiles/{provider}-partition/{z}/{x}/{y}.mvt, separate from the real chart set. +const partitionSuffix = "-partition" + +// Zoom range for the debug partition. z0..12 covers overview→approach across a whole +// provider quickly; harbor-level detail (z13+) multiplies the tile count ~4×/zoom, so +// the overlay overzooms past 12 rather than baking it. +const ( + partitionMinZoom = 0 + partitionMaxZoom = 12 +) + +// handleDebugPartition drives the ownership-partition DEBUG overlay. +// +// POST /api/debug/partition → (re)generate the partition for every provider, +// in the background; returns the provider list. +// GET /api/debug/partition → per provider: whether its partition tile set is +// ready + the tile URL template to overlay. +func (s *Server) handleDebugPartition(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + providers := s.installedProviders() + // Bake off the request goroutine — a whole provider is seconds, several add up. + go func() { + for _, p := range providers { + if err := s.bakeProviderPartition(p); err != nil { + log.Printf("partition-debug %s: %v", p, err) + } + } + }() + writeJSON(w, map[string]any{"ok": true, "providers": providers}) + case http.MethodGet: + type item struct { + Provider string `json:"provider"` + Ready bool `json:"ready"` + Tiles string `json:"tiles"` + } + items := []item{} + for _, p := range s.installedProviders() { + _, ready := s.sets.get(p + partitionSuffix) + items = append(items, item{ + Provider: p, + Ready: ready, + Tiles: "tiles/" + p + partitionSuffix + "/{z}/{x}/{y}.mvt", + }) + } + writeJSON(w, map[string]any{"partitions": items}) + default: + apiErr(w, http.StatusMethodNotAllowed, "POST or GET") + } +} + +// bakeProviderPartition bakes ONE provider's ENC_ROOT into an ownership-partition debug +// PMTiles (which cell renders which ground per band; NO portrayed content) and registers +// it as the "{provider}-partition" tile set. Reuses the same ENC_ROOT + output dir as the +// real bake; the partition math is entirely in the engine (tile57.BakePartitionDebug). +func (s *Server) bakeProviderPartition(provider string) error { + if len(s.providerDistricts(provider)) == 0 { + return fmt.Errorf("provider %q has no districts", provider) + } + encRoot := s.encRootDir(provider) + outPath := filepath.Join(s.setDir(provider), "tiles", provider+partitionSuffix+".pmtiles") + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return err + } + n, err := tile57.BakePartitionDebug(encRoot, outPath, partitionMinZoom, partitionMaxZoom, tile57.BandGoverning) + if err != nil { + return fmt.Errorf("bake partition: %w", err) + } + src, err := tilesource.Open(outPath) + if err != nil { + return fmt.Errorf("open partition pmtiles: %w", err) + } + s.sets.register(provider+partitionSuffix, src) + log.Printf("partition-debug: %q (%d cell(s)) → %s", provider, n, outPath) + return nil +} diff --git a/internal/engine/server/http.go b/internal/engine/server/http.go index 75fa4f3..013f06f 100644 --- a/internal/engine/server/http.go +++ b/internal/engine/server/http.go @@ -63,13 +63,18 @@ type Server struct { // that must survive a cache wipe — pass "" to default it to cacheDir (single-dir // mode). allowRemote is true when the bind host is not loopback (the operator // opted into network exposure), which skips the per-request Host-header check. -func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { +func New(assetsDir, cacheDir, dataDir string, allowRemote bool, engineCommit string) *Server { if dataDir == "" { dataDir = cacheDir } migrateLegacyENCRoot(dataDir) // one-time: retired flat ENC_ROOT → loose/cells (before indexing) migrateProviderEncRoot(dataDir, cacheDir) // one-time: per-district-pack layout → per-provider ENC_ROOT s := &Server{assetsDir: assetsDir, cacheDir: cacheDir, dataDir: dataDir, allowRemote: allowRemote, sets: newTileSets(), imports: newImportJobs(), auxIdx: newAuxIndex(), cellIdx: newCellIndex(dataDir)} + // The engine commit must be known BEFORE the boot registration below: + // registerLiveProviders/rebakeMissingProviders gate on the .enginever stamp, + // and an empty commit counts every kept archive as current (stale tiles + // would register and the self-heal re-bake would never fire). + s.EngineCommit = engineCommit s.cellIdx.build() // backfill cell bounds in the background (kick spawns its own goroutine) // Discover every baked pack on disk (provider trees + flat tiles/), then // register the ENABLED ones (disabled packs stay on disk but off the map). State @@ -92,7 +97,9 @@ func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { if len(s.packs) > 0 { log.Printf("tilesets: %d pack(s) on disk, %d enabled (from %s)", len(s.packs), n, cacheDir) } - s.rebakeMissingProviders() // self-heal: bake any provider with an ENC_ROOT but no bundle (post-migration) + s.registerLiveProviders() // re-register live runtime compositors from kept per-cell archives (survives restart) + s.registerLiveCompose() // dev/test: on-demand runtime compositor from TILE57_LIVE_COMPOSE (no-op if unset) + s.rebakeMissingProviders() // self-heal: bake any provider with an ENC_ROOT but no set (→ live prep) return s } @@ -104,6 +111,15 @@ func New(assetsDir, cacheDir, dataDir string, allowRemote bool) *Server { func (s *Server) rebakeMissingProviders() { var missing []string for _, prov := range s.installedProviders() { + if _, live := s.sets.get(prov); live { + // Serving, but from another engine build's archives: re-bake to a staging + // tree and swap when done (prepareLiveProvider) — the old tiles keep + // serving meanwhile, and registerProviderSet replaces the composer. + if !s.liveEngineCurrent(s.liveCellsDir(prov)) { + missing = append(missing, prov) + } + continue + } if _, ok := s.packPath(prov); !ok { missing = append(missing, prov) } @@ -325,6 +341,8 @@ func (s *Server) handleAPI(w http.ResponseWriter, r *http.Request) { s.handleDeleteDistrict(w, r) // DELETE: remove one district + re-bake the provider case r.URL.Path == "/api/proxy": s.serveProxy(w, r) // dumb CORS/Range passthrough for a NOAA URL (e.g. All_ENCs.zip) + case strings.HasPrefix(r.URL.Path, "/api/debug/partition"): + s.handleDebugPartition(w, r) // POST: bake ownership-partition overlays; GET: their status default: apiErr(w, http.StatusNotFound, "unknown endpoint") } diff --git a/internal/engine/server/import.go b/internal/engine/server/import.go index 33d5259..2d5401a 100644 --- a/internal/engine/server/import.go +++ b/internal/engine/server/import.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -33,19 +34,25 @@ import ( type importJob struct { ID string `json:"id"` Set string `json:"set"` - State string `json:"state"` // "running" | "done" | "error" - Phase string `json:"phase"` // "download" | "extract" | "bake" - Band string `json:"band"` // usage band being baked (e.g. "coastal"); "" outside the bake phase - Pack string `json:"pack"` // set key of the pack being processed now (multi-pack import); "" for a single set - PackNum int `json:"packNum"` // 1-based position of the current pack in the batch (0 = n/a) - PackTotal int `json:"packTotal"` // packs in the batch (0/1 = single, no "N of M" shown) - Note string `json:"note"` // human-readable current step (e.g. "downloading US5MD1MC") - Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) - Total int `json:"total"` // phase total (0 until known) - Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" - Cells int `json:"cells"` // cells successfully parsed - Err string `json:"error,omitempty"` - Started string `json:"started"` + State string `json:"state"` // "running" | "done" | "error" + Phase string `json:"phase"` // "download" | "extract" | "bake" + Band string `json:"band"` // usage band being baked (e.g. "coastal"); "" outside the bake phase + Pack string `json:"pack"` // set key of the pack being processed now (multi-pack import); "" for a single set + PackNum int `json:"packNum"` // 1-based position of the current pack in the batch (0 = n/a) + PackTotal int `json:"packTotal"` // packs in the batch (0/1 = single, no "N of M" shown) + Note string `json:"note"` // human-readable current step (e.g. "downloading US5MD1MC") + Done int `json:"done"` // phase units done (bytes/cells downloaded, then tiles emitted) + Total int `json:"total"` // phase total (0 until known) + ETA int `json:"eta,omitempty"` // seconds remaining in this phase (0 = unknown/none) + Unit string `json:"unit"` // what done/total count: "bytes" | "cells" | "tiles" | "" (opaque) + Cells int `json:"cells"` // cells successfully parsed + // Compose-step zoom progress: the ownership-partition compose walks the zoom ladder, so it + // reports "zoom N of M" instead of a chart count. Zoom == 0 means "not composing". + Zoom int `json:"-"` + ZoomMin int `json:"-"` + ZoomMax int `json:"-"` + Err string `json:"error,omitempty"` + Started string `json:"started"` } // importJobs is the (in-memory) registry of import jobs. @@ -114,6 +121,20 @@ func (j *importJobs) running() (importJob, bool) { return *best, true } +// runningFor reports whether a bake/import is in flight for `set` — its own job, or the pack being +// processed in a multi-pack batch. The tile server keeps caching OFF for a set while its content is +// in flux (a blank tile now may fill in when a cell finishes baking). +func (j *importJobs) runningFor(set string) bool { + j.mu.Lock() + defer j.mu.Unlock() + for _, job := range j.m { + if job.State == "running" && (job.Set == set || job.Pack == set) { + return true + } + } + return false +} + // handleImport routes the import endpoints (already past the /api host check). func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/import/status" { @@ -553,11 +574,7 @@ func (s *Server) looseCellsDir() string { // (/.cells.json). /api/cells?active reads these to return exactly the // installed cells, instead of every cached cell whose bounds overlap the pack's // (often global, for a worldwide-scattered import) bounding box. -func (s *Server) writeSetCells(set string, cells map[string]baker.CellData) error { - stems := make([]string, 0, len(cells)) - for n := range cells { - stems = append(stems, strings.TrimSuffix(n, ".000")) - } +func (s *Server) writeSetCells(set string, stems []string) error { sort.Strings(stems) dir := s.setDir(set) if err := os.MkdirAll(dir, 0o755); err != nil { @@ -585,16 +602,152 @@ func (s *Server) setCells(set string) ([]string, bool) { return stems, true } -// statusJSON renders a job snapshot as the status JSON line (shared by the polling -// endpoint and the SSE stream). +// statusJSON renders a job snapshot as the status JSON line (shared by the polling endpoint and +// the SSE stream). The server owns the human wording: `action` (the live step) and `detail` (the +// count/ETA beside it) are display-ready strings the client renders verbatim, and `frac` is the +// bar fill (0..1, or null for an indeterminate/opaque-step bar). The raw fields ride along for the +// client's own use (pack context, region title), but it must not re-derive the status from them. func (j importJob) statusJSON() string { - pct := 0 + action, detail, eta, frac := j.statusDisplay() + return fmt.Sprintf( + `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"cells":%d,"action":%q,"detail":%q,"eta":%q,"frac":%s,"error":%q}`, + j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, j.Cells, action, detail, eta, fracJSON(frac), j.Err) +} + +// statusDisplay composes the display-ready progress strings the client renders verbatim. `detail` +// (the count) and `eta` are kept SEPARATE so the client can pin each in its own fixed, right- +// aligned slot — glued together they change width and the whole line jitters as the numbers move: +// +// action — the live step ("Baking US5MD1MD", "Composing tiles", "Downloading charts") +// detail — the count beside it ("1,234 / 4,567 charts", "zoom 12 / 16") +// eta — the time remaining ("~2m 30s left"), or "" when unknown +// frac — bar fill 0..1, or a negative value for an indeterminate (opaque-step) bar. +func (j importJob) statusDisplay() (action, detail, eta string, frac float64) { + action = j.actionText() + + switch { + case j.Zoom > 0: // the compose step reports its zoom position, not a chart count + detail = fmt.Sprintf("zoom %d / %d", j.Zoom-j.ZoomMin+1, j.ZoomMax-j.ZoomMin+1) + case j.Unit == "bytes": + if j.Total > 0 { + detail = fmtBytes(j.Done) + " / " + fmtBytes(j.Total) + } else { + detail = fmtBytes(j.Done) + } + case j.Total > 0: + unit := j.Unit + if unit == "cells" { + unit = "charts" // "charts" reads friendlier than the internal "cells" + } + detail = commaInt(j.Done) + " / " + commaInt(j.Total) + " " + unit + } + if e := fmtEtaSecs(j.ETA); e != "" { + eta = e + " left" + } + + // A known total → a determinate bar (the compose weights or the chart count both work). An + // opaque step with no total (initial "Preparing", the compose warm-up) sweeps. if j.Total > 0 { - pct = j.Done * 100 / j.Total + frac = float64(j.Done) / float64(j.Total) + } else { + frac = -1 + } + return +} + +// actionText is the live step verb. The bake/meta phases carry a clean, self-contained note that +// names the step ("Baking charts", "Composing tiles", "Reading chart metadata"); use it verbatim. +// The download/extract notes echo a source filename ("Downloading US5MD1MC"), so those get a +// friendly generic verb instead. +func (j importJob) actionText() string { + switch j.Phase { + case "bake", "meta": + if j.Note != "" { + if j.Band != "" { + return j.Note + " " + j.Band // legacy per-band bake: "Generating tiles coastal" + } + return j.Note + } + case "download": + return "Downloading charts" + case "extract": + return "Extracting charts" + } + if j.Note != "" { + return j.Note + } + if j.Phase != "" { + return strings.ToUpper(j.Phase[:1]) + j.Phase[1:] + } + return "Working" +} + +// fracJSON renders a bar fill as a JSON number, or null for an indeterminate bar. +func fracJSON(frac float64) string { + if frac < 0 { + return "null" + } + if frac > 1 { + frac = 1 + } + return strconv.FormatFloat(frac, 'f', 4, 64) +} + +// fmtBytes renders a byte count as a compact "12 MB" / "1.4 KB". +func fmtBytes(n int) string { + if n <= 0 { + return "0 B" + } + units := []string{"B", "KB", "MB", "GB"} + f, i := float64(n), 0 + for f >= 1024 && i < len(units)-1 { + f /= 1024 + i++ + } + if f < 10 && i > 0 { + return fmt.Sprintf("%.1f %s", f, units[i]) + } + return fmt.Sprintf("%.0f %s", f, units[i]) +} + +// commaInt renders an int with thousands separators ("1234567" → "1,234,567"). +func commaInt(n int) string { + s := strconv.Itoa(n) + neg := strings.HasPrefix(s, "-") + if neg { + s = s[1:] + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if i > 0 && (len(s)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteByte(s[i]) + } + if neg { + return "-" + b.String() + } + return b.String() +} + +// fmtEtaSecs renders seconds-remaining as a compact "~2m 30s" / "~45s" / "~1h" (matches the +// client's old fmtEta); "" when unknown. +func fmtEtaSecs(sec int) string { + if sec <= 0 { + return "" + } + switch { + case sec >= 3600: + return fmt.Sprintf("~%dh", (sec+1800)/3600) + case sec >= 60: + m, r := sec/60, sec%60 + if r != 0 { + return fmt.Sprintf("~%dm %ds", m, r) + } + return fmt.Sprintf("~%dm", m) + default: + return fmt.Sprintf("~%ds", sec) } - return fmt.Sprintf( - `{"ok":true,"id":%q,"set":%q,"state":%q,"phase":%q,"band":%q,"pack":%q,"packNum":%d,"packTotal":%d,"note":%q,"done":%d,"total":%d,"unit":%q,"percent":%d,"cells":%d,"error":%q}`, - j.ID, j.Set, j.State, j.Phase, j.Band, j.Pack, j.PackNum, j.PackTotal, j.Note, j.Done, j.Total, j.Unit, pct, j.Cells, j.Err) } // importStatus returns a job's state as JSON (one-shot poll). diff --git a/internal/engine/server/import_meta_test.go b/internal/engine/server/import_meta_test.go index 35dceb0..f72f5f7 100644 --- a/internal/engine/server/import_meta_test.go +++ b/internal/engine/server/import_meta_test.go @@ -50,7 +50,7 @@ func buildExchangeZip(t *testing.T) []byte { // title falls back to the cell's dataset name; the client resolves a nicer name // where it can. func TestImport_NoCatalog(t *testing.T) { - s := New(t.TempDir(), t.TempDir(), t.TempDir(), false) + s := New(t.TempDir(), t.TempDir(), t.TempDir(), false, "") cell, err := os.ReadFile("../../../testdata/US5MD1MC.000") if err != nil { t.Fatal(err) @@ -94,7 +94,7 @@ func TestImport_NoCatalog(t *testing.T) { // post-bake metadata tail (ExtractCellMeta → sidecar) independently of a real bake. func TestImport_AutoNameAndMeta(t *testing.T) { cacheDir, dataDir := t.TempDir(), t.TempDir() - s := New(t.TempDir(), cacheDir, dataDir, false) + s := New(t.TempDir(), cacheDir, dataDir, false, "") zipData := buildExchangeZip(t) cells, _, cat, err := extractZipCells(zipData) diff --git a/internal/engine/server/import_packs_test.go b/internal/engine/server/import_packs_test.go index 87c6e0c..c1fa7ed 100644 --- a/internal/engine/server/import_packs_test.go +++ b/internal/engine/server/import_packs_test.go @@ -25,7 +25,7 @@ func TestImportPacks(t *testing.T) { t.Skipf("testdata cell absent: %v", err) } cacheDir, dataDir := t.TempDir(), t.TempDir() - s := New(t.TempDir(), cacheDir, dataDir, false) + s := New(t.TempDir(), cacheDir, dataDir, false, "") ts := httptest.NewServer(s) defer ts.Close() @@ -79,14 +79,28 @@ func TestImportPacks(t *testing.T) { t.Fatalf("job state=%q error=%q", st.State, st.Error) } - // ONE provider set ("noaa") registered, with a baked chart.pmtiles under the cache - // dir, and both districts' source cells kept under the data dir. + // ONE provider set ("noaa") registered, serving from the live-composite structure + // under the cache dir (per-cell tiles/.pmtiles + partition.tpart), and both + // districts' source cells kept under the data dir. if _, ok := s.sets.get("noaa"); !ok { t.Errorf("provider set %q not registered", "noaa") } - chart := filepath.Join(s.setDir("noaa"), "tiles", "chart.pmtiles") - if fi, err := os.Stat(chart); err != nil || fi.Size() == 0 { - t.Errorf("provider %q: no baked pmtiles (%v)", "noaa", err) + // The bake mirrors the ENC tree, so the cell's archive lives under its district subdir + // (tiles/d5/US5MD1MC.pmtiles) — walk for it. + var found bool + for _, p := range s.liveCellArchives("noaa") { + if strings.HasSuffix(p, "US5MD1MC.pmtiles") { + found = true + if fi, err := os.Stat(p); err != nil || fi.Size() == 0 { + t.Errorf("provider %q: empty archive %s (%v)", "noaa", p, err) + } + } + } + if !found { + t.Errorf("provider %q: no baked US5MD1MC archive in the mirrored tree", "noaa") + } + if _, err := os.Stat(filepath.Join(s.setDir("noaa"), "partition.tpart")); err != nil { + t.Errorf("provider %q: no partition sidecar (%v)", "noaa", err) } for _, dist := range []string{"d5", "d7"} { if _, err := os.Stat(filepath.Join(s.districtDir("noaa", dist), "US5MD1MC.000")); err != nil { @@ -97,4 +111,53 @@ func TestImportPacks(t *testing.T) { if stems, ok := s.setCells("noaa"); !ok || len(stems) != 1 || stems[0] != "US5MD1MC" { t.Errorf("provider cell manifest = %v (ok=%t), want [US5MD1MC]", stems, ok) } + + // A live provider is a FIRST-CLASS set, surfaced provider-centrically (no disk pack): + // /api/packs reports its coverage bounds from the registry. + pr, _ := http.Get(ts.URL + "/api/packs") + pb, _ := io.ReadAll(pr.Body) + pr.Body.Close() + var packs struct { + Packs []struct { + Name string `json:"name"` + Bounds []float64 `json:"bounds"` + } `json:"packs"` + } + json.Unmarshal(pb, &packs) + var noaaBounds []float64 + for _, p := range packs.Packs { + if p.Name == "noaa" { + noaaBounds = p.Bounds + } + } + if len(noaaBounds) != 4 { + t.Errorf("/api/packs: live provider noaa has no bounds: %v", noaaBounds) + } + + // disable → enable round-trips through the registry (re-opening the runtime + // compositor from kept per-cell archives), not just disk packs. + if resp, err := http.Post(ts.URL+"/api/set/disable?set=noaa", "", nil); err == nil { + resp.Body.Close() + } + if _, live := s.sets.get("noaa"); live { + t.Error("disabled live provider still registered") + } + if resp, err := http.Post(ts.URL+"/api/set/enable?set=noaa", "", nil); err == nil { + resp.Body.Close() + } + if _, live := s.sets.get("noaa"); !live { + t.Error("re-enabled live provider not re-registered from kept archives") + } + + // progressiveReKey re-opens + re-registers the set over the cells baked so far — the + // mechanism that fills the map in batch-by-batch during a long import — and stamps the + // content token so the client re-fetches. + s.sets.remove("noaa") + s.progressiveReKey("noaa") + if _, live := s.sets.get("noaa"); !live { + t.Error("progressiveReKey did not register the live set") + } + if s.packGen("noaa") <= 0 { + t.Error("progressiveReKey did not persist the content token") + } } diff --git a/internal/engine/server/import_test.go b/internal/engine/server/import_test.go index 24b75ea..e3221eb 100644 --- a/internal/engine/server/import_test.go +++ b/internal/engine/server/import_test.go @@ -76,7 +76,7 @@ func TestExtractZipCells(t *testing.T) { func TestImportValidation(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Bad set name → 400. @@ -121,7 +121,7 @@ func TestImportValidation(t *testing.T) { // real cell. No archive is registered. func TestImportJobErrorPath(t *testing.T) { dir := t.TempDir() - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") ts := httptest.NewServer(srv) defer ts.Close() @@ -175,7 +175,7 @@ func TestImportJobErrorPath(t *testing.T) { func TestImportFetchValidation(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() post := func(body string) int { @@ -210,7 +210,7 @@ func TestImportFetchDownloadOnly(t *testing.T) { if err := os.WriteFile(cp, []byte("cell-bytes"), 0o644); err != nil { t.Fatal(err) } - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") // The seeded USER/ENC_ROOT makes New() kick off a self-heal bake of "user" in the // background; drain it (srv.Close → bakeWG.Wait) before t.TempDir's RemoveAll, else // that bake's USER/assets write races cleanup ("directory not empty" on CI). @@ -266,7 +266,7 @@ func TestServeCells(t *testing.T) { } os.WriteFile(p, []byte("x"), 0o644) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/api/cells") body, _ := io.ReadAll(resp.Body) diff --git a/internal/engine/server/live_compose.go b/internal/engine/server/live_compose.go new file mode 100644 index 0000000..6b2f34e --- /dev/null +++ b/internal/engine/server/live_compose.go @@ -0,0 +1,59 @@ +package server + +import ( + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// registerLiveCompose registers a live runtime-composited tile set when TILE57_LIVE_COMPOSE names +// a directory of per-cell PMTiles (each from `tile57 compose --keep-cells`). It serves tiles by +// on-demand composition (the tile57 ComposeSource) instead of a prebaked archive. Optional env: +// +// TILE57_LIVE_PARTITION a partition sidecar (`tile57 compose --save-partition`) to load +// instead of building the ownership partition at open; +// TILE57_LIVE_NAME the set name (default "live"), served at /tiles//{z}/{x}/{y}. +// +// A dev/test hook for the on-demand compositor — a no-op when TILE57_LIVE_COMPOSE is unset. +func (s *Server) registerLiveCompose() { + dir := os.Getenv("TILE57_LIVE_COMPOSE") + if dir == "" { + return + } + entries, err := os.ReadDir(dir) + if err != nil { + log.Printf("live-compose: cannot read %s: %v", dir, err) + return + } + var paths []string + for _, e := range entries { + if e.IsDir() { + continue + } + if n := e.Name(); strings.HasSuffix(n, ".cell.tmp") || strings.HasSuffix(n, ".pmtiles") { + paths = append(paths, filepath.Join(dir, n)) + } + } + sort.Strings(paths) + if len(paths) == 0 { + log.Printf("live-compose: no per-cell archives (*.cell.tmp / *.pmtiles) in %s", dir) + return + } + name := os.Getenv("TILE57_LIVE_NAME") + if name == "" { + name = "live" + } + src, err := tilesource.NewComposer(paths, os.Getenv("TILE57_LIVE_PARTITION")) + if err != nil { + log.Printf("live-compose: open %s failed: %v", dir, err) + return + } + s.sets.register(name, src) + m := src.Meta() + log.Printf("live-compose: registered %q — %d archive(s), z%d..%d, bounds [%.3f, %.3f, %.3f, %.3f] (GET /tiles/%s/{z}/{x}/{y})", + name, len(paths), m.MinZoom, m.MaxZoom, m.W, m.S, m.E, m.N, name) +} diff --git a/internal/engine/server/live_provider.go b/internal/engine/server/live_provider.go new file mode 100644 index 0000000..aa0ac7a --- /dev/null +++ b/internal/engine/server/live_provider.go @@ -0,0 +1,299 @@ +package server + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "log" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/beetlebugorg/chartplotter/internal/engine/baker" + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// liveCellsDir is /tiles — the KEPT per-cell PMTiles the runtime compositor mmaps. +// This is the same layout `tile57 bake -o ` writes (/tiles/.pmtiles next to +// /partition.tpart), so a CLI-baked structure drops straight into a provider's set dir. +func (s *Server) liveCellsDir(provider string) string { + return filepath.Join(s.setDir(provider), "tiles") +} + +// livePartitionPath is /partition.tpart — the saved ownership-partition sidecar. +func (s *Server) livePartitionPath(provider string) string { + return filepath.Join(s.setDir(provider), "partition.tpart") +} + +// liveGenPath is /live.gen — it holds the provider's CONTENT cache-bust token (a +// decimal of the sha-of-shas over its per-cell archives; see liveGenToken), which packGen reads +// and stamps into tile URLs as ?g. It changes exactly when the cell set or any cell's content +// changes, so a no-op re-bake keeps the client's cached tiles. Its existence also marks "an +// import registered" — registerLiveProviders only re-serves a provider that has one, so a set +// left partial by an interrupted bake is completed by rebakeMissingProviders instead of served. +func (s *Server) liveGenPath(provider string) string { + return filepath.Join(s.setDir(provider), "live.gen") +} + +// liveGenToken is the provider's CONTENT cache-bust token: a sha-of-shas over its per-cell +// archives (each archive's content sha, from the .sha sidecar written at bake) plus the engine +// commit serving them. The lines are sorted so the token is order-independent, and the low 63 +// bits become the positive ?g int. It changes exactly when the set of cells, any cell's +// content, or the engine build changes — the engine composes live tiles at SERVE time, so its +// identity is part of a tile's content address: a serve-path fix must bust client caches even +// when the baked archives are byte-identical. +func (s *Server) liveGenToken(provider string) int64 { + paths := s.liveCellArchives(provider) + if len(paths) == 0 { + return 0 + } + lines := make([]string, 0, len(paths)+1) + if s.EngineCommit != "" { + lines = append(lines, "engine:"+s.EngineCommit) + } + for _, p := range paths { + stem := strings.TrimSuffix(filepath.Base(p), ".pmtiles") + sha, err := os.ReadFile(p + ".sha") + if err != nil { // no sidecar (shouldn't happen post-bake) — hash the archive itself + if b, e := os.ReadFile(p); e == nil { + sum := sha256.Sum256(b) + sha = []byte(hex.EncodeToString(sum[:])) + } + } + lines = append(lines, stem+":"+strings.TrimSpace(string(sha))) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return int64(binary.BigEndian.Uint64(sum[:8]) & 0x7fffffffffffffff) // 63 bits → non-negative +} + +// bumpLiveGen recomputes and persists the live provider's content token (writes live.gen). +// Called when an import registration completes. +func (s *Server) bumpLiveGen(provider string) { + p := s.liveGenPath(provider) + _ = os.MkdirAll(filepath.Dir(p), 0o755) + _ = os.WriteFile(p, []byte(strconv.FormatInt(s.liveGenToken(provider), 10)), 0o644) +} + +// liveCellArchives lists a provider's kept per-cell PMTiles paths (sorted). The bake mirrors the +// ENC tree, so the archives live in subdirs (/d1/US4CT1AA.pmtiles, …) — walk, don't ReadDir. +func (s *Server) liveCellArchives(provider string) []string { + return archivesUnder(s.liveCellsDir(provider)) +} + +// archivesUnder lists the per-cell PMTiles under any tree (sorted) — the kept live +// dir or an engine re-bake's staging dir. +func archivesUnder(root string) []string { + var paths []string + _ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() && strings.HasSuffix(p, ".pmtiles") { + paths = append(paths, p) + } + return nil + }) + sort.Strings(paths) + return paths +} + +// liveBakeWorkers is how many cells bake in parallel — a MEMORY bound (each concurrent bake holds a +// whole cell's parse+portray+raster working set), so the CPU count capped modestly, overridable via +// CHARTPLOTTER_BAKE_WORKERS. +func liveBakeWorkers() int { + if v := os.Getenv("CHARTPLOTTER_BAKE_WORKERS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + if n := runtime.NumCPU(); n < 8 { + return n + } + return 8 +} + +// openLiveComposer opens a runtime compositor over a provider's kept per-cell archives, loading the +// partition sidecar when present (else building it and saving one for next time). Returns nil when +// the provider has no live-cells dir yet. +func (s *Server) openLiveComposer(provider string) (*tilesource.Composer, error) { + paths := s.liveCellArchives(provider) + if len(paths) == 0 { + return nil, nil + } + sidecar := s.livePartitionPath(provider) + load := "" + if fi, err := os.Stat(sidecar); err == nil && fi.Size() > 0 { + load = sidecar + } + c, err := tilesource.NewComposer(paths, load) + if err != nil { + return nil, err + } + // Persist the (possibly rebuilt) partition so the on-disk sidecar always matches the current + // cell set: a progressive re-key or an added/removed district changes the inputs, the + // compositor rebuilds from a stale sidecar, and saving keeps the sidecar current for a fast + // boot. (When the sidecar was loaded intact this re-writes identical bytes.) + if err := c.SavePartition(sidecar); err != nil { + log.Printf("live %s: save partition sidecar: %v", provider, err) + } + return c, nil +} + +// progressiveReKey re-opens the live compositor over the cells baked SO FAR, registers it as the +// provider's (enabled) set, and advances the content token — so a long import fills in on the map +// batch by batch instead of appearing only when the whole provider finishes. Called synchronously +// from the bake loop (which is paused, so it never races the writer). Best-effort: a transient +// open failure just skips this batch; the next one (or the final register) catches up. +func (s *Server) progressiveReKey(provider string) { + c, err := s.openLiveComposer(provider) + if err != nil || c == nil { + if err != nil { + log.Printf("live %s: progressive re-key: %v", provider, err) + } + return + } + s.sets.register(provider, c) + s.prefs.setDisabled(provider, false) + s.bumpLiveGen(provider) +} + +// stagingSuffix names the engine re-bake staging tree beside the kept live dir +// (/tiles.next): a full re-bake lands there while the previous archives +// keep serving, and the swap into place is the LAST step. +const stagingSuffix = ".next" + +// prepareLiveProvider bakes the provider's cells to its kept live-cells dir and opens a runtime +// compositor over them — the live counterpart of composeProvider, with NO district compose pass +// (tiles compose on demand). Same-engine bakes are incremental IN PLACE (adding a district bakes +// only its new cells). An ENGINE change re-bakes everything into a staging tree (tiles.next) +// while the previous archives keep serving, and replaces the served tree only as the LAST step — +// a failed or interrupted re-bake leaves the old tiles in place (and the staging tree resumes +// incrementally on the next run under the same build). Returns the contributing cell count + +// the Composer. +func (s *Server) prepareLiveProvider(jobID, encRoot, provider string) (int, tilesource.TileSource, error) { + cellsDir := s.liveCellsDir(provider) + bakeDir := cellsDir + staged := !s.liveEngineCurrent(cellsDir) + if staged { + bakeDir = cellsDir + stagingSuffix + // A staging tree left by a DIFFERENT engine build is itself stale — start over. + if !s.liveEngineCurrent(bakeDir) { + _ = os.RemoveAll(bakeDir) + } + } + if err := os.MkdirAll(bakeDir, 0o755); err != nil { + return 0, nil, err + } + // Stamp the tree with the baking engine up front: an interrupted run resumes + // incrementally under the same build and restarts under a different one. + _ = os.WriteFile(filepath.Join(bakeDir, ".enginever"), []byte(s.EngineCommit), 0o644) + start := time.Now() + if _, err := baker.PrepareLive(encRoot, bakeDir, liveBakeWorkers(), func(done, total int) { + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Band, j.Zoom = "bake", "", 0 + eta := 0 + if done > 0 && total > done { + per := time.Since(start) / time.Duration(done) + eta = int((per * time.Duration(total-done)).Round(time.Second).Seconds()) + } + j.Unit, j.Note, j.Done, j.Total, j.ETA = "cells", "Baking charts", done, total, eta + }) + }); err != nil { + return 0, nil, err // old tiles untouched; a staging tree stays for resume + } + if staged { + // Everything baked — swap the staged tree into place. This is the ONLY point + // the previous archives are dropped. + if len(archivesUnder(bakeDir)) == 0 { + _ = os.RemoveAll(bakeDir) // staged bake produced nothing — keep serving the old tiles + return 0, nil, nil + } + if err := os.RemoveAll(cellsDir); err != nil { + return 0, nil, err + } + if err := os.Rename(bakeDir, cellsDir); err != nil { + return 0, nil, err + } + } + // The mirrored tree holds every kept per-cell archive (baked + reused). + paths := s.liveCellArchives(provider) + if len(paths) == 0 { + _ = os.RemoveAll(cellsDir) // nothing valid parsed — clean the empty tree + return 0, nil, nil + } + c, err := s.openLiveComposer(provider) + if err != nil { + return 0, nil, err + } + if c == nil { + return 0, nil, nil + } + return len(paths), c, nil +} + +// liveEngineCurrent reports whether the per-cell archives in a tree were baked by the +// RUNNING engine (the .enginever stamp matches the linked tile57 commit). An unstamped +// tree counts as stale — prepareLiveProvider re-bakes it to staging and swaps. With no +// engine commit linked in, everything counts as current (nothing to compare against). +func (s *Server) liveEngineCurrent(cellsDir string) bool { + if s.EngineCommit == "" { + return true + } + b, err := os.ReadFile(filepath.Join(cellsDir, ".enginever")) + return err == nil && string(b) == s.EngineCommit +} + +// registerLiveProviders re-registers, at boot, a runtime compositor for every installed provider +// that has kept per-cell archives (a live provider from a previous run) and isn't disabled — so +// live layers survive a restart without re-baking. A batch pack registered for the same provider is +// replaced (live wins). Runs before rebakeMissingProviders, which then skips the ones registered here. +func (s *Server) registerLiveProviders() { + for _, prov := range s.installedProviders() { + // Finish an interrupted engine re-bake swap: a crash between dropping the old + // tree and renaming the staged one leaves tiles/ missing with tiles.next + // present (the staged tree was complete — the swap only runs after a full + // bake). Move it into place; dropping live.gen routes the set through the + // self-heal bake (a fast incremental no-op) before it re-registers. + cellsDir := s.liveCellsDir(prov) + if _, err := os.Stat(cellsDir); os.IsNotExist(err) { + if _, e := os.Stat(cellsDir + stagingSuffix); e == nil { + log.Printf("live %s: recovering an interrupted engine re-bake swap", prov) + _ = os.Rename(cellsDir+stagingSuffix, cellsDir) + _ = os.Remove(s.liveGenPath(prov)) + } + } + if s.prefs.isDisabled(prov) || len(s.liveCellArchives(prov)) == 0 { + continue + } + // Only re-serve a provider whose import COMPLETED (live.gen written at registration). A set + // left partial by an interrupted bake has tiles/ but no live.gen → skip it here, and + // rebakeMissingProviders finishes it (incremental) instead of serving a partial map. + if _, err := os.Stat(s.liveGenPath(prov)); err != nil { + continue + } + // Archives baked by an older engine are stale (a new engine portrays + // different tiles) — but stale tiles beat NO tiles: keep serving them while + // rebakeMissingProviders re-bakes to staging and swaps when done. + if !s.liveEngineCurrent(s.liveCellsDir(prov)) { + log.Printf("live %s: kept archives are from another engine build — serving them while the re-bake runs", prov) + } + c, err := s.openLiveComposer(prov) + if err != nil || c == nil { + if err != nil { + log.Printf("live %s: reopen at boot: %v", prov, err) + } + continue + } + s.sets.register(prov, c) + // Re-persist the token: the archives are unchanged, but the serving + // engine may not be the one that wrote live.gen, and the token embeds it. + s.bumpLiveGen(prov) + m := c.Meta() + log.Printf("live: registered %q from kept per-cell archives (z%d..%d)", prov, m.MinZoom, m.MaxZoom) + } +} diff --git a/internal/engine/server/live_stage_test.go b/internal/engine/server/live_stage_test.go new file mode 100644 index 0000000..9bebd69 --- /dev/null +++ b/internal/engine/server/live_stage_test.go @@ -0,0 +1,83 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + + "github.com/beetlebugorg/chartplotter/internal/engine/tilesource" +) + +// TestEngineRebakeStagesAndSwapsLast: an engine change re-bakes into the staging tree +// (tiles.next) while the previously baked archives stay in place, and replaces them only +// as the LAST step, after the whole bake succeeded — a staged bake that produces nothing +// leaves the served tiles (and their stamp) untouched. +func TestEngineRebakeStagesAndSwapsLast(t *testing.T) { + cell, err := os.ReadFile("../../../testdata/US5MD1MC.000") + if err != nil { + t.Skipf("testdata cell absent: %v", err) + } + cacheDir, dataDir := t.TempDir(), t.TempDir() + s := New(t.TempDir(), cacheDir, dataDir, false, "engine-B") + + // The provider's source ENC_ROOT holds the real cell. + src := filepath.Join(s.districtDir("noaa", "d5"), "US5MD1MC.000") + if err := os.MkdirAll(filepath.Dir(src), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(src, cell, 0o644); err != nil { + t.Fatal(err) + } + + // A previously served tree, baked by another engine build. + tiles := s.liveCellsDir("noaa") + old := filepath.Join(tiles, "d5", "OLD.pmtiles") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(old, []byte("previous archive"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tiles, ".enginever"), []byte("engine-A"), 0o644); err != nil { + t.Fatal(err) + } + + // A staged bake over an ENC root with no cells produces nothing: the served + // tree must survive, stamp and all, and the staging tree must be cleaned up. + n, ts, err := s.prepareLiveProvider("job-empty", t.TempDir(), "noaa") + if err != nil { + t.Fatalf("empty staged bake errored: %v", err) + } + if n != 0 || ts != nil { + t.Fatalf("empty staged bake: n=%d src=%v, want 0/nil", n, ts) + } + if _, err := os.Stat(old); err != nil { + t.Error("failed staged bake dropped the served tiles") + } + if b, _ := os.ReadFile(filepath.Join(tiles, ".enginever")); string(b) != "engine-A" { + t.Errorf("served tree re-stamped by a failed staged bake: %q", b) + } + if _, err := os.Stat(tiles + stagingSuffix); !os.IsNotExist(err) { + t.Error("empty staging tree not cleaned up") + } + + // The real staged bake fills tiles.next and swaps it into place — only now do + // the old archives disappear, replaced by the fresh tree with the new stamp. + n, ts, err = s.prepareLiveProvider("job-real", s.encRootDir("noaa"), "noaa") + if err != nil || n == 0 || ts == nil { + t.Fatalf("staged re-bake: n=%d err=%v", n, err) + } + defer func() { _ = tilesource.Close(ts) }() + if _, err := os.Stat(old); !os.IsNotExist(err) { + t.Error("old archive survived the swap") + } + if _, err := os.Stat(filepath.Join(tiles, "d5", "US5MD1MC.pmtiles")); err != nil { + t.Errorf("swapped tree missing the fresh archive: %v", err) + } + if b, _ := os.ReadFile(filepath.Join(tiles, ".enginever")); string(b) != "engine-B" { + t.Errorf("swapped tree stamp = %q, want engine-B", b) + } + if _, err := os.Stat(tiles + stagingSuffix); !os.IsNotExist(err) { + t.Error("staging tree left behind after the swap") + } +} diff --git a/internal/engine/server/packs_test.go b/internal/engine/server/packs_test.go index 9aa59ac..e308508 100644 --- a/internal/engine/server/packs_test.go +++ b/internal/engine/server/packs_test.go @@ -36,7 +36,7 @@ func TestProviderKeys(t *testing.T) { // installed districts read from the ENC_ROOT folder listing (not per-district sets). func TestHandlePacksProviders(t *testing.T) { dir := t.TempDir() - s := New(dir, dir, dir, false) + s := New(dir, dir, dir, false, "") // Create ENC_ROOT district folders (each with a placeholder .000) for two providers. for _, dd := range []struct{ prov, dist string }{ diff --git a/internal/engine/server/prefs.go b/internal/engine/server/prefs.go index 4cc63ab..13e7d83 100644 --- a/internal/engine/server/prefs.go +++ b/internal/engine/server/prefs.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "sync" ) @@ -95,33 +96,35 @@ func (p *prefs) setDisabled(set string, off bool) { } } -// scanPacks walks the cache and returns every baked provider bundle keyed by set name -// (= provider) → path. The tile57 bundle writes //tiles/chart.pmtiles -// (one archive per provider, provider-enc-root), so the set name is the provider dir — -// otherwise every bundle collapses to "chart" and the chart library can't list them -// after a restart. +// scanPacks discovers STANDALONE tile archives — prebaked .pmtiles/.mbtiles hand-dropped +// into the flat /tiles dir — keyed set name (the file basename) → path. This is the +// narrow overlay path for hand-placed archives. +// +// It scans ONLY the flat tiles dir, never the whole cache: a live runtime-compositor +// provider owns //tiles/*.pmtiles + partition.tpart, and those per-cell +// archives are compositor INPUTS discovered provider-centrically by registerLiveProviders — +// never scavenged here (else every cell would surface as its own phantom set, and an +// interrupted import that hasn't saved its partition yet would leak cells as packs). func scanPacks(cacheDir string) map[string]string { out := map[string]string{} - _ = filepath.WalkDir(cacheDir, func(path string, d os.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - if !strings.HasSuffix(path, ".pmtiles") && !strings.HasSuffix(path, ".mbtiles") { - return nil + dir := tilesDir(cacheDir) + entries, err := os.ReadDir(dir) + if err != nil { + return out + } + for _, e := range entries { + if e.IsDir() { + continue } - name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) - // tile57 bundle: //tiles/chart.pmtiles → the provider name. - if name == "chart" && filepath.Base(filepath.Dir(path)) == "tiles" { - provider := filepath.Base(filepath.Dir(filepath.Dir(path))) - if provider != "" && provider != "." { - name = strings.ToLower(provider) - } + name := e.Name() + if !strings.HasSuffix(name, ".pmtiles") && !strings.HasSuffix(name, ".mbtiles") { + continue } - if isSetName(name) { - out[name] = path + set := strings.TrimSuffix(name, filepath.Ext(name)) + if isSetName(set) { + out[set] = filepath.Join(dir, name) } - return nil - }) + } return out } @@ -157,6 +160,14 @@ func (s *Server) packGen(set string) int64 { return fi.ModTime().UnixNano() } } + // Live runtime-compositor provider (no disk pack): the CONTENT token stored in live.gen (a + // sha-of-shas over its per-cell archives), so its tiles are content-addressed by ?g — a no-op + // re-bake keeps the token, and each progressive re-key advances it as cells fill in. + if b, err := os.ReadFile(s.liveGenPath(set)); err == nil { + if n, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64); err == nil { + return n + } + } return 0 } diff --git a/internal/engine/server/prefs_test.go b/internal/engine/server/prefs_test.go new file mode 100644 index 0000000..6a8c2d3 --- /dev/null +++ b/internal/engine/server/prefs_test.go @@ -0,0 +1,123 @@ +package server + +import ( + "os" + "path/filepath" + "testing" +) + +// TestScanPacksStandaloneArchivesOnly: scanPacks discovers only STANDALONE archives dropped +// into the flat /tiles dir (keyed by basename). A live runtime-compositor provider owns +// /tiles/*.pmtiles + partition.tpart — compositor INPUTS discovered +// provider-centrically by registerLiveProviders — and must NEVER be scavenged here (no phantom +// per-cell sets), even when its partition sidecar isn't saved yet. +func TestScanPacksStandaloneArchivesOnly(t *testing.T) { + cache := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(cache, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + // Standalone archives in the flat /tiles dir → registered by basename. + write("tiles/charts.pmtiles", "pm") + write("tiles/overlay.mbtiles", "mb") + + // A live provider's per-cell inputs under /tiles — WITH a partition sidecar... + write("NOAA/tiles/US5MD1MC.pmtiles", "pm") + write("NOAA/partition.tpart", "part") + // ...and one mid-import (archives present, partition not saved yet). + write("IENC/tiles/US4MD81M.pmtiles", "pm") + + got := scanPacks(cache) + + for _, want := range []string{"charts", "overlay"} { + if _, ok := got[want]; !ok { + t.Errorf("standalone archive %q not discovered: %v", want, got) + } + } + for _, phantom := range []string{"noaa", "ienc", "us5md1mc", "us4md81m", "US5MD1MC", "US4MD81M", "partition"} { + if p, ok := got[phantom]; ok { + t.Errorf("live-provider input mis-registered as set %q -> %q", phantom, p) + } + } + if len(got) != 2 { + t.Errorf("scanPacks = %v, want {charts, overlay}", got) + } +} + +// TestLiveGenTokenContentAddressed: a live provider's ?g token is a content sha-of-shas over its +// per-cell archives — deterministic (same content → same token), order-independent, and it moves +// only when a cell's content changes or the set gains/loses a cell. A no-op re-bake keeps the +// token so the client's cached tiles stay valid. +func TestLiveGenTokenContentAddressed(t *testing.T) { + s := &Server{cacheDir: t.TempDir()} + dir := s.liveCellsDir("noaa") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + cell := func(stem, sha string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, stem+".pmtiles"), []byte("pm"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, stem+".pmtiles.sha"), []byte(sha), 0o644); err != nil { + t.Fatal(err) + } + } + + cell("US5MD1MC", "aaaa") + cell("US4MD81M", "bbbb") + base := s.liveGenToken("noaa") + if base <= 0 { + t.Fatalf("token = %d, want positive", base) + } + if got := s.liveGenToken("noaa"); got != base { // deterministic + t.Errorf("token not stable: %d vs %d", got, base) + } + + cell("US4MD81M", "cccc") // a cell's content changed → token changes + if got := s.liveGenToken("noaa"); got == base { + t.Errorf("token unchanged after a cell's sha changed") + } + cell("US4MD81M", "bbbb") // restore → original token (content-addressed, history-independent) + if got := s.liveGenToken("noaa"); got != base { + t.Errorf("token not restored: %d vs %d", got, base) + } + + cell("US3MD01M", "dddd") // adding a cell changes it; removing restores + if got := s.liveGenToken("noaa"); got == base { + t.Errorf("token unchanged after adding a cell") + } + _ = os.Remove(filepath.Join(dir, "US3MD01M.pmtiles")) + _ = os.Remove(filepath.Join(dir, "US3MD01M.pmtiles.sha")) + if got := s.liveGenToken("noaa"); got != base { + t.Errorf("token not restored after removing the cell: %d vs %d", got, base) + } + + // The engine composes live tiles at serve time, so its commit is part of the + // content address: a rebuilt engine must move the token even over identical + // archives (serve-path fixes bust client caches), deterministically. + s.EngineCommit = "abc123" + withEngine := s.liveGenToken("noaa") + if withEngine == base { + t.Errorf("token unchanged after engine commit set") + } + if got := s.liveGenToken("noaa"); got != withEngine { + t.Errorf("engine-stamped token not stable: %d vs %d", got, withEngine) + } + s.EngineCommit = "def456" + if got := s.liveGenToken("noaa"); got == withEngine { + t.Errorf("token unchanged after engine commit changed") + } + s.EngineCommit = "" + + if got := (&Server{cacheDir: t.TempDir()}).liveGenToken("noaa"); got != 0 { // empty set → 0 + t.Errorf("empty token = %d, want 0", got) + } +} diff --git a/internal/engine/server/provider.go b/internal/engine/server/provider.go index 97259c4..5926f31 100644 --- a/internal/engine/server/provider.go +++ b/internal/engine/server/provider.go @@ -54,7 +54,7 @@ func (s *Server) providerDataDir(provider string) string { } // encRootDir is //ENC_ROOT/ — the bake input, walked recursively by -// tile57.BakeBundle for every *.000 across all installed district subfolders. It +// tile57.BakeTree for every *.000 across all installed district subfolders. It // lives under the DATA dir (persistent, survives a cache wipe: it is the downloads' // home + the bake input), so ClearCache never touches it. func (s *Server) encRootDir(provider string) string { @@ -111,15 +111,15 @@ func (s *Server) installedProviders() []string { } // districtCatFile is the per-district sidecar of parsed CATALOG.031 titles (the raw -// catalogue isn't kept at the ENC_ROOT root — that would flip BakeBundle into -// catalog-only mode — so titles are stashed here and re-gathered per provider bake). +// catalogue isn't kept at the ENC_ROOT root, so titles are stashed here and +// re-gathered per provider bake). const districtCatFile = "_catalog.json" // cacheDistrict writes one district's downloaded exchange-set content into its // ENC_ROOT subfolder: each cell FLAT as .000 (+ .001… updates), aux content // files (TXTDSC/PICREP) flat beside them, and a _catalog.json of parsed titles. The // bake reads the whole provider ENC_ROOT from here (no temp staging); the flat layout -// is transparent to BakeBundle's recursive *.000 walk. Best-effort; errors logged. +// is transparent to BakeTree's recursive *.000 walk. Best-effort; errors logged. func (s *Server) cacheDistrict(provider, district string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry) { dir := s.districtDir(provider, district) if err := os.MkdirAll(dir, 0o755); err != nil { @@ -174,54 +174,6 @@ func (s *Server) cacheDistrict(provider, district string, cells map[string]baker } } -// providerCellData reads every base cell (+ its .001… updates) under a provider's -// ENC_ROOT, DE-DUPLICATED by stem (a boundary cell shared by two districts is read -// once), for the bake's metadata sidecar + cell manifest. The bake itself reads the -// tree directly (BakeBundle); this is only the in-memory view the register tail needs. -func (s *Server) providerCellData(provider string) map[string]baker.CellData { - root := s.encRootDir(provider) - cells := map[string]baker.CellData{} // stem+".000" → base - updates := map[string]map[string][]byte{} // stem → update-name → bytes - _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - ext := encExtServer(d.Name()) - if ext == "" { - return nil - } - stem := strings.TrimSuffix(d.Name(), filepath.Ext(d.Name())) - if !isCellName(stem) { - return nil - } - if ext == ".000" { - if _, seen := cells[stem+".000"]; seen { - return nil // dedup: first district's copy wins - } - if b, e := os.ReadFile(path); e == nil { - cells[stem+".000"] = baker.CellData{Base: b} - } - return nil - } - if updates[stem] == nil { - updates[stem] = map[string][]byte{} - } - if _, seen := updates[stem][d.Name()]; !seen { - if b, e := os.ReadFile(path); e == nil { - updates[stem][d.Name()] = b - } - } - return nil - }) - for name, cd := range cells { - if u := updates[strings.TrimSuffix(name, ".000")]; len(u) > 0 { - cd.Updates = u - cells[name] = cd - } - } - return cells -} - // providerAux gathers the aux content files (TXTDSC/PICREP text + pictures) across a // provider's ENC_ROOT, de-duplicated by upper-cased basename, for the provider's one // companion aux.zip. diff --git a/internal/engine/server/security_test.go b/internal/engine/server/security_test.go index 3b0f6b9..442b5ae 100644 --- a/internal/engine/server/security_test.go +++ b/internal/engine/server/security_test.go @@ -63,7 +63,7 @@ func TestCrossSiteWrite(t *testing.T) { // A cross-site POST to a state-changing endpoint must be rejected before any // handler runs, and security headers must be present on every response. func TestHandlerBlocksCrossSiteAndSetsHeaders(t *testing.T) { - s := New("", t.TempDir(), t.TempDir(), false) + s := New("", t.TempDir(), t.TempDir(), false, "") defer s.Close() r := httptest.NewRequest("POST", "http://127.0.0.1:8080/api/settings", nil) @@ -85,7 +85,7 @@ func TestHandlerBlocksCrossSiteAndSetsHeaders(t *testing.T) { // The proxy must refuse a non-provider URL (SSRF). func TestProxyRejectsNonProviderURL(t *testing.T) { - s := New("", t.TempDir(), t.TempDir(), false) + s := New("", t.TempDir(), t.TempDir(), false, "") defer s.Close() r := httptest.NewRequest("GET", "http://127.0.0.1:8080/api/proxy?url=http://169.254.169.254/", nil) diff --git a/internal/engine/server/server_test.go b/internal/engine/server/server_test.go index b43c4e1..ff1d91b 100644 --- a/internal/engine/server/server_test.go +++ b/internal/engine/server/server_test.go @@ -19,7 +19,7 @@ func TestServeStaticAndRange(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "data.pmtiles"), body, 0o644); err != nil { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Root serves index.html. @@ -62,7 +62,7 @@ func TestServeCell(t *testing.T) { if err := os.WriteFile(cp, cell, 0o644); err != nil { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/api/cell/US5MD1MC") @@ -87,7 +87,7 @@ func TestServeCell(t *testing.T) { func TestAPIHealthAndHostCheck(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) // loopback-only + ts := httptest.NewServer(New(dir, dir, dir, false, "")) // loopback-only defer ts.Close() resp, _ := http.Get(ts.URL + "/api/health") @@ -115,7 +115,7 @@ func TestAPIHealthAndHostCheck(t *testing.T) { // disk (share.json persistence). func TestShareAndUpload(t *testing.T) { dir := t.TempDir() - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // No snapshot yet → 404. @@ -166,7 +166,7 @@ func TestShareAndUpload(t *testing.T) { } // A fresh Server over the same cache dir reloads the snapshot from share.json. - ts2 := httptest.NewServer(New(dir, dir, dir, false)) + ts2 := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts2.Close() resp, _ = http.Get(ts2.URL + "/api/share") got, _ = io.ReadAll(resp.Body) diff --git a/internal/engine/server/tile57_bake.go b/internal/engine/server/tile57_bake.go index 37b2069..d178c0d 100644 --- a/internal/engine/server/tile57_bake.go +++ b/internal/engine/server/tile57_bake.go @@ -13,58 +13,43 @@ import ( tile57 "github.com/beetlebugorg/tile57/bindings/go" ) -// bakeProgress returns a per-band progress callback for a provider bake. BakeBundle -// fires stage-0 (portraying a band's cells) then stage-1 (writing that band's tiles) -// events per navigational-purpose band. A calm per-band bar: it fills 0→100% per band -// and resets at each band — clamped monotonic WITHIN a band (the engine double-counts a -// band's tiles across its parallel-gen + serial-write phases, which would otherwise -// rewind). The multi-district DOWNLOAD phase sets pack "N of M" on the job separately. -func (s *Server) bakeProgress(jobID string) func(tile57.BakeProgress) { - curBand, bandDoneMax := -1, 0 - return func(p tile57.BakeProgress) { - s.imports.update(jobID, func(j *importJob) { - j.Phase, j.Band = "bake", p.BandName - if p.Stage == 0 { // portraying this band's cells - j.Unit, j.Note, j.Done, j.Total = "cells", "Preparing charts", p.Done, p.Total - return - } - if p.BandIndex != curBand { // new band → reset the within-band floor - curBand, bandDoneMax = p.BandIndex, 0 - } - if p.Done > bandDoneMax { - bandDoneMax = p.Done - } - j.Unit, j.Note, j.Done, j.Total = "tiles", "Generating tiles", bandDoneMax, p.Total - }) - } -} - -// registerBakedSet registers a freshly-baked PROVIDER's chart.pmtiles as the live tile -// set and writes its tail — bake stamps, cell manifest, companion aux.zip, metadata -// sidecar — so a tile57 provider is as complete in the chart library as any pack. -// Returns false (recording a job error) if the bundle can't be opened. `set` is the -// provider name (one archive per provider). -func (s *Server) registerBakedSet(jobID, set string, cells map[string]baker.CellData, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { +// registerProviderSet registers `src` as provider `set`'s live tile set and writes its tail (bake +// stamps, cell manifest, companion aux, metadata sidecar) so the provider is as complete in the +// chart library as any pack. `packPath` is the on-disk chart.pmtiles of a BATCH-baked provider +// (recorded for the pack list + stamped with the bake/engine version), or "" for a LIVE +// runtime-compositor provider (no disk archive — its bounds come from the TileSource Meta/TileJSON, +// and it is surfaced in /api/packs straight from the registry). +func (s *Server) registerProviderSet(jobID, set string, src tilesource.TileSource, packPath string, aux map[string][]byte, cat []tile57.CatalogEntry, created string) bool { outDir := s.setDir(set) - chart := filepath.Join(outDir, "tiles", "chart.pmtiles") - src, err := tilesource.Open(chart) - if err != nil { - log.Printf("import %s (%s): open baked bundle: %v", jobID, set, err) - return false - } s.sets.register(set, src) - s.packAdd(set, chart) s.prefs.setDisabled(set, false) - if s.Version != "" { - _ = os.WriteFile(chart+bakeVerExt, []byte(s.Version), 0o644) + if packPath != "" { + s.packAdd(set, packPath) + if s.Version != "" { + _ = os.WriteFile(packPath+bakeVerExt, []byte(s.Version), 0o644) + } + // Bake-time engine stamp (.enginever): the tile57 commit THIS binary links, + // recorded beside the pack so the set's TileJSON can report which engine baked + // these tiles even after the binary is upgraded. Best-effort, like .bakever. + if s.EngineCommit != "" { + _ = os.WriteFile(packPath+engineVerExt, []byte(s.EngineCommit), 0o644) + } + } else { + // Live compositor: advance the generation token so the client re-fetches with a fresh ?g, + // invalidating any tiles (incl. blank 204s) it cached against a previous, less-complete set. + s.bumpLiveGen(set) } - // Bake-time engine stamp (.enginever): the tile57 commit THIS binary links, - // recorded beside the pack so the set's TileJSON can report which engine baked - // these tiles even after the binary is upgraded. Best-effort, like .bakever. - if s.EngineCommit != "" { - _ = os.WriteFile(chart+engineVerExt, []byte(s.EngineCommit), 0o644) + // Per-cell metadata straight from the provider's on-disk ENC tree — the whole + // point of the path-based read is that a catalogue-sized import never holds + // cell bytes in memory (the previous in-RAM staging peaked in the gigabytes). + cellMeta := baker.ExtractCellMetaDir(s.encRootDir(set), func(name string, e error) { + log.Printf("import %s: meta skip %s: %v", jobID, name, e) + }) + stems := make([]string, 0, len(cellMeta)) + for n := range cellMeta { + stems = append(stems, n) } - if err := s.writeSetCells(set, cells); err != nil { + if err := s.writeSetCells(set, stems); err != nil { log.Printf("import %s: cell manifest %q: %v", jobID, set, err) } @@ -81,9 +66,6 @@ func (s *Server) registerBakedSet(jobID, set string, cells map[string]baker.Cell // Per-pack metadata sidecar for the chart library (per-cell scale/edition/date/ // agency/coverage + catalogue titles). - cellMeta := baker.ExtractCellMeta(cells, func(name string, e error) { - log.Printf("import %s: meta skip %s: %v", jobID, name, e) - }) meta := buildSetMeta(set, cellMeta, cat) meta.Imported = created if err := s.writeSetMeta(set, meta); err != nil { @@ -122,25 +104,32 @@ func (s *Server) bakeProvider(jobID, provider string) bool { j.Phase, j.Band, j.Unit, j.Note, j.Done, j.Total = "bake", "", "cells", "Preparing charts", 0, 0 }) created := time.Now().UTC().Format(time.RFC3339) - // MaxZoom 24 = the ABI's "no clamp" (each cell's full native band); MaxZoom 0 would - // clamp every band down to z0 — an EMPTY archive. - n, bbox, err := tile57.BakeBundle(encRoot, outDir, tile57.BakeOpts{Created: created, MaxZoom: 24}, s.bakeProgress(jobID)) + + // Live runtime compositor — the ONLY tile-production path. Bake each cell to its own + // native-scale PMTiles (KEPT under /tiles), build + save the ownership-partition + // sidecar, and register a Composer that composes tiles on demand. Adding a district re-bakes + // only its new cells. There is no district compose pass and no chart.pmtiles bundle; the + // MapLibre style + sprite/glyph/colortable assets are served at runtime (tile57.Style + the + // asset endpoints), not emitted per provider. + n, liveSrc, err := s.prepareLiveProvider(jobID, encRoot, provider) if err != nil { return fail(err) } - // An inverted/empty bbox (or zero cells) means nothing valid parsed. Treat it as a - // failed import (don't register an empty pack) and drop the stub bundle it wrote. - if n == 0 || bbox[2] <= bbox[0] || bbox[3] <= bbox[1] { - os.RemoveAll(outDir) - return fail(fmt.Errorf("import produced no coverage (%d cell(s), no valid S-57 data)", n)) + // Zero cells means nothing valid parsed → a failed import (don't register an empty set). + // prepareLiveProvider already cleaned whichever tree it baked (a fresh dir, or a staged + // engine re-bake that never replaced the served tiles); the set dir itself survives — in + // single-dir mode it also holds the source ENC_ROOT. + if n == 0 { + return fail(fmt.Errorf("import produced no coverage (no valid S-57 data)")) } - s.imports.update(jobID, func(j *importJob) { j.Phase, j.Note = "meta", "Reading chart metadata" }) - cells := s.providerCellData(provider) + s.imports.update(jobID, func(j *importJob) { + j.Phase, j.Note, j.Zoom, j.Unit, j.Done, j.Total, j.ETA = "meta", "Reading chart metadata", 0, "", 0, 0, 0 + }) aux := s.providerAux(provider) cat := s.providerCatalog(provider) - if !s.registerBakedSet(jobID, provider, cells, aux, cat, created) { - return fail(fmt.Errorf("could not register baked bundle for %q", provider)) + if !s.registerProviderSet(jobID, provider, liveSrc, "", aux, cat, created) { + return fail(fmt.Errorf("could not register set for %q", provider)) } s.imports.update(jobID, func(j *importJob) { j.Cells = n }) log.Printf("import %s: baked provider %q (%d cell(s)) → %s", jobID, provider, n, outDir) diff --git a/internal/engine/server/tileserve.go b/internal/engine/server/tileserve.go index 4b0c171..f4c973a 100644 --- a/internal/engine/server/tileserve.go +++ b/internal/engine/server/tileserve.go @@ -3,6 +3,7 @@ package server import ( "compress/gzip" "fmt" + "log" "net/http" "os" "strconv" @@ -60,27 +61,42 @@ func (s *Server) serveTileSet(w http.ResponseWriter, r *http.Request) { apiErr(w, http.StatusNotFound, "unknown tile set") return } - body, err := src.Tile(uint8(z), x, y) + // Fetch the tile, plus — where the backend can tell us (the runtime compositor) — whether the + // ownership partition says a cell SHOULD render here. `owned` lets us tell a transient/erroneous + // empty (a cell owns this ground but produced nothing) from true empty ocean. + var body []byte + var owned bool + var err error + if ot, ok := src.(tilesource.OwnershipTiler); ok { + body, owned, err = ot.TileOwned(uint8(z), x, y) + } else { + body, err = src.Tile(uint8(z), x, y) + } if err != nil { apiErr(w, http.StatusInternalServerError, err.Error()) return } - // Cache policy first, so it rides the 204 too: an EMPTY tile at a given ?g is - // as content-addressed as a full one (emptiness is baked into that generation), - // and empty ocean tiles are the MAJORITY of a viewport's grid — caching those - // 204s saves the most round-trips. Tiles are immutable per bake generation: the - // ?g token in the URL (the pack archive's mtime) changes on every re-bake, so a - // given tile URL always maps to identical bytes/emptiness and can cache forever. - // The live/dynamic set carries no generation (?g absent or 0) and regenerates on - // demand, so it stays no-cache. Keying off the token — not pack-vs-live plumbing - // — ties the policy exactly to the content-addressing guarantee. - if g := r.URL.Query().Get("g"); g != "" && g != "0" { + + // Cache policy (rides the 204 too). A tile is immutable-per-generation ONLY when it is stable: + // it has content, or it is truly empty (no cell owns this ground — open ocean, the majority of a + // viewport). Two cases must NEVER cache, so they re-fetch once content lands (the ?g bumps on + // bake completion): the set is still baking (its content is in flux), or the partition says a + // cell SHOULD render here but it came back blank (its per-cell bake hasn't produced the tile + // yet). Emptiness at a given ?g is otherwise as content-addressed as bytes are. + blank := len(body) == 0 + generating := s.imports.runningFor(set) + shouldFill := blank && owned // expected content, not here yet + if g := r.URL.Query().Get("g"); g != "" && g != "0" && !generating && !shouldFill { w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") } else { w.Header().Set("Cache-Control", "no-cache") } - if len(body) == 0 { - w.WriteHeader(http.StatusNoContent) // blank/missing tile (still cacheable per ?g) + if shouldFill && !generating { + // Bakes are done, yet a tile a cell owns rendered nothing — a missing or failed cell. + log.Printf("tiles %s: z%d/%d/%d owned but empty (missing or failed cell?)", set, z, x, y) + } + if blank { + w.WriteHeader(http.StatusNoContent) // blank/missing tile return } // Tiles serve BYTES-VERBATIM in the set's stored encoding (no transcode). diff --git a/internal/engine/server/tileserve_test.go b/internal/engine/server/tileserve_test.go index fe11cb5..937e537 100644 --- a/internal/engine/server/tileserve_test.go +++ b/internal/engine/server/tileserve_test.go @@ -36,7 +36,7 @@ func TestServeTileSet(t *testing.T) { dir := t.TempDir() body := writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // A present tile → 200 with the MVT body and the vector-tile content type. @@ -120,7 +120,7 @@ func TestServeTileSet(t *testing.T) { func TestServeTileGzip(t *testing.T) { dir := t.TempDir() body := writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // Go's transport transparently decodes gzip unless we set Accept-Encoding @@ -148,7 +148,7 @@ func TestServeTileGzip(t *testing.T) { func TestServeTileJSONAndList(t *testing.T) { dir := t.TempDir() writeTestPMTiles(t, dir, "charts", 8, 10, 20) - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() // TileJSON descriptor. @@ -205,7 +205,7 @@ func TestServeTileJSONEngineStamp(t *testing.T) { t.Fatal(err) } - srv := New(dir, dir, dir, false) + srv := New(dir, dir, dir, false, "") srv.EngineCommit = "fff999000" // the RUNNING binary's engine // A dynamic set: registered without a pack path (plugin tiles). live, err := tilesource.Open(filepath.Join(tilesDir(dir), "legacy.pmtiles")) @@ -263,7 +263,7 @@ func TestServeTileSetMLT(t *testing.T) { t.Fatal(err) } - ts := httptest.NewServer(New(dir, dir, dir, false)) + ts := httptest.NewServer(New(dir, dir, dir, false, "")) defer ts.Close() resp, _ := http.Get(ts.URL + "/tiles/mltcharts.json") diff --git a/internal/engine/server/tilesets.go b/internal/engine/server/tilesets.go index 0f49c23..9b9ff5f 100644 --- a/internal/engine/server/tilesets.go +++ b/internal/engine/server/tilesets.go @@ -176,6 +176,12 @@ func (s *Server) handlePacks(w http.ResponseWriter, r *http.Request) { for _, prov := range s.installedProviders() { seen[prov] = true } + // Runtime-registered sets with no disk pack (e.g. the live runtime compositor) — so a set + // that only lives in the registry still surfaces as a first-class, toggleable map layer. + // Its bounds/metadata come from the TileJSON (src.Meta()), not a disk pack. + for _, name := range s.sets.names() { + seen[providerOf(name)] = true + } providers := make([]string, 0, len(seen)) for p := range seen { providers = append(providers, p) @@ -189,18 +195,26 @@ func (s *Server) handlePacks(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, ",") } fmt.Fprintf(w, `{"name":%q,"enabled":%t`, prov, !s.prefs.isDisabled(prov)) - // Bounds straight from the baked archive on disk — works even while DISABLED (its - // tiles aren't served, but the boundary still marks "you have this chart here"). + meta, hasMeta := s.readSetMeta(prov) + // Coverage bounds, so the client marks "you have this chart here" even while a set is + // DISABLED. A standalone archive reads them off the file; a live runtime-compositor + // provider (no disk archive) takes them from the registry when enabled, else from its + // meta sidecar — surfaced first-class from provider structure, not a pack path. if path, ok := s.packPath(prov); ok { if src, err := tilesource.Open(path); err == nil { m := src.Meta() _ = tilesource.Close(src) fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, m.W, m.S, m.E, m.N) } + } else if src, live := s.sets.get(prov); live { + m := src.Meta() + fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, m.W, m.S, m.E, m.N) + } else if hasMeta && len(meta.BBox) == 4 { + fmt.Fprintf(w, `,"bounds":[%g,%g,%g,%g]`, meta.BBox[0], meta.BBox[1], meta.BBox[2], meta.BBox[3]) } // Extracted metadata (title/agency/scale range/counts/imported date), from the // .meta.json sidecar. Cells omitted here — GET /api/pack/. - if m, ok := s.readSetMeta(prov); ok { + if m := meta; hasMeta { if m.Title != "" { fmt.Fprintf(w, `,"title":%q`, m.Title) } @@ -267,14 +281,22 @@ func (s *Server) handleSetEnabled(w http.ResponseWriter, r *http.Request) { enable := strings.HasSuffix(r.URL.Path, "/enable") s.prefs.setDisabled(provider, !enable) if enable { - if path, ok := s.packPath(provider); ok { - if _, live := s.sets.get(provider); !live { + if _, live := s.sets.get(provider); !live { + if path, ok := s.packPath(provider); ok { + // Standalone/overlay archive: re-open the disk pack. if src, err := tilesource.Open(path); err == nil { s.sets.register(provider, src) } else { apiErr(w, http.StatusInternalServerError, err.Error()) return } + } else if c, err := s.openLiveComposer(provider); err != nil { + apiErr(w, http.StatusInternalServerError, err.Error()) + return + } else if c != nil { + // Live runtime-compositor provider (no disk pack): re-open the compositor + // from its kept per-cell archives + partition sidecar. + s.sets.register(provider, c) } } } else { @@ -352,6 +374,22 @@ func (s *Server) enabledPackCells() (map[string]bool, [][4]float64) { legacy = append(legacy, [4]float64{m.W, m.S, m.E, m.N}) } } + // Live runtime-compositor providers (no disk pack): their cells come from the SAME + // per-provider cell manifest (writeSetCells at import), keyed provider-centrically — + // so a live provider's charts count as "on the map" for ?active just like a pack's. + for _, prov := range s.installedProviders() { + if s.prefs.isDisabled(prov) { + continue + } + if _, isPack := s.packPath(prov); isPack { + continue // already covered above + } + if stems, ok := s.setCells(prov); ok { + for _, st := range stems { + cells[st] = true + } + } + } return cells, legacy } diff --git a/internal/engine/tilesource/composer.go b/internal/engine/tilesource/composer.go new file mode 100644 index 0000000..797ad6c --- /dev/null +++ b/internal/engine/tilesource/composer.go @@ -0,0 +1,70 @@ +package tilesource + +import ( + tile57 "github.com/beetlebugorg/tile57/bindings/go" +) + +// Composer is a TileSource backed by the tile57 runtime compositor: the per-cell PMTiles +// held mmap'd and the ownership partition resident, composing each tile on demand. It is the +// live counterpart of a prebaked .pmtiles archive — no district bake; tiles are built as the +// camera asks (a classify + one decode/clip or a decompress per tile). Serve is serialised +// inside tile57.ComposeSource, so this satisfies TileSource with no extra locking. +type Composer struct { + src *tile57.ComposeSource + meta TileMeta +} + +// NewComposer opens a runtime compositor over the per-cell PMTiles at paths (each from +// `tile57 compose --keep-cells` / tile57.BakeChart). partitionPath (or "") names a partition +// sidecar (`tile57 compose --save-partition`) to load and skip the owned-face build. Close it +// when done — callers must not Close while any request can still call Tile. +func NewComposer(paths []string, partitionPath string) (*Composer, error) { + src, err := tile57.OpenCompose(paths, partitionPath) + if err != nil { + return nil, err + } + m := src.Meta() + return &Composer{ + src: src, + meta: TileMeta{ + MinZoom: m.MinZoom, + MaxZoom: m.MaxZoom, + W: m.West, + S: m.South, + E: m.East, + N: m.North, + Gzipped: false, // Serve returns decompressed MLT; the HTTP layer gzips on the wire + TileType: "mlt", + }, + }, nil +} + +// OwnershipTiler is a TileSource that also reports tile OWNERSHIP: whether its data model says a +// cell SHOULD render at (z,x,y). A blank (nil body) from an OWNED tile is transient (a cell's bake +// is still running) or an error (bake done) — the HTTP layer must not cache it, so it re-fetches +// once content lands; a blank from an UNOWNED tile is true empty ocean (safe to cache). The runtime +// Composer implements this; prebaked archives do not (their blanks are always true empty). +type OwnershipTiler interface { + TileOwned(z uint8, x, y uint32) (body []byte, owned bool, err error) +} + +// Tile composes (z,x,y) on demand, returning decompressed MLT, or (nil, nil) for a blank tile. +func (c *Composer) Tile(z uint8, x, y uint32) ([]byte, error) { + body, _, err := c.src.Tile(z, x, y) + return body, err +} + +// TileOwned is Tile plus the ownership flag (implements OwnershipTiler). +func (c *Composer) TileOwned(z uint8, x, y uint32) (body []byte, owned bool, err error) { + return c.src.Tile(z, x, y) +} + +// Meta returns the compositor's display metadata (zoom range + coverage bounds). +func (c *Composer) Meta() TileMeta { return c.meta } + +// SavePartition persists the resident ownership partition to path, so a later NewComposer can load +// it (as partitionPath) and skip the owned-face build. +func (c *Composer) SavePartition(path string) error { return c.src.SavePartition(path) } + +// Close releases the compositor (io.Closer, so tilesource.Close finds it). +func (c *Composer) Close() error { return c.src.Close() } diff --git a/tile57 b/tile57 index a2022b6..4b6d73c 160000 --- a/tile57 +++ b/tile57 @@ -1 +1 @@ -Subproject commit a2022b6758dc26258bc83fc9a4b3c22beb5eba75 +Subproject commit 4b6d73c5ae23955156a9799d62f3ef6abea4ce69 diff --git a/web/src/chartplotter.mjs b/web/src/chartplotter.mjs index 0421001..8779424 100644 --- a/web/src/chartplotter.mjs +++ b/web/src/chartplotter.mjs @@ -1010,7 +1010,7 @@ export class ChartPlotter extends HTMLElement { if (!j || j.state !== "running" || !j.id) return; this._task = { kind: "download", status: "running" }; const name = this._reattachName(j.set); - this._setProgress({ label: "Working", pill: `Working on ${name}`, sub: "", frac: j.percent ? j.percent / 100 : null }); + this._setProgress({ label: "Working", pill: `Working on ${name}`, sub: j.action ? `${j.action}…` : "", frac: (j.frac === null || j.frac === undefined) ? null : j.frac }); this._pollImport(j.id, (p) => this._setProgress(p), name).catch(() => {}).then(async () => { this._task = null; this._setProgress(null); @@ -1875,9 +1875,11 @@ export class ChartPlotter extends HTMLElement { const title = p.label || p.pill || ""; const action = p.error ? String(p.error) : (p.sub || ""); const count = p.error ? "" : (p.detail || ""); + const eta = p.error ? "" : (p.eta || ""); r.getElementById("db-prog-title").textContent = title; r.getElementById("db-prog-action").textContent = action; r.getElementById("db-prog-count").textContent = count; + r.getElementById("db-prog-eta").textContent = eta; const fill = r.getElementById("db-prog-fill"); fill.style.width = p.frac != null ? `${Math.round(p.frac * 100)}%` : "100%"; fill.classList.toggle("indet", p.frac == null && !done); // slow sweep when no fraction diff --git a/web/src/chartplotter.view.mjs b/web/src/chartplotter.view.mjs index daa1cd1..e11458d 100644 --- a/web/src/chartplotter.view.mjs +++ b/web/src/chartplotter.view.mjs @@ -462,7 +462,11 @@ export const STYLE = ` .db-prog-status { display:flex; align-items:baseline; gap:10px; font:500 11.5px/1.3 system-ui,sans-serif; color:var(--ui-text-dim); } .db-prog-action { flex:1; min-width:0; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } - .db-prog-count { flex:none; font-variant-numeric:tabular-nums; } + /* Count + ETA are fixed, right-aligned tabular slots so the text stays put as the numbers + change width (the ETA reserves a min-width; both right-align so their right edge is pinned). */ + .db-prog-count { flex:none; text-align:right; font-variant-numeric:tabular-nums; } + .db-prog-eta { flex:none; min-width:7em; text-align:right; font-variant-numeric:tabular-nums; } + .db-prog-eta:empty { display:none; } .db-prog-status:empty { display:none; } .db-prog.error .db-prog-title, .db-prog.error .db-prog-action { color:#c0392b; } .db-prog-track { position:relative; width:100%; height:6px; border-radius:3px; overflow:hidden; background:var(--ui-surface-2); } @@ -611,6 +615,7 @@ export const CHROME = `
+
diff --git a/web/src/data/chart-service.mjs b/web/src/data/chart-service.mjs index 87cc28a..7aaca40 100644 --- a/web/src/data/chart-service.mjs +++ b/web/src/data/chart-service.mjs @@ -16,26 +16,6 @@ // filenames. Methods throw on hard failure; pollJob resolves with the final // status object or rejects on error/timeout. -// Job phase → [verb, noun] for the progress label, e.g. bake → "Generating … tiles". -// The noun is what's being acted on at that phase (source charts while fetching / -// reading; tiles while baking; nothing while finishing). -const PHASE = { - download: ["Downloading", "charts"], fetch: ["Downloading", "charts"], dl: ["Downloading", "charts"], - upload: ["Uploading", "charts"], - extract: ["Extracting", "charts"], unzip: ["Extracting", "charts"], expand: ["Extracting", "charts"], - parse: ["Reading", "charts"], read: ["Reading", "charts"], import: ["Reading", "charts"], - bake: ["Generating", "tiles"], tiles: ["Generating", "tiles"], render: ["Generating", "tiles"], - register: ["Finishing", "up"], finalize: ["Finishing", "up"], index: ["Finishing", "up"], -}; - -// Bytes → compact "12 MB" / "1.4 KB" (local copy so this module is self-contained). -function fmtBytes(n) { - if (!n) return "0 B"; - const u = ["B", "KB", "MB", "GB"]; let i = 0; - while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; } - return `${n.toFixed(n < 10 && i > 0 ? 1 : 0)} ${u[i]}`; -} - export class ChartService { constructor({ assets = "" } = {}) { this._assets = assets; @@ -156,25 +136,21 @@ export class ChartService { // tiles", "12 MB / 45 MB"). The action carries the band but not the noun, so the // unit isn't repeated — "Generating coastal…" + "1,234 / 4,567 tiles". _formatStatus(s, name) { - const m = s.phase ? PHASE[s.phase] : null; - let verb = m ? m[0] : (s.phase ? s.phase[0].toUpperCase() + s.phase.slice(1) : "Working"); - // The bake phase has two visible stages the server distinguishes by unit: - // "cells" while a band's charts are parsed + portrayed (the long gap before - // any tile emits), then "tiles" while that band's tiles are generated. - if (m && m[1] === "tiles" && s.unit === "cells") verb = "Preparing"; - const sub = [verb, s.band].filter(Boolean).join(" ") + "…"; // "Preparing coastal…" - // Count, with the unit named in every stage so a bare number is never ambiguous. - let detail = ""; - if (s.unit === "bytes") detail = s.total ? `${fmtBytes(s.done)} / ${fmtBytes(s.total)}` : fmtBytes(s.done); - else if (s.total) { - const u = s.unit === "cells" ? "charts" : (s.unit || ""); - detail = `${s.done.toLocaleString()} / ${s.total.toLocaleString()} ${u}`.trim(); - } - const frac = s.total ? s.done / s.total : (s.percent ? s.percent / 100 : null); - // pack/packNum/packTotal identify WHICH pack a multi-pack batch is on right now (the - // caller resolves pack — a set key — to a friendly name), so progress reads - // "Mid-Atlantic (2 of 4) · Generating coastal…" instead of looping on one line. - return { label: name || "", sub, detail, frac, pack: s.pack || "", packNum: s.packNum || 0, packTotal: s.packTotal || 0 }; + // The SERVER owns the wording: `action` is the live step and `detail` the count/ETA beside + // it, both display-ready — the client renders them verbatim rather than re-deriving the + // status from the raw phase/unit/note fields. `frac` is the bar fill (0..1, or null → the + // shell sweeps an indeterminate bar). The client only supplies the region title (label) and + // the pack context (pack/packNum/packTotal, resolved to a friendly name by the caller), so + // progress reads "Mid-Atlantic (2 of 4) · Composing tiles…". + const frac = (s.frac === null || s.frac === undefined) ? null : s.frac; + return { + label: name || "", + sub: s.action ? `${s.action}…` : "", + detail: s.detail || "", + eta: s.eta || "", // kept separate from detail so each pins in its own fixed slot + frac, + pack: s.pack || "", packNum: s.packNum || 0, packTotal: s.packTotal || 0, + }; } // GET /api/packs — the installed-pack registry (single source of truth, incl. diff --git a/web/src/plugins/chart-library.mjs b/web/src/plugins/chart-library.mjs index 02e33a5..7576b69 100644 --- a/web/src/plugins/chart-library.mjs +++ b/web/src/plugins/chart-library.mjs @@ -738,7 +738,10 @@ export class ChartLibrary extends HTMLElement { if (name === "import") return "Imported charts"; const ie = /^ienc-(.+)$/.exec(name); if (ie) return `IENC · ${ie[1]}`; - return name; + // A bare provider set (the live-composite model registers ONE set per provider, + // e.g. "noaa") → its branded display name ("NOAA"), not the lowercase routing key. + const prov = this._providers().find((p) => p.id === this._providerKey(name)); + return prov ? prov.name : name; } _wirePacks() { this._wireMillerRows(); this._wireDetailButtons(); } @@ -906,10 +909,16 @@ export class ChartLibrary extends HTMLElement { return (pack ? `${pack}${count} · ` : "") + ((p && p.sub) || "Downloading…"); } + // "1,234 / 4,567 charts · ~2m left" — the count and (when present) the ETA, which the server + // reports separately so the card can pin each; the banner is one line, so re-join them here. + _batchDetail(p) { + return (p && p.detail ? p.detail : "") + (p && p.eta ? ` · ${p.eta}` : ""); + } + // The in-dialog banner descriptor for the packs header (bar + text). _batchBanner() { const p = this._batchStatus; - return { busy: true, sub: this._batchSub(p), detail: (p && p.detail) || "", frac: (p && p.frac) || 0 }; + return { busy: true, sub: this._batchSub(p), detail: this._batchDetail(p), frac: (p && p.frac) || 0 }; } // Update the banner in place from a formatted status (no re-render → the bar animates). @@ -921,7 +930,7 @@ export class ChartLibrary extends HTMLElement { const sub = r.querySelector(".dl-banner .dl-sub"); if (sub) sub.textContent = this._batchSub(p); const detail = r.querySelector(".dl-banner .dl-detail"); - if (detail) detail.textContent = p.detail || ""; + if (detail) detail.textContent = this._batchDetail(p); } // Find-a-chart search box. diff --git a/web/src/plugins/dev-tools.mjs b/web/src/plugins/dev-tools.mjs index 66d0aad..bbcb01a 100644 --- a/web/src/plugins/dev-tools.mjs +++ b/web/src/plugins/dev-tools.mjs @@ -91,6 +91,10 @@ export class DevTools { this._inspectLastKey = ""; this._areaCleanup = null; // mirrors the shell's box-download guard (kept null here) this._busy = false; // a rebake is running (in addition to the injected isBusy) + // Ownership-partition debug overlay state. + this._partitions = null; // [{provider, ready, tiles}] from GET /api/debug/partition (null = not loaded) + this._partOn = new Set(); // providers whose partition overlay is currently on the map + this._partGen = false; // a generate run is in flight this._registerSelf(deps.registry); this._wireMap(); @@ -128,12 +132,16 @@ export class DevTools { const busy = this._isBusy(); // The dialog's shadow has its own sheet; inject our chrome once per render so // the dev-tools + inspector classes resolve inside it. - host.innerHTML = `${devToolsPanel(busy, this._inspectMode, this._selectingArea)}`; + const part = { list: this._partitions, on: this._partOn, gen: this._partGen }; + host.innerHTML = `${devToolsPanel(busy, this._inspectMode, this._selectingArea, part)}`; const q = (id) => host.querySelector("#" + id); const rebuild = q("dev-rebuild"); if (rebuild && !rebuild.disabled) rebuild.onclick = (e) => this._rebuildAllPerBand(e.currentTarget); const inspect = q("dev-inspect"); if (inspect) inspect.onclick = () => this.setInspectMode(!this._inspectMode); const area = q("dev-area"); if (area && !area.disabled) area.onclick = () => this._toggleSelectArea(); const feat = q("dev-feat"); if (feat && !feat.disabled) feat.onclick = (e) => this._copyInspectDebug(e.currentTarget); + const pgen = q("dev-part-gen"); if (pgen && !pgen.disabled) pgen.onclick = () => this._generatePartitions(); + host.querySelectorAll("[data-part-toggle]").forEach((el) => (el.onclick = () => this._togglePartitionOverlay(el.dataset.partToggle, !this._partOn.has(el.dataset.partToggle)))); + if (this._partitions == null && !this._partLoading) this._loadPartitions(); // lazy: fetch status once // If inspect is on with a live selection, repaint the result panel. if (this._inspectMode && this._inspectFeats.length) this._renderInspect(); else if (this._inspectMode) this._inspectHint(INSPECT_HINT); @@ -195,6 +203,82 @@ export class DevTools { if (btn) flashBtn(btn, `✓ rebuilt ${todo.length}`); } + // --- ownership-partition debug overlay ----------------------------------- + // GET the per-provider partition status (is each provider's partition tile set baked + // and ready to overlay?) and repaint the panel. + async _loadPartitions() { + this._partLoading = true; + const assets = this._d.assets || ""; + try { + const j = await fetch(`${assets}api/debug/partition?t=${Date.now()}`).then((r) => (r.ok ? r.json() : null)); + this._partitions = (j && j.partitions) || []; + } catch (e) { this._partitions = []; } + this._partLoading = false; + this._refreshPanel(); + } + + // POST to (re)generate every provider's partition PMTiles on the server, then poll the + // status until they're all ready (or give up), repainting as each lands. + async _generatePartitions() { + if (this._partGen) return; + const assets = this._d.assets || ""; + this._partGen = true; + this._refreshPanel(); + try { await fetch(`${assets}api/debug/partition`, { method: "POST" }); } catch (e) { console.warn("[partition] generate", e); } + for (let i = 0; i < 90; i++) { + await new Promise((r) => setTimeout(r, 2000)); + await this._loadPartitions(); + const ps = this._partitions || []; + if (ps.length && ps.every((p) => p.ready)) break; + } + this._partGen = false; + this._refreshPanel(); + } + + // Show/hide a provider's partition overlay on the chart map. + _togglePartitionOverlay(provider, on) { + if (!provider) return; + if (on) { this._partOn.add(provider); this._addPartitionLayers(provider); } + else { this._partOn.delete(provider); this._removePartitionLayers(provider); } + this._refreshPanel(); + } + + _partSourceId(provider) { return `dbg-part-${provider}`; } + + // Add the vector source + fill (by face colour) + symbol (cell name) layers for a + // provider's partition, pointing at the server's /tiles/{provider}-partition set. + _addPartitionLayers(provider) { + const map = this._map; + if (!map || !map.isStyleLoaded || !map.isStyleLoaded()) return; + this._removePartitionLayers(provider); // idempotent re-add + const sid = this._partSourceId(provider); + const base = new URL(`${this._d.assets || ""}tiles/${provider}-partition/`, document.baseURI).href; + try { + map.addSource(sid, { type: "vector", tiles: [base + "{z}/{x}/{y}.mvt"], minzoom: 0, maxzoom: 12 }); + map.addLayer({ + id: sid + "-fill", type: "fill", source: sid, "source-layer": "partition", + paint: { "fill-color": ["get", "color"], "fill-opacity": 0.35, "fill-outline-color": "#000000" }, + }); + map.addLayer({ + id: sid + "-label", type: "symbol", source: sid, "source-layer": "labels", + layout: { "text-field": ["get", "cell"], "text-font": ["Noto Sans Regular"], "text-size": 11, "text-allow-overlap": false }, + paint: { "text-color": "#ffffff", "text-halo-color": "#000000", "text-halo-width": 1.4 }, + }); + } catch (e) { console.warn("[partition] add layers", provider, e); } + } + + _removePartitionLayers(provider) { + const map = this._map; + if (!map) return; + const sid = this._partSourceId(provider); + for (const id of [sid + "-fill", sid + "-label"]) { if (map.getLayer && map.getLayer(id)) map.removeLayer(id); } + if (map.getSource && map.getSource(sid)) map.removeSource(sid); + } + + // The chart style is rebuilt on mariner changes (setStyle drops every source); re-add + // any active partition overlays once the new style has loaded. + _reapplyPartitions() { for (const p of this._partOn) this._addPartitionLayers(p); } + // --- feature inspector --------------------------------------------------- // Add the inspector's map listeners ONCE. They all no-op unless inspect mode is // on (so the shell's own click handler need only defer to `inspecting`). The @@ -273,6 +357,9 @@ export class DevTools { c.addEventListener("pointerup", this._onPointerUp); c.addEventListener("pointercancel", this._onPointerUp); map.on("click", this._onClick); + // A mariner change rebuilds the style (setStyle), dropping our overlay sources — + // re-add any that are toggled on once the new style loads. + map.on("style.load", this._onStyleLoad = () => this._reapplyPartitions()); } _unwireMap() { @@ -285,6 +372,8 @@ export class DevTools { if (this._onPointerUp) { c.removeEventListener("pointerup", this._onPointerUp); c.removeEventListener("pointercancel", this._onPointerUp); } } if (this._onClick) map.off("click", this._onClick); + if (this._onStyleLoad) { map.off("style.load", this._onStyleLoad); this._onStyleLoad = null; } + try { this._partOn.forEach((p) => this._removePartitionLayers(p)); } catch (e) { /* map gone */ } if (this._hoverRaf) { cancelAnimationFrame(this._hoverRaf); this._hoverRaf = 0; } } diff --git a/web/src/plugins/dev-tools.view.mjs b/web/src/plugins/dev-tools.view.mjs index 207b762..412dc5a 100644 --- a/web/src/plugins/dev-tools.view.mjs +++ b/web/src/plugins/dev-tools.view.mjs @@ -91,10 +91,36 @@ export function inspectorSection(inspecting, selectingArea) { `; } -// The whole dev-tools panel skeleton: rebake + inspector sections + the +// The ownership-partition debug section: a "Generate" button (the server bakes one +// partition PMTiles per provider) + a per-provider "Show/Hide overlay" toggle. `part` +// = { list, on, gen }: list = [{provider, ready, tiles}] (null while loading), on = the +// Set of providers whose overlay is shown, gen = a generate run is in flight. +export function partitionSection(part, busy) { + const list = part && part.list; + const on = (part && part.on) || new Set(); + const gen = part && part.gen; + let rows; + if (list == null) rows = `

Loading…

`; + else if (!list.length) rows = `

No providers installed.

`; + else rows = list.map((p) => { + const shown = on.has(p.provider); + const ctrl = p.ready + ? `` + : `generating…`; + return `
${esc(p.provider)}${ctrl}
`; + }).join(""); + return `
+
Ownership partition (debug)
+ + ${rows} +

A PMTiles per provider showing which cell owns each region per band (finer cells win; coarser fill the gaps). Enable a provider's overlay to see the composite quilt on the chart — each face is coloured and labelled with its cell.

+
`; +} + +// The whole dev-tools panel skeleton: rebake + partition + inspector sections + the // inspect-result container (filled separately by the logic on hover/click). -export function devToolsPanel(busy, inspecting, selectingArea) { - return `
${rebakeSection(busy)}${inspectorSection(inspecting, selectingArea)}
`; +export function devToolsPanel(busy, inspecting, selectingArea, part) { + return `
${rebakeSection(busy)}${partitionSection(part, busy)}${inspectorSection(inspecting, selectingArea)}
`; } // One inspected feature card. `label`/`acr`/`named` come from the logic's injected