From d7d25c95fffbfecdf214c86ff7bda68bedd25070 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Sun, 28 Jun 2026 10:31:33 +1000 Subject: [PATCH 01/52] feat(solvers): custom geometric-MG prolongation hook (scalar) Add set_custom_mg() + utilities/custom_mg.py: drive PCMG geometric multigrid with a prolongation we build ourselves (barycentric or RBF) from independent, possibly non-nested coarse meshes. Coarse operators come from Galerkin RAP (P^T A P), so only the prolongation list is supplied. This decouples geometric multigrid from a nested refine() hierarchy. Mechanism: inject the custom P before the first PCSetUp (avoids the MatProductReplaceMats shape error a live matrix swap triggers under Galerkin), and KSPSetDMActive(OPERATOR, False) so PETSc uses our explicit P instead of re-deriving DM-hierarchy interpolation. Finest level reduced to the BC-eliminated global ordering; coarse levels use full DOF coords. Validated (scalar Poisson, refinement=2 box): custom barycentric P from independent coarse meshes reaches FMG iteration counts (1-3 vs FMG 2) and the same solution. On a mesh with NO refine hierarchy (FMG unavailable -> GAMG, 13 iters) custom-P geometric MG converges in ~9 iters. test_1015 (4 tests); test_1014 (11) still pass. Single-field (scalar/vector) only; Stokes velocity-block path is a follow-up. Underworld development team with AI support from Claude Code --- .../cython/petsc_generic_snes_solvers.pyx | 46 ++++ src/underworld3/utilities/custom_mg.py | 208 ++++++++++++++++++ tests/test_1015_custom_mg_prolongation.py | 77 +++++++ 3 files changed, 331 insertions(+) create mode 100644 src/underworld3/utilities/custom_mg.py create mode 100644 tests/test_1015_custom_mg_prolongation.py diff --git a/src/underworld3/cython/petsc_generic_snes_solvers.pyx b/src/underworld3/cython/petsc_generic_snes_solvers.pyx index 296e162e..35c57ae3 100644 --- a/src/underworld3/cython/petsc_generic_snes_solvers.pyx +++ b/src/underworld3/cython/petsc_generic_snes_solvers.pyx @@ -93,6 +93,45 @@ class SolverBaseClass(uw_object): # smoother / coarse-solver options with the framework FMG bundle. self._pc_user_override = False + # Custom multigrid prolongation hierarchy (see set_custom_mg / + # utilities.custom_mg). None => standard FMG/GAMG path, unchanged. + self._custom_mg = None + + def set_custom_mg(self, coarse_meshes, kind="barycentric", verbose=False): + r"""Drive geometric multigrid with a prolongation we build ourselves. + + Supplies a sequence of (possibly **non-nested**) coarse meshes from which + a barycentric or RBF prolongation ``P`` is assembled and installed into + the PCMG via ``PC.setMGInterpolation``; coarse operators are formed by + Galerkin RAP. This decouples geometric multigrid from a nested + ``refine()`` hierarchy — it works even when the solver mesh has no + refinement hierarchy at all. + + Parameters + ---------- + coarse_meshes : list of Mesh + Coarsest-first list of coarse meshes (the finest level is the + solver's own mesh). Need not be nested with the solver mesh. + kind : {"barycentric", "rbf"} + Prolongation builder. ``barycentric`` is FE-exact; ``rbf`` is a + polyharmonic RBF (Shepard-normalised). Default ``barycentric``. + verbose : bool + Print the per-level DOF counts at injection. + + Notes + ----- + Single-field (scalar/vector) solvers only for now; the Stokes + velocity-block path is a separate increment. See + :mod:`underworld3.utilities.custom_mg`. + """ + if kind not in ("barycentric", "rbf"): + raise ValueError("kind must be 'barycentric' or 'rbf'") + if not coarse_meshes: + raise ValueError("coarse_meshes must be a non-empty coarsest-first list") + self._custom_mg = {"coarse_meshes": list(coarse_meshes), + "kind": kind, "verbose": verbose} + self.is_setup = False + def add_update_callback(self, callback): r"""Register a callback fired at the start of every nonlinear (SNES) iteration. @@ -2662,6 +2701,13 @@ class SNES_Scalar(SolverBaseClass): # ``constant_nullspace`` was set. self._attach_constant_nullspace() + # Custom multigrid prolongation: inject our P hierarchy before the + # first PCSetUp (so the Galerkin coarse operators are built from it). + # No-op unless set_custom_mg() was called. + if self._custom_mg is not None: + from underworld3.utilities.custom_mg import inject_custom_mg + inject_custom_mg(self) + # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py new file mode 100644 index 00000000..9ffb5c62 --- /dev/null +++ b/src/underworld3/utilities/custom_mg.py @@ -0,0 +1,208 @@ +""" +Custom geometric-multigrid prolongation injection. + +Build the multigrid prolongation ``P`` **ourselves** (barycentric or RBF) from a +sequence of independent — possibly *non-nested* — coarse meshes, and install it +into a solver's PETSc ``PCMG`` via ``PC.setMGInterpolation``. The coarse +operators are then formed by Galerkin RAP (``Pᵀ A P``), exactly as UW3's FMG +already does. + +Why +--- +UW3's geometric FMG requires a nested ``refine()`` hierarchy with PETSc's exact +nested interpolation. That couples multigrid to uniform refinement and rules out +local adaptation / a dynamic node budget. PETSc's ``PCMG`` does **not** require +its transfer operator to come from the DM hierarchy, however: any prolongation +supplied with ``setMGInterpolation`` is used, and ``pc_mg_galerkin=both`` builds +consistent coarse operators from it. The prolongation "evaluate the coarse FE +function at the fine DOF coordinates" is exactly what the nested path computes — +so we can build it ourselves by barycentric (FE-exact) or RBF interpolation for +*any* coarse/fine pair. This decouples *how the mesh refines* from *what transfer +multigrid uses*. + +Validated (scalar Poisson, refinement=2 box): custom barycentric ``P`` from +independent coarse meshes reaches FMG iteration counts (1–3 vs FMG 2). On a mesh +with **no** refine hierarchy (FMG unavailable → GAMG, 13 iters) custom-P +geometric MG converges in ~9 iters. + +Notes +----- +* The finest level is the solver's real (BC-eliminated) operator space; its + ``P`` rows are reduced to the global/MG ordering. Coarse levels use the full + DOF coordinates of each coarse mesh (no BC reduction needed — Galerkin RAP and + the finest reduction keep the coarse correction consistent). +* Injection happens at solver build time, *before* the first ``PCSetUp``: the + first Galerkin assembly is built from our ``P`` directly, avoiding PETSc's + ``MatProductReplaceMats`` shape error that a live matrix swap triggers. +* ``KSPSetDMActive(OPERATOR, False)`` stops PETSc re-deriving interpolation / + operators from the DM hierarchy, so our explicit ``P`` is used. +""" + +import numpy as np +from petsc4py import PETSc + +__all__ = ["barycentric_prolongation", "rbf_prolongation", "inject_custom_mg"] + + +# --------------------------------------------------------------------------- # +# Prolongation builders (return scipy CSR matrices) +# --------------------------------------------------------------------------- # +def barycentric_prolongation(coarse_coords, fine_coords): + """FE-exact prolongation: each fine DOF interpolated by the coarse element + that contains it (barycentric weights). Partition of unity (row sums = 1); + fine points outside the coarse hull fall back to the nearest coarse DOF.""" + import scipy.sparse as sp + from scipy.spatial import Delaunay, cKDTree + + tri = Delaunay(coarse_coords) + simp = tri.find_simplex(fine_coords) + tree = cKDTree(coarse_coords) + dim = coarse_coords.shape[1] + rows, cols, vals = [], [], [] + for i in range(fine_coords.shape[0]): + s = simp[i] + if s < 0: # outside coarse hull → nearest coarse DOF + rows.append(i); cols.append(int(tree.query(fine_coords[i])[1])); vals.append(1.0) + continue + verts = tri.simplices[s] + T = tri.transform[s] + bary = T[:dim].dot(fine_coords[i] - T[dim]) + w = np.append(bary, 1.0 - bary.sum()) + for k in range(dim + 1): + rows.append(i); cols.append(int(verts[k])); vals.append(float(w[k])) + return sp.csr_matrix((vals, (rows, cols)), + shape=(fine_coords.shape[0], coarse_coords.shape[0])) + + +def rbf_prolongation(coarse_coords, fine_coords, smooth=0.0): + """RBF prolongation: polyharmonic (r² log r) kernel + affine polynomial tail + (reproduces linear fields), Shepard row-normalised to a partition of unity. + Works for arbitrary (non-nested) point sets; software-equivalent to the + barycentric builder as an MG transfer operator.""" + import scipy.sparse as sp + from scipy.spatial.distance import cdist + + def phi(r): + r = np.where(r == 0.0, 1e-30, r) + return r ** 2 * np.log(r) + + nc, dim = coarse_coords.shape + Pc = np.hstack([np.ones((nc, 1)), coarse_coords]) # affine tail + Acc = phi(cdist(coarse_coords, coarse_coords)) + smooth * np.eye(nc) + M = np.block([[Acc, Pc], [Pc.T, np.zeros((dim + 1, dim + 1))]]) + Minv = np.linalg.inv(M) + B = np.hstack([phi(cdist(fine_coords, coarse_coords)), + np.ones((fine_coords.shape[0], 1)), fine_coords]) + Praw = (B @ Minv)[:, :nc] + rs = Praw.sum(axis=1, keepdims=True) + rs[np.abs(rs) < 1e-12] = 1.0 + return sp.csr_matrix(Praw / rs) + + +_BUILDERS = {"barycentric": barycentric_prolongation, "rbf": rbf_prolongation} + + +# --------------------------------------------------------------------------- # +# Coordinate helpers +# --------------------------------------------------------------------------- # +def _to_petsc_aij(csr): + csr = csr.tocsr() + M = PETSc.Mat().createAIJ( + size=csr.shape, + csr=(csr.indptr.astype("int32"), csr.indices.astype("int32"), csr.data), + ) + M.assemble() + return M + + +def _reduce_to_global(dm, full_coords): + """Reduce full local DOF coordinates to the solver's global (BC-eliminated, + MG-ordered) layout by scattering each component through ``localToGlobal`` + (which drops constrained boundary DOFs and reorders to global indices).""" + lvec = dm.getLocalVec() + gvec = dm.getGlobalVec() + cdim = full_coords.shape[1] + if lvec.getLocalSize() != full_coords.shape[0]: + dm.restoreLocalVec(lvec); dm.restoreGlobalVec(gvec) + raise RuntimeError( + f"custom_mg: local DOF count {lvec.getLocalSize()} != coord count " + f"{full_coords.shape[0]} — degree/continuity mismatch") + out = np.zeros((gvec.getSize(), cdim)) + for c in range(cdim): + lvec.array[:] = full_coords[:, c] + dm.localToGlobal(lvec, gvec, addv=False) + out[:, c] = gvec.array + dm.restoreLocalVec(lvec) + dm.restoreGlobalVec(gvec) + return out + + +# --------------------------------------------------------------------------- # +# Injection +# --------------------------------------------------------------------------- # +def inject_custom_mg(solver): + """Configure ``solver``'s PCMG to use our custom prolongation hierarchy. + + Called from the solver's ``solve()`` (after ``_build``, before the SNES + solve) when ``solver._custom_mg`` is set via ``set_custom_mg``. Scalar / + single-field solvers only for now (the Stokes velocity-block path is a + separate increment). + """ + cfg = solver._custom_mg + coarse_meshes = cfg["coarse_meshes"] + kind = cfg["kind"] + builder = _BUILDERS[kind] + + var = solver.Unknowns.u + degree = var.degree + continuous = getattr(var, "continuous", True) + + if solver._pc_option_prefix not in ("", None): + raise NotImplementedError( + "custom_mg currently supports single-field (scalar/vector) solvers; " + f"solver uses PC prefix '{solver._pc_option_prefix}' (e.g. Stokes " + "velocity block) which is a separate increment.") + + # --- per-level DOF coordinates --------------------------------------- # + fine = _reduce_to_global(solver.dm, + solver.mesh._get_coords_for_basis(degree, continuous)) + levels = [m._get_coords_for_basis(degree, continuous) for m in coarse_meshes] + levels.append(fine) + nlev = len(levels) + if nlev < 2: + raise ValueError("custom_mg needs at least one coarse mesh") + + # --- build prolongations P[l]: level l-1 -> level l ------------------ # + Ps = [None] + [_to_petsc_aij(builder(levels[l - 1], levels[l])) + for l in range(1, nlev)] + + # --- configure the geometric MG bundle on the managed block ---------- # + opts = solver.petsc_options + pfx = solver._pc_option_prefix or "" + opts[pfx + "pc_type"] = "mg" + opts[pfx + "pc_mg_type"] = "full" + opts[pfx + "pc_mg_galerkin"] = "both" # coarse operators = PᵀAP + opts[pfx + "mg_levels_ksp_type"] = "richardson" + opts[pfx + "mg_levels_pc_type"] = "sor" + opts[pfx + "mg_coarse_pc_type"] = "redundant" + opts[pfx + "mg_coarse_redundant_pc_type"] = "lu" + for key in ("pc_gamg_type", "pc_gamg_repartition", "pc_gamg_agg_nsmooths"): + opts.delValue(pfx + key) + + # --- inject before the first PCSetUp --------------------------------- # + solver.snes.setUp() + ksp = solver.snes.getKSP() + # Stop PETSc re-deriving interpolation/operators from the DM hierarchy. + ksp.setDMActive(PETSc.KSP.DMActive.OPERATOR, False) + pc = ksp.getPC() + pc.setType("mg") + pc.setMGLevels(nlev) + pc.setMGType(PETSc.PC.MGType.FULL) + for l in range(1, nlev): + pc.setMGInterpolation(l, Ps[l]) + pc.setFromOptions() + + if cfg.get("verbose"): + from underworld3 import mpi + mpi.pprint(f"[{solver.name}] custom_mg ({kind}): levels " + f"{[lv.shape[0] for lv in levels]}") diff --git a/tests/test_1015_custom_mg_prolongation.py b/tests/test_1015_custom_mg_prolongation.py new file mode 100644 index 00000000..ea975df1 --- /dev/null +++ b/tests/test_1015_custom_mg_prolongation.py @@ -0,0 +1,77 @@ +""" +Custom multigrid prolongation hook (set_custom_mg / utilities.custom_mg). + +Geometric multigrid driven by a prolongation we build ourselves (barycentric or +RBF) from independent, non-nested coarse meshes — decoupling geometric MG from a +nested refine() hierarchy. See docs/developer/design and the study notes. +""" +import numpy as np +import pytest +import underworld3 as uw + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + + +def _poisson(mesh): + p = uw.systems.Poisson(mesh) + p.constitutive_model = uw.constitutive_models.DiffusionModel + p.constitutive_model.Parameters.diffusivity = 1 + p.f = 0.0 + p.add_dirichlet_bc(0.0, "Bottom") + p.add_dirichlet_bc(1.0, "Top") + return p + + +def _box(cellSize, refinement=None): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0), + cellSize=cellSize, refinement=refinement, qdegree=2) + + +def test_custom_mg_barycentric_matches_fmg(): + """Custom barycentric P (independent coarse meshes) must converge and agree + with the nested-FMG solution on the same fine problem.""" + fmg = _poisson(_box(0.25, refinement=2)) + fmg.solve() + assert fmg.snes.getConvergedReason() > 0 + + cust = _poisson(_box(0.25, refinement=2)) + cust.set_custom_mg([_box(0.285), _box(0.142)], kind="barycentric") + cust.solve() + assert cust.snes.getKSP().getPC().getType() == "mg" + assert cust.snes.getConvergedReason() > 0 + # same fine problem -> same solution (Galerkin RAP from our P) + rel = (np.linalg.norm(cust.Unknowns.u.data - fmg.Unknowns.u.data) + / (np.linalg.norm(fmg.Unknowns.u.data) + 1e-30)) + assert rel < 1e-4 + # competitive with nested FMG (not pathologically worse) + assert cust.snes.getKSP().getIterationNumber() <= fmg.snes.getKSP().getIterationNumber() + 5 + + +def test_custom_mg_rbf_drives_solver(): + """The RBF builder must also drive a converging geometric MG solve.""" + cust = _poisson(_box(0.25, refinement=2)) + cust.set_custom_mg([_box(0.285), _box(0.142)], kind="rbf") + cust.solve() + assert cust.snes.getKSP().getPC().getType() == "mg" + assert cust.snes.getConvergedReason() > 0 + + +def test_custom_mg_without_refine_hierarchy(): + """Key capability: geometric MG on a mesh with NO refine() hierarchy + (FMG unavailable -> normally GAMG). Custom P supplies the hierarchy.""" + mesh = _box(0.08) # no refinement -> single-level + assert len(mesh.dm_hierarchy) == 1 + cust = _poisson(mesh) + cust.set_custom_mg([_box(0.3), _box(0.16)], kind="barycentric") + cust.solve() + assert cust.snes.getKSP().getPC().getType() == "mg" + assert cust.snes.getConvergedReason() > 0 + + +def test_set_custom_mg_validation(): + p = _poisson(_box(0.5)) + with pytest.raises(ValueError): + p.set_custom_mg([_box(0.8)], kind="not-a-builder") + with pytest.raises(ValueError): + p.set_custom_mg([], kind="barycentric") From 76298f359d7d508a2b968e6fa0ca0cead159e430 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 06:18:39 +1000 Subject: [PATCH 02/52] feat(custom_mg): Layer-1 generalized FMG hierarchy (BC-per-level) + scoped SBR Phase 1 of hardening the custom-prolongation work into a generalized FMG hierarchy. - CustomMGHierarchy: adapter-agnostic multi-level hierarchy that builds custom-P transfers (barycentric/rbf) with essential BCs applied at EVERY level (transfers map reduced->reduced). This is the load-bearing invariant: on an exactly-nested hierarchy, omitting per-level reduction makes a coarse boundary DOF coincide with a BC-removed fine DOF -> zero column -> singular Galerkin coarse operator. build() guards against zero columns. - set_custom_fmg(): register a hierarchy + per-level solver factory; built and installed at solve() time (build-time injection, DMActive(OPERATOR,False), Galerkin RAP). Scalar / single-field-vector (top-level PC); Stokes velocity block is Phase 2. - sbr_refine / sbr_refine_where: local Skeleton-Based Refinement (no MMG, on-rank, conforming) with SCOPED dm_plex_transform_type (leaking it globally breaks UW's uniform refine() with err73). - Legacy finest-only path kept for back-compat (test_1015). Validated: scalar jump-coefficient Poisson on a 5-level (3 uniform + 2 SBR) hierarchy converges in 3 FMG iters vs GAMG 46, solution matches to 2.4e-8. test_1016 (3 tests); test_1015 (4) + test_1014 (11) still pass. Design: docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 120 ++++++++ src/underworld3/utilities/custom_mg.py | 285 +++++++++++++++--- tests/test_1016_custom_mg_hierarchy.py | 60 ++++ 3 files changed, 424 insertions(+), 41 deletions(-) create mode 100644 docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md create mode 100644 tests/test_1016_custom_mg_hierarchy.py diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md new file mode 100644 index 00000000..cb2908cc --- /dev/null +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -0,0 +1,120 @@ +# Generalized FMG hierarchy + adapt-on-top + +Status: design (hardening the custom-prolongation prototype). 2026-06-28. + +## Context + +UW3's geometric FMG (`Mesh(refinement=N)` → `dm_hierarchy`, `preconditioner="fmg"`) +requires a **uniform nested** hierarchy with PETSc's canonical nested interpolation. +That couples multigrid to uniform refinement and rules out local adaptation. + +A prototype established that we can drive geometric multigrid with a prolongation we +**build ourselves** (barycentric or local-RBF) and install via `PC.setMGInterpolation`, +with Galerkin RAP forming the coarse operators. Validated: +- FMG-equivalent convergence on non-nested *and* nested hierarchies (SolCx velocity + block, η-jump 1e6: 8 iters, vs GAMG 157 — geometric MG where FMG cannot otherwise exist); +- a 5-level hierarchy (3 uniform + 2 SBR-targeted at the jump) solves at 8 iters; +- local SBR refinement is a ~4-line stock-petsc4py call (no MMG): mark cells in a label, + `dm_plex_transform_type=refine_sbr`, `dm.adaptLabel("name")` — conforming, on-rank. + +**Load-bearing correctness lesson:** essential BCs must be applied at **every** level, +transfers mapping reduced→reduced. PETSc native FMG does this automatically (each level's +`PetscSection` carries the Dirichlet constraints); custom-P from raw coordinates must +replicate it. Exact nesting makes the omission fatal (coarse boundary-normal DOFs coincide +with BC-removed fine DOFs → zero columns → singular coarse operator). + +## Two-layer architecture + +### Layer 1 — generalized FMG hierarchy (adapter-agnostic) +**The existing `refinement=N` uniform hierarchy STANDS** and is the FMG base. Layer 1 +generalizes the *transfer machinery* so the hierarchy may contain levels that are **not** +uniform refinements, by carrying **supplied prolongations** instead of relying on canonical +nesting. + +Layer 1 knows nothing about *how* a level was produced. Its contract is a sequence of +levels, each `(dm, prolongation_to_next)`, plus: +- **BC-per-level reduction** — transfers map reduced→reduced (the invariant above), derived + from the owning solver's essential BCs via each level DM's global section. +- **Transfer builders** — pluggable: `nested` (PETSc canonical, for uniform levels), + `barycentric` (FE-exact, needs a triangulation), `rbf` (local kNN, scattered-point + robust, reuses the swarm-proxy RBF kernel). Default per level: `nested` where the level + is a genuine uniform refinement, else `barycentric`. +- **Installation** — set the transfers on the velocity-block (Stokes) or top (scalar/vector) + PCMG via `setMGInterpolation`, before first `PCSetUp`, with `KSPSetDMActive(OPERATOR,False)` + and `pc_mg_galerkin=both`. Re-attach nullspaces (pressure / rigid-body) after rebuild. + +Home: `src/underworld3/utilities/custom_mg.py` (grow the committed module). + +Generality requirement: Layer 1's interface must admit **any** future level source +(MMG/parmmg, edge/face-point injection, knockout-as-coarsening, independent meshes) and +**any** transfer builder. No SBR-specific assumptions in Layer 1. + +### Layer 2 — adapt-on-top (first concrete application) +On top of the uniform FMG base, add refined levels via the **existing adaptive-meshing +pattern**: choose an **adapter** + a **metric**, call **periodically**; field +transfer/redistribution handled by the existing remesh machinery +(`remesh_with_field_transfer`, `on_remesh`, `mesh.adapt`). + +This pass's adapter is **SBR refine-on-top**: +- metric → mark cells (e.g. |∇field| threshold, or distance-to-feature) → `adaptLabel` + with `refine_sbr` → a new finest level on top of the current finest; +- **non-load-balancing** (on-rank, no redistribution) — which **bounds the number of + extra levels** (imbalance grows with depth); the adapter exposes/limits this; +- registers `(new level DM, custom-P transfer)` into the Layer-1 hierarchy. + +Home: an adapter consistent with `adaptivity.py` / `mesh.adapt(metric, adapter=...)`, +calling into Layer 1. Future adapters (load-balancing MMG, etc.) plug in the same way. + +## Parallel strategy (parallel from the start) + +The supported parallel path is the **nested, co-partitioned** hierarchy: +- The uniform base is co-partitioned by construction (`dm.refine()` preserves partition); + SBR is on-rank → levels stay co-located (a fine cell's parent coarse cell is on the + same rank). So each rank builds its block of `P` from its **local** coarse cells — + point-location is rank-local; only a thin ghost layer at partition boundaries. +- **BC-per-level reduction is parallel-correct for free**: it rides the DM global section + (`localToGlobal` gives the reduced global ordering across ranks). +- `P` is assembled as an MPIAIJ matrix (fine local rows; coarse local + ghost columns). + +**Non-nested / independent-mesh custom-P** (cross-rank point-location) is **serial-only / +experimental** — it is not on the parallel-supported path. The production case +(targeted refinement = nested) needs only the rank-local path. + +Non-load-balancing SBR → bound the added refinement depth (configurable cap); document the +imbalance/level trade-off. + +## Correctness invariants (must hold, all levels) +1. BCs applied at every level; transfers reduced→reduced (no zero columns). +2. Each prolongation reproduces constants (row-sums = 1 — partition of unity). +3. `pc_mg_galerkin=both` (coarse operators = PᵀAP; UW installs no coarse residual/Jacobian). +4. Nullspaces (constant-pressure / rigid-body) re-attached after any rebuild. +5. No silent fallback: if a level lacks a valid transfer, error (don't degrade to GAMG silently). + +## Test matrix +- Hierarchy build: uniform-only; uniform + N SBR levels; nesting + label survival. +- Per-level reduction: zero-column count == 0 on all transfers (scalar + free-slip vector). +- Transfer: partition-of-unity; barycentric vs rbf; nested-exact vs general. +- Solve: FMG + V-cycle converge on nested and (serial) non-nested; match uniform-FMG iters; + beat GAMG; regression — existing FMG/GAMG paths unchanged. +- Parallel (np>1): co-located transfers, reduced sections, FMG convergence; bounded levels. + +## Phased roadmap +1. **Layer 1 core** — generalized hierarchy + auto BC-per-level reduction + transfer builders + (nested/barycentric/rbf) + install into PCMG; scalar + free-slip vector; serial-validated, + designed parallel-correct (rank-local construction). +2. **Stokes integration** — `set_custom_mg`/hierarchy on the velocity block; nullspace + re-attach; guards; SolCx end-to-end. +3. **Parallel validation** — np>1 tests; co-location/ghost handling; reduced-section + correctness; fix the known rank-local-accumulator pitfalls. +4. **Layer 2 adapter** — SBR refine-on-top following `mesh.adapt` (metric→mark→refine→register), + non-load-balancing, bounded depth, field transfer via existing remesh. +5. **Tests + design doc finalize + tier classification.** + +Later (not this pass; do not block): dynamic per-step reallocation loop; efficiency +(transfer caching, exact-nested combinatorial P, operator reuse); load-balancing adapters; +non-nested parallel transfers; knockout-as-coarsening as an alternative Layer-2 strategy. + +## Explicit non-goals for this pass +- Knockout (shown to pay full-fine assembly; structural value only) — not pursued now. +- Non-nested custom-P in parallel — serial/experimental only. +- Dynamic time-stepping convection integration — later phase. diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 9ffb5c62..b6fea9c3 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -41,7 +41,8 @@ import numpy as np from petsc4py import PETSc -__all__ = ["barycentric_prolongation", "rbf_prolongation", "inject_custom_mg"] +__all__ = ["barycentric_prolongation", "rbf_prolongation", "inject_custom_mg", + "CustomMGHierarchy", "set_custom_fmg", "sbr_refine", "sbr_refine_where"] # --------------------------------------------------------------------------- # @@ -102,6 +103,57 @@ def phi(r): _BUILDERS = {"barycentric": barycentric_prolongation, "rbf": rbf_prolongation} +# --------------------------------------------------------------------------- # +# Local Skeleton-Based Refinement (no MMG; on-rank; conforming) +# --------------------------------------------------------------------------- # +_DM_ADAPT_REFINE = 1 # PETSc DM_ADAPT_REFINE + + +def _sbr_apply(dm, mark): + """Clone ``dm``, let ``mark(clone, adapt_label)`` flag cells, and SBR-refine. + Conforming (edge bisection + propagation, no hanging nodes), on-rank (no + redistribution), no external mesh libraries. Cell geometry / marking is done + on the CLONE (the un-cloned mesh DM raises err73 from computeCellGeometryFVM). + + IMPORTANT: ``dm_plex_transform_type`` is a *global* PETSc option; leaving it as + ``refine_sbr`` breaks UW3's uniform ``dm.refine()`` (FMG hierarchy build) with + PETSc error 73. It is set only for the duration of the call and restored.""" + opts = PETSc.Options() + had = opts.hasName("dm_plex_transform_type") + prev = opts.getString("dm_plex_transform_type") if had else None + opts.setValue("dm_plex_transform_type", "refine_sbr") + try: + d = dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + mark(d, lab) + return d.adaptLabel("adapt") + finally: + if had: + opts.setValue("dm_plex_transform_type", prev) + else: + opts.delValue("dm_plex_transform_type") + + +def sbr_refine(dm, cells): + """SBR-refine an explicit list of ``cells``.""" + def mark(d, lab): + for c in cells: + lab.setValue(int(c), _DM_ADAPT_REFINE) + return _sbr_apply(dm, mark) + + +def sbr_refine_where(dm, predicate): + """SBR-refine cells whose centroid satisfies ``predicate(centroid)``.""" + def mark(d, lab): + cs, ce = d.getHeightStratum(0) + for c in range(cs, ce): + if predicate(d.computeCellGeometryFVM(c)[1]): + lab.setValue(c, _DM_ADAPT_REFINE) + return _sbr_apply(dm, mark) + + # --------------------------------------------------------------------------- # # Coordinate helpers # --------------------------------------------------------------------------- # @@ -138,50 +190,75 @@ def _reduce_to_global(dm, full_coords): # --------------------------------------------------------------------------- # -# Injection +# Layer 1 — generalized FMG hierarchy (BC-reduced, adapter-agnostic) +# +# INVARIANT (load-bearing): essential BCs are applied at EVERY level, so every +# transfer maps reduced->reduced. PETSc native FMG does this automatically via +# each level's constrained PetscSection; custom-P built from raw coordinates must +# replicate it. Omitting it is fatal on an EXACTLY-nested hierarchy: a coarse +# boundary DOF coincides with a BC-removed fine DOF -> zero column -> singular +# Galerkin coarse operator. (See docs/developer/design/ +# GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md.) # --------------------------------------------------------------------------- # -def inject_custom_mg(solver): - """Configure ``solver``'s PCMG to use our custom prolongation hierarchy. +def _field_subdm(dm, field_id): + """Return the sub-DM for ``field_id`` (whole dm if single-field / None).""" + if field_id is None: + return dm + try: + if dm.getNumFields() <= 1: + return dm + except Exception: + return dm + _iset, sub = dm.createSubDM(field_id) + return sub - Called from the solver's ``solve()`` (after ``_build``, before the SNES - solve) when ``solver._custom_mg`` is set via ``set_custom_mg``. Scalar / - single-field solvers only for now (the Stokes velocity-block path is a - separate increment). - """ - cfg = solver._custom_mg - coarse_meshes = cfg["coarse_meshes"] - kind = cfg["kind"] - builder = _BUILDERS[kind] - var = solver.Unknowns.u - degree = var.degree - continuous = getattr(var, "continuous", True) +def _reduced_map(dm, field_id=None): + """Reduced->full DOF map for a field on a BUILT DM (serial). Index ``k`` of + the returned array is the local DOF that maps to global/MG index ``k`` — + i.e. the BC-eliminated, MG-ordered layout the operator lives on. - if solver._pc_option_prefix not in ("", None): - raise NotImplementedError( - "custom_mg currently supports single-field (scalar/vector) solvers; " - f"solver uses PC prefix '{solver._pc_option_prefix}' (e.g. Stokes " - "velocity block) which is a separate increment.") + NOTE: serial-correct (uses local indices). Parallel correctness — mapping to + *global* numbering across ranks — is a Phase-3 item (the supported parallel + path keeps levels co-partitioned so this stays rank-local).""" + sub = _field_subdm(dm, field_id) + lv = sub.getLocalVec() + n_full = lv.getLocalSize() + lv.array[:] = np.arange(n_full) + gv = sub.getGlobalVec() + sub.localToGlobal(lv, gv, addv=False) + r2f = gv.array.astype(int).copy() + sub.restoreLocalVec(lv) + sub.restoreGlobalVec(gv) + return r2f, n_full - # --- per-level DOF coordinates --------------------------------------- # - fine = _reduce_to_global(solver.dm, - solver.mesh._get_coords_for_basis(degree, continuous)) - levels = [m._get_coords_for_basis(degree, continuous) for m in coarse_meshes] - levels.append(fine) - nlev = len(levels) - if nlev < 2: - raise ValueError("custom_mg needs at least one coarse mesh") - # --- build prolongations P[l]: level l-1 -> level l ------------------ # - Ps = [None] + [_to_petsc_aij(builder(levels[l - 1], levels[l])) - for l in range(1, nlev)] +def _reduced_transfer(coarse_coords, fine_coords, r2f_c, r2f_f, ncomp, builder): + """Build one prolongation reduced(coarse) -> reduced(fine): + node-level scalar P -> interleave ``ncomp`` components -> drop BC rows/cols.""" + import scipy.sparse as sp + Pn = builder(coarse_coords, fine_coords) # (n_f_nodes, n_c_nodes) + Pv = sp.kron(Pn, sp.eye(ncomp), format="csr") # interleaved full vector + Pr = Pv.tocsr()[r2f_f, :][:, r2f_c] # reduced -> reduced + return Pr + - # --- configure the geometric MG bundle on the managed block ---------- # +def _install_transfers(solver, Ps, verbose=False): + """Configure the managed PCMG block to use the supplied prolongations. + Build-time injection (before first PCSetUp): set DMActive(OPERATOR,False) so + PETSc does not re-derive interpolation from the DM, Galerkin RAP for coarse + operators. Scalar / single-field-vector (top-level PC: ``_pc_option_prefix`` + is ``""``); the Stokes velocity-block path is Phase 2.""" + nlev = len(Ps) + 1 opts = solver.petsc_options pfx = solver._pc_option_prefix or "" + if pfx not in ("",): + raise NotImplementedError( + "Layer-1 install supports top-level PC (scalar / single-field vector). " + f"PC prefix '{pfx}' (e.g. Stokes velocity block) is Phase 2.") opts[pfx + "pc_type"] = "mg" opts[pfx + "pc_mg_type"] = "full" - opts[pfx + "pc_mg_galerkin"] = "both" # coarse operators = PᵀAP + opts[pfx + "pc_mg_galerkin"] = "both" opts[pfx + "mg_levels_ksp_type"] = "richardson" opts[pfx + "mg_levels_pc_type"] = "sor" opts[pfx + "mg_coarse_pc_type"] = "redundant" @@ -189,20 +266,146 @@ def inject_custom_mg(solver): for key in ("pc_gamg_type", "pc_gamg_repartition", "pc_gamg_agg_nsmooths"): opts.delValue(pfx + key) - # --- inject before the first PCSetUp --------------------------------- # solver.snes.setUp() ksp = solver.snes.getKSP() - # Stop PETSc re-deriving interpolation/operators from the DM hierarchy. ksp.setDMActive(PETSc.KSP.DMActive.OPERATOR, False) pc = ksp.getPC() pc.setType("mg") pc.setMGLevels(nlev) pc.setMGType(PETSc.PC.MGType.FULL) for l in range(1, nlev): - pc.setMGInterpolation(l, Ps[l]) + pc.setMGInterpolation(l, Ps[l - 1]) pc.setFromOptions() - - if cfg.get("verbose"): + if verbose: from underworld3 import mpi - mpi.pprint(f"[{solver.name}] custom_mg ({kind}): levels " - f"{[lv.shape[0] for lv in levels]}") + mpi.pprint(f"[{solver.name}] custom FMG installed: {nlev} levels, " + f"P sizes {[tuple(P.getSize()) for P in Ps]}") + + +class CustomMGHierarchy: + """A generalized FMG hierarchy: a sequence of level meshes (coarsest..finest) + whose transfers are built by a pluggable builder, with BCs applied at every + level. Adapter-agnostic — it consumes meshes + a way to get each level's + BC-reduced DOF map; it does not know how the levels were produced. + + Parameters + ---------- + level_meshes : list of Mesh + Coarsest-first; the LAST entry must be the solver's own mesh. + builder : {"barycentric", "rbf"} + Per-level node prolongation builder. + field_id : int or None + Field index for multi-field solvers (e.g. 0 = velocity); None = single field. + """ + + def __init__(self, level_meshes, builder="barycentric", field_id=None): + if builder not in _BUILDERS: + raise ValueError("builder must be 'barycentric' or 'rbf'") + if len(level_meshes) < 2: + raise ValueError("need at least 2 levels (>=1 coarse + finest)") + self.level_meshes = list(level_meshes) + self.builder = _BUILDERS[builder] + self.builder_name = builder + self.field_id = field_id + self.transfers = None + + def build(self, solver, level_solver_factory): + """Build the BC-reduced prolongations. ``solver`` is the (built) finest + solver; ``level_solver_factory(mesh) -> built solver`` provides a + same-discretisation solver on each COARSE level so its constrained + section gives that level's reduced map. The finest level uses ``solver``.""" + var = solver.Unknowns.u + degree = var.degree + continuous = getattr(var, "continuous", True) + nlev = len(self.level_meshes) + + coords, r2f, ncomp = [], [], [] + for k, mesh in enumerate(self.level_meshes): + c = np.asarray(mesh._get_coords_for_basis(degree, continuous)) + if k == nlev - 1: + rmap, nfull = _reduced_map(solver.dm, self.field_id) + else: + tmp = level_solver_factory(mesh) + tmp._build(False, False, None) + tmp.snes.setUp() + rmap, nfull = _reduced_map(tmp.dm, self.field_id) + nc = nfull // c.shape[0] + if nfull % c.shape[0] != 0: + raise RuntimeError( + f"level {k}: full DOFs {nfull} not divisible by nodes {c.shape[0]}") + coords.append(c); r2f.append(rmap); ncomp.append(nc) + + if len(set(ncomp)) != 1: + raise RuntimeError(f"inconsistent component counts across levels: {ncomp}") + nc = ncomp[0] + + Ps = [] + for l in range(1, nlev): + Pr = _reduced_transfer(coords[l - 1], coords[l], r2f[l - 1], r2f[l], + nc, self.builder) + zc = int((np.asarray((Pr != 0).sum(axis=0)).ravel() == 0).sum()) + if zc: + raise RuntimeError( + f"transfer {l-1}->{l} has {zc} zero columns (coarse DOFs with no " + f"fine image) — BC-per-level reduction failed; coarse operator " + f"would be singular.") + Ps.append(_to_petsc_aij(Pr)) + self.transfers = Ps + return Ps + + def install(self, solver, verbose=False): + if self.transfers is None: + raise RuntimeError("call build() before install()") + _install_transfers(solver, self.transfers, verbose=verbose) + + +# --------------------------------------------------------------------------- # +# Entry points +# --------------------------------------------------------------------------- # +def set_custom_fmg(solver, coarse_meshes, *, level_solver_factory, + builder="barycentric", field_id=None, verbose=False): + """Generalized custom-P FMG with BC-per-level reduction (the correct path). + + Registers a :class:`CustomMGHierarchy` on the solver so that the next + ``solve()`` builds and installs it (build-time injection). The hierarchy is + ``[*coarse_meshes, solver.mesh]``; ``level_solver_factory(mesh)`` must return + a same-discretisation solver on a coarse level (used only to read its + BC-constrained section).""" + solver._custom_mg = { + "mode": "hierarchy", + "hierarchy": CustomMGHierarchy(list(coarse_meshes) + [solver.mesh], + builder=builder, field_id=field_id), + "factory": level_solver_factory, + "verbose": verbose, + } + solver.is_setup = False + + +def inject_custom_mg(solver): + """Build + install the custom-P FMG. Called from ``solve()`` (after ``_build``, + before the SNES solve) when ``solver._custom_mg`` is set. Dispatches: + - ``mode == "hierarchy"`` -> BC-per-level reduced path (correct, general); + - legacy dict ``{coarse_meshes, kind}`` -> finest-only reduction (kept for + back-compat; valid only when coarse levels are non-nested / unconstrained).""" + cfg = solver._custom_mg + + if isinstance(cfg, dict) and cfg.get("mode") == "hierarchy": + h = cfg["hierarchy"] + h.build(solver, cfg["factory"]) + h.install(solver, verbose=cfg.get("verbose", False)) + return + + # ---- legacy finest-only path (back-compat) ------------------------------ + coarse_meshes = cfg["coarse_meshes"] + builder = _BUILDERS[cfg["kind"]] + var = solver.Unknowns.u + degree = var.degree + continuous = getattr(var, "continuous", True) + if solver._pc_option_prefix not in ("", None): + raise NotImplementedError("legacy custom_mg path is scalar/single-field only") + fine = _reduce_to_global(solver.dm, + solver.mesh._get_coords_for_basis(degree, continuous)) + levels = [m._get_coords_for_basis(degree, continuous) for m in coarse_meshes] + levels.append(fine) + Ps = [_to_petsc_aij(builder(levels[l - 1], levels[l])) for l in range(1, len(levels))] + _install_transfers(solver, Ps, verbose=cfg.get("verbose", False)) diff --git a/tests/test_1016_custom_mg_hierarchy.py b/tests/test_1016_custom_mg_hierarchy.py new file mode 100644 index 00000000..60c873b2 --- /dev/null +++ b/tests/test_1016_custom_mg_hierarchy.py @@ -0,0 +1,60 @@ +"""Layer-1 generalized FMG hierarchy: CustomMGHierarchy + set_custom_fmg with +BC-per-level reduction, over a uniform + SBR-targeted nested hierarchy.""" +import numpy as np, sympy, pytest, underworld3 as uw +from petsc4py import PETSc +from underworld3.utilities import custom_mg + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + +def _sbr(dm, band=0.2): + # scoped SBR (restores dm_plex_transform_type so uniform refine() still works) + return custom_mg.sbr_refine_where(dm, lambda cen: abs(cen[0]-0.5) < band) +def _wrap(dm,m0): + return uw.discretisation.Mesh(dm.clone(), simplex=True, + coordinate_system_type=m0.CoordinateSystem.coordinate_type, qdegree=3, boundaries=m0.boundaries) +def _poisson(mesh): + p=uw.systems.Poisson(mesh); p.constitutive_model=uw.constitutive_models.DiffusionModel + x,y=mesh.X + p.constitutive_model.Parameters.diffusivity=sympy.Piecewise((1.0,x<0.5),(1000.0,True)) + p.f=8*sympy.pi**2*sympy.sin(2*sympy.pi*x)*sympy.sin(2*sympy.pi*y) + for b in ("Bottom","Top","Left","Right"): p.add_dirichlet_bc(0.0,b) + p.petsc_options["ksp_rtol"]=1e-8; p.petsc_options["ksp_type"]="cg" + return p + +def _hierarchy(): + m0=uw.meshing.UnstructuredSimplexBox(minCoords=(0,0),maxCoords=(1,1),cellSize=0.5,regular=True,qdegree=3) + dm0=m0.dm; dm1=dm0.refine(); dm2=_sbr(dm1) # 2 uniform-ish + 1 SBR + return m0, [_wrap(dm0,m0), _wrap(dm1,m0)], _wrap(dm2,m0) + +def test_custom_fmg_hierarchy_converges(): + m0, coarse, fine = _hierarchy() + s=_poisson(fine) + custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_poisson, builder="barycentric") + s.solve() + assert s.snes.getKSP().getPC().getType()=="mg" + assert s.snes.getKSP().getPC().getMGLevels()==3 + assert s.snes.getConvergedReason()>0 + # matches a GAMG solve of the same fine problem + g=_poisson(_wrap(fine.dm,m0)); g.preconditioner="gamg"; g.solve() + rel=np.linalg.norm(s.Unknowns.u.data-g.Unknowns.u.data)/np.linalg.norm(g.Unknowns.u.data) + assert rel < 1e-4 + +def test_bc_per_level_reduction_no_zero_columns(): + """build() must produce transfers with no zero columns (BCs at every level).""" + m0, coarse, fine = _hierarchy() + s=_poisson(fine); s._build(False,False,None); s.snes.setUp() + h=custom_mg.CustomMGHierarchy(coarse+[fine], builder="barycentric") + Ps=h.build(s, _poisson) # raises if any transfer has zero columns + assert len(Ps)==2 + for P in Ps: + Pc=P.getValuesCSR(); import scipy.sparse as sp + M=sp.csr_matrix(Pc[::-1], shape=P.getSize()) + assert int((np.asarray((M!=0).sum(axis=0)).ravel()==0).sum())==0 + +def test_rbf_builder_also_works(): + m0, coarse, fine = _hierarchy() + s=_poisson(fine) + custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_poisson, builder="rbf") + s.solve() + assert s.snes.getKSP().getPC().getType()=="mg" + assert s.snes.getConvergedReason()>0 From 6eb0b340d8e48d6762971c9c49d269b26eab51a8 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 06:36:58 +1000 Subject: [PATCH 03/52] docs(custom_mg): Layer-1 status + remaining hardening before merge Layer 1 (generalized FMG hierarchy) is an independent, working capability (arbitrary nested/non-nested coarse grids -> geometric FMG, BC-per-level). Current scope experimental: scalar/single-field, serial, factory API. Remaining before merge-ready general feature: Stokes velocity-block, drop the factory (label-based BC reduction), parallel. Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index cb2908cc..e0016959 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -114,6 +114,31 @@ Later (not this pass; do not block): dynamic per-step reallocation loop; efficie (transfer caching, exact-nested combinatorial P, operator reuse); load-balancing adapters; non-nested parallel transfers; knockout-as-coarsening as an alternative Layer-2 strategy. +## Status (2026-06-29) + +**Layer 1 is an independent, working capability** — arbitrary coarse grids +(nested *or* non-nested, uniform *or* SBR-refined) → geometric FMG via custom-P +with BC-per-level reduction. It does not depend on Layer 2. + +Landed (committed, tested — `test_1014/1015/1016`, 18 pass): +- `CustomMGHierarchy`, `set_custom_fmg`, `sbr_refine`/`sbr_refine_where`; +- automatic BC-per-level reduction + zero-column guard; +- validated: scalar jump-coeff Poisson, 5-level (3 uniform + 2 SBR), 3 FMG iters + vs GAMG 46. + +**Current scope = experimental:** scalar / single-field-vector (top-level PC), +**serial**, and `set_custom_fmg` takes a `level_solver_factory`. + +Remaining hardening **before** this is a merge-ready general feature (the +relaunch's first task, in order): +1. **Stokes / saddle-point** (velocity-block injection; `_install_transfers` + rejects `fieldsplit_velocity_`). Blocker: velocity sub-PC unreachable pre-solve + → use Galerkin-off + `MatPtAP` coarse ops after first solve. + nullspace re-attach. +2. **Drop the factory** — derive per-level constrained DOFs from boundary labels + + BC components (no throwaway solver) for a clean API. +3. **Parallel (np>1)** — `_reduced_map` is serial; validate co-located rank-local + transfers (fast-follow). + ## Explicit non-goals for this pass - Knockout (shown to pay full-fine assembly; structural value only) — not pursued now. - Non-nested custom-P in parallel — serial/experimental only. From 2dc60e3333e5bd51c9025bab8745c994436cbf79 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:14:18 +1000 Subject: [PATCH 04/52] feat(custom_mg): Stokes velocity-block custom-P FMG (Phase 1, Step 1) Integrate the generalized FMG hierarchy into the saddle-point solver's velocity sub-block. set_custom_fmg(..., field_id=0) now drives geometric multigrid on the velocity block with our supplied (barycentric / RBF) prolongations + Galerkin RAP. The velocity sub-PC is unreachable until the monolithic Jacobian is assembled (PCFieldSplit forms A_vv via MatCreateSubMatrix; snes.setUp builds structure only -> err73). So _install_velocity_block_transfers: forces a Jacobian assembly (computeFunction + computeJacobian at the zero guess; throwaway max_it=0 fallback), reaches the velocity sub-PC, reset()s it and rebuilds a FRESH PCMG from our P (mirrors the proven standalone recipe; sidesteps the MatProductReplaceMats live-swap bug), re-attaches the coupled Stokes nullspace. _configure_pcmg now derives the options prefix from pc.getOptionsPrefix() so both the scalar top-level PC and the velocity sub-PC are configured correctly. inject_custom_mg is wired into SNES_Stokes_SaddlePt.solve (guarded no-op unless set_custom_fmg was called), injected after setFromOptions / nullspace, before the real solve. Validated (SolCx eta_B=1e6, 3-level nested hierarchy, in-solver via solver.solve()): velocity block converges in 6 MG iters vs GAMG 198, solution matches the GAMG reference to 1.5e-9. test_1017 (barycentric + rbf). test_1014/1015/1016 unchanged (20 pass total). Underworld development team with AI support from Claude Code --- .../cython/petsc_generic_snes_solvers.pyx | 14 ++ src/underworld3/utilities/custom_mg.py | 138 ++++++++++++++---- tests/test_1017_custom_mg_stokes.py | 119 +++++++++++++++ 3 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 tests/test_1017_custom_mg_stokes.py diff --git a/src/underworld3/cython/petsc_generic_snes_solvers.pyx b/src/underworld3/cython/petsc_generic_snes_solvers.pyx index 35c57ae3..58f21ab1 100644 --- a/src/underworld3/cython/petsc_generic_snes_solvers.pyx +++ b/src/underworld3/cython/petsc_generic_snes_solvers.pyx @@ -7458,6 +7458,13 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): self.petsc_options.setValue("snes_max_it", snes_max_it) self.snes.setFromOptions() self._attach_stokes_nullspace() + # Custom geometric-MG prolongation on the velocity block (if registered + # via set_custom_fmg). Injected here — after setFromOptions/nullspace, + # before the real solve — because the velocity sub-PC is only reachable + # once the monolithic Jacobian is assembled (see custom_mg). + if self._custom_mg is not None: + from underworld3.utilities.custom_mg import inject_custom_mg + inject_custom_mg(self) self._snes_solve_with_retries(gvec, divergence_retries, verbose) else: @@ -7468,6 +7475,13 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): self.petsc_options.setValue("snes_max_it", snes_max_it) self.snes.setFromOptions() self._attach_stokes_nullspace() + # Custom geometric-MG prolongation on the velocity block (if registered + # via set_custom_fmg). Injected here — after setFromOptions/nullspace, + # before the real solve — because the velocity sub-PC is only reachable + # once the monolithic Jacobian is assembled (see custom_mg). + if self._custom_mg is not None: + from underworld3.utilities.custom_mg import inject_custom_mg + inject_custom_mg(self) self._snes_solve_with_retries(gvec, divergence_retries, verbose) # Project the rigid-body rotation gauge out of the converged solution. diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index b6fea9c3..187634db 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -243,42 +243,128 @@ def _reduced_transfer(coarse_coords, fine_coords, r2f_c, r2f_f, ncomp, builder): return Pr -def _install_transfers(solver, Ps, verbose=False): - """Configure the managed PCMG block to use the supplied prolongations. - Build-time injection (before first PCSetUp): set DMActive(OPERATOR,False) so - PETSc does not re-derive interpolation from the DM, Galerkin RAP for coarse - operators. Scalar / single-field-vector (top-level PC: ``_pc_option_prefix`` - is ``""``); the Stokes velocity-block path is Phase 2.""" +def _configure_pcmg(pc, Ps): + """Reconfigure ``pc`` as a fresh PCMG (FMG F-cycle) driven by the supplied + reduced->reduced prolongations ``Ps``, Galerkin RAP for coarse operators. + + Writes the MG bundle into the options DB under the PC's OWN options prefix + (``pc.getOptionsPrefix()`` — e.g. ``Solver_N_`` for the scalar top-level PC, + ``Solver_N_fieldsplit_velocity_`` for the Stokes velocity sub-PC) and removes + any gamg keys BEFORE ``setFromOptions`` — otherwise ``setFromOptions`` re-reads + a lingering ``pc_type=gamg`` and reverts ``setType("mg")``. ``setMGInterpolation`` + persists through ``setFromOptions``; the first ``PCSetUp`` builds the coarse + operators from our P (no ``MatProductReplaceMats`` shape bug, since the PCMG is + fresh and P's size is fixed).""" nlev = len(Ps) + 1 - opts = solver.petsc_options - pfx = solver._pc_option_prefix or "" - if pfx not in ("",): - raise NotImplementedError( - "Layer-1 install supports top-level PC (scalar / single-field vector). " - f"PC prefix '{pfx}' (e.g. Stokes velocity block) is Phase 2.") - opts[pfx + "pc_type"] = "mg" - opts[pfx + "pc_mg_type"] = "full" - opts[pfx + "pc_mg_galerkin"] = "both" - opts[pfx + "mg_levels_ksp_type"] = "richardson" - opts[pfx + "mg_levels_pc_type"] = "sor" - opts[pfx + "mg_coarse_pc_type"] = "redundant" - opts[pfx + "mg_coarse_redundant_pc_type"] = "lu" + prefix = pc.getOptionsPrefix() or "" + opts = PETSc.Options() + opts.setValue(prefix + "pc_type", "mg") + opts.setValue(prefix + "pc_mg_type", "full") + opts.setValue(prefix + "pc_mg_galerkin", "both") + opts.setValue(prefix + "mg_levels_ksp_type", "richardson") + opts.setValue(prefix + "mg_levels_pc_type", "sor") + opts.setValue(prefix + "mg_coarse_pc_type", "redundant") + opts.setValue(prefix + "mg_coarse_redundant_pc_type", "lu") for key in ("pc_gamg_type", "pc_gamg_repartition", "pc_gamg_agg_nsmooths"): - opts.delValue(pfx + key) - - solver.snes.setUp() - ksp = solver.snes.getKSP() - ksp.setDMActive(PETSc.KSP.DMActive.OPERATOR, False) - pc = ksp.getPC() + opts.delValue(prefix + key) pc.setType("mg") pc.setMGLevels(nlev) pc.setMGType(PETSc.PC.MGType.FULL) for l in range(1, nlev): pc.setMGInterpolation(l, Ps[l - 1]) pc.setFromOptions() + + +def _install_transfers(solver, Ps, verbose=False): + """Configure the managed PCMG block to use the supplied prolongations. + + Two paths, keyed by ``solver._pc_option_prefix``: + + * **scalar / single-field vector** (top-level PC, prefix ``""``): build-time + injection *before* the first ``PCSetUp`` — the first Galerkin assembly is + built from our P directly. + * **Stokes velocity block** (prefix ``"fieldsplit_velocity_"``): the velocity + sub-PC is unreachable until the monolithic Jacobian is assembled + (``PCFieldSplit`` forms ``A_vv`` via ``MatCreateSubMatrix``; ``snes.setUp`` + builds structure only -> err73). So force a Jacobian assembly, reach the + velocity sub-PC, ``reset`` it and rebuild a fresh PCMG from our P, then + re-attach the coupled Stokes nullspace. + + Either way ``KSPSetDMActive(OPERATOR, False)`` stops PETSc re-deriving the + interpolation from the DM hierarchy.""" + nlev = len(Ps) + 1 + pfx = solver._pc_option_prefix or "" + + if pfx == "": + ksp = solver.snes.getKSP() + solver.snes.setUp() + ksp.setDMActive(PETSc.KSP.DMActive.OPERATOR, False) + _configure_pcmg(ksp.getPC(), Ps) + if verbose: + from underworld3 import mpi + mpi.pprint(f"[{solver.name}] custom FMG installed: {nlev} levels, " + f"P sizes {[tuple(P.getSize()) for P in Ps]}") + return + + if pfx != "fieldsplit_velocity_": + raise NotImplementedError( + f"custom_mg install: unsupported PC prefix '{pfx}'.") + + _install_velocity_block_transfers(solver, Ps, verbose=verbose) + + +def _install_velocity_block_transfers(solver, Ps, verbose=False): + """Stokes velocity-block install (mechanism A: reset + fresh PCMG). + + Preconditions: ``solver._build`` + ``setFromOptions`` + ``_attach_stokes_nullspace`` + have run (the call site in ``SNES_Stokes_SaddlePt.solve`` guarantees this), so + the SNES / DM exist but the Jacobian VALUES are not yet assembled.""" + snes = solver.snes + snes.setUp() + + # 1. force monolithic Jacobian assembly so the fieldsplit can form A_vv + x0 = solver.dm.getGlobalVec() + x0.set(0.0) + J = snes.getJacobian()[0] + Pmat = snes.getJacobian()[1] + try: + f = J.createVecLeft() + snes.computeFunction(x0, f) + snes.computeJacobian(x0, J, Pmat) + except PETSc.Error: + # fallback: throwaway max_it=0 solve assembles + splits the operator + saved = (solver.petsc_options.getString("snes_max_it") + if solver.petsc_options.hasName("snes_max_it") else None) + solver.petsc_options["snes_max_it"] = 0 + snes.setFromOptions() + snes.solve(None, x0) + if saved is not None: + solver.petsc_options["snes_max_it"] = saved + snes.setFromOptions() + solver.dm.restoreGlobalVec(x0) + + # 2. split -> reach the velocity sub-KSP / sub-PC (field 0) + outer_pc = snes.getKSP().getPC() + outer_pc.setUp() + vel_ksp = outer_pc.getFieldSplitSubKSP()[0] + vel_pc = vel_ksp.getPC() + sub = vel_pc.getOptionsPrefix() or "fieldsplit_velocity_" + A_vv, P_vv = vel_pc.getOperators() # capture before reset (reset drops them) + + # 3. fresh PCMG on the velocity sub-block from our Ps + vel_ksp.setDMActive(PETSc.KSP.DMActive.OPERATOR, False) + vel_pc.reset() + vel_pc.setOperators(A_vv, P_vv) + _configure_pcmg(vel_pc, Ps) + vel_pc.setUp() + + # 4. re-attach the coupled Stokes nullspace (operator state was touched) + solver._attach_stokes_nullspace() + if verbose: from underworld3 import mpi - mpi.pprint(f"[{solver.name}] custom FMG installed: {nlev} levels, " + mpi.pprint(f"[{solver.name}] custom FMG installed on velocity block: " + f"{len(Ps) + 1} levels, sub-prefix {sub!r}, " f"P sizes {[tuple(P.getSize()) for P in Ps]}") diff --git a/tests/test_1017_custom_mg_stokes.py b/tests/test_1017_custom_mg_stokes.py new file mode 100644 index 00000000..5abe7b42 --- /dev/null +++ b/tests/test_1017_custom_mg_stokes.py @@ -0,0 +1,119 @@ +"""Layer-1 generalized FMG hierarchy on the STOKES velocity block (Phase-1 Step 1). + +Custom-built prolongations (barycentric / RBF) drive geometric multigrid on the +velocity sub-block of the saddle-point solver, via set_custom_fmg(field_id=0). +Validated on SolCx (eta_B=1e6): the velocity block converges in a handful of MG +iterations (vs ~200 for GAMG) and the solution matches a GAMG reference. + +The hard part (see custom_mg._install_velocity_block_transfers): the velocity +sub-PC is unreachable until the monolithic Jacobian is assembled, so the install +forces a Jacobian assembly, reaches the velocity sub-PC, and rebuilds a fresh PCMG +from our supplied prolongations. +""" +import numpy as np +import pytest +import underworld3 as uw +from underworld3.function import analytic as A +from underworld3.utilities import custom_mg + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + + +def _wrap(dm, m0): + return uw.discretisation.Mesh( + dm.clone(), simplex=True, + coordinate_system_type=m0.CoordinateSystem.coordinate_type, + qdegree=3, boundaries=m0.boundaries) + + +def _hierarchy(): + """cellSize 0.25 base, two uniform refinements -> 3 nested levels.""" + m0 = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.25, regular=True, qdegree=3) + dm0 = m0.dm + dm1 = dm0.refine() + dm2 = dm1.refine() + meshes = [_wrap(dm0, m0), _wrap(dm1, m0), _wrap(dm2, m0)] + return meshes[:-1], meshes[-1] + + +def _walls(s): + s.add_dirichlet_bc((0.0, None), "Left") + s.add_dirichlet_bc((0.0, None), "Right") + s.add_dirichlet_bc((None, 0.0), "Bottom") + s.add_dirichlet_bc((None, 0.0), "Top") + s.petsc_use_pressure_nullspace = True + s.petsc_options["snes_type"] = "ksponly" + + +def _coarse_factory(mesh): + """Same-discretisation coarse-level Stokes (only its constrained section is + read by CustomMGHierarchy.build to derive the BC-reduced DOF map).""" + s = uw.systems.Stokes(mesh) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = 1.0 + _walls(s) + return s + + +def _solcx(mesh, eta_B=1e6): + sol = A.SolCx(mesh, eta_A=1.0, eta_B=eta_B, x_c=0.5, n=1) + s = uw.systems.Stokes(mesh) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.saddle_preconditioner = 1.0 / sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + _walls(s) + s.tolerance = 1e-8 + return s, sol + + +def _vel_ksp(s): + return s.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + + +def test_custom_fmg_velocity_block_barycentric(): + """Custom barycentric P drives geometric MG on the SolCx velocity block, + converging in few iterations and matching a GAMG reference solution.""" + coarse, fine = _hierarchy() + + # GAMG reference + sg, solg = _solcx(fine) + sg.preconditioner = "gamg" + sg.solve() + verr_g = solg.velocity_error(sg.u) + iters_g = _vel_ksp(sg).getIterationNumber() + + # custom-P geometric MG on the velocity block + s, sol = _solcx(fine) + custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_coarse_factory, + builder="barycentric", field_id=0) + s.solve() + vksp = _vel_ksp(s) + + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGLevels() == len(coarse) + 1 + # geometric MG crushes GAMG on the eta-jump velocity block + assert vksp.getIterationNumber() <= 15 + assert vksp.getIterationNumber() < iters_g + # correct solution: matches analytic to GAMG's accuracy, and matches GAMG itself + verr = sol.velocity_error(s.u) + assert verr < 2.0 * verr_g + 1e-6 + rel = np.linalg.norm(s.u.data - sg.u.data) / (np.linalg.norm(sg.u.data) + 1e-30) + assert rel < 1e-4 + + +def test_custom_fmg_velocity_block_rbf(): + """The RBF builder must also drive a converging velocity-block MG solve.""" + coarse, fine = _hierarchy() + s, sol = _solcx(fine) + custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_coarse_factory, + builder="rbf", field_id=0) + s.solve() + vksp = _vel_ksp(s) + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGLevels() == len(coarse) + 1 + assert vksp.getIterationNumber() <= 25 + assert sol.velocity_error(s.u) < 1e-2 From cb77289e52c7347f5dc5a06265116f4584004303 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:14:44 +1000 Subject: [PATCH 05/52] docs(custom_mg): mark Stokes velocity-block integration done (Phase 1, Step 1) Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index e0016959..19bbc72b 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -126,14 +126,26 @@ Landed (committed, tested — `test_1014/1015/1016`, 18 pass): - validated: scalar jump-coeff Poisson, 5-level (3 uniform + 2 SBR), 3 FMG iters vs GAMG 46. -**Current scope = experimental:** scalar / single-field-vector (top-level PC), -**serial**, and `set_custom_fmg` takes a `level_solver_factory`. - -Remaining hardening **before** this is a merge-ready general feature (the -relaunch's first task, in order): -1. **Stokes / saddle-point** (velocity-block injection; `_install_transfers` - rejects `fieldsplit_velocity_`). Blocker: velocity sub-PC unreachable pre-solve - → use Galerkin-off + `MatPtAP` coarse ops after first solve. + nullspace re-attach. +**Current scope = experimental:** **serial**, and `set_custom_fmg` takes a +`level_solver_factory`. Scalar / single-field-vector **and** Stokes velocity-block +are now supported. + +Remaining hardening **before** this is a merge-ready general feature (in order): +1. ~~**Stokes / saddle-point** (velocity-block injection).~~ **DONE** (2026-06-29). + `set_custom_fmg(..., field_id=0)` drives custom-P geometric MG on the velocity + sub-block. The sub-PC is unreachable until the monolithic Jacobian is assembled, + so `_install_velocity_block_transfers` forces a Jacobian assembly + (`computeFunction`+`computeJacobian` at the zero guess; `max_it=0` fallback), + reaches the velocity sub-PC, `reset`s it and rebuilds a **fresh PCMG** from our P + (mechanism A — mirrors the proven standalone recipe and sidesteps the + `MatProductReplaceMats` live-swap bug; the Galerkin-off + `MatPtAP` path remains + the documented fallback), then re-attaches the coupled Stokes nullspace. Wired + into `SNES_Stokes_SaddlePt.solve` as a guarded no-op. Validated on SolCx + (η-jump 1e6, 3-level nested, in-solver): velocity block 6 MG iters vs GAMG 198, + solution matches GAMG to 1.5e-9 (`test_1017`). NOTE: free-slip velocity + rigid-body modes on `A_vv` + coarse ops are **not yet** handled (reusing + `_attach_stokes_nullspace` covers the SolCx pressure-nullspace case only) — a + Phase-2.5 follow-up. 2. **Drop the factory** — derive per-level constrained DOFs from boundary labels + BC components (no throwaway solver) for a clean API. 3. **Parallel (np>1)** — `_reduced_map` is serial; validate co-located rank-local From e0181ba0017f873833af3585eac7d31d8f913a31 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:37:18 +1000 Subject: [PATCH 06/52] refactor(custom_mg): drop level_solver_factory (Phase 1, Step 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each coarse level's BC-constrained reduced map is now derived directly from the coarse mesh DM via _coarse_reduced_map: clone the DM, copy the (built) finest solver's fields + DS onto it, createDS, read the global section. The DS carries UW's exact essential-BC definitions and is a topology-independent discretisation spec, so it constrains the matching boundary DOFs on ANY coarse mesh that shares the solver's boundary labels (nested or not). This removes the throwaway-solver factory (heavy, leak-prone, and an API wart): set_custom_fmg and CustomMGHierarchy.build no longer take a level_solver_factory. Leak-free — DM ops only, no SNES / JIT. Validated identical to the old factory path (reduced maps byte-equal: 126 + 510 DOFs on the SolCx 3-level hierarchy). test_1014/1015/1016/1017 all green (20). custom-P vs native FMG unchanged (6 vs 5 iters, sol match 3.7e-10). Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 50 ++++++++++++++++++-------- tests/test_1016_custom_mg_hierarchy.py | 6 ++-- tests/test_1017_custom_mg_stokes.py | 16 ++------- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 187634db..611638f6 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -233,6 +233,27 @@ def _reduced_map(dm, field_id=None): return r2f, n_full +def _coarse_reduced_map(solver, coarse_mesh, field_id=None): + """A COARSE level's BC-constrained reduced map, with NO throwaway solver. + + Clone the coarse mesh DM and copy the (built) finest ``solver``'s fields + DS + onto it. The DS carries UW's exact essential-BC definitions (the custom + essential-field boundaries), and is a topology-independent discretisation spec, + so ``createDS`` constrains the matching boundary DOFs on ANY coarse mesh that + carries the same boundary labels (nested or not). The resulting global section + gives the same reduced map a full same-discretisation solver would — validated + identical to the old ``level_solver_factory`` path. Leak-free: DM ops only, no + SNES / JIT. + + NOTE: serial, like ``_reduced_map`` (uses local indices); parallel correctness + is a Phase-3 item.""" + cdm = coarse_mesh.dm.clone() + solver.dm.copyFields(cdm) + solver.dm.copyDS(cdm) + cdm.createDS() + return _reduced_map(cdm, field_id) + + def _reduced_transfer(coarse_coords, fine_coords, r2f_c, r2f_f, ncomp, builder): """Build one prolongation reduced(coarse) -> reduced(fine): node-level scalar P -> interleave ``ncomp`` components -> drop BC rows/cols.""" @@ -395,11 +416,12 @@ def __init__(self, level_meshes, builder="barycentric", field_id=None): self.field_id = field_id self.transfers = None - def build(self, solver, level_solver_factory): + def build(self, solver): """Build the BC-reduced prolongations. ``solver`` is the (built) finest - solver; ``level_solver_factory(mesh) -> built solver`` provides a - same-discretisation solver on each COARSE level so its constrained - section gives that level's reduced map. The finest level uses ``solver``.""" + solver. Each COARSE level's BC-constrained reduced map is derived directly + from the coarse mesh DM by copying the finest solver's fields + DS onto it + (``_coarse_reduced_map``) — no throwaway solver. The finest level reads its + map from ``solver.dm``.""" var = solver.Unknowns.u degree = var.degree continuous = getattr(var, "continuous", True) @@ -411,10 +433,7 @@ def build(self, solver, level_solver_factory): if k == nlev - 1: rmap, nfull = _reduced_map(solver.dm, self.field_id) else: - tmp = level_solver_factory(mesh) - tmp._build(False, False, None) - tmp.snes.setUp() - rmap, nfull = _reduced_map(tmp.dm, self.field_id) + rmap, nfull = _coarse_reduced_map(solver, mesh, self.field_id) nc = nfull // c.shape[0] if nfull % c.shape[0] != 0: raise RuntimeError( @@ -448,20 +467,21 @@ def install(self, solver, verbose=False): # --------------------------------------------------------------------------- # # Entry points # --------------------------------------------------------------------------- # -def set_custom_fmg(solver, coarse_meshes, *, level_solver_factory, - builder="barycentric", field_id=None, verbose=False): +def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", + field_id=None, verbose=False): """Generalized custom-P FMG with BC-per-level reduction (the correct path). Registers a :class:`CustomMGHierarchy` on the solver so that the next ``solve()`` builds and installs it (build-time injection). The hierarchy is - ``[*coarse_meshes, solver.mesh]``; ``level_solver_factory(mesh)`` must return - a same-discretisation solver on a coarse level (used only to read its - BC-constrained section).""" + ``[*coarse_meshes, solver.mesh]``; each coarse level's BC-constrained reduced + map is derived directly from its DM by copying the solver's fields + DS + (``_coarse_reduced_map``), so ``coarse_meshes`` need only carry the same + boundary labels as the solver's mesh. For a saddle-point (Stokes) solver pass + ``field_id=0`` to target the velocity sub-block.""" solver._custom_mg = { "mode": "hierarchy", "hierarchy": CustomMGHierarchy(list(coarse_meshes) + [solver.mesh], builder=builder, field_id=field_id), - "factory": level_solver_factory, "verbose": verbose, } solver.is_setup = False @@ -477,7 +497,7 @@ def inject_custom_mg(solver): if isinstance(cfg, dict) and cfg.get("mode") == "hierarchy": h = cfg["hierarchy"] - h.build(solver, cfg["factory"]) + h.build(solver) h.install(solver, verbose=cfg.get("verbose", False)) return diff --git a/tests/test_1016_custom_mg_hierarchy.py b/tests/test_1016_custom_mg_hierarchy.py index 60c873b2..402ee9c4 100644 --- a/tests/test_1016_custom_mg_hierarchy.py +++ b/tests/test_1016_custom_mg_hierarchy.py @@ -29,7 +29,7 @@ def _hierarchy(): def test_custom_fmg_hierarchy_converges(): m0, coarse, fine = _hierarchy() s=_poisson(fine) - custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_poisson, builder="barycentric") + custom_mg.set_custom_fmg(s, coarse, builder="barycentric") s.solve() assert s.snes.getKSP().getPC().getType()=="mg" assert s.snes.getKSP().getPC().getMGLevels()==3 @@ -44,7 +44,7 @@ def test_bc_per_level_reduction_no_zero_columns(): m0, coarse, fine = _hierarchy() s=_poisson(fine); s._build(False,False,None); s.snes.setUp() h=custom_mg.CustomMGHierarchy(coarse+[fine], builder="barycentric") - Ps=h.build(s, _poisson) # raises if any transfer has zero columns + Ps=h.build(s) # raises if any transfer has zero columns assert len(Ps)==2 for P in Ps: Pc=P.getValuesCSR(); import scipy.sparse as sp @@ -54,7 +54,7 @@ def test_bc_per_level_reduction_no_zero_columns(): def test_rbf_builder_also_works(): m0, coarse, fine = _hierarchy() s=_poisson(fine) - custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_poisson, builder="rbf") + custom_mg.set_custom_fmg(s, coarse, builder="rbf") s.solve() assert s.snes.getKSP().getPC().getType()=="mg" assert s.snes.getConvergedReason()>0 diff --git a/tests/test_1017_custom_mg_stokes.py b/tests/test_1017_custom_mg_stokes.py index 5abe7b42..3ee4954e 100644 --- a/tests/test_1017_custom_mg_stokes.py +++ b/tests/test_1017_custom_mg_stokes.py @@ -46,16 +46,6 @@ def _walls(s): s.petsc_options["snes_type"] = "ksponly" -def _coarse_factory(mesh): - """Same-discretisation coarse-level Stokes (only its constrained section is - read by CustomMGHierarchy.build to derive the BC-reduced DOF map).""" - s = uw.systems.Stokes(mesh) - s.constitutive_model = uw.constitutive_models.ViscousFlowModel - s.constitutive_model.Parameters.shear_viscosity_0 = 1.0 - _walls(s) - return s - - def _solcx(mesh, eta_B=1e6): sol = A.SolCx(mesh, eta_A=1.0, eta_B=eta_B, x_c=0.5, n=1) s = uw.systems.Stokes(mesh) @@ -86,8 +76,7 @@ def test_custom_fmg_velocity_block_barycentric(): # custom-P geometric MG on the velocity block s, sol = _solcx(fine) - custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_coarse_factory, - builder="barycentric", field_id=0) + custom_mg.set_custom_fmg(s, coarse, builder="barycentric", field_id=0) s.solve() vksp = _vel_ksp(s) @@ -108,8 +97,7 @@ def test_custom_fmg_velocity_block_rbf(): """The RBF builder must also drive a converging velocity-block MG solve.""" coarse, fine = _hierarchy() s, sol = _solcx(fine) - custom_mg.set_custom_fmg(s, coarse, level_solver_factory=_coarse_factory, - builder="rbf", field_id=0) + custom_mg.set_custom_fmg(s, coarse, builder="rbf", field_id=0) s.solve() vksp = _vel_ksp(s) assert s.snes.getConvergedReason() > 0 From 2e2c4f942ddc33b467e74ea73a2f4e228cc95df7 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:37:48 +1000 Subject: [PATCH 07/52] docs(custom_mg): mark factory removal done (Phase 1, Step 2) Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index 19bbc72b..c64f396e 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -126,9 +126,8 @@ Landed (committed, tested — `test_1014/1015/1016`, 18 pass): - validated: scalar jump-coeff Poisson, 5-level (3 uniform + 2 SBR), 3 FMG iters vs GAMG 46. -**Current scope = experimental:** **serial**, and `set_custom_fmg` takes a -`level_solver_factory`. Scalar / single-field-vector **and** Stokes velocity-block -are now supported. +**Current scope = experimental:** **serial**. Scalar / single-field-vector **and** +Stokes velocity-block are supported; the throwaway-solver factory has been removed. Remaining hardening **before** this is a merge-ready general feature (in order): 1. ~~**Stokes / saddle-point** (velocity-block injection).~~ **DONE** (2026-06-29). @@ -146,10 +145,16 @@ Remaining hardening **before** this is a merge-ready general feature (in order): rigid-body modes on `A_vv` + coarse ops are **not yet** handled (reusing `_attach_stokes_nullspace` covers the SolCx pressure-nullspace case only) — a Phase-2.5 follow-up. -2. **Drop the factory** — derive per-level constrained DOFs from boundary labels + - BC components (no throwaway solver) for a clean API. -3. **Parallel (np>1)** — `_reduced_map` is serial; validate co-located rank-local - transfers (fast-follow). +2. ~~**Drop the factory**.~~ **DONE** (2026-06-29). Each coarse level's + BC-constrained reduced map is derived directly from its DM via + `_coarse_reduced_map`: clone the coarse DM, `copyFields` + `copyDS` from the + finest solver, `createDS`. The DS carries UW's exact essential-BC definitions + and is topology-independent, so it constrains any coarse mesh sharing the + solver's boundary labels — validated byte-identical to the old factory path, + leak-free (no SNES / JIT). `set_custom_fmg` / `build` no longer take a + `level_solver_factory`. +3. **Parallel (np>1)** — `_reduced_map` / `_coarse_reduced_map` are serial (local + indices); validate co-located rank-local transfers (fast-follow). ## Explicit non-goals for this pass - Knockout (shown to pay full-fine assembly; structural value only) — not pursued now. From ecabfed08617ffc99e00a235fcaafbfd26f39d95 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:43:23 +1000 Subject: [PATCH 08/52] feat(custom_mg): loud serial-only guard for np>1 (Phase 1, Step 3 partial) The reduced maps use rank-local DOF indices and the prolongations assemble as serial AIJ, so at np>1 custom-P would silently build wrong P (or hit a cryptic broadcast error). Add _require_serial: set_custom_fmg and inject_custom_mg now raise a clear NotImplementedError on >1 ranks, pointing to preconditioner='fmg'/'gamg'. Serial behaviour unchanged. Parallel test tests/parallel/test_1017_custom_mg_serial_guard_mpi.py (np=2) asserts both the set-time and legacy solve-time guards fire. Full parallel custom-P (nested co-partitioned, rank-local point location + ghosting, MPIAIJ assembly with global-section reduction) remains a designed fast-follow. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 16 ++++++ .../test_1017_custom_mg_serial_guard_mpi.py | 52 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/parallel/test_1017_custom_mg_serial_guard_mpi.py diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 611638f6..11c4a122 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -103,6 +103,20 @@ def phi(r): _BUILDERS = {"barycentric": barycentric_prolongation, "rbf": rbf_prolongation} +def _require_serial(where): + """Custom-P transfers are serial-only (experimental). The reduced maps use + rank-local DOF indices and the prolongations assemble as serial AIJ; at np>1 + they would silently build wrong P / mis-numbered transfers. Parallel support + (nested co-partitioned, rank-local P + MPIAIJ, global-section reduction) is a + designed fast-follow — until then, fail loudly rather than wrong.""" + from underworld3 import mpi + if mpi.size > 1: + raise NotImplementedError( + f"custom_mg ({where}): custom-P geometric MG is serial-only " + f"(running on {mpi.size} ranks). Parallel (np>1) support is not yet " + f"implemented; use preconditioner='fmg' (nested) or 'gamg' in parallel.") + + # --------------------------------------------------------------------------- # # Local Skeleton-Based Refinement (no MMG; on-rank; conforming) # --------------------------------------------------------------------------- # @@ -478,6 +492,7 @@ def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", (``_coarse_reduced_map``), so ``coarse_meshes`` need only carry the same boundary labels as the solver's mesh. For a saddle-point (Stokes) solver pass ``field_id=0`` to target the velocity sub-block.""" + _require_serial("set_custom_fmg") solver._custom_mg = { "mode": "hierarchy", "hierarchy": CustomMGHierarchy(list(coarse_meshes) + [solver.mesh], @@ -493,6 +508,7 @@ def inject_custom_mg(solver): - ``mode == "hierarchy"`` -> BC-per-level reduced path (correct, general); - legacy dict ``{coarse_meshes, kind}`` -> finest-only reduction (kept for back-compat; valid only when coarse levels are non-nested / unconstrained).""" + _require_serial("inject_custom_mg") cfg = solver._custom_mg if isinstance(cfg, dict) and cfg.get("mode") == "hierarchy": diff --git a/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py b/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py new file mode 100644 index 00000000..f9862e65 --- /dev/null +++ b/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py @@ -0,0 +1,52 @@ +"""Parallel guard for custom-P geometric MG (custom_mg). + +Custom-P transfers are serial-only (experimental): the reduced maps use rank-local +DOF indices and the prolongations assemble as serial AIJ, so at np>1 they would +silently build wrong P. set_custom_fmg / inject must therefore FAIL LOUDLY in +parallel rather than produce incorrect results. Parallel (np>1) support is a +designed fast-follow (nested co-partitioned, rank-local P + MPIAIJ). + +Run: + mpirun -n 2 python -m pytest --with-mpi tests/parallel/test_1017_custom_mg_serial_guard_mpi.py +""" +import pytest +import sympy +import underworld3 as uw +from underworld3.utilities import custom_mg + +pytestmark = [pytest.mark.mpi(min_size=2), pytest.mark.timeout(120)] + + +def _box(cellSize, refinement=None): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0), + cellSize=cellSize, refinement=refinement, qdegree=2) + + +def _poisson(mesh): + p = uw.systems.Poisson(mesh) + p.constitutive_model = uw.constitutive_models.DiffusionModel + p.constitutive_model.Parameters.diffusivity = 1 + p.f = 0.0 + p.add_dirichlet_bc(0.0, "Bottom"); p.add_dirichlet_bc(1.0, "Top") + return p + + +@pytest.mark.mpi(min_size=2) +def test_set_custom_fmg_raises_in_parallel(): + """set_custom_fmg must raise NotImplementedError (not silently mis-build) np>1.""" + assert uw.mpi.size > 1 + s = _poisson(_box(0.25, refinement=2)) + with pytest.raises(NotImplementedError): + custom_mg.set_custom_fmg(s, [_box(0.285), _box(0.142)], builder="barycentric") + + +@pytest.mark.mpi(min_size=2) +def test_inject_guard_blocks_legacy_path_in_parallel(): + """The legacy set_custom_mg path must also fail loudly at solve() in parallel.""" + assert uw.mpi.size > 1 + s = _poisson(_box(0.25, refinement=2)) + # legacy setter writes _custom_mg directly (no module guard); inject must catch it + s.set_custom_mg([_box(0.285), _box(0.142)], kind="barycentric") + with pytest.raises(NotImplementedError): + s.solve() From caff202014876e8bfb53b8f1d94a5a80ef4068de Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 13:43:42 +1000 Subject: [PATCH 09/52] docs(custom_mg): record np>1 serial guard (Phase 1, Step 3) Underworld development team with AI support from Claude Code --- .../design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index c64f396e..901ff5c9 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -154,7 +154,13 @@ Remaining hardening **before** this is a merge-ready general feature (in order): leak-free (no SNES / JIT). `set_custom_fmg` / `build` no longer take a `level_solver_factory`. 3. **Parallel (np>1)** — `_reduced_map` / `_coarse_reduced_map` are serial (local - indices); validate co-located rank-local transfers (fast-follow). + indices) and `P` assembles as serial AIJ. **Guarded** (2026-06-29): + `set_custom_fmg` / `inject_custom_mg` raise a clear `NotImplementedError` at + np>1 (`_require_serial`) so parallel cannot silently mis-build — test + `tests/parallel/test_1017_custom_mg_serial_guard_mpi.py`. The full + parallel-correct implementation (nested co-partitioned, rank-local point + location + ghost layer, MPIAIJ assembly with global-section reduction) remains + the designed fast-follow. ## Explicit non-goals for this pass - Knockout (shown to pay full-fine assembly; structural value only) — not pursued now. From 19b778c783aaa590f237e9fa1ae885500311a6d9 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 14:03:51 +1000 Subject: [PATCH 10/52] feat(custom_mg): parallel (np>1) custom-P FMG, nested co-partitioned (Phase 1, Step 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hierarchy path now builds parallel-correct transfers. Each rank builds its block of P rank-locally: ghost-inclusive coarse coords mean every owned fine node lands in a local coarse simplex (verified 0 misses np=2/4), and the reduced global numbering rides the DM global section via a clean local->global-reduced map (_level_dof_layout scatters owned global indices out through globalToLocal; constrained DOFs stay -1, ghosts resolve to the owner's global index). Transfers assemble as MPIAIJ (owned fine rows, global coarse cols incl. off-rank); constrained coarse DOFs drop -> reduced->reduced. Parallel zero-column guard via P^T·1 + allreduce. _coarse_dof_layout reuses the leak-free copyDS trick for coarse levels. CustomMGHierarchy.build branches serial (scipy CSR) vs parallel (MPIAIJ). The guard now blocks only the legacy finest-only path (still serial). Validated np=1/2/4: scalar Poisson custom-P 4 iters and Stokes SolCx velocity block 6 iters, both matching the GAMG reference (and each other across rank counts). New tests/parallel/test_1017_custom_mg_parallel_mpi.py (scalar + Stokes correctness + legacy serial-guard). Serial 20 unchanged. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 150 +++++++++++++++--- .../test_1017_custom_mg_parallel_mpi.py | 113 +++++++++++++ .../test_1017_custom_mg_serial_guard_mpi.py | 52 ------ 3 files changed, 243 insertions(+), 72 deletions(-) create mode 100644 tests/parallel/test_1017_custom_mg_parallel_mpi.py delete mode 100644 tests/parallel/test_1017_custom_mg_serial_guard_mpi.py diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 11c4a122..f8863940 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -278,6 +278,99 @@ def _reduced_transfer(coarse_coords, fine_coords, r2f_c, r2f_f, ncomp, builder): return Pr +# --------------------------------------------------------------------------- # +# Parallel (np>1) — nested co-partitioned, rank-local P + MPIAIJ +# +# Supported path: the levels are co-partitioned (uniform refine() / on-rank SBR +# keep a fine cell's parent coarse cell on the same rank), so each rank builds +# its block of P from its LOCAL (ghost-inclusive) coarse coords — point-location +# is rank-local. The reduced global numbering rides the DM global section. +# --------------------------------------------------------------------------- # +def _level_dof_layout(dm, field_id=None): + """Parallel DOF layout for one level: ``(l2g, rstart, rend, n_full)``. + + ``l2g[i]`` is the GLOBAL reduced index of local DOF ``i`` — ghost-resolved to + the owner's global index, and ``-1`` for a BC-constrained DOF. Built by + scattering each owned global index out to the local (incl. ghost) layout via + ``globalToLocal`` (constrained local DOFs have no global source, so they keep + the pre-set ``-1``). ``[rstart, rend)`` is this rank's owned global range.""" + sub = _field_subdm(dm, field_id) + gv = sub.getGlobalVec() + lv = sub.getLocalVec() + rstart, rend = gv.getOwnershipRange() + gv.array[:] = np.arange(rstart, rend, dtype=float) + lv.set(-1.0) + sub.globalToLocal(gv, lv, addv=PETSc.InsertMode.INSERT_VALUES) + l2g = np.rint(lv.array).astype(np.int64).copy() + n_full = lv.getLocalSize() + sub.restoreGlobalVec(gv) + sub.restoreLocalVec(lv) + return l2g, rstart, rend, n_full + + +def _coarse_dof_layout(solver, coarse_mesh, field_id=None): + """Parallel coarse-level DOF layout, no throwaway solver — same copyDS trick + as :func:`_coarse_reduced_map` but returning the parallel ``l2g`` layout.""" + cdm = coarse_mesh.dm.clone() + solver.dm.copyFields(cdm) + solver.dm.copyDS(cdm) + cdm.createDS() + return _level_dof_layout(cdm, field_id) + + +def _build_parallel_transfer(cc, fc, lay_c, lay_f, ncomp, builder, comm): + """One reduced->reduced prolongation as an MPIAIJ matrix. + + Node-level barycentric/RBF weights are built rank-locally (coarse LOCAL coords + incl. ghosts -> every owned fine node lands in a local coarse simplex). Each + fine OWNED DOF becomes a global row; its coarse contributions map through the + coarse ``l2g`` to global columns (off-rank columns are fine for MPIAIJ). + Constrained coarse DOFs (``l2g == -1``) drop out -> reduced->reduced.""" + l2g_c, cstart, cend, _ = lay_c + l2g_f, fstart, fend, _ = lay_f + Pn = builder(cc, fc).tocsr() # (n_f_nodes, n_c_nodes), local + nloc_f = fend - fstart + nloc_c = cend - cstart + + P = PETSc.Mat().create(comm=comm) + P.setSizes(((nloc_f, None), (nloc_c, None))) + P.setType("aij") + P.setUp() + for i in range(Pn.shape[0]): # fine local node + js = Pn.indices[Pn.indptr[i]:Pn.indptr[i + 1]] + ws = Pn.data[Pn.indptr[i]:Pn.indptr[i + 1]] + for c in range(ncomp): + grow = int(l2g_f[i * ncomp + c]) + if grow < fstart or grow >= fend: # set OWNED rows only + continue + gcols, vals = [], [] + for jj, w in zip(js.tolist(), ws.tolist()): + gcol = int(l2g_c[jj * ncomp + c]) + if gcol >= 0: # skip constrained coarse DOFs + gcols.append(gcol) + vals.append(w) + if gcols: + P.setValues([grow], gcols, vals, addv=PETSc.InsertMode.INSERT_VALUES) + P.assemble() + return P + + +def _assert_no_zero_columns_parallel(P, comm): + """Parallel zero-column guard: a coarse DOF with no fine image -> singular + Galerkin coarse operator. Column sums via P^T·1 (weights are positive, so a + zero sum means an empty column).""" + ones_f = P.createVecLeft(); ones_f.set(1.0) + colsum = P.createVecRight() + P.multTranspose(ones_f, colsum) + nzero_local = int((colsum.array == 0.0).sum()) + nzero = comm.tompi4py().allreduce(nzero_local) + ones_f.destroy(); colsum.destroy() + if nzero: + raise RuntimeError( + f"parallel transfer has {nzero} zero columns (coarse DOFs with no fine " + f"image) — BC-per-level reduction failed; coarse operator would be singular.") + + def _configure_pcmg(pc, Ps): """Reconfigure ``pc`` as a fresh PCMG (FMG F-cycle) driven by the supplied reduced->reduced prolongations ``Ps``, Galerkin RAP for coarse operators. @@ -434,25 +527,36 @@ def build(self, solver): """Build the BC-reduced prolongations. ``solver`` is the (built) finest solver. Each COARSE level's BC-constrained reduced map is derived directly from the coarse mesh DM by copying the finest solver's fields + DS onto it - (``_coarse_reduced_map``) — no throwaway solver. The finest level reads its - map from ``solver.dm``.""" + (no throwaway solver); the finest level reads its map from ``solver.dm``. + + Serial: scipy reduced->reduced CSR. Parallel (np>1): rank-local node-level + weights assembled into MPIAIJ transfers with global-section reduced + numbering (nested co-partitioned path).""" + from underworld3 import mpi var = solver.Unknowns.u degree = var.degree continuous = getattr(var, "continuous", True) nlev = len(self.level_meshes) + parallel = mpi.size > 1 - coords, r2f, ncomp = [], [], [] + coords, maps, ncomp = [], [], [] for k, mesh in enumerate(self.level_meshes): c = np.asarray(mesh._get_coords_for_basis(degree, continuous)) - if k == nlev - 1: - rmap, nfull = _reduced_map(solver.dm, self.field_id) + finest = (k == nlev - 1) + if parallel: + lay = (_level_dof_layout(solver.dm, self.field_id) if finest + else _coarse_dof_layout(solver, mesh, self.field_id)) + nfull = lay[3] + maps.append(lay) else: - rmap, nfull = _coarse_reduced_map(solver, mesh, self.field_id) + rmap, nfull = (_reduced_map(solver.dm, self.field_id) if finest + else _coarse_reduced_map(solver, mesh, self.field_id)) + maps.append(rmap) nc = nfull // c.shape[0] if nfull % c.shape[0] != 0: raise RuntimeError( f"level {k}: full DOFs {nfull} not divisible by nodes {c.shape[0]}") - coords.append(c); r2f.append(rmap); ncomp.append(nc) + coords.append(c); ncomp.append(nc) if len(set(ncomp)) != 1: raise RuntimeError(f"inconsistent component counts across levels: {ncomp}") @@ -460,15 +564,22 @@ def build(self, solver): Ps = [] for l in range(1, nlev): - Pr = _reduced_transfer(coords[l - 1], coords[l], r2f[l - 1], r2f[l], - nc, self.builder) - zc = int((np.asarray((Pr != 0).sum(axis=0)).ravel() == 0).sum()) - if zc: - raise RuntimeError( - f"transfer {l-1}->{l} has {zc} zero columns (coarse DOFs with no " - f"fine image) — BC-per-level reduction failed; coarse operator " - f"would be singular.") - Ps.append(_to_petsc_aij(Pr)) + if parallel: + P = _build_parallel_transfer(coords[l - 1], coords[l], + maps[l - 1], maps[l], nc, + self.builder, solver.dm.comm) + _assert_no_zero_columns_parallel(P, solver.dm.comm) + Ps.append(P) + else: + Pr = _reduced_transfer(coords[l - 1], coords[l], maps[l - 1], + maps[l], nc, self.builder) + zc = int((np.asarray((Pr != 0).sum(axis=0)).ravel() == 0).sum()) + if zc: + raise RuntimeError( + f"transfer {l-1}->{l} has {zc} zero columns (coarse DOFs with " + f"no fine image) — BC-per-level reduction failed; coarse " + f"operator would be singular.") + Ps.append(_to_petsc_aij(Pr)) self.transfers = Ps return Ps @@ -492,7 +603,6 @@ def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", (``_coarse_reduced_map``), so ``coarse_meshes`` need only carry the same boundary labels as the solver's mesh. For a saddle-point (Stokes) solver pass ``field_id=0`` to target the velocity sub-block.""" - _require_serial("set_custom_fmg") solver._custom_mg = { "mode": "hierarchy", "hierarchy": CustomMGHierarchy(list(coarse_meshes) + [solver.mesh], @@ -508,16 +618,16 @@ def inject_custom_mg(solver): - ``mode == "hierarchy"`` -> BC-per-level reduced path (correct, general); - legacy dict ``{coarse_meshes, kind}`` -> finest-only reduction (kept for back-compat; valid only when coarse levels are non-nested / unconstrained).""" - _require_serial("inject_custom_mg") cfg = solver._custom_mg if isinstance(cfg, dict) and cfg.get("mode") == "hierarchy": h = cfg["hierarchy"] - h.build(solver) + h.build(solver) # parallel-capable (nested co-partitioned) h.install(solver, verbose=cfg.get("verbose", False)) return - # ---- legacy finest-only path (back-compat) ------------------------------ + # ---- legacy finest-only path (back-compat, serial only) ----------------- + _require_serial("legacy custom_mg (set_custom_mg)") coarse_meshes = cfg["coarse_meshes"] builder = _BUILDERS[cfg["kind"]] var = solver.Unknowns.u diff --git a/tests/parallel/test_1017_custom_mg_parallel_mpi.py b/tests/parallel/test_1017_custom_mg_parallel_mpi.py new file mode 100644 index 00000000..e57f2c3b --- /dev/null +++ b/tests/parallel/test_1017_custom_mg_parallel_mpi.py @@ -0,0 +1,113 @@ +"""Parallel (np>1) correctness for custom-P geometric MG (custom_mg). + +The nested co-partitioned hierarchy path is parallel-capable: rank-local point +location (ghost-inclusive coarse coords) + global-section reduced numbering + +MPIAIJ transfers. These tests assert that, on >1 ranks, custom-P drives a +converging geometric-MG solve whose result matches a GAMG reference — for both a +scalar Poisson and the Stokes (SolCx) velocity block. The legacy finest-only +path remains serial-only and must still fail loudly. + +Run: + mpirun -n 2 python -m pytest --with-mpi tests/parallel/test_1017_custom_mg_parallel_mpi.py +""" +import numpy as np +import pytest +import underworld3 as uw +from underworld3.function import analytic as A +from underworld3.utilities import custom_mg + +pytestmark = [pytest.mark.mpi(min_size=2), pytest.mark.timeout(180)] + + +def _wrap(dm, m0, q=2): + return uw.discretisation.Mesh( + dm.clone(), simplex=True, + coordinate_system_type=m0.CoordinateSystem.coordinate_type, + qdegree=q, boundaries=m0.boundaries) + + +def _box(cellSize, refinement=None, q=2): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0., 0.), maxCoords=(1., 1.), + cellSize=cellSize, regular=True, refinement=refinement, qdegree=q) + + +def _rel_l2(a, b): + """Global relative L2 difference of two field-data arrays (rank-reduced).""" + da = np.asarray(a) - np.asarray(b) + num = uw.mpi.comm.allreduce(float(np.sum(da**2))) + den = uw.mpi.comm.allreduce(float(np.sum(np.asarray(b)**2))) + return (num**0.5) / (den**0.5 + 1e-30) + + +def _poisson(mesh): + p = uw.systems.Poisson(mesh) + p.constitutive_model = uw.constitutive_models.DiffusionModel + p.constitutive_model.Parameters.diffusivity = 1 + p.f = 0.0 + p.add_dirichlet_bc(0.0, "Bottom"); p.add_dirichlet_bc(1.0, "Top") + p.petsc_options["ksp_rtol"] = 1e-8; p.petsc_options["ksp_type"] = "cg" + return p + + +@pytest.mark.mpi(min_size=2) +def test_parallel_custom_fmg_scalar(): + """Scalar custom-P drives a converging geometric MG solve in parallel and + matches a GAMG solve of the same problem.""" + assert uw.mpi.size > 1 + m0 = _box(0.25); dm1 = m0.dm.refine(); dm2 = dm1.refine() + coarse = [_wrap(m0.dm, m0), _wrap(dm1, m0)]; fine = _wrap(dm2, m0) + + g = _poisson(_wrap(dm2, m0)); g.preconditioner = "gamg"; g.solve() + c = _poisson(fine) + custom_mg.set_custom_fmg(c, coarse, builder="barycentric") + c.solve() + + assert c.snes.getKSP().getPC().getType() == "mg" + assert c.snes.getConvergedReason() > 0 + assert _rel_l2(c.Unknowns.u.data, g.Unknowns.u.data) < 1e-4 + + +@pytest.mark.mpi(min_size=2) +def test_parallel_custom_fmg_stokes_velocity_block(): + """Custom-P on the SolCx velocity block converges in few MG iters in parallel + and matches the analytic velocity to the GAMG reference's accuracy.""" + assert uw.mpi.size > 1 + fine = _box(0.25, refinement=2, q=3) + coarse = [_wrap(d, fine, q=3) for d in fine.dm_hierarchy[:-1]] + + def solcx(mesh): + sol = A.SolCx(mesh, eta_A=1.0, eta_B=1e6, x_c=0.5, n=1) + s = uw.systems.Stokes(mesh) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.saddle_preconditioner = 1.0 / sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_dirichlet_bc((0.0, None), "Left"); s.add_dirichlet_bc((0.0, None), "Right") + s.add_dirichlet_bc((None, 0.0), "Bottom"); s.add_dirichlet_bc((None, 0.0), "Top") + s.petsc_use_pressure_nullspace = True; s.tolerance = 1e-8 + s.petsc_options["snes_type"] = "ksponly" + return s, sol + + sg, solg = solcx(fine); sg.preconditioner = "gamg"; sg.solve() + sc, sol = solcx(fine) + custom_mg.set_custom_fmg(sc, coarse, builder="barycentric", field_id=0) + sc.solve() + vksp = sc.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + + assert sc.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGLevels() == len(coarse) + 1 + assert vksp.getIterationNumber() <= 15 # geometric MG, not GAMG (~200) + assert sol.velocity_error(sc.u) < 2.0 * solg.velocity_error(sg.u) + 1e-6 + + +@pytest.mark.mpi(min_size=2) +def test_legacy_path_serial_guard(): + """The legacy finest-only set_custom_mg path is serial-only and must raise + loudly in parallel (it has no parallel reduced map).""" + assert uw.mpi.size > 1 + s = _poisson(_box(0.25, refinement=2)) + s.set_custom_mg([_box(0.285), _box(0.142)], kind="barycentric") + with pytest.raises(NotImplementedError): + s.solve() diff --git a/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py b/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py deleted file mode 100644 index f9862e65..00000000 --- a/tests/parallel/test_1017_custom_mg_serial_guard_mpi.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Parallel guard for custom-P geometric MG (custom_mg). - -Custom-P transfers are serial-only (experimental): the reduced maps use rank-local -DOF indices and the prolongations assemble as serial AIJ, so at np>1 they would -silently build wrong P. set_custom_fmg / inject must therefore FAIL LOUDLY in -parallel rather than produce incorrect results. Parallel (np>1) support is a -designed fast-follow (nested co-partitioned, rank-local P + MPIAIJ). - -Run: - mpirun -n 2 python -m pytest --with-mpi tests/parallel/test_1017_custom_mg_serial_guard_mpi.py -""" -import pytest -import sympy -import underworld3 as uw -from underworld3.utilities import custom_mg - -pytestmark = [pytest.mark.mpi(min_size=2), pytest.mark.timeout(120)] - - -def _box(cellSize, refinement=None): - return uw.meshing.UnstructuredSimplexBox( - minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0), - cellSize=cellSize, refinement=refinement, qdegree=2) - - -def _poisson(mesh): - p = uw.systems.Poisson(mesh) - p.constitutive_model = uw.constitutive_models.DiffusionModel - p.constitutive_model.Parameters.diffusivity = 1 - p.f = 0.0 - p.add_dirichlet_bc(0.0, "Bottom"); p.add_dirichlet_bc(1.0, "Top") - return p - - -@pytest.mark.mpi(min_size=2) -def test_set_custom_fmg_raises_in_parallel(): - """set_custom_fmg must raise NotImplementedError (not silently mis-build) np>1.""" - assert uw.mpi.size > 1 - s = _poisson(_box(0.25, refinement=2)) - with pytest.raises(NotImplementedError): - custom_mg.set_custom_fmg(s, [_box(0.285), _box(0.142)], builder="barycentric") - - -@pytest.mark.mpi(min_size=2) -def test_inject_guard_blocks_legacy_path_in_parallel(): - """The legacy set_custom_mg path must also fail loudly at solve() in parallel.""" - assert uw.mpi.size > 1 - s = _poisson(_box(0.25, refinement=2)) - # legacy setter writes _custom_mg directly (no module guard); inject must catch it - s.set_custom_mg([_box(0.285), _box(0.142)], kind="barycentric") - with pytest.raises(NotImplementedError): - s.solve() From af8e734b431da181cf8d5f208323678227937c30 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 14:16:37 +1000 Subject: [PATCH 11/52] docs(custom_mg): record parallel np>1 done (Phase 1, Step 3) Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index 901ff5c9..2cb670fc 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -120,16 +120,21 @@ non-nested parallel transfers; knockout-as-coarsening as an alternative Layer-2 (nested *or* non-nested, uniform *or* SBR-refined) → geometric FMG via custom-P with BC-per-level reduction. It does not depend on Layer 2. -Landed (committed, tested — `test_1014/1015/1016`, 18 pass): +Landed (committed, tested — `test_1014/1015/1016/1017` serial 20 + +`tests/parallel/test_1017_custom_mg_parallel_mpi.py` np2): - `CustomMGHierarchy`, `set_custom_fmg`, `sbr_refine`/`sbr_refine_where`; - automatic BC-per-level reduction + zero-column guard; -- validated: scalar jump-coeff Poisson, 5-level (3 uniform + 2 SBR), 3 FMG iters - vs GAMG 46. +- Stokes velocity-block injection; leak-free per-level reduction (copyDS, no + factory); parallel (np>1) nested co-partitioned transfers; +- validated: scalar jump-coeff Poisson 5-level (3 uniform + 2 SBR) 3 FMG iters + vs GAMG 46; SolCx velocity block 6 iters vs GAMG ~198, np=1/2/4. -**Current scope = experimental:** **serial**. Scalar / single-field-vector **and** -Stokes velocity-block are supported; the throwaway-solver factory has been removed. +**Current scope:** scalar / single-field-vector **and** Stokes velocity-block; +**serial and parallel** (nested co-partitioned). Non-nested custom-P is serial-only +(experimental). Hardening steps 1–3 are complete; what remains is test +tier-classification + undrafting PR #290. -Remaining hardening **before** this is a merge-ready general feature (in order): +Hardening steps (all complete as of 2026-06-29): 1. ~~**Stokes / saddle-point** (velocity-block injection).~~ **DONE** (2026-06-29). `set_custom_fmg(..., field_id=0)` drives custom-P geometric MG on the velocity sub-block. The sub-PC is unreachable until the monolithic Jacobian is assembled, @@ -153,14 +158,21 @@ Remaining hardening **before** this is a merge-ready general feature (in order): solver's boundary labels — validated byte-identical to the old factory path, leak-free (no SNES / JIT). `set_custom_fmg` / `build` no longer take a `level_solver_factory`. -3. **Parallel (np>1)** — `_reduced_map` / `_coarse_reduced_map` are serial (local - indices) and `P` assembles as serial AIJ. **Guarded** (2026-06-29): - `set_custom_fmg` / `inject_custom_mg` raise a clear `NotImplementedError` at - np>1 (`_require_serial`) so parallel cannot silently mis-build — test - `tests/parallel/test_1017_custom_mg_serial_guard_mpi.py`. The full - parallel-correct implementation (nested co-partitioned, rank-local point - location + ghost layer, MPIAIJ assembly with global-section reduction) remains - the designed fast-follow. +3. ~~**Parallel (np>1)**.~~ **DONE** (2026-06-29). The hierarchy path builds + parallel-correct transfers on the nested co-partitioned hierarchy: each rank + builds its block of `P` rank-locally (ghost-inclusive coarse coords → every + owned fine node lands in a local coarse simplex, verified 0 misses np=2/4), + the reduced global numbering rides the DM global section + (`_level_dof_layout` scatters owned global indices out via `globalToLocal` — + constrained DOFs `-1`, ghosts resolve to the owner's global), and transfers + assemble as MPIAIJ (owned fine rows, global coarse cols incl. off-rank; + constrained coarse DOFs drop → reduced→reduced). Parallel zero-column guard + via `Pᵀ·1` + allreduce. Validated np=1/2/4: scalar Poisson 4 iters + Stokes + SolCx velocity block 6 iters, matching the GAMG reference and each other + across rank counts (`tests/parallel/test_1017_custom_mg_parallel_mpi.py`). The + **legacy** finest-only path (`set_custom_mg` / `_reduce_to_global`) stays + serial-only and raises loudly at np>1. Non-nested custom-P in parallel remains + out of scope (cross-rank point location). ## Explicit non-goals for this pass - Knockout (shown to pay full-fine assembly; structural value only) — not pursued now. From 3686b7b84ca2491b848a2b22335922f3d36e28cd Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 14:43:10 +1000 Subject: [PATCH 12/52] feat(custom_mg): cover all solver families (SNES_Vector hook; Vector + Constrained tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify custom-P works for every solver family that consumes the mesh, so a mesh-owned adapted hierarchy drives MG uniformly: - SNES_Vector.solve was MISSING the inject_custom_mg hook (only SNES_Scalar and SNES_Stokes_SaddlePt had it) — so Vector_Projection silently stayed on GAMG. Add the hook (mirrors the scalar one). All 8 solver-subclass solve() overrides delegate to a base solve(), so this completes coverage: scalar branch -> Poisson/Darcy/Projection/AdvDiff/Diffusion; vector branch -> Vector_Projection; velocity-block -> Stokes/VE/Constrained/NS. - test_1017: add SNES_Vector (Vector_Projection, top-level vector PC, field_id=None) and Stokes_Constrained (free-slip multipliers, grouped [p,h] split, field_id=0) serial tests — both drive custom-P 'mg', constrained matches analytic SolCx to 4.4e-4. - parallel test: add Vector (passes np=2) and Constrained. The Constrained parallel test is SKIPPED: Stokes_Constrained is not parallel-safe yet — it segfaults at np>1 independently of custom-P (canonical test_1062_constrained_solcx also segfaults at np=2 under plain GAMG). It auto-enables when the constrained solver becomes parallel-ready. Serial: custom_mg 22 + projections + constrained green. Parallel np=2: scalar + Stokes-velocity + vector + legacy-guard pass. Underworld development team with AI support from Claude Code --- .../cython/petsc_generic_snes_solvers.pyx | 6 ++ .../test_1017_custom_mg_parallel_mpi.py | 59 ++++++++++++++++++ tests/test_1017_custom_mg_stokes.py | 61 +++++++++++++++++-- 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/src/underworld3/cython/petsc_generic_snes_solvers.pyx b/src/underworld3/cython/petsc_generic_snes_solvers.pyx index 58f21ab1..7c680a88 100644 --- a/src/underworld3/cython/petsc_generic_snes_solvers.pyx +++ b/src/underworld3/cython/petsc_generic_snes_solvers.pyx @@ -3724,6 +3724,12 @@ class SNES_Vector(SolverBaseClass): # Update constants (e.g. changed material params) before solve self._update_constants() + # Custom geometric-MG prolongation on the (top-level vector) PC, if + # registered via set_custom_fmg. Mirrors the SNES_Scalar hook. + if self._custom_mg is not None: + from underworld3.utilities.custom_mg import inject_custom_mg + inject_custom_mg(self) + # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) diff --git a/tests/parallel/test_1017_custom_mg_parallel_mpi.py b/tests/parallel/test_1017_custom_mg_parallel_mpi.py index e57f2c3b..0437a59e 100644 --- a/tests/parallel/test_1017_custom_mg_parallel_mpi.py +++ b/tests/parallel/test_1017_custom_mg_parallel_mpi.py @@ -102,6 +102,65 @@ def solcx(mesh): assert sol.velocity_error(sc.u) < 2.0 * solg.velocity_error(sg.u) + 1e-6 +@pytest.mark.mpi(min_size=2) +def test_parallel_custom_fmg_vector(): + """SNES_Vector (Vector_Projection) custom-P drives a top-level vector PCMG in + parallel (field_id=None, ncomp=dim). Fine mesh has NO native hierarchy, so the + 'mg' PC can only come from our custom-P.""" + import sympy + assert uw.mpi.size > 1 + m0 = _box(0.25); dm2 = m0.dm.refine().refine() + coarse = [_wrap(m0.dm, m0), _wrap(m0.dm.refine(), m0)] + fine = _wrap(dm2, m0) + x, y = fine.X + v = uw.discretisation.MeshVariable("Ucpp", fine, fine.dim, degree=2) + proj = uw.systems.Vector_Projection(fine, v) + proj.uw_function = sympy.Matrix([[sympy.sin(sympy.pi * x) * sympy.cos(sympy.pi * y), + sympy.cos(sympy.pi * x) * sympy.sin(sympy.pi * y)]]) + proj.smoothing = 1.0e-3 + proj.add_dirichlet_bc((0.0, 0.0), "Bottom") + proj.add_dirichlet_bc((0.0, 0.0), "Top") + custom_mg.set_custom_fmg(proj, coarse, builder="barycentric") + proj.solve() + assert proj.snes.getKSP().getPC().getType() == "mg" + assert proj.snes.getConvergedReason() > 0 + + +@pytest.mark.skip(reason="Stokes_Constrained is not parallel-safe yet — it " + "segfaults at np>1 independently of custom-P (the canonical " + "test_1062_constrained_solcx also segfaults at np=2, plain GAMG). " + "custom-P on the constrained velocity block works in SERIAL " + "(see test_1017_custom_mg_stokes.test_custom_fmg_stokes_constrained); " + "this auto-enables once the constrained solver is parallel-ready.") +@pytest.mark.mpi(min_size=2) +def test_parallel_custom_fmg_stokes_constrained(): + """Stokes_Constrained (free-slip multipliers, grouped [p,h] split) custom-P on + the velocity block in parallel — velocity must match the analytic SolCx.""" + import sympy + assert uw.mpi.size > 1 + m0 = _box(0.25, q=3); dm2 = m0.dm.refine().refine() + coarse = [_wrap(m0.dm, m0, q=3), _wrap(m0.dm.refine(), m0, q=3)] + fine = _wrap(dm2, m0, q=3) + sol = A.SolCx(fine, eta_A=1.0, eta_B=1.0e6, x_c=0.5, n=1) + s = uw.systems.Stokes_Constrained(fine) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_constraint_bc("Left", g=0.0, normal=sympy.Matrix([[-1.0, 0.0]])) + s.add_constraint_bc("Right", g=0.0, normal=sympy.Matrix([[1.0, 0.0]])) + s.add_constraint_bc("Bottom", g=0.0, normal=sympy.Matrix([[0.0, -1.0]])) + s.add_constraint_bc("Top", g=0.0, normal=sympy.Matrix([[0.0, 1.0]])) + s.petsc_use_pressure_nullspace = True + s.tolerance = 1.0e-9 + s.petsc_options["snes_type"] = "ksponly" + custom_mg.set_custom_fmg(s, coarse, builder="barycentric", field_id=0) + s.solve() + vksp = s.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert sol.velocity_error(s.u) < 5.0e-3 + + @pytest.mark.mpi(min_size=2) def test_legacy_path_serial_guard(): """The legacy finest-only set_custom_mg path is serial-only and must raise diff --git a/tests/test_1017_custom_mg_stokes.py b/tests/test_1017_custom_mg_stokes.py index 3ee4954e..5046d71d 100644 --- a/tests/test_1017_custom_mg_stokes.py +++ b/tests/test_1017_custom_mg_stokes.py @@ -1,9 +1,12 @@ -"""Layer-1 generalized FMG hierarchy on the STOKES velocity block (Phase-1 Step 1). +"""Layer-1 generalized FMG hierarchy across the solver families that consume a mesh. -Custom-built prolongations (barycentric / RBF) drive geometric multigrid on the -velocity sub-block of the saddle-point solver, via set_custom_fmg(field_id=0). -Validated on SolCx (eta_B=1e6): the velocity block converges in a handful of MG -iterations (vs ~200 for GAMG) and the solution matches a GAMG reference. +Custom-built prolongations (barycentric / RBF) drive geometric multigrid via +set_custom_fmg, validated here on each solver-PC topology so the same adapted-mesh +hierarchy works for every solver running on it: + - STOKES velocity sub-block (set_custom_fmg field_id=0) — SolCx eta_B=1e6; + - SNES_Vector (Vector_Projection) — top-level vector PC (field_id=None); + - Stokes_Constrained (free-slip via in-saddle multipliers) — velocity block under + the grouped [p, h] fieldsplit. The hard part (see custom_mg._install_velocity_block_transfers): the velocity sub-PC is unreachable until the monolithic Jacobian is assembled, so the install @@ -105,3 +108,51 @@ def test_custom_fmg_velocity_block_rbf(): assert vksp.getPC().getMGLevels() == len(coarse) + 1 assert vksp.getIterationNumber() <= 25 assert sol.velocity_error(s.u) < 1e-2 + + +def test_custom_fmg_vector_solver(): + """SNES_Vector (Vector_Projection): custom-P on the top-level vector PC + (field_id=None) — the same scalar/single-field-vector path, ncomp=dim.""" + import sympy + coarse, fine = _hierarchy() + x, y = fine.X + v = uw.discretisation.MeshVariable("Ucp", fine, fine.dim, degree=2) + proj = uw.systems.Vector_Projection(fine, v) + proj.uw_function = sympy.Matrix([[sympy.sin(sympy.pi * x) * sympy.cos(sympy.pi * y), + sympy.cos(sympy.pi * x) * sympy.sin(sympy.pi * y)]]) + proj.smoothing = 1.0e-3 + proj.add_dirichlet_bc((0.0, 0.0), "Bottom") + proj.add_dirichlet_bc((0.0, 0.0), "Top") + custom_mg.set_custom_fmg(proj, coarse, builder="barycentric") # field_id=None + proj.solve() + assert proj.snes.getKSP().getPC().getType() == "mg" + assert proj.snes.getKSP().getPC().getMGLevels() == len(coarse) + 1 + assert proj.snes.getConvergedReason() > 0 + + +def test_custom_fmg_stokes_constrained(): + """Stokes_Constrained (free-slip via in-saddle multipliers): custom-P on the + velocity block under the grouped [p, h] fieldsplit. Velocity must still match + the analytic SolCx free-slip solution.""" + import sympy + coarse, fine = _hierarchy() + sol = A.SolCx(fine, eta_A=1.0, eta_B=1.0e6, x_c=0.5, n=1) + s = uw.systems.Stokes_Constrained(fine) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_constraint_bc("Left", g=0.0, normal=sympy.Matrix([[-1.0, 0.0]])) + s.add_constraint_bc("Right", g=0.0, normal=sympy.Matrix([[1.0, 0.0]])) + s.add_constraint_bc("Bottom", g=0.0, normal=sympy.Matrix([[0.0, -1.0]])) + s.add_constraint_bc("Top", g=0.0, normal=sympy.Matrix([[0.0, 1.0]])) + s.petsc_use_pressure_nullspace = True + s.tolerance = 1.0e-9 + s.petsc_options["snes_type"] = "ksponly" + custom_mg.set_custom_fmg(s, coarse, builder="barycentric", field_id=0) + s.solve() + vksp = _vel_ksp(s) + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGLevels() == len(coarse) + 1 + assert vksp.getIterationNumber() <= 15 + assert sol.velocity_error(s.u) < 5.0e-3 From 32fec5adb3584d54254d6b1f994c7acb6425e3f0 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 14:43:39 +1000 Subject: [PATCH 13/52] docs(custom_mg): solver-family coverage matrix + constrained-parallel limitation Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index 2cb670fc..8da85492 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -134,6 +134,23 @@ Landed (committed, tested — `test_1014/1015/1016/1017` serial 20 + (experimental). Hardening steps 1–3 are complete; what remains is test tier-classification + undrafting PR #290. +**Solver-family coverage** — the install keys off the PC topology, so one of two +branches covers every solver that consumes the mesh (all `solve()` overrides +delegate to a hooked base `solve()`; the `inject_custom_mg` hook lives in +`SNES_Scalar.solve`, `SNES_Vector.solve`, `SNES_Stokes_SaddlePt.solve`): + +| Solver family | Branch (`field_id`) | Serial | Parallel | +|---|---|---|---| +| `SNES_Scalar` (Poisson, Darcy, Projection, AdvDiff, Diffusion) | top-level (`None`) | ✓ | ✓ | +| `SNES_Vector` (Vector_Projection, displacement) | top-level (`None`) | ✓ | ✓ | +| `SNES_Stokes` / VE / NavierStokes | velocity block (`0`) | ✓ | ✓ | +| `SNES_Stokes_Constrained` (in-saddle multipliers) | velocity block (`0`) | ✓ | **skip** ¹ | + +¹ The constrained solver is **not parallel-safe today** — it segfaults at np>1 +*independently of custom-P* (the canonical `test_1062_constrained_solcx` also +segfaults at np=2 under plain GAMG). custom-P on the constrained velocity block +works in serial; the parallel test auto-enables once the solver is parallel-ready. + Hardening steps (all complete as of 2026-06-29): 1. ~~**Stokes / saddle-point** (velocity-block injection).~~ **DONE** (2026-06-29). `set_custom_fmg(..., field_id=0)` drives custom-P geometric MG on the velocity From d7a4ca17e0e78c5a233c89b008805a8ce559956b Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 15:01:36 +1000 Subject: [PATCH 14/52] docs(custom_mg): link constrained-parallel segfault to issue #291 Underworld development team with AI support from Claude Code --- .../design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 8 +++++--- tests/parallel/test_1017_custom_mg_parallel_mpi.py | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index 8da85492..4380f1d0 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -147,9 +147,11 @@ delegate to a hooked base `solve()`; the `inject_custom_mg` hook lives in | `SNES_Stokes_Constrained` (in-saddle multipliers) | velocity block (`0`) | ✓ | **skip** ¹ | ¹ The constrained solver is **not parallel-safe today** — it segfaults at np>1 -*independently of custom-P* (the canonical `test_1062_constrained_solcx` also -segfaults at np=2 under plain GAMG). custom-P on the constrained velocity block -works in serial; the parallel test auto-enables once the solver is parallel-ready. +*independently of custom-P*, in the interior-multiplier section reduction +(`_constrain_interior_multipliers_in_section`; **issue #291**; the canonical +`test_1062_constrained_solcx` also segfaults at np=2 under plain GAMG; workaround +`_reduce_interior_multiplier = False`). custom-P on the constrained velocity block +works in serial; the parallel test auto-enables once #291 is fixed. Hardening steps (all complete as of 2026-06-29): 1. ~~**Stokes / saddle-point** (velocity-block injection).~~ **DONE** (2026-06-29). diff --git a/tests/parallel/test_1017_custom_mg_parallel_mpi.py b/tests/parallel/test_1017_custom_mg_parallel_mpi.py index 0437a59e..3a3e29a5 100644 --- a/tests/parallel/test_1017_custom_mg_parallel_mpi.py +++ b/tests/parallel/test_1017_custom_mg_parallel_mpi.py @@ -127,11 +127,12 @@ def test_parallel_custom_fmg_vector(): @pytest.mark.skip(reason="Stokes_Constrained is not parallel-safe yet — it " - "segfaults at np>1 independently of custom-P (the canonical " + "segfaults at np>1 independently of custom-P, in the " + "interior-multiplier section reduction (issue #291; canonical " "test_1062_constrained_solcx also segfaults at np=2, plain GAMG). " "custom-P on the constrained velocity block works in SERIAL " "(see test_1017_custom_mg_stokes.test_custom_fmg_stokes_constrained); " - "this auto-enables once the constrained solver is parallel-ready.") + "this auto-enables once #291 is fixed.") @pytest.mark.mpi(min_size=2) def test_parallel_custom_fmg_stokes_constrained(): """Stokes_Constrained (free-slip multipliers, grouped [p,h] split) custom-P on From 932e8d7d457f1fbd6fac10442143bdb6cabcafc6 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 17:59:15 +1000 Subject: [PATCH 15/52] docs(layer2): SBR adapt-on-top design (mesh-owned custom-P hierarchy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures the Layer-2 design: integrate a nested on-rank SBR adapter into mesh.adapt(adapter="sbr", max_levels, node_budget) with the existing isotropic-metric interface; re-adapt (non-cumulative, no node translation — unlike MMPDE) discarding the SBR top and re-marking from the static base each step; mesh-owned hierarchy that ALL solvers consume; Eulerian REMAP field transfer via the existing remesh machinery; no-redistribution as a correctness requirement (custom-P parallel needs the finest co-partitioned with the static coarse tail); static-coarse/transient-fine data model with top-only transfer rebuild + coarse-transfer caching; marker-sidecar checkpoint scheme (designed, deferred). Phased plan + invariants. Underworld development team with AI support from Claude Code --- .../design/LAYER2_SBR_ADAPT_ON_TOP.md | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md diff --git a/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md new file mode 100644 index 00000000..610db569 --- /dev/null +++ b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md @@ -0,0 +1,173 @@ +# Layer 2 — SBR adapt-on-top (mesh-owned custom-P hierarchy) + +Status: design (2026-06-29). Builds on Layer 1 +(`docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md`, +`utilities/custom_mg.py`, PR #290). Branch `feature/adapt-on-top`. + +## Context + +Layer 1 gave us geometric multigrid on **arbitrary** (nested *or* non-nested, +uniform *or* SBR-refined) hierarchies via custom-built prolongations + Galerkin RAP, +with BC-per-level reduction, for every solver family (scalar / vector / Stokes +velocity block), serial and parallel. Layer 2 is the **first concrete application**: +locally refine the mesh where the solution needs it, **on top of** a static uniform +base, and have **every solver that uses the mesh** drive geometric MG on the result. + +The defining property (L.M.): **this is adapt / re-adapt, not node movement.** There +is *no node translation and no cumulative refinement* — fundamentally unlike MMPDE. +Each adapt **discards** the previous refined level(s) and **re-marks from the static +base**. The refined top is therefore a pure function of the current metric, fully +rebuildable and fully described by its marker set. + +## Model + +``` + base finest ── SBR(marker_t) ─▶ refined finest at step t (discard at t+1) + base finest ── SBR(marker_{t+1}) ─▶ refined finest at step t+1 +``` + +- **Base** = the existing `Mesh(refinement=N)` uniform hierarchy. **Static for the + whole run** (built once, never moves, sidecar-reconstructable). It supplies the MG + coarse levels. +- **Refined top** = up to `max_levels` SBR levels applied to the **base finest**, + marked from an isotropic metric, capped by a node budget. Transient: discarded and + rebuilt each adapt. SBR cannot coarsen — re-marking from base IS the coarsening. +- The solver operator lives on the **refined finest**; the MG hierarchy is + `[base L0 … base finest] + [SBR level(s)]`. + +Because nodes do not move, the inter-adapt field transfer is plain **Eulerian REMAP** +(evaluate the old finest field at the new finest DOF coords) — no ALE / CARRY +semantics. (Contrast MMPDE, where nodes move with the material and history needs ALE +carry; not applicable here.) + +## API — integrate into `mesh.adapt` + +A new **nested, on-rank adapter mode** alongside today's MMG path: + +```python +mesh.adapt(metric, adapter="sbr", max_levels=2, node_budget=None) # new (this work) +mesh.adapt(metric) # = adapter="mmg" (today) +``` + +- `metric` — the existing isotropic metric interface (`adaptivity.create_metric`, + `metric_from_gradient`, `metric_from_field`): a scalar MeshVariable carrying + `M = 1/h²` (target edge length h). Reused unchanged. +- `adapter="sbr"` — nested skeleton-based refinement on top of the base finest. + `adapter="mmg"` (default) keeps today's topology-changing/redistributing behaviour. +- `max_levels` — cap on SBR depth (bounds the non-load-balanced imbalance). +- `node_budget` — cap on added DOFs: mark the highest-metric cells first until the + budget is hit (so refinement concentrates where the metric is largest). + +Marking: convert the metric to a per-cell target h, mark cells whose current size +exceeds target (SBR `adaptLabel`/`refine_sbr`, via `custom_mg.sbr_refine_where`). + +## Mesh-owned hierarchy — all solvers consume it + +Layer 1 registers custom-P **per solver** (`solver._custom_mg` via +`set_custom_fmg`). Layer 2 moves the hierarchy's home to the **mesh** so every solver +on that mesh consumes the same refined hierarchy with no per-solver call — directly +realising "force the adaptivity into the mesh; all solvers consume it". + +- The mesh holds the current `[static coarse meshes] + refined finest` and the + per-level transfer builders. +- A solver's existing custom-P injection (`inject_custom_mg`, already wired into every + base `solve()`) learns to pick up a **mesh-owned** hierarchy if present (in addition + to a solver-set one). `field_id` is inferred per solver topology (0 for the Stokes + velocity block, None for scalar/vector) exactly as today. +- `mesh.adapt(adapter="sbr")` updates the mesh-owned hierarchy and invalidates + registered solvers (`is_setup=False`, the existing mechanism) so each rebuilds on + the new finest at next `solve()`. + +## Field transfer + +Reuse the existing remesh machinery — **no new transfer code**: +- The adapt wraps the SBR move in the existing var-transfer path + (`remesh_with_field_transfer` / the `mesh.adapt` var-reset+`global_evaluate` + REMAP). Eulerian REMAP is correct here (no node translation). +- `global_evaluate` is swarm-migration based → **parallel-safe and partition-agnostic** + for the field values, independent of the mesh partition. +- Operator `on_remesh` hooks fire as usual; SLCN/DuDt history transfers per its + policy. (Re-adapt ⇒ REMAP fallback is the right semantics, not ALE carry.) + +## No redistribution — a CORRECTNESS requirement, not just cost + +Adapted layers are **on-rank**; load balancing is not required and the imbalance is +accepted (bounded by `max_levels`). But redistribution must be actively prevented, +because **custom-P's parallel path requires the finest to stay co-partitioned with the +coarse levels** (rank-local point location: every owned fine node lands in a *local* +coarse simplex). Redistributing the refined finest would diverge its partition from the +static coarse tail and **break parallel custom-P** — not merely slow it down. + +Guards: +- Wrap the SBR'd DM with **`distribute=False`**; never call `DMPlexDistribute` / + redistribute on adapted layers (the MMG path's `redistribute` flag must not reach the + `sbr` path). +- `dm.refine()` (base) and `adaptLabel`/SBR (top) both preserve partition by + construction; the only redistribution risk is the Mesh-wrap of the new finest → + assert per-rank ownership of the static coarse tail is unchanged after an adapt. + +## Data model — static coarse tail, transient fine head + +- **Coarse levels: allocate once, reuse forever.** Uniform + static ⇒ DMs, sections, + coordinates and their (nested) transfers never change. Small (coarse uniform) ⇒ + negligible standing memory, zero per-adapt cost. +- **Only the finest is transient:** each adapt frees the old finest's field vecs + top + transfer and allocates new ones (new DOF count). Memory churn is confined to one + level. +- **Transfer caching (efficiency lever):** coarse→coarse transfers are constant → + build once, cache. Only the **top** transfer (static coarse finest → new SBR finest) + is rebuilt per adapt. Base-level transfers, being genuine uniform refinements, can use + PETSc **nested** interpolation (exact, cheap, cached); the SBR-top uses **barycentric** + custom-P. (`CustomMGHierarchy.build` currently rebuilds all levels — add a per-level + cache keyed on level identity.) +- **Galerkin coarse operators still recompute** each solve (PᵀAP from the new fine A — + the physics/coefficients changed); intrinsic, not avoidable by keeping coarse grids. + But the transfers feeding RAP are mostly cached. + +## Checkpointing (designed here; implemented in a follow-up) + +Adapted meshes are restartable by storing **markers, not meshes** — consistent with the +existing FMG sidecar philosophy and exploiting SBR determinism: + +- **Base**: existing sidecar (coarsest DM + refine count) → reconstruct the static + uniform hierarchy bit-identically (canonical `refine()` numbering). +- **Adapted levels**: store **one cell-marker label per SBR level**, in that level's + (deterministic) cell numbering. SBR (`adaptLabel`/`refine_sbr`) is deterministic given + the marker ⇒ replaying rebuilds each refined level bit-identically. Markers are tiny + (a label/IS) ⇒ checkpoint size stays dominated by field data. +- **Fields**: stored on the finest as today. +- **Reload**: base sidecar → re-refine uniform → replay SBR markers per level → load + fields onto the reconstructed finest. + +Robustness note: **custom-P sidesteps the `err77` canonical-nested-numbering fragility +that bit native-FMG checkpoint reconstruction** ([[project_fmg_checkpoint_hierarchy]]). +Our transfers are built from *coordinates*, not PETSc parent-child maps, so reconstruction +only needs the finest's coordinates/topology to match — far more forgiving than native +nested interp. Store the realized marker (not the metric), since the marker is what +produced the current mesh. + +## Correctness invariants (Layer 2) + +1. Base hierarchy is immutable for the run; only the SBR top changes. +2. Adapted layers are on-rank — **no redistribution** (partition of the static coarse + tail is invariant across adapts). +3. Re-adapt is non-cumulative: each adapt re-marks from the base finest; previous SBR + levels are discarded (no node translation, no accumulation). +4. Field transfer is Eulerian REMAP via `global_evaluate` (parallel-safe). +5. All Layer-1 invariants hold per level (BCs at every level, transfers reduced→reduced, + partition-of-unity, `pc_mg_galerkin=both`, nullspaces re-attached, no silent GAMG + fallback). + +## Phased plan + +1. **Live adapt path (this increment)** — `mesh.adapt(metric, adapter="sbr", + max_levels, node_budget)`: metric→mark→SBR(`distribute=False`)→wrap→mesh-owned + custom-P hierarchy→REMAP field transfer→solver auto-pickup→invalidate. Validate a + solve converges via custom-P on the refined mesh and a moving-feature re-adapt loop + carries fields; serial then np=2 (co-partitioned, no redistribute). +2. **Transfer caching** — cache static coarse-level transfers; rebuild only the top. +3. **Checkpointing** — marker-sidecar store + reconstruct (per the scheme above). +4. **Driver/example + tests + tier classification.** + +Non-goals this pass: cumulative refinement / node movement (explicitly out — that's +MMPDE's domain); load-balancing the adapted layers; MMG-path changes. From e6b4c09bc911f8da36578468793ce57ada813eae Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 18:10:14 +1000 Subject: [PATCH 16/52] docs(layer2): record adapt-on-top prototype validation (serial + np=2) SBR adapt-on-top + custom-P + REMAP + non-cumulative re-adapt loop proven end-to-end with Layer-1 code only; np=2 confirms the no-redistribute / co-partitioned correctness (refined finest stays co-partitioned with the static base coarse levels). 4 MG iters, matches GAMG. Underworld development team with AI support from Claude Code --- .../design/LAYER2_SBR_ADAPT_ON_TOP.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md index 610db569..5cd891ef 100644 --- a/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md +++ b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md @@ -171,3 +171,27 @@ produced the current mesh. Non-goals this pass: cumulative refinement / node movement (explicitly out — that's MMPDE's domain); load-balancing the adapted layers; MMG-path changes. + +## Prototype validation (2026-06-29, before touching mesh.adapt) + +The full mechanic is de-risked in a study script +(`~/+Simulations/layer2_adapt_on_top_study/proto_sbr_adapt_on_top.py`) using **only +Layer-1 code** (`sbr_refine_where` + `set_custom_fmg`), serial **and** np=2: + +- **A. SBR adapt-on-top + custom-P** — static base (refinement=2, finest 512 cells) + → SBR-refine near a feature (→1023) → wrap → custom-P hierarchy + `[L0,L1,base-finest,SBR-finest]` → Poisson solve: **pc=mg, 4 levels, 4 iters**, + matches GAMG to 2.4e-8. +- **B. REMAP field transfer** base-finest → SBR-finest via `global_evaluate`: rel err + vs analytic 3.9e-4 (P2 interpolation floor). +- **C. Re-adapt (non-cumulative)** — feature moves 0.7→0.5→0.35; each step discards the + SBR top, re-marks from the **same static base finest**, re-solves (4 iters, pc=mg), + carries the field (~5e-4). Base finest stays 512 cells (unchanged) throughout. +- **np=2 (no-redistribute correctness)** — each rank SBR-refines its OWN cells + (rank0 256→502, rank1 256→521; on-rank imbalance accepted), the refined finest stays + **co-partitioned with the base coarse levels**, so parallel custom-P works (pc=mg, + 4 iters, matches GAMG). REMAP + re-adapt loop parallel-safe. + +Conclusion: the design holds end-to-end. Remaining work is wiring it into +`mesh.adapt(adapter="sbr")` with the mesh-owned hierarchy + solver auto-pickup +(no per-solver `set_custom_fmg`), then transfer caching, then checkpointing. From 6d6d85ebdccbeba0a0839ad2ea2a45091b97e1e9 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 21:09:42 +1000 Subject: [PATCH 17/52] fix(custom_mg): wire assembled Jacobian into KSP before velocity-block fieldsplit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Stokes velocity-block injection forces computeJacobian, then reaches the fieldsplit BEFORE the SNES solve. SNESSolve normally wires the freshly-assembled Jacobian into the KSP/outer-PC; doing it ourselves was missing, so for some configurations (e.g. Stokes on an SBR-refined child mesh with a mesh-variable bodyforce) the outer PC carried an UNASSEMBLED operator and PCSetUp failed with "Matrix must be set first" (err73). Fix: snes.getKSP().setOperators(J, Pmat) after forcing assembly, before outer_pc.setUp(). Found via the Layer-2 adapt-on-top prototype (Stokes on a fault-refined child). Diagnosed: ksp.A set but asm=False while J.assembled=True -> operator not wired. New regression test_custom_fmg_stokes_on_sbr_child (Stokes velocity-block custom-P on an SBR child with a mesh-variable bodyforce) — fails pre-fix, passes after. test_1014/1015/1016/1017 green (serial); parallel np2 unchanged. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 6 +++++ tests/test_1017_custom_mg_stokes.py | 34 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index f8863940..32023429 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -471,6 +471,12 @@ def _install_velocity_block_transfers(solver, Ps, verbose=False): snes.setFromOptions() solver.dm.restoreGlobalVec(x0) + # Wire the freshly-assembled Jacobian into the KSP/outer-PC. SNESSolve does + # this lazily, but we reach the fieldsplit BEFORE the solve — without it the + # outer PC can carry an unassembled operator and PCSetUp fails with + # "Matrix must be set first" (err73) for some configurations. + snes.getKSP().setOperators(J, Pmat) + # 2. split -> reach the velocity sub-KSP / sub-PC (field 0) outer_pc = snes.getKSP().getPC() outer_pc.setUp() diff --git a/tests/test_1017_custom_mg_stokes.py b/tests/test_1017_custom_mg_stokes.py index 5046d71d..bab16eb0 100644 --- a/tests/test_1017_custom_mg_stokes.py +++ b/tests/test_1017_custom_mg_stokes.py @@ -156,3 +156,37 @@ def test_custom_fmg_stokes_constrained(): assert vksp.getPC().getMGLevels() == len(coarse) + 1 assert vksp.getIterationNumber() <= 15 assert sol.velocity_error(s.u) < 5.0e-3 + + +def test_custom_fmg_stokes_on_sbr_child(): + """Stokes velocity-block custom-P on an SBR-refined CHILD mesh whose bodyforce + depends on a mesh variable (the Layer-2 adapt-on-top scenario). Regression for + the operator-not-wired bug: forcing the Jacobian before the fieldsplit left the + KSP carrying an unassembled operator -> PCSetUp 'Matrix must be set first' + (err73). Fixed by wiring the assembled Jacobian into the KSP before reaching + the velocity sub-PC.""" + import sympy + base = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.25, regular=True, + refinement=2, qdegree=3) + child = _wrap(custom_mg.sbr_refine_where( + base.dm_hierarchy[-1], lambda c: abs(c[0] - 0.5) < 0.15), base) + coarse = [_wrap(d, base) for d in base.dm_hierarchy] # static base levels + + Tp = uw.discretisation.MeshVariable("Tp", child, 1, degree=2) # proxy field + Tp.data[:, 0] = 1.0 + s = uw.systems.Stokes(child) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = 1.0 + s.saddle_preconditioner = 1.0 + s.bodyforce = sympy.Matrix([[0.0], [1.0e4 * Tp.sym[0]]]) # mesh-variable bodyforce + s.add_dirichlet_bc((0.0, 0.0), "Bottom"); s.add_dirichlet_bc((0.0, 0.0), "Top") + s.add_dirichlet_bc((0.0, None), "Left"); s.add_dirichlet_bc((0.0, None), "Right") + s.petsc_use_pressure_nullspace = True + s.petsc_options["snes_type"] = "ksponly" + custom_mg.set_custom_fmg(s, coarse, builder="barycentric", field_id=0) + s.solve() + vksp = _vel_ksp(s) + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGLevels() == len(coarse) + 1 From fa082c2792f7a6d2bcaba22e8a80cd445eccbdf3 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 21:39:45 +1000 Subject: [PATCH 18/52] feat(layer2): mesh.adapt SBR adapt-on-top returns a refined child MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the Layer-2 nested adapt-on-top path into the mesh API (de-risked prototype now integrated). The naming settled as: adapt RETURNS a refined child; the old in-place MMG adapt is renamed remesh. mesh.adapt(metric, max_levels=2, node_budget=None, builder, adapter="sbr"): - marks base-finest cells whose size exceeds the metric target, up to max_levels SBR passes (on-rank, distribute=False; node_budget keeps the highest-metric cells first); metric sampled by evaluate (serial) / global_evaluate (np>1); - wraps the refined finest as a child (child.parent = self, _relationship_kind="refinement", registered in _registered_children); the base mesh is untouched (re-adapt is non-cumulative); - the child owns the static coarse tail (_custom_mg_coarse_meshes, cached on the parent) so solvers auto-pickup geometric MG. Solver auto-pickup: custom_mg.maybe_inject_custom_mg (called from the four solve hooks) builds a CustomMGHierarchy from a mesh-owned coarse tail when no solver-set hierarchy is present (field_id 0 for the Stokes velocity block, None for scalar/vector) — every solver on an adapted mesh drives geometric MG with no per-solver set_custom_fmg. Clean no-op for plain (unadapted) solves. copy_into/add_into dispatch on the child kind: refinement children prolongate parent->child (FE-exact barycentric custom-P / global_evaluate REMAP at np>1) and restrict child->parent (nearest-node injection / global_evaluate). Submesh children keep the existing subpoint_is path. mesh.remesh(metric) is the renamed in-place MMG path (byte-identical body); mesh.adapt(adapter="mmg") is a deprecation shim that warns and forwards. Migrated the MMG regression tests (test_0810/0830, ptest_0763) to remesh and hardened test_0810 with a runtime backend probe (skips cleanly where MMG is non-functional). Validation: tests/test_0835_sbr_adapt_on_top.py (7 tests, tier_b; SBR needs no MMG build) + study script validate_mesh_adapt_integration.py (serial + np=2, all OK): child link + finer, solver auto-pickup pc=mg 4 iters == GAMG, copy_into both directions, non-cumulative re-adapt, node_budget localises, shim warns. Submesh + custom-mg regressions (test_0771/0772/1015/1017) green. Docs: LAYER2_SBR_ADAPT_ON_TOP.md "Implementation" section (as-built API); submesh-solver-architecture.md reframed under the parent/child DAG (submesh and refinement are two kinds of child). Underworld development team with AI support from Claude Code --- .../design/LAYER2_SBR_ADAPT_ON_TOP.md | 54 +++- .../design/submesh-solver-architecture.md | 23 +- .../cython/petsc_generic_snes_solvers.pyx | 34 +- .../discretisation/discretisation_mesh.py | 295 +++++++++++++++++- .../discretisation/enhanced_variables.py | 86 ++--- src/underworld3/utilities/custom_mg.py | 25 ++ .../ptest_0763_mesh_adapt_transfer.py | 2 +- ...est_0810_amr_swarm_migration_regression.py | 32 +- .../test_0830_mesh_adapt_variable_transfer.py | 12 +- tests/test_0835_sbr_adapt_on_top.py | 154 +++++++++ 10 files changed, 642 insertions(+), 75 deletions(-) create mode 100644 tests/test_0835_sbr_adapt_on_top.py diff --git a/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md index 5cd891ef..96929a97 100644 --- a/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md +++ b/docs/developer/design/LAYER2_SBR_ADAPT_ON_TOP.md @@ -1,6 +1,7 @@ # Layer 2 — SBR adapt-on-top (mesh-owned custom-P hierarchy) -Status: design (2026-06-29). Builds on Layer 1 +Status: **Phase 1 implemented (2026-06-29)** — design below; see "Implementation" +at the end for the as-built API and what landed. Builds on Layer 1 (`docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md`, `utilities/custom_mg.py`, PR #290). Branch `feature/adapt-on-top`. @@ -195,3 +196,54 @@ Layer-1 code** (`sbr_refine_where` + `set_custom_fmg`), serial **and** np=2: Conclusion: the design holds end-to-end. Remaining work is wiring it into `mesh.adapt(adapter="sbr")` with the mesh-owned hierarchy + solver auto-pickup (no per-solver `set_custom_fmg`), then transfer caching, then checkpointing. + +## Implementation (Phase 1, 2026-06-29) + +The live-adapt path is wired into the mesh API. The naming settled differently +from the early `adapter=` sketch above: `adapt` now **returns a child**, and the +old in-place MMG `adapt` was **renamed `remesh`**. + +**`mesh.adapt(metric, max_levels=2, node_budget=None, builder="barycentric", +adapter="sbr", verbose=False) -> child`** (`discretisation_mesh.py`, +`_adapt_sbr`): +- Marks base-finest cells whose characteristic size `h ≈ (dim!·vol)^(1/dim)` + exceeds the metric target `1/√M` at the centroid; up to `max_levels` SBR passes; + `node_budget` keeps the highest-metric cells first (approximate DOF cap). +- Metric sampled by `uw.function.evaluate` (serial) / `global_evaluate` (np>1, + partition-agnostic). SBR via `custom_mg.sbr_refine` (`distribute=False`, on-rank). +- Wraps the refined finest as a child: `child.parent = self`, + `child._relationship_kind = "refinement"`, registered in + `parent._registered_children`. The base mesh is untouched (re-adapt is + non-cumulative). +- The child owns the static coarse tail `child._custom_mg_coarse_meshes` + (`= parent._coarse_level_meshes()`, built once and cached) + `_custom_mg_builder`. + +**Solver auto-pickup** (`custom_mg.maybe_inject_custom_mg`, called from the four +solve hooks in `petsc_generic_snes_solvers.pyx`): when a solver's `_custom_mg` is +unset but its mesh carries `_custom_mg_coarse_meshes`, it lazily builds +`CustomMGHierarchy([*coarse, solver.mesh], field_id=…)` (0 for the Stokes +velocity block, None for scalar/vector) — so *every* solver on an adapted mesh +drives geometric MG with no per-solver `set_custom_fmg`. A solver-set hierarchy +still wins. + +**`copy_into` / `add_into`** (`enhanced_variables.py`) dispatch on the child kind +via `mesh._refine_prolongate` / `_refine_restrict`: parent→child is FE-exact +barycentric custom-P (serial) or `global_evaluate` REMAP (np>1); child→parent is +nearest-node injection (serial) / `global_evaluate` (np>1). Submesh children keep +the existing `subpoint_is` restrict/prolongate. + +**`mesh.remesh(metric)`** is the renamed in-place MMG path (byte-identical body). +`mesh.adapt(adapter="mmg")` is a deprecation shim → `DeprecationWarning` + forwards +to `remesh`. The MMG regression tests (`test_0810/0830`, `ptest_0763`) were +migrated to `remesh`. + +**Validation**: `tests/test_0835_sbr_adapt_on_top.py` (7 tests, tier_b) + the study +script `~/+Simulations/layer2_adapt_on_top_study/validate_mesh_adapt_integration.py` +(serial + np=2, ALL OK): child link + finer; solver auto-pickup pc=mg 4 iters == +GAMG; copy_into both directions; non-cumulative re-adapt; node_budget localises; +shim warns. SBR needs no MMG/pragmatic PETSc build (unlike `remesh`). + +**Not yet** (follow-ups, unchanged from the phased plan): transfer caching +(reuse cached coarse-level transfers; rebuild only the SBR top); marker-sidecar +checkpointing (`child._adapt_markers` already stashes the per-level markers); +chaining `adapt` on an already-adapted child; load-balancing the adapted layers. diff --git a/docs/developer/design/submesh-solver-architecture.md b/docs/developer/design/submesh-solver-architecture.md index 70ea7e51..2f6de699 100644 --- a/docs/developer/design/submesh-solver-architecture.md +++ b/docs/developer/design/submesh-solver-architecture.md @@ -21,14 +21,33 @@ Underworld3 needs to support solving different equations on different subsets of 5. **Normalised `Gamma_N`** (merged) — `mesh.Gamma_N` now returns a unit normal. Penalty and Nitsche BCs are mesh-independent. -## Three Submesh Flavours: Subdomain, Resolution Level, Surface +## The parent/child model: submeshes and adapted meshes are both *children* + +```{note} +**Unified terminology (parent/child DAG).** Every mesh derived from another +carries a `parent` link and a `_relationship_kind`, forming a directed +derivation graph. A *child* comes in two **kinds**: + +- **submesh** (`_relationship_kind == "submesh"`) — a *subset* of the parent + produced by `extract_region` / `extract_surface`. DOFs **coincide** with the + parent (`subpoint_is`); transfer is exact injection. +- **refinement** (`_relationship_kind == "refinement"`) — a *finer* mesh + produced by `mesh.adapt(metric, …)` (SBR adapt-on-top). The child is + **bigger** than the parent; DOFs do **not** coincide, so transfer is FE-exact + custom-P prolongation (parent→child) and injection restriction (child→parent). + See [`LAYER2_SBR_ADAPT_ON_TOP.md`](LAYER2_SBR_ADAPT_ON_TOP.md). + +`copy_into` / `add_into` dispatch on the kind, so the same call works for both: +`parent_var.copy_into(child_var)` and `child_var.copy_into(parent_var)`. +The "submesh flavours" below are all the **submesh kind** of child. +``` A *submesh* in UW3 is any mesh pulled out of a parent mesh that retains a lineage link (`parent`, registration in `parent._registered_submeshes`) and supports explicit field transfer back and forth. There are three flavours, and they share one usage pattern: -> **get a submesh → build a solver on it → map fields back and forth** +> **get a child → build a solver on it → map fields back and forth** | | Subdomain | Resolution level | Surface | |---|---|---|---| diff --git a/src/underworld3/cython/petsc_generic_snes_solvers.pyx b/src/underworld3/cython/petsc_generic_snes_solvers.pyx index 7c680a88..8ccaabf9 100644 --- a/src/underworld3/cython/petsc_generic_snes_solvers.pyx +++ b/src/underworld3/cython/petsc_generic_snes_solvers.pyx @@ -2703,10 +2703,10 @@ class SNES_Scalar(SolverBaseClass): # Custom multigrid prolongation: inject our P hierarchy before the # first PCSetUp (so the Galerkin coarse operators are built from it). - # No-op unless set_custom_mg() was called. - if self._custom_mg is not None: - from underworld3.utilities.custom_mg import inject_custom_mg - inject_custom_mg(self) + # Picks up a solver-set (set_custom_mg) OR a mesh-owned (adapt child) + # hierarchy. No-op unless one is present. + from underworld3.utilities.custom_mg import maybe_inject_custom_mg + maybe_inject_custom_mg(self, field_id=None) # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) @@ -3725,10 +3725,10 @@ class SNES_Vector(SolverBaseClass): self._update_constants() # Custom geometric-MG prolongation on the (top-level vector) PC, if - # registered via set_custom_fmg. Mirrors the SNES_Scalar hook. - if self._custom_mg is not None: - from underworld3.utilities.custom_mg import inject_custom_mg - inject_custom_mg(self) + # registered via set_custom_fmg or owned by an adapt() mesh. Mirrors the + # SNES_Scalar hook. + from underworld3.utilities.custom_mg import maybe_inject_custom_mg + maybe_inject_custom_mg(self, field_id=None) # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) @@ -7467,10 +7467,11 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): # Custom geometric-MG prolongation on the velocity block (if registered # via set_custom_fmg). Injected here — after setFromOptions/nullspace, # before the real solve — because the velocity sub-PC is only reachable - # once the monolithic Jacobian is assembled (see custom_mg). - if self._custom_mg is not None: - from underworld3.utilities.custom_mg import inject_custom_mg - inject_custom_mg(self) + # once the monolithic Jacobian is assembled (see custom_mg). Picks up + # a solver-set (set_custom_fmg field_id=0) or mesh-owned (adapt child) + # hierarchy on the velocity block. + from underworld3.utilities.custom_mg import maybe_inject_custom_mg + maybe_inject_custom_mg(self, field_id=0) self._snes_solve_with_retries(gvec, divergence_retries, verbose) else: @@ -7484,10 +7485,11 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): # Custom geometric-MG prolongation on the velocity block (if registered # via set_custom_fmg). Injected here — after setFromOptions/nullspace, # before the real solve — because the velocity sub-PC is only reachable - # once the monolithic Jacobian is assembled (see custom_mg). - if self._custom_mg is not None: - from underworld3.utilities.custom_mg import inject_custom_mg - inject_custom_mg(self) + # once the monolithic Jacobian is assembled (see custom_mg). Picks up + # a solver-set (set_custom_fmg field_id=0) or mesh-owned (adapt child) + # hierarchy on the velocity block. + from underworld3.utilities.custom_mg import maybe_inject_custom_mg + maybe_inject_custom_mg(self, field_id=0) self._snes_solve_with_retries(gvec, divergence_retries, verbose) # Project the rigid-body rotation gauge out of the converged solution. diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index 7628e503..bdc0d996 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -347,6 +347,7 @@ def __init__( self._registered_swarms = weakref.WeakSet() self._registered_surfaces = weakref.WeakSet() # Surfaces using this mesh self._registered_submeshes = weakref.WeakSet() # Submeshes from extract_region + self._registered_children = weakref.WeakSet() # SBR refinement children from adapt() # _mesh_update_lock: Re-entrant lock to coordinate mesh deformation. # Held by mesh_update_callback during _deform_mesh(). Checked by @@ -510,8 +511,20 @@ class replacement_boundaries(Enum): self._bounding_surfaces = {} self.boundary_normals = boundary_normals self.regions = regions - self.parent = None # Set by extract_region() for submeshes + self.parent = None # Set by extract_region()/adapt() for children self.subpoint_is = None # IS mapping submesh points -> parent points + # How this mesh was derived from ``self.parent`` (the parent/child DAG): + # None -- a base mesh (no parent) + # "submesh" -- a subset extracted via extract_region (DOFs coincide) + # "refinement"-- an SBR adapt-on-top child (parent ⊂ child, DOFs differ) + # copy_into/restrict/prolongate dispatch on this kind. + self._relationship_kind = None + # Mesh-owned custom-P geometric-MG hierarchy (set on refinement children + # by adapt()): the static coarse-mesh tail every solver on this mesh + # consumes. ``None`` => no mesh-owned hierarchy (solvers fall back to + # their own preconditioner / an explicit set_custom_fmg). + self._custom_mg_coarse_meshes = None + self._custom_mg_builder = "barycentric" # Wrapped imported DMPlex meshes may only expose generic Gmsh labels # such as "Face Sets". Rebuild named boundary labels from those sets so @@ -1703,6 +1716,7 @@ def extract_region(self, label_name, label_value=None): # Store lineage sub_mesh.parent = self + sub_mesh._relationship_kind = "submesh" sub_mesh.subpoint_is = subpoint_is sub_mesh._parent_mesh_version = self._mesh_version sub_mesh._extract_label_name = label_name @@ -1877,6 +1891,7 @@ def extract_surface(self, label_name, label_value=None, verbose=False): # Submesh lineage — same shape as extract_region surf_mesh.parent = self + surf_mesh._relationship_kind = "submesh" surf_mesh.subpoint_is = subpoint_is surf_mesh._parent_mesh_version = self._mesh_version surf_mesh._extract_label_name = label_name @@ -2224,6 +2239,75 @@ def prolongate(self, sub_var, parent_var, mode="replace"): parent_var._data_is_dirty = True + # ------------------------------------------------------------------ # + # Refinement-child transfers (SBR adapt-on-top; self is the CHILD) + # + # Unlike the submesh restrict/prolongate above (DOFs coincide via + # ``subpoint_is``), an adapt() child is *bigger* than its parent and the + # DOFs do not coincide. Direction therefore flips: + # parent (coarse) -> child (fine) = PROLONGATE (FE-exact custom-P) + # child (fine) -> parent (coarse) = RESTRICT (injection at shared nodes) + # Serial uses the structured barycentric custom-P / nearest-node injection + # (FE-exact, the quality path); parallel (np>1) falls back to the + # partition-agnostic ``global_evaluate`` REMAP (swarm-migration based). + # ------------------------------------------------------------------ # + def _refine_prolongate(self, parent_var, child_var, mode="replace"): + """Coarse parent -> fine child interpolation (this mesh is the child).""" + if self._relationship_kind != "refinement": + raise ValueError("_refine_prolongate requires an adapt() refinement child") + pv = getattr(parent_var, "_base_var", parent_var) + cv = getattr(child_var, "_base_var", child_var) + + if uw.mpi.size > 1: + out = numpy.asarray( + uw.function.global_evaluate(pv.sym, numpy.asarray(cv.coords)) + ).reshape(cv.data.shape) + else: + from underworld3.utilities import custom_mg + cc = numpy.asarray(self.parent._get_coords_for_basis(pv.degree, pv.continuous)) + fc = numpy.asarray(self._get_coords_for_basis(cv.degree, cv.continuous)) + P = custom_mg.barycentric_prolongation(cc, fc) + out = (P @ numpy.asarray(pv.data)).reshape(cv.data.shape) + + new = numpy.array(cv.data) + if mode == "replace": + new[:] = out + elif mode == "add": + new[:] += out + else: + raise ValueError(f"mode must be 'replace' or 'add', got '{mode}'") + cv.pack_raw_data_to_petsc(new, sync=True) + + def _refine_restrict(self, child_var, parent_var, mode="replace"): + """Fine child -> coarse parent injection (this mesh is the child).""" + if self._relationship_kind != "refinement": + raise ValueError("_refine_restrict requires an adapt() refinement child") + pv = getattr(parent_var, "_base_var", parent_var) + cv = getattr(child_var, "_base_var", child_var) + + if uw.mpi.size > 1: + out = numpy.asarray( + uw.function.global_evaluate(cv.sym, numpy.asarray(pv.coords)) + ).reshape(pv.data.shape) + else: + from scipy.spatial import cKDTree + cc = numpy.asarray(self.parent._get_coords_for_basis(pv.degree, pv.continuous)) + fc = numpy.asarray(self._get_coords_for_basis(cv.degree, cv.continuous)) + # nested SBR: every coarse DOF coincides with a fine DOF (P1) or sits + # on a fine element edge (P2) -> nearest fine node is exact / near-exact. + _, idx = cKDTree(fc).query(cc) + out = numpy.asarray(cv.data)[idx].reshape(pv.data.shape) + + new = numpy.array(pv.data) + if mode == "replace": + new[:] = out + elif mode == "add": + new[:] += out + else: + raise ValueError(f"mode must be 'replace' or 'add', got '{mode}'") + pv.pack_raw_data_to_petsc(new, sync=True) + pv._data_is_dirty = True + def nuke_coords_and_rebuild( self, verbose=False, @@ -5905,13 +5989,204 @@ def OT_adapt_reset_reference(self, coords=None): self._ot_adapt_reference_coords = numpy.asarray(coords).copy() @timing.routine_timer_decorator - def adapt(self, metric_field, verbose=False): + def _wrap_coarse_level(self, dm): + """Wrap a (static) coarse-hierarchy DM as a UW Mesh carrying this mesh's + boundary labels — a coarse level for the custom-P geometric-MG hierarchy.""" + return Mesh( + dm.clone(), + simplex=self.dm.isSimplex(), + coordinate_system_type=self.CoordinateSystem.coordinate_type, + qdegree=self.qdegree, + boundaries=self.boundaries, + verbose=False, + ) + + def _coarse_level_meshes(self): + """The static coarse-mesh tail (one Mesh per base hierarchy level, + coarsest..base-finest), built once and cached — they never change + because the base hierarchy is static across adapts.""" + cached = getattr(self, "_coarse_level_meshes_cache", None) + if cached is None: + cached = [self._wrap_coarse_level(d) for d in self.dm_hierarchy] + self._coarse_level_meshes_cache = cached + return cached + + def adapt(self, metric_field, max_levels=2, node_budget=None, + builder="barycentric", adapter="sbr", verbose=False): r""" - Adapt the mesh discretization based on a metric field. + Nested **SBR adapt-on-top**: return a refined **child** mesh. + + Skeleton-based-refinement (SBR) the static base finest where the metric + demands resolution, **on top of** the existing uniform hierarchy, and + return a new child mesh (``child.parent is self``). The base mesh is + **not modified** — this is *adapt / re-adapt*, not node movement: each + call re-marks from the static base finest, so successive adapts are + non-cumulative (cf. :meth:`remesh`, which regenerates the mesh in place + via MMG and may redistribute). - This method refines or coarsens the mesh in place, automatically - transferring all attached MeshVariables, updating Surfaces, and - marking Solvers for rebuild on their next solve() call. + The child owns a custom-P geometric-MG hierarchy + (``[base coarse levels … base finest] + child``) so every solver built on + it drives geometric multigrid on the refined operator with no per-solver + setup. + + Parameters + ---------- + metric_field : MeshVariable + Scalar metric ``M = 1/h²`` (target edge length ``h``); larger ⇒ finer. + Same interface as :meth:`remesh` / ``adaptivity.create_metric``. + max_levels : int + Maximum SBR depth applied on top of the base finest (bounds the + on-rank imbalance). Each level re-marks against the metric. + node_budget : int or None + Optional cap on the number of cells refined **per level** (highest- + metric cells first). Approximate DOF control; ``None`` ⇒ uncapped. + builder : {"barycentric", "rbf"} + Per-level node-prolongation builder for the child's custom-P hierarchy. + adapter : {"sbr", "mmg"} + ``"sbr"`` (default) is this nested path. ``"mmg"`` is a **deprecated + shim** that forwards to :meth:`remesh` (in-place, returns ``self``). + verbose : bool + + Returns + ------- + Mesh + The refined child (or ``self`` when ``adapter='mmg'``). + """ + import warnings + + if adapter == "mmg": + warnings.warn( + "mesh.adapt(adapter='mmg') is deprecated; the in-place MMG " + "remesher is now mesh.remesh(). Call mesh.remesh(metric) instead.", + DeprecationWarning, stacklevel=2, + ) + self.remesh(metric_field, verbose=verbose) + return self + if adapter != "sbr": + raise ValueError(f"adapter must be 'sbr' or 'mmg', got {adapter!r}") + + return self._adapt_sbr( + metric_field, max_levels=max_levels, node_budget=node_budget, + builder=builder, verbose=verbose, + ) + + def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, + builder="barycentric", verbose=False): + """Core SBR adapt-on-top. See :meth:`adapt`.""" + import math + from underworld3.utilities import custom_mg + + if self.parent is not None: + raise NotImplementedError( + "adapt(adapter='sbr') refines a BASE mesh; chaining adapt on an " + "already-adapted child is not yet supported." + ) + if getattr(self, "dm_hierarchy", None) is None or len(self.dm_hierarchy) < 2: + raise RuntimeError( + "SBR adapt-on-top needs a base mesh built with refinement>=1 (a " + "dm_hierarchy of coarse levels supplies the geometric-MG tail). " + "Build the mesh with e.g. refinement=2." + ) + + dim = self.dim + edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex + + current_dm = self.dm_hierarchy[-1] # static base finest + markers_per_level = [] + + for level in range(max_levels): + cs, ce = current_dm.getHeightStratum(0) + ncells = ce - cs + if ncells == 0: + break + centroids = numpy.empty((ncells, self.cdim)) + cur_h = numpy.empty(ncells) + for i, c in enumerate(range(cs, ce)): + vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] + centroids[i] = numpy.asarray(cen)[: self.cdim] + cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) + + # Metric M = 1/h_target² evaluated at the cell centroids (parent field). + # Parallel: use the swarm-migration-based global_evaluate (partition- + # agnostic, collective-safe); serial: the cheaper local evaluate. + if uw.mpi.size > 1: + M = numpy.asarray( + uw.function.global_evaluate(metric_field.sym, centroids) + ).reshape(-1) + else: + M = numpy.asarray( + uw.function.evaluate(metric_field.sym, centroids) + ).reshape(-1) + M = numpy.clip(M, 1e-30, None) + h_target = 1.0 / numpy.sqrt(M) + + refine = numpy.where(cur_h > h_target)[0] + if refine.size == 0: + if verbose: + uw.pprint(0, f"[adapt] level {level}: no cells need refinement") + break + + if node_budget is not None and refine.size > node_budget: + # keep the highest-metric (finest-demand) cells first + order = numpy.argsort(M[refine])[::-1] + refine = refine[order[:node_budget]] + + cell_ids = [int(cs + j) for j in refine] + markers_per_level.append(cell_ids) + if verbose: + uw.pprint(0, f"[adapt] level {level}: refining {len(cell_ids)} " + f"of {ncells} cells") + current_dm = custom_mg.sbr_refine(current_dm, cell_ids) + + if verbose: + base_n = self.dm_hierarchy[-1].getHeightStratum(0) + fin_n = current_dm.getHeightStratum(0) + uw.pprint(0, f"[adapt] base finest {base_n[1]-base_n[0]} -> " + f"child {fin_n[1]-fin_n[0]} cells " + f"({len(markers_per_level)} SBR level(s))") + + # Wrap the refined finest as the child mesh (on-rank; no redistribute). + child = Mesh( + current_dm.clone(), + simplex=self.dm.isSimplex(), + coordinate_system_type=self.CoordinateSystem.coordinate_type, + qdegree=self.qdegree, + boundaries=self.boundaries, + verbose=False, + ) + + # Lineage (parent/child DAG) and mesh-owned custom-P hierarchy. + child.parent = self + child._relationship_kind = "refinement" + child.regions = self.regions + child._parent_mesh_version = self._mesh_version + # Markers per SBR level (in each level's cell numbering) — the + # checkpoint-by-marker payload (design only; storage is a follow-up). + child._adapt_markers = markers_per_level + # Static coarse tail (incl. base finest); the child appends itself when a + # solver builds the hierarchy. Reuses the parent's cached wrapped levels. + child._custom_mg_coarse_meshes = self._coarse_level_meshes() + child._custom_mg_builder = builder + + self._registered_children.add(child) + return child + + def remesh(self, metric_field, verbose=False): + r""" + Re-mesh (regenerate) the discretization in place from a metric field. + + This is the **MMG / topology-changing** remesher: it regenerates the + mesh in place (cells are created/destroyed and the partition may change), + automatically transferring all attached MeshVariables, updating Surfaces, + and marking Solvers for rebuild on their next solve() call. + + Contrast :meth:`adapt`, which performs *nested* skeleton-based refinement + on top of the static base and **returns a refined child** (the mesh is + not modified in place). ``remesh`` is the in-place, redistributing path; + prefer ``adapt`` when you want a parent/child geometric-MG hierarchy. + + This method was formerly called ``adapt``; ``adapt`` now performs the + nested SBR adapt-on-top. Parameters ---------- @@ -5940,7 +6215,7 @@ def adapt(self, metric_field, verbose=False): >>> with mesh.access(metric): ... # Smaller H near fault, larger far away ... metric.data[:, 0] = 0.01 + 0.09 * fault.distance_from(mesh.data) - >>> mesh.adapt(metric, verbose=True) + >>> mesh.remesh(metric, verbose=True) >>> stokes.solve() # Solver rebuilds automatically """ import underworld3 as uw @@ -6173,8 +6448,12 @@ def mesh_update_callback(array, change_context): if hasattr(self, '_dminterpolation_cache'): self._dminterpolation_cache.invalidate_all(reason="mesh_adaptation") - # Re-extract registered submeshes from the adapted parent + # Re-extract registered submeshes from the re-meshed parent. Only true + # subset submeshes (extract_region) can be re-filtered; SBR refinement + # children (adapt) have a different lineage and are skipped. for submesh in list(self._registered_submeshes): + if getattr(submesh, "_relationship_kind", "submesh") != "submesh": + continue try: submesh._re_extract_from_parent(verbose=verbose) except Exception as e: diff --git a/src/underworld3/discretisation/enhanced_variables.py b/src/underworld3/discretisation/enhanced_variables.py index 207dae80..0eaf5dc0 100644 --- a/src/underworld3/discretisation/enhanced_variables.py +++ b/src/underworld3/discretisation/enhanced_variables.py @@ -514,38 +514,33 @@ def read_timestep(self, *args, **kwargs): return self._base_var.read_timestep(*args, **kwargs) def copy_into(self, target): - """Copy this variable's data into a variable on a related mesh. + """Copy this variable's data into a variable on a related (child/parent) mesh. - Detects the parent/submesh relationship and calls restrict or - prolongate as appropriate. Both meshes must be related via - ``extract_region``. + Detects the parent/child relationship and dispatches restrict or + prolongate as appropriate. The two meshes must be related via + ``extract_region`` (submesh child) or ``adapt`` (SBR refinement child). + + The operation depends on the **kind** of child: + + - **submesh** (``extract_region``): the child is a *subset*, DOFs + coincide. parent → child is restrict; child → parent is prolongate. + - **refinement** (``adapt``): the child is *finer*, DOFs differ. parent → + child is prolongate (FE-exact custom-P); child → parent is restrict + (injection at shared nodes). Parameters ---------- target : MeshVariable - Destination variable. Must be on the parent or a submesh - of this variable's mesh. + Destination variable, on the parent or a child of this variable's mesh. Examples -------- - >>> v_full.copy_into(v_rock) # restrict: parent → submesh - >>> v_rock.copy_into(v_full) # prolongate: submesh → parent + >>> v_full.copy_into(v_rock) # submesh: restrict parent → child + >>> v_rock.copy_into(v_full) # submesh: prolongate child → parent + >>> v_base.copy_into(v_child) # refinement: prolongate parent → child + >>> v_child.copy_into(v_base) # refinement: restrict child → parent """ - src_mesh = self._base_var.mesh - tgt_mesh = target._base_var.mesh if hasattr(target, '_base_var') else target.mesh - - if hasattr(tgt_mesh, 'parent') and tgt_mesh.parent is src_mesh: - # target is submesh of source → restrict - tgt_mesh.restrict(self, target, mode="replace") - elif hasattr(src_mesh, 'parent') and src_mesh.parent is tgt_mesh: - # source is submesh of target → prolongate - src_mesh.prolongate(self, target, mode="replace") - else: - raise ValueError( - "copy_into requires a parent/submesh relationship between " - "the two variables' meshes. Use uw.function.evaluate() " - "for unrelated meshes." - ) + self._copy_into(target, mode="replace") def add_into(self, target): """Add this variable's data into a variable on a related mesh. @@ -553,29 +548,44 @@ def add_into(self, target): Like ``copy_into`` but uses ADD_VALUES — adds to existing values in the target rather than replacing them. - Parameters - ---------- - target : MeshVariable - Destination variable. Must be on the parent or a submesh - of this variable's mesh. - Examples -------- >>> v_rock.add_into(v_full) # prolongate with ADD """ - src_mesh = self._base_var.mesh - tgt_mesh = target._base_var.mesh if hasattr(target, '_base_var') else target.mesh - - if hasattr(tgt_mesh, 'parent') and tgt_mesh.parent is src_mesh: - tgt_mesh.restrict(self, target, mode="add") - elif hasattr(src_mesh, 'parent') and src_mesh.parent is tgt_mesh: - src_mesh.prolongate(self, target, mode="add") + self._copy_into(target, mode="add") + + def _copy_into(self, target, mode): + """Shared parent/child dispatch for copy_into / add_into.""" + src_var = self + tgt_var = target + src_mesh = src_var._base_var.mesh + tgt_mesh = tgt_var._base_var.mesh if hasattr(tgt_var, '_base_var') else tgt_var.mesh + + # Identify which mesh is the child (its .parent is the other). + if getattr(tgt_mesh, 'parent', None) is src_mesh: + child, parent_is_src = tgt_mesh, True # src=parent, tgt=child + elif getattr(src_mesh, 'parent', None) is tgt_mesh: + child, parent_is_src = src_mesh, False # src=child, tgt=parent else: raise ValueError( - "add_into requires a parent/submesh relationship between " - "the two variables' meshes." + "copy_into requires a parent/child relationship between the two " + "variables' meshes (extract_region or adapt). Use " + "uw.function.evaluate() for unrelated meshes." ) + kind = getattr(child, "_relationship_kind", "submesh") + + if kind == "refinement": + if parent_is_src: # parent → child + child._refine_prolongate(src_var, tgt_var, mode=mode) + else: # child → parent + child._refine_restrict(src_var, tgt_var, mode=mode) + else: # submesh (DOFs coincide) + if parent_is_src: # parent → child: restrict + child.restrict(src_var, tgt_var, mode=mode) + else: # child → parent: prolongate + child.prolongate(src_var, tgt_var, mode=mode) + def stats(self, *args, **kwargs): """Get statistics for the variable.""" return self._base_var.stats(*args, **kwargs) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 32023429..6a2407bc 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -618,6 +618,31 @@ def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", solver.is_setup = False +def maybe_inject_custom_mg(solver, field_id=None): + """Solve-hook entry: inject custom-P FMG from either a solver-set hierarchy + (``set_custom_fmg``) or a **mesh-owned** one (``mesh.adapt`` refinement child). + + A refinement child carries ``mesh._custom_mg_coarse_meshes`` (the static + coarse tail). The first time a solver on such a mesh solves, we lazily build a + :class:`CustomMGHierarchy` ``[*coarse, solver.mesh]`` targeting ``field_id`` + (0 for the Stokes velocity block, None for scalar/vector) and register it on + the solver — so every solver on an adapted mesh drives geometric MG with no + per-solver call. A solver-set hierarchy (if present) always wins. + """ + if solver._custom_mg is None: + coarse = getattr(solver.mesh, "_custom_mg_coarse_meshes", None) + if coarse is None: + return # nothing to inject + builder = getattr(solver.mesh, "_custom_mg_builder", "barycentric") + solver._custom_mg = { + "mode": "hierarchy", + "hierarchy": CustomMGHierarchy(list(coarse) + [solver.mesh], + builder=builder, field_id=field_id), + "verbose": False, + } + inject_custom_mg(solver) + + def inject_custom_mg(solver): """Build + install the custom-P FMG. Called from ``solve()`` (after ``_build``, before the SNES solve) when ``solver._custom_mg`` is set. Dispatches: diff --git a/tests/parallel/ptest_0763_mesh_adapt_transfer.py b/tests/parallel/ptest_0763_mesh_adapt_transfer.py index aee1bc37..e099f112 100644 --- a/tests/parallel/ptest_0763_mesh_adapt_transfer.py +++ b/tests/parallel/ptest_0763_mesh_adapt_transfer.py @@ -45,7 +45,7 @@ def test_adapt_preserves_degree_2_field_in_parallel(): H.coords[:, 0] > 0.5, 1.0 / 0.05 ** 2, 1.0 / 0.1 ** 2 ) - mesh.adapt(H) + mesh.remesh(H) T2 = mesh.vars["T"] expected = _smooth_field(T2.coords) diff --git a/tests/test_0810_amr_swarm_migration_regression.py b/tests/test_0810_amr_swarm_migration_regression.py index 727c17d8..5f2df09d 100644 --- a/tests/test_0810_amr_swarm_migration_regression.py +++ b/tests/test_0810_amr_swarm_migration_regression.py @@ -18,12 +18,38 @@ from test_0800_optional_modules import requires_amr -pytestmark = [pytest.mark.level_1, requires_amr] +def _has_petsc_adaptation_backend(): + """Runtime probe: try a trivial in-place ``remesh`` (MMG). ``requires_amr`` + only greps petscvariables for 'pragmatic', which can be present while the + runtime adapter is non-functional (e.g. the custom-MG PETSc build); this + actually exercises ``adaptMetric`` and skips if it raises.""" + try: + m = uw.meshing.UnstructuredSimplexBox( + minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0), cellSize=0.5) + H = uw.discretisation.MeshVariable("_probe810", m, 1, degree=1) + H.data[:, 0] = 100.0 + m.remesh(H) + return True + except Exception: + return False + + +_petsc_has_adaptation = _has_petsc_adaptation_backend() + +pytestmark = [ + pytest.mark.level_1, + requires_amr, + pytest.mark.skipif( + not _petsc_has_adaptation, + reason="PETSc mesh-adaptation backend (MMG/pragmatic) is non-functional " + "at runtime; remesh() cannot run here.", + ), +] @requires_amr def test_swarm_migration_after_adapt_does_not_raise(): - """mesh.adapt() then swarm._force_migration_after_mesh_change() must not + """mesh.remesh() then swarm._force_migration_after_mesh_change() must not raise IndexError on the failing case from issue #135. """ mesh = uw.meshing.UnstructuredSimplexBox( @@ -45,7 +71,7 @@ def test_swarm_migration_after_adapt_does_not_raise(): h_vals = uw.function.evaluate(h_subbed, mesh.X.coords) metric = uw.adaptivity.create_metric(mesh, h_vals) - mesh.adapt(metric) + mesh.remesh(metric) cells_after = mesh.dm.getHeightStratum(0)[1] assert cells_after != cells_before, ( diff --git a/tests/test_0830_mesh_adapt_variable_transfer.py b/tests/test_0830_mesh_adapt_variable_transfer.py index 1aaabc7a..8ec8386b 100644 --- a/tests/test_0830_mesh_adapt_variable_transfer.py +++ b/tests/test_0830_mesh_adapt_variable_transfer.py @@ -41,7 +41,7 @@ def _has_petsc_adaptation_backend(): H.array[:, 0, 0] = 100.0 # Use the real adapt() entry point but with a coarse metric; # if PETSc raises, we capture and skip. - m.adapt(H) + m.remesh(H) return True except Exception: return False @@ -87,7 +87,7 @@ def test_adapt_preserves_degree_1_continuous(mesh): T.array[:, 0, 0] = _smooth_field(T.coords) metric = _refinement_metric(mesh) - mesh.adapt(metric) + mesh.remesh(metric) T2 = mesh.vars["T"] expected = _smooth_field(T2.coords) @@ -113,7 +113,7 @@ def test_adapt_preserves_degree_2_continuous(mesh): T.array[:, 0, 0] = _smooth_field(T.coords) metric = _refinement_metric(mesh) - mesh.adapt(metric) + mesh.remesh(metric) T2 = mesh.vars["T"] expected = _smooth_field(T2.coords) @@ -135,7 +135,7 @@ def test_adapt_preserves_vector_field(mesh): V.array[:, 0, 1] = -_smooth_field(V.coords) metric = _refinement_metric(mesh) - mesh.adapt(metric) + mesh.remesh(metric) V2 = mesh.vars["V"] expected_x = _smooth_field(V2.coords) @@ -155,7 +155,7 @@ def test_adapt_preserves_constant_field(mesh): T.array[:, 0, 0] = 3.14 metric = _refinement_metric(mesh) - mesh.adapt(metric) + mesh.remesh(metric) T2 = mesh.vars["T"] actual = np.asarray(T2.array[:, 0, 0]) @@ -177,7 +177,7 @@ def test_adapt_preserves_discontinuous_field(mesh): T.array[:, 0, 0] = _smooth_field(T.coords) metric = _refinement_metric(mesh) - mesh.adapt(metric) + mesh.remesh(metric) T2 = mesh.vars["T_dg"] expected = _smooth_field(T2.coords) diff --git a/tests/test_0835_sbr_adapt_on_top.py b/tests/test_0835_sbr_adapt_on_top.py new file mode 100644 index 00000000..695fd392 --- /dev/null +++ b/tests/test_0835_sbr_adapt_on_top.py @@ -0,0 +1,154 @@ +"""Layer-2 SBR adapt-on-top: mesh.adapt(adapter='sbr') returns a refined CHILD. + +The new nested adapter (no MMG, on-rank, no redistribute) refines the static base +finest where a metric demands resolution and returns a child mesh that owns a +custom-P geometric-MG hierarchy. Validated here: + + - adapt() returns a refinement child (parent link, finer than the base); + - a solver built on the child auto-picks-up the mesh-owned custom-P hierarchy + (pc=mg, no per-solver set_custom_fmg) and matches GAMG; + - copy_into prolongates parent->child (FE-exact) and restricts child->parent; + - re-adapt is non-cumulative (the base finest is unchanged); + - node_budget localises refinement; + - mesh.remesh is the renamed MMG path and adapt(adapter='mmg') is a deprecated + shim that warns. + +SBR uses only PETSc's refine_sbr transform + scipy custom-P, so (unlike the MMG +remesh tests) these do NOT require an mmg/pragmatic PETSc build. +""" +import warnings +import numpy as np +import pytest +import sympy +import underworld3 as uw + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + + +def _ev(fn, coords): + return np.asarray(uw.function.evaluate(fn, np.asarray(coords))).reshape(-1) + + +def _ncell(mesh): + cs, ce = mesh.dm.getHeightStratum(0) + return ce - cs + + +def _base(): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.3, regular=True, + refinement=2, qdegree=3) + + +def _metric(base, center, h_coarse=0.0625, h_fine=0.025, width=0.15): + """M = 1/h^2 with a fine band around x=center (anchored at the base size).""" + M = uw.discretisation.MeshVariable(f"M_{int(center*100)}", base, 1, degree=1) + band = sympy.exp(-(((base.N.x - center) / width) ** 2)) + M.data[:, 0] = _ev(1.0 / (h_coarse + (h_fine - h_coarse) * band) ** 2, M.coords) + return M + + +def _poisson(mesh): + p = uw.systems.Poisson(mesh) + p.constitutive_model = uw.constitutive_models.DiffusionModel + p.constitutive_model.Parameters.diffusivity = 1 + p.f = 0.0 + p.add_dirichlet_bc(0.0, "Bottom") + p.add_dirichlet_bc(1.0, "Top") + p.petsc_options["ksp_rtol"] = 1e-8 + p.petsc_options["ksp_type"] = "cg" + return p + + +def test_adapt_returns_refinement_child(): + base = _base() + n0 = _ncell(base) + child = base.adapt(_metric(base, 0.7), max_levels=1) + + assert child is not base + assert child.parent is base + assert child._relationship_kind == "refinement" + assert _ncell(child) > n0 + # base is untouched (adapt is not in-place) + assert _ncell(base) == n0 + + +def test_child_solver_auto_picks_up_custom_mg(): + base = _base() + child = base.adapt(_metric(base, 0.7), max_levels=1) + + s = _poisson(child) + s.solve() # NO set_custom_fmg + assert s.snes.getKSP().getPC().getType() == "mg" + assert s.snes.getConvergedReason() > 0 + + g = _poisson(child) + g.preconditioner = "gamg" + g.solve() + rel = np.linalg.norm(s.Unknowns.u.data - g.Unknowns.u.data) / ( + np.linalg.norm(g.Unknowns.u.data) + 1e-30) + assert rel < 1e-4 + + +def test_copy_into_prolongate_and_restrict(): + base = _base() + child = base.adapt(_metric(base, 0.7), max_levels=1) + fn = sympy.sin(2 * sympy.pi * base.N.x) * sympy.cos(sympy.pi * base.N.y) + + Tb = uw.discretisation.MeshVariable("Tb", base, 1, degree=2) + Tb.data[:, 0] = _ev(fn, Tb.coords) + Tc = uw.discretisation.MeshVariable("Tc", child, 1, degree=2) + + Tb.copy_into(Tc) # prolongate parent -> child + exact_c = _ev(fn, Tc.coords) + assert np.linalg.norm(Tc.data[:, 0] - exact_c) / ( + np.linalg.norm(exact_c) + 1e-30) < 1e-2 + + Tb2 = uw.discretisation.MeshVariable("Tb2", base, 1, degree=2) + Tc.copy_into(Tb2) # restrict child -> parent (injection) + exact_b = _ev(fn, Tb2.coords) + assert np.linalg.norm(Tb2.data[:, 0] - exact_b) / ( + np.linalg.norm(exact_b) + 1e-30) < 1e-2 + + +def test_readapt_is_non_cumulative(): + base = _base() + n0 = _ncell(base) + c1 = base.adapt(_metric(base, 0.7), max_levels=1) + n1 = _ncell(c1) + c2 = base.adapt(_metric(base, 0.35), max_levels=1) + # each adapt re-marks from the SAME static base finest + assert _ncell(base) == n0 + assert _ncell(c2) == n1 # symmetric band -> same count, different place + assert c2 is not c1 + + +def test_node_budget_localises_refinement(): + base = _base() + sharp = _metric(base, 0.7, width=0.06) + full = base.adapt(sharp, max_levels=1) + budgeted = base.adapt(sharp, max_levels=1, node_budget=40) + assert _ncell(budgeted) < _ncell(full) + + +def test_adapt_mmg_shim_warns(): + base = _base() + H = uw.discretisation.MeshVariable("Hshim", base, 1, degree=1) + H.data[:, 0] = 1.0 / 0.1**2 + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + try: + base.adapt(H, adapter="mmg") # forwards to remesh (MMG) + except Exception: + pass # MMG backend may be absent + assert any(issubclass(x.category, DeprecationWarning) for x in w) + + +def test_adapt_requires_base_hierarchy(): + # a mesh without a refinement hierarchy cannot supply the MG coarse tail + flat = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.3, regular=True, qdegree=3) + H = uw.discretisation.MeshVariable("Hflat", flat, 1, degree=1) + H.data[:, 0] = 1.0 / 0.02**2 + with pytest.raises(RuntimeError): + flat.adapt(H, max_levels=1) From b70dbc781db1f72ddf669f2c8fddb515e0100f19 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 21:51:45 +1000 Subject: [PATCH 19/52] test(layer2): Stokes velocity-block FMG auto-pickup on an adapt() child Canonical SolCx (eta-jump 1e6) Stokes on an SBR-adapted child refined near the viscosity jump (128/512 cells -> child 992, genuinely local). The velocity block auto-picks-up the mesh-owned custom-P FMG (field_id=0, full cycle, fgmres outer) with NO set_custom_fmg, converges in 6 iters == GAMG, matches the GAMG solution to rel 0 and the analytic velocity to GAMG accuracy. Validated serial + np=2 via demo_stokes_on_adapted_child.py. Underworld development team with AI support from Claude Code --- tests/test_0835_sbr_adapt_on_top.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_0835_sbr_adapt_on_top.py b/tests/test_0835_sbr_adapt_on_top.py index 695fd392..84ab43f3 100644 --- a/tests/test_0835_sbr_adapt_on_top.py +++ b/tests/test_0835_sbr_adapt_on_top.py @@ -21,6 +21,7 @@ import pytest import sympy import underworld3 as uw +from underworld3.function import analytic as A pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] @@ -144,6 +145,60 @@ def test_adapt_mmg_shim_warns(): assert any(issubclass(x.category, DeprecationWarning) for x in w) +def test_stokes_velocity_block_auto_picks_up_fmg(): + """Canonical case: SolCx Stokes (eta jump 1e6) on an adapt() child, refined + near the viscosity jump. The velocity block must auto-pick-up the mesh-owned + custom-P FMG (field_id=0, fgmres outer, full cycle) with NO set_custom_fmg, + converge, and match a GAMG reference. The refinement must be genuinely local.""" + base = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.25, regular=True, + refinement=2, qdegree=3) + n_uniform = _ncell(base) * 4 # one global SBR level = x4 + + # metric: refine only near the x=0.5 jump (h_coarse just above the base size) + M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) + band = sympy.exp(-(((base.N.x - 0.5) / 0.08) ** 2)) + M.data[:, 0] = _ev(1.0 / (0.07 + (1.0 / 80 - 0.07) * band) ** 2, M.coords) + child = base.adapt(M, max_levels=1) + assert _ncell(child) < n_uniform # local, not a uniform refine + + def _solcx(mesh): + sol = A.SolCx(mesh, eta_A=1.0, eta_B=1.0e6, x_c=0.5, n=1) + s = uw.systems.Stokes(mesh) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.saddle_preconditioner = 1.0 / sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_dirichlet_bc((0.0, None), "Left") + s.add_dirichlet_bc((0.0, None), "Right") + s.add_dirichlet_bc((None, 0.0), "Bottom") + s.add_dirichlet_bc((None, 0.0), "Top") + s.petsc_use_pressure_nullspace = True + s.petsc_options["snes_type"] = "ksponly" + s.tolerance = 1e-8 + return s, sol + + sg, solg = _solcx(child) + sg.preconditioner = "gamg" + sg.solve() + it_g = sg.snes.getKSP().getPC().getFieldSplitSubKSP()[0].getIterationNumber() + + s, sol = _solcx(child) + assert s._custom_mg is None # nothing registered on the solver + s.solve() # velocity block auto-picks-up FMG + vksp = s.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGType() == 2 # PETSc PC.MGType.FULL == FMG + assert vksp.getPC().getMGLevels() == len(base.dm_hierarchy) + 1 + assert vksp.getType() == "fgmres" + assert vksp.getIterationNumber() <= it_g # FMG matches/beats GAMG + rel = np.linalg.norm(s.u.data - sg.u.data) / (np.linalg.norm(sg.u.data) + 1e-30) + assert rel < 1e-4 + assert sol.velocity_error(s.u) < 2.0 * solg.velocity_error(sg.u) + 1e-6 + + def test_adapt_requires_base_hierarchy(): # a mesh without a refinement hierarchy cannot supply the MG coarse tail flat = uw.meshing.UnstructuredSimplexBox( From 13a1a2e2a828b64618e191d15599aae2cb716ce8 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Mon, 29 Jun 2026 22:30:07 +1000 Subject: [PATCH 20/52] feat(layer2): one MG level per SBR refinement step (not a single jump) Previously max_levels>1 collapsed all SBR levels into a single base-finest -> child custom-P transfer, so the whole refinement got just one MG level (the coarse-grid correction made one big jump). Now each SBR pass becomes its own MG level: the child's mesh-owned tail is [base L0 ... base finest] + [SBR level 1 ... SBR level n-1] and the solver appends its own mesh (the finest SBR level = child), so transfers between consecutive levels each span a single refinement. _adapt_sbr keeps every intermediate SBR DM, wraps it as a coarse-level mesh (transient, on the child), and appends them after the static base tail. Effect: MG levels now scale with depth (4/5/6 for max_levels 1/2/3), matching the design doc's intended [base levels] + [SBR levels] hierarchy. The 2-level box-fault Stokes solve improves 4 levels/8 iters -> 5 levels/5 iters; np=2 max_levels=2 Poisson drives 5 levels, converges in 2 iters (intermediate SBR meshes co-partition correctly, no redistribute). test_0835::test_each_sbr_level_is_its_own_mg_level asserts MG levels = base levels + n_sbr and that a 2-level solve drives all of them. 9 tests pass. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 26 ++++++++++++++----- tests/test_0835_sbr_adapt_on_top.py | 20 ++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index bdc0d996..01476b80 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6024,10 +6024,12 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, non-cumulative (cf. :meth:`remesh`, which regenerates the mesh in place via MMG and may redistribute). - The child owns a custom-P geometric-MG hierarchy - (``[base coarse levels … base finest] + child``) so every solver built on - it drives geometric multigrid on the refined operator with no per-solver - setup. + The child owns a custom-P geometric-MG hierarchy with **one level per SBR + refinement step** — ``[base L0 … base finest, SBR-1, …, SBR-n(child)]`` — + so every solver built on it drives geometric multigrid on the refined + operator with no per-solver setup. (Each ``max_levels`` SBR pass adds its + own MG level; the transfers between consecutive levels each span a single + refinement.) Parameters ---------- @@ -6093,6 +6095,7 @@ def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, current_dm = self.dm_hierarchy[-1] # static base finest markers_per_level = [] + sbr_level_dms = [] # one DM per SBR level (level 1 … n) for level in range(max_levels): cs, ce = current_dm.getHeightStratum(0) @@ -6137,6 +6140,7 @@ def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, uw.pprint(0, f"[adapt] level {level}: refining {len(cell_ids)} " f"of {ncells} cells") current_dm = custom_mg.sbr_refine(current_dm, cell_ids) + sbr_level_dms.append(current_dm) if verbose: base_n = self.dm_hierarchy[-1].getHeightStratum(0) @@ -6163,9 +6167,17 @@ def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, # Markers per SBR level (in each level's cell numbering) — the # checkpoint-by-marker payload (design only; storage is a follow-up). child._adapt_markers = markers_per_level - # Static coarse tail (incl. base finest); the child appends itself when a - # solver builds the hierarchy. Reuses the parent's cached wrapped levels. - child._custom_mg_coarse_meshes = self._coarse_level_meshes() + # Mesh-owned custom-P geometric-MG tail. EVERY SBR level is its own MG + # level (one custom-P transfer per refinement step), not a single + # base-finest -> child jump: the tail is + # [base L0 … base finest] + [SBR level 1 … SBR level n-1] + # and the solver appends its own mesh (the finest SBR level = child). + # Each intermediate SBR level is wrapped here (transient, lives on the + # child); the static base levels reuse the parent's cached wraps. + intermediate_sbr = [ + self._wrap_coarse_level(d) for d in sbr_level_dms[:-1] + ] + child._custom_mg_coarse_meshes = self._coarse_level_meshes() + intermediate_sbr child._custom_mg_builder = builder self._registered_children.add(child) diff --git a/tests/test_0835_sbr_adapt_on_top.py b/tests/test_0835_sbr_adapt_on_top.py index 84ab43f3..e7dd1133 100644 --- a/tests/test_0835_sbr_adapt_on_top.py +++ b/tests/test_0835_sbr_adapt_on_top.py @@ -199,6 +199,26 @@ def _solcx(mesh): assert sol.velocity_error(s.u) < 2.0 * solg.velocity_error(sg.u) + 1e-6 +def test_each_sbr_level_is_its_own_mg_level(): + """Every SBR refinement step must add its OWN custom-P MG level (not collapse + into a single base-finest -> child jump): MG levels = base levels + n_sbr.""" + base = _base() # refinement=2 -> 3 base hierarchy levels + n_base = len(base.dm_hierarchy) + sharp = _metric(base, 0.5, h_fine=1.0 / 150, width=0.06) + for ml, want_levels in [(1, n_base + 1), (2, n_base + 2)]: + child = base.adapt(sharp, max_levels=ml) + # coarse tail + the child itself + assert len(child._custom_mg_coarse_meshes) + 1 == want_levels + + # a Poisson solve on a 2-level child must actually drive all n_base+2 levels + child2 = base.adapt(sharp, max_levels=2) + s = _poisson(child2) + s.solve() + assert s.snes.getKSP().getPC().getType() == "mg" + assert s.snes.getKSP().getPC().getMGLevels() == n_base + 2 + assert s.snes.getConvergedReason() > 0 + + def test_adapt_requires_base_hierarchy(): # a mesh without a refinement hierarchy cannot supply the MG coarse tail flat = uw.meshing.UnstructuredSimplexBox( From c0296b6f3146b7218aa878da8587ba8993fe2c33 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 07:36:00 +1000 Subject: [PATCH 21/52] docs+test(layer2): funnel is metric-shape, not node_budget; document honestly Investigation (recorded in the docstrings): a per-level node_budget cannot make the finest SBR level hug a feature, because SBR's conforming closure re-refines the whole connected patch from any seed (measured: 30/60/120/240 seeds -> identical output). The funnel is controlled by the METRIC SHAPE: a wedge (size growing with distance, Surface.refinement_metric(..., profile="linear")) makes each level's band ~half the previous width so the finest level concentrates at the feature with bounded per-level growth; a flat-core Gaussian refines the whole core uniformly at every level (no funnel). - adapt(): new Notes section on controlling the grading/funnel via the metric; node_budget docstring corrected to state it caps SEEDS only (not output DOFs) and points to the metric profile for the funnel. - test_0835::test_wedge_metric_funnels_finest_level_to_feature asserts the wedge metric's per-level marked-cell growth (finest/coarsest) is milder than a flat gaussian's on the same fault (the funnel signature). 10 tests pass. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 30 ++++++++++++++-- tests/test_0835_sbr_adapt_on_top.py | 34 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index 01476b80..abe6f474 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6040,8 +6040,17 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, Maximum SBR depth applied on top of the base finest (bounds the on-rank imbalance). Each level re-marks against the metric. node_budget : int or None - Optional cap on the number of cells refined **per level** (highest- - metric cells first). Approximate DOF control; ``None`` ⇒ uncapped. + Optional cap on the number of *seed* cells marked per level (highest- + metric first). **Caveat:** this caps the marked seeds, *not* the + resulting DOFs — SBR's conforming closure re-refines the whole + connected patch from any seed in it, so a seed cap does not bound the + added DOFs and cannot, on its own, concentrate the finest level near a + feature. To make the **finest level hug a feature** (a funnel) with + bounded per-level growth, shape the **metric** so element size grows + with distance (e.g. ``Surface.refinement_metric(..., profile="linear")``, + a wedge), not this budget. A *flat-core* metric (e.g. a Gaussian) + refines the whole core uniformly at every level (no funnel). + ``None`` ⇒ uncapped. builder : {"barycentric", "rbf"} Per-level node-prolongation builder for the child's custom-P hierarchy. adapter : {"sbr", "mmg"} @@ -6053,6 +6062,23 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, ------- Mesh The refined child (or ``self`` when ``adapter='mmg'``). + + Notes + ----- + **Controlling the grading (funnel toward a feature).** Each SBR pass + refines every cell that is still coarser than the metric target, so the + final grading *is* whatever the metric specifies. To make the **finest** + level hug a feature (a funnel), use a metric whose target size grows with + distance from the feature — a *wedge*, e.g. + ``Surface.refinement_metric(h_near, h_far, width, profile="linear")`` with + a small ``width``. A flat-core metric (Gaussian) instead refines the whole + core uniformly at every level, so every level has the same width. + + SBR's conforming closure re-refines the whole connected patch from any + marked seed, so the funnel must come from the metric shape, not from + capping seed counts (see ``node_budget``). For a 1-D feature in 2-D the + per-level DOFs still grow (the along-feature resolution doubles each + level); the wedge keeps the total finite and feature-concentrated. """ import warnings diff --git a/tests/test_0835_sbr_adapt_on_top.py b/tests/test_0835_sbr_adapt_on_top.py index e7dd1133..9452b55e 100644 --- a/tests/test_0835_sbr_adapt_on_top.py +++ b/tests/test_0835_sbr_adapt_on_top.py @@ -219,6 +219,40 @@ def test_each_sbr_level_is_its_own_mg_level(): assert s.snes.getConvergedReason() > 0 +def test_wedge_metric_funnels_finest_level_to_feature(): + """A wedge metric (size ~ distance, profile='linear') concentrates the FINEST + SBR level near the feature with bounded per-level growth — whereas a flat-core + (gaussian) metric refines the whole core uniformly at every level. We assert + the wedge's per-level marked-cell growth is much milder than the gaussian's. + """ + base = _base() + fxy = np.column_stack([0.25 + np.linspace(0, 0.85, 41) * np.cos(np.deg2rad(35)), + 0.92 - np.linspace(0, 0.85, 41) * np.sin(np.deg2rad(35))]) + fault = uw.meshing.Surface("fault_funnel", base, + np.column_stack([fxy, np.zeros(41)]), symbol="Ff") + fault.discretize() + + wedge = fault.refinement_metric(h_near=0.0625 / 8, h_far=0.07, width=0.05, + profile="linear", name="wedge_m") + gauss = fault.refinement_metric(h_near=0.0625 / 8, h_far=0.07, width=0.05, + profile="gaussian", name="gauss_m") + + cw = base.adapt(wedge, max_levels=3) + cg = base.adapt(gauss, max_levels=3) + # per-level marked counts are stashed for checkpoint-by-marker + nw = [len(m) for m in cw._adapt_markers] + ng = [len(m) for m in cg._adapt_markers] + assert len(nw) == 3 and len(ng) == 3 + + # growth = finest-level marked / coarsest-level marked. The wedge funnels + # (finest band narrower -> mild growth); the flat gaussian keeps the wide + # core at every level (finest level marks far more) -> steeper growth. This + # bounded per-level growth IS the funnel signature. + growth_wedge = nw[-1] / max(nw[0], 1) + growth_gauss = ng[-1] / max(ng[0], 1) + assert growth_wedge < growth_gauss + + def test_adapt_requires_base_hierarchy(): # a mesh without a refinement hierarchy cannot supply the MG coarse tail flat = uw.meshing.UnstructuredSimplexBox( From 4fd7359b78a3bae14e5da18220a038432816dd83 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 12:31:01 +1000 Subject: [PATCH 22/52] docs(layer2): design note for newest-vertex bisection (graded adapt-on-top) Records the SBR/longest-edge grading limitation (uniform-patch refill, measured) and designs the fix: newest-vertex bisection (NVB), whose closure is provably bounded (O(#marked)) and which produces the graded level+1/level+2 staircase keeping simplices (p4est ruled out). Covers: NVB rule (refinement edge / newest-vertex), bounded conforming completion, initial labelling, two implementation routes (Route A = serial numpy on the triangulation + DMPlex-from-cells, proposed first; Route B = native PETSc DMPlexTransform, parallel, deferred), integration seam (engine="nvb" in the existing _adapt_nested; custom-P hierarchy unchanged since transfers are coordinate-built), parallel/checkpoint considerations, a complexity table, a 4-step phasing, and alternatives (red-green, longest-edge, p4est). docs/developer/design/NVB_GRADED_ADAPT.md. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 223 ++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 docs/developer/design/NVB_GRADED_ADAPT.md diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md new file mode 100644 index 00000000..e8839e79 --- /dev/null +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -0,0 +1,223 @@ +# Newest-Vertex Bisection (NVB) for graded adapt-on-top + +Status: design (2026-06-30). Follows the Layer-2 investigation in +[`LAYER2_SBR_ADAPT_ON_TOP.md`](LAYER2_SBR_ADAPT_ON_TOP.md). Goal: replace the +refinement *engine* under `mesh.adapt(adapter="sbr")` so that successive levels +produce a **graded** mesh (a level+1 ring, a level+2 sub-ring dividing *some* +of those cells, …) instead of a uniform-finest patch — keeping simplices, +without p4est/DMForest. + +## Problem (why `refine_sbr` cannot grade) + +PETSc's `refine_sbr` is **longest-edge bisection (LEB)**. Its conforming closure +is *unbounded for region marking* — the "longest-edge propagation path" (LEPP) +chains across a quasi-uniform mesh. We measured this directly (study scripts in +`~/+Simulations/layer2_adapt_on_top_study/`): + +- **One triangle** → local on a structured base (refined cells stay within ~1 + ring), but on an unstructured base a single mark sends a 1-cell-wide chain + clear across the domain (centre cell → refined cells out to r=0.583). +- **A dense marked region** cascades to a *constant* extent regardless of how + much is marked (r<0.05 disk and r<0.30 disk both refined out to r=0.583 in one + pass). +- **Cumulative multi-level** marking (nested disks / shrinking fault bands) + **refills to uniform finest** with only single-cell level+1/level+2 transition + rings at the patch boundary. Bullseye level histogram: `216 / 104 / 240 / 7408` + — 95 % of refined cells at the finest level; level-3 cells spread to r=0.337 + when only r<0.12 was marked. +- A **structured base** helps the *first* level (closure is local) but each + refinement destroys the local longest-edge structure, so deeper levels cascade + again; the finest band fills. Unstructured is strictly worse (more cells, + ragged transition, and *detached* refined cells where a LEPP chain jumped the + gap). + +Conclusion: the uniform-patch behaviour is a property of the **engine**, not of +the metric, mesh type, or marking. No tuning fixes it. + +Important scope note: this is purely a **DOF-efficiency / grading** problem. The +uniform-patch SBR meshes are perfectly valid for the *MG hierarchy* — the +custom-P FMG converges on them (4–6 iters). NVB is about spending DOFs where they +are needed, not about solver correctness. + +## Why NVB + +**Newest-vertex bisection** has a *provably bounded* closure: the number of +elements added by conforming completion is bounded by a constant times the number +of marked elements (Binev–Dahmen–DeVore 2004; Stevenson 2008), given a compatible +initial edge labelling. It produces conforming, 2:1-balanced, shape-regular +graded meshes — exactly the level+1-ring / level+2-divides-some staircase — and +in 2D yields at most **4 similarity classes** of triangle from each parent, so no +element degenerates under arbitrarily deep refinement. It is the standard engine +of adaptive FEM for this reason. + +## NVB primer + +Each triangle carries a **refinement edge** (equivalently a "newest vertex" — the +refinement edge is the edge *opposite* the newest vertex). Refining a triangle: + +1. add the **midpoint M** of its refinement edge; +2. connect M to the opposite vertex, splitting the triangle into two children; +3. **M becomes the newest vertex of both children** — so each child's refinement + edge is the parent edge incident to the apex (the two edges that shared the + opposite vertex). + +``` + v3 (apex) v3 + /\ /|\ + / \ / | \ + / \ refine / | \ + / \ ───────▶ / | \ + v1──────v2 v1───M───v2 + (refinement edge v1–v2) children: (v1,M,v3) ref-edge v1–v3 + (M,v2,v3) ref-edge v2–v3 +``` + +Cycling the refinement edge through the parent's edges this way is what bounds +the recursion (4 similarity classes), unlike LEB which re-picks the geometric +longest edge each time and can chain. + +**Conforming completion.** Bisecting refinement edge `e = v1–v2` of `T` adds M on +`e`. The neighbour `T'` sharing `e` must also split at M: +- if `e` is *also* `T'`'s refinement edge → both bisect at M → conforming in one + step (a *compatible* edge); +- if not → first refine `T'` (bisect *its* refinement edge), then revisit. This + recursion is the bounded part: it terminates because each step makes progress + in the marked-edge structure, and globally the added work is O(#marked). + +## Algorithm + +``` +refine(mesh, marked_cells): + queue = marked_cells + while queue: + T = queue.pop() + e = refinement_edge(T) + T' = neighbour_across(T, e) # None if boundary + if T' is not None and e is not refinement_edge(T'): + queue.push(T) # defer T + queue.push(T') # make e compatible first + continue + bisect_pair(T, T') # add midpoint, 4 (or 2) children + # children inherit refinement edges by the newest-vertex rule +``` + +**Initial labelling.** Assign each base-mesh triangle a refinement edge so the +mesh is *compatibly divisible* (every interior edge is the refinement edge of +both its triangles, or the labelling admits a compatible completion). Two +standard choices: +- **Longest-edge initial labelling** (label each triangle's longest edge): always + conforming-terminating in 2D; simple; the labelling is only the *seed* — NVB's + bounded propagation then takes over (this is *not* the same as running LEB, + which re-picks longest edges every step). +- **Compatible (paired) labelling** via an edge ordering / matching — needed for + the sharp O(#marked) complexity bound; can be a follow-up. + +For a first cut the longest-edge seed is sufficient and robust. + +## Representation and DMPlex construction + +Two implementation routes: + +**Route A — Python on the triangulation (proposed first).** Maintain the mesh as +numpy arrays: `cells (N,3) int` vertex indices, `coords (V,2)`, and a per-cell +**refinement-edge** encoded by *vertex ordering convention* (newest vertex = +local index 0 ⇒ refinement edge = local vertices (1,2)). Refinement and closure +are pure-numpy on these arrays; after a refinement pass, build the PETSc DMPlex +from the cell list (`DMPlexCreateFromCellListPetsc` / the UW wrapper used for +imported meshes) and wrap as a `uw.discretisation.Mesh` — exactly how +`custom_mg.sbr_refine` returns a DM today. Serial first. Pros: tractable, +debuggable, no PETSc-C; integrates with the existing custom-P hierarchy unchanged +(transfers are built from *coordinates*, so a graded mesh is fine). Cons: rebuilds +the DM each pass; serial; we own the boundary-label transfer. + +**Route B — a PETSc `DMPlexTransform`.** Implement NVB as a transform type +alongside `refine_sbr`. Pros: native, parallel, incremental. Cons: deep PETSc-C, +marked-edge state must live in the transform, long lead time. Defer until Route A +proves the grading + hierarchy end-to-end. + +The marked-edge bookkeeping is the crux either way: children's refinement edges +must be set by the newest-vertex rule, and boundary labels / region labels must +be carried onto the children (Route A: re-tag by edge midpoint membership of the +parent's labelled faces). + +## Integration with Layer-2 + +Minimal surface change — NVB slots in where SBR is today: + +- `custom_mg.nvb_refine(dm_or_mesh, marked_cells) -> refined` mirroring + `sbr_refine`, plus a stateful `NVBMesh` carrying the marked-edge labelling + across levels (needed because the labelling propagates parent→child). +- `_adapt_sbr` (rename → `_adapt_nested`) gains `engine="nvb"|"sbr"`: same + metric → marking loop, but each level calls `nvb_refine` and keeps every + intermediate level for the custom-P tail (already implemented — one MG level + per refinement step). The **finest level is now graded**, so the marked count + per level genuinely shrinks toward the feature. +- The mesh-owned custom-P hierarchy, REMAP transfer, parent/child lineage, + copy_into prolong/restrict, no-redistribute guard — **all unchanged** (they are + engine-agnostic; transfers come from coordinates). + +## Parallel + +Route A is serial (rebuild-from-cell-list, on-rank). The no-redistribute +correctness requirement (custom-P needs the finest co-partitioned with the coarse +tail) is satisfied if NVB runs on each rank's owned cells with a halo for closure +across rank boundaries — the same on-rank story as SBR, but closure that crosses a +partition boundary needs a parallel completion step (exchange marked edges on +shared faces until no new marks). This is the main parallel risk and argues for +Route B (a transform handles it natively) once the serial design is proven. + +## Checkpointing + +NVB is deterministic given the **initial labelling + the marked-cell set per +level**. Store those (tiny) — replay reconstructs every level bit-identically, +consistent with the marker-sidecar scheme already designed for SBR. The +coordinate-built custom-P sidesteps the canonical-numbering fragility (same as +SBR). + +## Complexity assessment + +| Piece | Route A (Python) | Risk | +|---|---|---| +| Marked-edge data model + newest-vertex rule | small | low (well-specified) | +| `refine` + bounded conforming completion | medium | medium (termination/closure correctness — needs careful tests) | +| Initial labelling (longest-edge seed) | small | low | +| DMPlex build-from-cells + boundary/region label transfer | medium | medium (label re-tagging, coordinate DM, UW Mesh wrap) | +| Wire into `_adapt_nested` + custom-P | small | low (engine-agnostic seam exists) | +| Parallel completion across ranks | — (serial first) | high (defer to Route B) | +| Route B transform (native, parallel) | large | high (PETSc-C) | + +A serial Route-A prototype that produces a graded fault funnel + drives the +custom-P FMG is the milestone that tells us whether to invest in Route B. + +## Phasing + +1. **Serial NVB core** (numpy): marked-edge model, `refine`, closure; unit-test + conformity (no hanging nodes), 2:1 balance, bounded closure (#added ≤ C·#marked), + shape regularity (similarity-class count bounded) on stress cases. +2. **DMPlex wrap**: build-from-cells + label transfer; `nvb_refine` mirroring + `sbr_refine`; render the graded bullseye + fault funnel coloured by level + (the acceptance picture: distributed rings, *not* a uniform core). +3. **Layer-2 wiring**: `engine="nvb"` in `_adapt_nested`; custom-P FMG on the + graded child (Poisson + SolCx Stokes), serial. +4. **Decide on Route B** (native transform) for parallel + incremental, informed + by 1–3. + +## Alternatives considered + +- **Longest-edge bisection (`refine_sbr`)** — current; unbounded closure, no + grading (this whole investigation). +- **Red–green refinement** — bounded but introduces "green" closure elements that + must be removed/re-refined between passes (green cells aren't themselves + refined), complicating a multi-level hierarchy; NVB's uniform treatment of all + cells is cleaner for our level-per-refinement custom-P tail. +- **p4est / DMForest octree** — natural 2:1 grading, but quad/hex: a complete UW3 + design change. **Ruled out** by L.M. + +## References + +- E. Bänsch, *Local mesh refinement in 2 and 3 dimensions* (1991). +- W. F. Mitchell, *Adaptive refinement by bisection* — newest-vertex rule. +- P. Binev, W. Dahmen, R. DeVore, *Adaptive FEM with convergence rates* (2004) — + bounded-closure complexity. +- R. Stevenson, *The completion of locally refined simplicial partitions created + by bisection* (2008) — initial-labelling compatibility + O(#marked) bound. From 7552dbdc38f3a8dde0cf851de17cae0c605d8da2 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 13:30:15 +1000 Subject: [PATCH 23/52] =?UTF-8?q?docs(layer2):=20NVB=20note=20=E2=80=94=20?= =?UTF-8?q?interior-interface=20intuition=20+=203D=20generalisation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the conceptual resolution of "why can we build an embedded uniform patch but not refine inside it without draining to the edge": graded meshes are valid and constructible; longest-edge bisection just has no local termination in a uniform bisected patch, so its closure drains to the existing outer interface and cannot carve a new interior interface (measured: 1 centre cell of an r<0.25 patch -> +3592 cells out to the patch edge, both structured and unstructured). NVB's combinatorial marked edge restores local termination -> bounded interior interfaces = grading. Add a 3D section: NVB generalises to tetrahedra (Bansch/Maubach/Stevenson, bounded closure in n-D); longest-edge is worse in 3D; Route A carries over to (N,4) cells; octree stays hex-only (ruled out). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 53 +++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index e8839e79..03579f15 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -34,6 +34,33 @@ chains across a quasi-uniform mesh. We measured this directly (study scripts in Conclusion: the uniform-patch behaviour is a property of the **engine**, not of the metric, mesh type, or marking. No tuning fixes it. +### The interior-interface intuition (why creating a patch works but grading inside it doesn't) + +A revealing experiment: build an embedded uniformly-refined patch (refine the +whole disk r<0.25), then refine **one** cell at its centre. That single cell +drains the entire patch — +3592 cells, refined out to r=0.28 ≈ the patch edge — +on *both* structured and unstructured bases (whereas one cell on the *pristine* +base is local, +10). + +The resolution of the apparent paradox: + +- The **graded mesh you want is valid and constructible** (a level+1 ring around a + level+2 sub-ring around a level+3 core is a fine 2:1-conforming mesh — gmsh + could build it directly from a size field). Graded meshes are not impossible. +- **Creating the patch works because you refine the *whole* region at once** — + every cell steps up together, so the *only* coarse/fine interface created is at + the patch's *outer* edge, closed locally in one clean step. No interior cell + ever neighbours a different resolution. +- **Grading needs a *new* interface in the *interior*** (the finer core must sit + inside the coarser ring). Longest-edge bisection has **no local termination in a + uniform bisected patch** — all edges are ~equal, so the closure's propagation + path chains until it meets an edge-length contrast, i.e. the patch's *existing* + outer interface. It can only push an interface outward, never carve one inside. +- **NVB's combinatorial marked edge restores local termination**: refine one cell + deep in a patch → O(1) cells added with a small local transition, *which is* + an interior interface. Same mesh space; only NVB can reach the graded states + with bounded local moves. + Important scope note: this is purely a **DOF-efficiency / grading** problem. The uniform-patch SBR meshes are perfectly valid for the *MG hierarchy* — the custom-P FMG converges on them (4–6 iters). NVB is about spending DOFs where they @@ -174,6 +201,32 @@ consistent with the marker-sidecar scheme already designed for SBR. The coordinate-built custom-P sidesteps the canonical-numbering fragility (same as SBR). +## 3D (tetrahedra) + +NVB is the right engine in 3D too, and the design generalises with no change of +approach: + +- Each **tetrahedron** carries a *refinement edge*; bisecting adds its midpoint + and splits the tet into two children, with the marked-edge propagation rule + carried over (Bänsch 1991; Maubach 1995; Arnold–Mukherjee–Pouly). Stevenson + (2008) proved **bounded closure + termination in n dimensions** given a + compatible initial labelling, with a finite number of similarity classes (no + sliver degeneration) — the same guarantees we rely on in 2D. +- **Longest-edge bisection is worse in 3D** (each refinement edge is shared by + more tets, so the drain-to-interface chaining is more severe) — additional + motivation to switch rather than patch `refine_sbr`. +- **Route A carries over directly**: cells become `(N,4)` vertex indices, the + refinement edge is encoded by vertex ordering, bisection splits one edge into + two children, and the bounded-closure completion is the same loop. DMPlex + build-from-cells works for tets unchanged. +- Octree (DMForest/p4est) in 3D is **hexes** — ruled out (UW3 design change), + same as 2D. + +The marked-edge labelling and similarity-class bookkeeping are the only parts +that carry extra 3D subtlety (the compatible initial labelling is more involved); +the integration seam, custom-P hierarchy, and DM construction are +dimension-agnostic. + ## Complexity assessment | Piece | Route A (Python) | Risk | From bd0328b2f3828ffbbce8a13837efabb9972bc264 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 13:36:20 +1000 Subject: [PATCH 24/52] =?UTF-8?q?feat(layer2):=20NVB=20serial=20core=20pro?= =?UTF-8?q?totype=20=E2=80=94=20graded=20AMR=20proven?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-1 core of the newest-vertex-bisection design: ~135 lines of numpy (nvb_prototype_2d.py, NVBMesh) implementing the newest-vertex data model and the recursive compatible-bisection with bounded conforming closure. Validated against the SBR longest-edge pathology on the same nested-disk bullseye: - one cell deep in a uniform patch -> +2 cells, LOCAL (r 0.016-0.030), where SBR drained +3592 cells to the patch edge; - the finest band stays confined (gen 6 at r<0.126) where SBR refilled to r=0.337; - conforming at every step (0 hanging nodes, 0 over-shared edges); - 3024 cells vs SBR's 7968 for the same target; - renders as clean concentric generation rings (true grading, not a uniform core). So the hard part (the closure algorithm) works. Remaining Phase-1 = DMPlex wrap + boundary/region label transfer (engineering). Design note updated with the result and the prototype reference. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 27 ++++- docs/developer/design/nvb_prototype_2d.py | 135 ++++++++++++++++++++++ 2 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 docs/developer/design/nvb_prototype_2d.py diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 03579f15..6fd6c035 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -242,11 +242,32 @@ dimension-agnostic. A serial Route-A prototype that produces a graded fault funnel + drives the custom-P FMG is the milestone that tells us whether to invest in Route B. +## Prototype result (Phase 1 core — DONE, 2026-06-30) + +The serial NVB core is implemented and validated in ~135 lines of numpy: +[`nvb_prototype_2d.py`](nvb_prototype_2d.py) (`NVBMesh`: newest-vertex data model, +recursive compatible-bisection `bisect`, `refine`, conformity check). Measured +against the SBR pathology, same nested-disk bullseye: + +| | NVB | SBR (`refine_sbr`) | +|---|---|---| +| one cell deep in a uniform patch | **+2 cells, local** (r 0.016–0.030) | +3592, drained to r=0.28 | +| finest band (gen 6) | confined **r < 0.126** | refilled to r=0.337 | +| conformity (hanging nodes / over-shared edges) | **0 / 0** every step | — | +| total cells (same bullseye target) | **3024** | 7968 | + +The bullseye renders as clean concentric generation rings (not a uniform core). +So the core algorithm — the genuinely hard part — works. Remaining Phase-1 work +is the DMPlex wrap + label transfer (engineering, not algorithm risk). NVB bisects +1→2 (area halves); a full isotropic SBR "level" (1→4) = two generations. + ## Phasing -1. **Serial NVB core** (numpy): marked-edge model, `refine`, closure; unit-test - conformity (no hanging nodes), 2:1 balance, bounded closure (#added ≤ C·#marked), - shape regularity (similarity-class count bounded) on stress cases. +1. **Serial NVB core** (numpy) — **DONE** (`nvb_prototype_2d.py`): marked-edge + model, `refine`, recursive compatible-closure; conformity (no hanging nodes, + 0 over-shared edges) and bounded-closure (1 cell → +2, local) verified; graded + bullseye rendered. TODO before src: similarity-class/shape-regularity assertion + and a #added ≤ C·#marked stress test. 2. **DMPlex wrap**: build-from-cells + label transfer; `nvb_refine` mirroring `sbr_refine`; render the graded bullseye + fault funnel coloured by level (the acceptance picture: distributed rings, *not* a uniform core). diff --git a/docs/developer/design/nvb_prototype_2d.py b/docs/developer/design/nvb_prototype_2d.py new file mode 100644 index 00000000..c82f3533 --- /dev/null +++ b/docs/developer/design/nvb_prototype_2d.py @@ -0,0 +1,135 @@ +"""Phase-1 prototype: serial newest-vertex bisection (NVB) on a 2D triangulation. + +Goal: discover the complexity and PROVE that NVB gives a GRADED mesh (a level+1 +ring, level+2 sub-ring dividing SOME of them, ...) where longest-edge SBR gave a +uniform-finest patch. + +Data model (newest-vertex convention): + cell = (peak, b0, b1) -- 'peak' is the newest vertex; the REFINEMENT EDGE is + {b0,b1} (opposite the peak). Bisecting splits {b0,b1} at midpoint m and makes m + the peak of both children: (m, peak, b0) and (m, peak, b1). + +Conforming completion (recursive): to bisect a cell whose refinement edge is not +shared as the refinement edge of its neighbour, first bisect the neighbour +(recursively) until the edge becomes compatible, then bisect the pair. This is +the BOUNDED closure (cf. longest-edge's unbounded drain-to-interface). +""" +import numpy as np + + +def fs(a, b): + return (a, b) if a < b else (b, a) + + +class NVBMesh: + def __init__(self, coords, tris): + self.coords = [np.asarray(p, float) for p in coords] + self.cells = {} # cid -> (peak, b0, b1) + self.edge2cells = {} # (a,b) sorted -> set(cid) + self.edge2mid = {} # (a,b) sorted -> midpoint vertex id + self._next = 0 + # initial labelling: refinement edge = LONGEST edge (a seed; NVB's + # marked-edge propagation takes over from here) + for t in tris: + self._add_cell(self._longest_edge_order(t)) + + # ---- vertex / cell bookkeeping ---------------------------------------- + def _longest_edge_order(self, t): + a, b, c = t + P = self.coords + e = {fs(a, b): (a, b, c), fs(b, c): (b, c, a), fs(c, a): (c, a, b)} + lens = {k: np.linalg.norm(P[k[0]] - P[k[1]]) for k in e} + kmax = max(lens, key=lens.get) + # want refinement edge {b0,b1} opposite peak -> order (peak, b0, b1) + b0, b1 = kmax + peak = ({a, b, c} - {b0, b1}).pop() + return (peak, b0, b1) + + def _add_cell(self, pbr): + cid = self._next + self._next += 1 + self.cells[cid] = pbr + p, b0, b1 = pbr + for e in (fs(p, b0), fs(b0, b1), fs(b1, p)): + self.edge2cells.setdefault(e, set()).add(cid) + return cid + + def _del_cell(self, cid): + p, b0, b1 = self.cells.pop(cid) + for e in (fs(p, b0), fs(b0, b1), fs(b1, p)): + self.edge2cells[e].discard(cid) + if not self.edge2cells[e]: + del self.edge2cells[e] + + def _midpoint(self, e): + if e not in self.edge2mid: + a, b = e + self.coords.append(0.5 * (self.coords[a] + self.coords[b])) + self.edge2mid[e] = len(self.coords) - 1 + return self.edge2mid[e] + + def _ref_edge(self, cid): + p, b0, b1 = self.cells[cid] + return fs(b0, b1) + + def _neighbour(self, cid, e): + others = self.edge2cells.get(e, set()) - {cid} + return next(iter(others)) if others else None + + # ---- the bisection ----------------------------------------------------- + def _split(self, cid, m): + p, b0, b1 = self.cells[cid] + self._del_cell(cid) + self._add_cell((m, p, b0)) # newest vertex m, ref edge {p,b0} + self._add_cell((m, p, b1)) # newest vertex m, ref edge {p,b1} + + def bisect(self, cid, _depth=0): + if _depth > 100000: + raise RuntimeError("NVB closure did not terminate (LEPP-like loop)") + e = self._ref_edge(cid) + nb = self._neighbour(cid, e) + # make compatible: neighbour must share e as ITS refinement edge + while nb is not None and self._ref_edge(nb) != e: + self.bisect(nb, _depth + 1) + nb = self._neighbour(cid, e) # a child of the old neighbour + m = self._midpoint(e) + self._split(cid, m) + if nb is not None: + self._split(nb, m) + + def refine(self, marked): + # marked is a set of current cell ids; refine each if still alive + for cid in list(marked): + if cid in self.cells: + self.bisect(cid) + + # ---- export / diagnostics --------------------------------------------- + def arrays(self): + coords = np.array(self.coords) + tris = np.array([(p, b0, b1) for (p, b0, b1) in self.cells.values()]) + return coords, tris + + def check_conforming(self): + """No hanging nodes: no live cell may carry an edge that has been + bisected (i.e. that has a midpoint).""" + bad = 0 + for cid, (p, b0, b1) in self.cells.items(): + for e in (fs(p, b0), fs(b0, b1), fs(b1, p)): + if e in self.edge2mid: + bad += 1 + # every interior edge shared by exactly 2 cells, boundary by 1 + overshared = sum(1 for e, cs in self.edge2cells.items() if len(cs) > 2) + return bad, overshared + + def cell_centroids_levels(self, base_area): + """Returns (centroids, generation, area). 'generation' counts BISECTIONS + (NVB bisects 1->2, so area halves each bisection); a full isotropic SBR + 'level' (1->4) is two generations.""" + coords = np.array(self.coords) + tris = np.array([(p, b0, b1) for (p, b0, b1) in self.cells.values()]) + cen = coords[tris].mean(axis=1) + a = coords[tris[:, 0]]; b = coords[tris[:, 1]]; c = coords[tris[:, 2]] + area = 0.5 * np.abs((b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1]) - + (c[:, 0] - a[:, 0]) * (b[:, 1] - a[:, 1])) + gen = np.round(np.log2(base_area / area)).astype(int) + return cen, gen, area From e097fef9f26903caf60c9e6291e35b932332c45f Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 15:14:03 +1000 Subject: [PATCH 25/52] =?UTF-8?q?feat(layer2):=20NVB=20prototype=20?= =?UTF-8?q?=E2=80=94=20track=20per-cell=20refinement=20depth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add depth bookkeeping (children inherit parent depth+1) and a depths() accessor, so refinement level can be coloured cleanly on a NON-UNIFORM base (area-derived generation conflates base size variation with refinement). Validated: NVB fault funnel on an irregular Delaunay base grades cleanly by depth (0..7), conforming (0 hanging nodes) and bounded (no cascade) at every step — where longest-edge SBR cascaded across the domain on an unstructured mesh. Underworld development team with AI support from Claude Code --- docs/developer/design/nvb_prototype_2d.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/developer/design/nvb_prototype_2d.py b/docs/developer/design/nvb_prototype_2d.py index c82f3533..89359dda 100644 --- a/docs/developer/design/nvb_prototype_2d.py +++ b/docs/developer/design/nvb_prototype_2d.py @@ -25,6 +25,7 @@ class NVBMesh: def __init__(self, coords, tris): self.coords = [np.asarray(p, float) for p in coords] self.cells = {} # cid -> (peak, b0, b1) + self.depth = {} # cid -> bisection depth from its base cell self.edge2cells = {} # (a,b) sorted -> set(cid) self.edge2mid = {} # (a,b) sorted -> midpoint vertex id self._next = 0 @@ -45,10 +46,11 @@ def _longest_edge_order(self, t): peak = ({a, b, c} - {b0, b1}).pop() return (peak, b0, b1) - def _add_cell(self, pbr): + def _add_cell(self, pbr, depth=0): cid = self._next self._next += 1 self.cells[cid] = pbr + self.depth[cid] = depth p, b0, b1 = pbr for e in (fs(p, b0), fs(b0, b1), fs(b1, p)): self.edge2cells.setdefault(e, set()).add(cid) @@ -56,6 +58,7 @@ def _add_cell(self, pbr): def _del_cell(self, cid): p, b0, b1 = self.cells.pop(cid) + self.depth.pop(cid, None) for e in (fs(p, b0), fs(b0, b1), fs(b1, p)): self.edge2cells[e].discard(cid) if not self.edge2cells[e]: @@ -79,9 +82,10 @@ def _neighbour(self, cid, e): # ---- the bisection ----------------------------------------------------- def _split(self, cid, m): p, b0, b1 = self.cells[cid] + d = self.depth[cid] + 1 self._del_cell(cid) - self._add_cell((m, p, b0)) # newest vertex m, ref edge {p,b0} - self._add_cell((m, p, b1)) # newest vertex m, ref edge {p,b1} + self._add_cell((m, p, b0), d) # newest vertex m, ref edge {p,b0} + self._add_cell((m, p, b1), d) # newest vertex m, ref edge {p,b1} def bisect(self, cid, _depth=0): if _depth > 100000: @@ -109,6 +113,11 @@ def arrays(self): tris = np.array([(p, b0, b1) for (p, b0, b1) in self.cells.values()]) return coords, tris + def depths(self): + """Per-cell bisection depth from its base ancestor, in cells order + (clean refinement colouring on a non-uniform base, unlike area-derived).""" + return np.array([self.depth[cid] for cid in self.cells]) + def check_conforming(self): """No hanging nodes: no live cell may carry an edge that has been bisected (i.e. that has a midpoint).""" From 4b1ffd854789782992d5903078efcf122637d445 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 17:17:40 +1000 Subject: [PATCH 26/52] =?UTF-8?q?docs(layer2):=20NVB=20parallel=20?= =?UTF-8?q?=E2=80=94=20decomposition=20preservation=20decides=20Route=20A?= =?UTF-8?q?=20vs=20B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sharpen the parallel section: the binding requirement is preserving the parent's decomposition (co-partitioned finest, no redistribute) so custom-P's rank-local point location holds — not load balance. This DISQUALIFIES Route A (createFromCellList) for parallel: it loses the parent point-SF (serial DM or fresh DMPlexDistribute partition); preserving it would need manual SF rebuild + consistent cross-rank marked-edge bisection + closure exchange. SBR preserves it today via in-place adaptLabel (SF propagated by the transform framework) — the model to match. So Route A = serial stepping stone; Route B (native DMPlexTransform) is the real parallel implementation, justified by decomposition preservation. Adds the np>1 acceptance test (child centroids in locally-owned base cells; coarse-tail partition bit-identical). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 41 ++++++++++++++++++----- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 6fd6c035..9da15dad 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -183,15 +183,38 @@ Minimal surface change — NVB slots in where SBR is today: copy_into prolong/restrict, no-redistribute guard — **all unchanged** (they are engine-agnostic; transfers come from coordinates). -## Parallel - -Route A is serial (rebuild-from-cell-list, on-rank). The no-redistribute -correctness requirement (custom-P needs the finest co-partitioned with the coarse -tail) is satisfied if NVB runs on each rank's owned cells with a halo for closure -across rank boundaries — the same on-rank story as SBR, but closure that crosses a -partition boundary needs a parallel completion step (exchange marked edges on -shared faces until no new marks). This is the main parallel risk and argues for -Route B (a transform handles it natively) once the serial design is proven. +## Parallel — preserving the inherited decomposition (decides Route A vs B) + +The hard parallel requirement is **not** load balance — it is preserving the +parent's decomposition: custom-P's parallel path needs the finest co-partitioned +with the coarse tail (rank *r* owns the refinements of rank *r*'s base cells) so +rank-local point location holds. A re-partition *breaks* it, it does not merely +slow it. This is the same no-redistribute invariant the SBR adapt-on-top path +already satisfies. + +**This requirement disqualifies Route A for parallel.** `createFromCellList` +builds a serial DM (rank 0) or, after `DMPlexDistribute`, a *freshly* partitioned +one — the parent's **point-SF** (shared/owned-vertex star-forest) is lost. +Preserving the decomposition via Route A would require manually rebuilding the SF +(ownership of every new rank-boundary midpoint), guaranteeing both ranks bisect +each shared edge *identically* (consistent marked-edge → consistent midpoint), and +a cross-rank closure-completion exchange. Fragile. + +**How SBR preserves it today (the model to match):** `sbr_refine` calls +`adaptLabel` on the *already-distributed* DM — an in-place transform, so the +partition and SF are preserved by construction (why the SBR np=2 path works). The +transform framework propagates the SF. + +Conclusion — the serial/parallel split is therefore: +- **Route A is serial-only**: a stepping stone to prove the NVB logic, label + transfer, and custom-P integration (no decomposition to preserve at np=1). +- **Route B (native `DMPlexTransform`) is the real parallel implementation**, + justified specifically by decomposition preservation — it inherits the same + SF-propagation machinery as `refine_sbr`, not just speed. + +**Acceptance test (np>1):** after an NVB adapt, every child cell's centroid lies +in a *locally-owned* base cell, and the coarse-tail partition is bit-identical +before/after — the same invariant the SBR path checks. ## Checkpointing From f4895abe72ce4bb1ab1600ff43957f9b8fcd00ce Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 17:36:51 +1000 Subject: [PATCH 27/52] =?UTF-8?q?feat(layer2):=20NVB=20graded=20adapt=20en?= =?UTF-8?q?gine=20=E2=80=94=20serial=20Route=20A=20in=20src?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the validated NVB serial core into src as utilities/nvb.py (NVBMesh: newest-vertex data model, recursive bounded compatible-bisection, from_dm / to_dm with boundary-edge + region-cell label transport, conformity / similarity-class diagnostics). createFromCellList needs consistent CCW winding (the (peak,b0,b1) refinement-edge order is geometry-agnostic) — reorient the exported cell list only; labels matched by vertex pair (coordinate, never index). custom_mg.nvb_refine mirrors sbr_refine (single-shot). Mesh._adapt_sbr renamed to _adapt_nested with engine='sbr'|'nvb' (default sbr, behaviour unchanged). The NVB path drives a persistent NVBMesh across 2*max_levels generations (1->2 area halving) so the refinement-edge labelling — hence the similarity-class bound — propagates parent->child; each generation snapshots a DM for the engine-agnostic coordinate-based custom-P FMG tail. NotImplementedError for engine='nvb' at np>1 (cell-list rebuild loses the parent decomposition the custom-P parallel path needs — that is Route B) and for dim!=2. Validated end-to-end serial: graded NVB child (fewer DOFs than the SBR uniform patch), Poisson FMG (5 levels) matches GAMG bit-identically (exact err 3.5e-10), SolCx Stokes (eta jump 1e6) velocity-block FMG matches GAMG iter-for-iter (8) and bit-identically. Existing SBR tests (test_0835) all pass. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 226 ++++++++---- src/underworld3/utilities/custom_mg.py | 29 +- src/underworld3/utilities/nvb.py | 341 ++++++++++++++++++ 3 files changed, 520 insertions(+), 76 deletions(-) create mode 100644 src/underworld3/utilities/nvb.py diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index abe6f474..21681123 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6012,17 +6012,32 @@ def _coarse_level_meshes(self): return cached def adapt(self, metric_field, max_levels=2, node_budget=None, - builder="barycentric", adapter="sbr", verbose=False): + builder="barycentric", adapter="sbr", engine="sbr", verbose=False): r""" - Nested **SBR adapt-on-top**: return a refined **child** mesh. - - Skeleton-based-refinement (SBR) the static base finest where the metric - demands resolution, **on top of** the existing uniform hierarchy, and - return a new child mesh (``child.parent is self``). The base mesh is - **not modified** — this is *adapt / re-adapt*, not node movement: each - call re-marks from the static base finest, so successive adapts are - non-cumulative (cf. :meth:`remesh`, which regenerates the mesh in place - via MMG and may redistribute). + Nested **adapt-on-top**: return a refined **child** mesh. + + Locally refine the static base finest where the metric demands resolution, + **on top of** the existing uniform hierarchy, and return a new child mesh + (``child.parent is self``). The base mesh is **not modified** — this is + *adapt / re-adapt*, not node movement: each call re-marks from the static + base finest, so successive adapts are non-cumulative (cf. :meth:`remesh`, + which regenerates the mesh in place via MMG and may redistribute). + + Two refinement **engines** (``engine=``): + + * ``"sbr"`` (default) — PETSc skeleton-based (longest-edge) bisection. Each + pass refines marked cells isotropically (1→4). Its conforming closure is + *unbounded for region marking*, so it produces a **uniform-finest patch**, + not a graded mesh (a marked cell drains the longest-edge path to the patch + edge). Robust and fine for the MG hierarchy. + * ``"nvb"`` — newest-vertex bisection (:mod:`underworld3.utilities.nvb`), + a **graded** engine with a *bounded* conforming closure: a marked cell + adds O(1) cells locally, so successive levels grade (a level+1 ring around + a finer core) and DOFs concentrate near the feature. Serial only this pass + (``NotImplementedError`` at np>1 — the parallel decomposition cannot be + preserved through the cell-list DM rebuild; a native transform is the + parallel path). Bisects 1→2, so one isotropic-equivalent ``max_levels`` is + run as **two** NVB generations. The child owns a custom-P geometric-MG hierarchy with **one level per SBR refinement step** — ``[base L0 … base finest, SBR-1, …, SBR-n(child)]`` — @@ -6054,8 +6069,13 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, builder : {"barycentric", "rbf"} Per-level node-prolongation builder for the child's custom-P hierarchy. adapter : {"sbr", "mmg"} - ``"sbr"`` (default) is this nested path. ``"mmg"`` is a **deprecated - shim** that forwards to :meth:`remesh` (in-place, returns ``self``). + ``"sbr"`` (default) is the nested adapt-on-top path (the refinement + engine is then chosen by ``engine``). ``"mmg"`` is a **deprecated shim** + that forwards to :meth:`remesh` (in-place, returns ``self``). + engine : {"sbr", "nvb"} + The nested refinement engine (ignored when ``adapter="mmg"``). ``"sbr"`` + (default) is longest-edge bisection (uniform patch); ``"nvb"`` is + newest-vertex bisection (graded, serial only). See above. verbose : bool Returns @@ -6092,88 +6112,141 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, return self if adapter != "sbr": raise ValueError(f"adapter must be 'sbr' or 'mmg', got {adapter!r}") + if engine not in ("sbr", "nvb"): + raise ValueError(f"engine must be 'sbr' or 'nvb', got {engine!r}") - return self._adapt_sbr( + return self._adapt_nested( metric_field, max_levels=max_levels, node_budget=node_budget, - builder=builder, verbose=verbose, + builder=builder, engine=engine, verbose=verbose, ) - def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, - builder="barycentric", verbose=False): - """Core SBR adapt-on-top. See :meth:`adapt`.""" + def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, + builder="barycentric", engine="sbr", verbose=False): + """Core nested adapt-on-top (SBR or NVB engine). See :meth:`adapt`.""" import math from underworld3.utilities import custom_mg if self.parent is not None: raise NotImplementedError( - "adapt(adapter='sbr') refines a BASE mesh; chaining adapt on an " - "already-adapted child is not yet supported." + "adapt() refines a BASE mesh; chaining adapt on an already-adapted " + "child is not yet supported." ) if getattr(self, "dm_hierarchy", None) is None or len(self.dm_hierarchy) < 2: raise RuntimeError( - "SBR adapt-on-top needs a base mesh built with refinement>=1 (a " + "Nested adapt-on-top needs a base mesh built with refinement>=1 (a " "dm_hierarchy of coarse levels supplies the geometric-MG tail). " "Build the mesh with e.g. refinement=2." ) + if engine == "nvb": + if uw.mpi.size > 1: + raise NotImplementedError( + "adapt(engine='nvb') is serial-only: the cell-list DMPlex " + "rebuild does not preserve the parent's parallel decomposition " + "(required by the custom-P parallel path). Use engine='sbr' at " + "np>1, or the native-transform (Route B) NVB when available." + ) + if self.dim != 2: + raise NotImplementedError( + "adapt(engine='nvb') is 2D only this pass (tets are a follow-up)." + ) dim = self.dim edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex - current_dm = self.dm_hierarchy[-1] # static base finest markers_per_level = [] - sbr_level_dms = [] # one DM per SBR level (level 1 … n) - - for level in range(max_levels): - cs, ce = current_dm.getHeightStratum(0) - ncells = ce - cs - if ncells == 0: - break - centroids = numpy.empty((ncells, self.cdim)) - cur_h = numpy.empty(ncells) - for i, c in enumerate(range(cs, ce)): - vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] - centroids[i] = numpy.asarray(cen)[: self.cdim] - cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) - - # Metric M = 1/h_target² evaluated at the cell centroids (parent field). - # Parallel: use the swarm-migration-based global_evaluate (partition- - # agnostic, collective-safe); serial: the cheaper local evaluate. - if uw.mpi.size > 1: - M = numpy.asarray( - uw.function.global_evaluate(metric_field.sym, centroids) - ).reshape(-1) - else: - M = numpy.asarray( - uw.function.evaluate(metric_field.sym, centroids) - ).reshape(-1) - M = numpy.clip(M, 1e-30, None) - h_target = 1.0 / numpy.sqrt(M) - - refine = numpy.where(cur_h > h_target)[0] - if refine.size == 0: + level_dms = [] # one DM per refinement level + + if engine == "nvb": + # Persistent NVBMesh: the refinement-edge labelling propagates + # parent→child across generations, which preserves the similarity-class + # (shape-regularity) bound. Each generation bisects 1→2 (h /√2), so we + # run up to 2·max_levels generations to reach the same isotropic- + # equivalent target as the SBR path's max_levels passes (1→4 each). + from underworld3.utilities.nvb import NVBMesh + carry = [(b.name, b.value) for b in self.boundaries + if b.name not in ("Null_Boundary", "All_Boundaries")] + rcarry = ([(r.name, r.value) for r in self.regions] + if self.regions is not None else []) + nvb = NVBMesh.from_dm(self.dm_hierarchy[-1], boundaries=carry, + regions=rcarry) + n_gen = 2 * max_levels + for level in range(n_gen): + centroids, cur_h, cids = nvb.centroids_h() + M = numpy.clip( + numpy.asarray( + uw.function.evaluate(metric_field.sym, centroids) + ).reshape(-1), 1e-30, None) + h_target = 1.0 / numpy.sqrt(M) + sel = numpy.where(cur_h > h_target)[0] + if sel.size == 0: + if verbose: + uw.pprint(0, f"[adapt] nvb gen {level}: nothing to refine") + break + if node_budget is not None and sel.size > node_budget: + order = numpy.argsort(M[sel])[::-1] + sel = sel[order[:node_budget]] + marked = [int(cids[j]) for j in sel] + markers_per_level.append(marked) + nvb.refine(set(marked)) + level_dms.append(nvb.to_dm(boundaries=carry, regions=rcarry, + comm=self.dm.comm)) if verbose: - uw.pprint(0, f"[adapt] level {level}: no cells need refinement") - break + uw.pprint(0, f"[adapt] nvb gen {level}: marked {len(marked)} " + f"-> {len(nvb.cells)} cells") + current_dm = level_dms[-1] if level_dms else self.dm_hierarchy[-1].clone() + else: + current_dm = self.dm_hierarchy[-1] # static base finest + for level in range(max_levels): + cs, ce = current_dm.getHeightStratum(0) + ncells = ce - cs + if ncells == 0: + break + centroids = numpy.empty((ncells, self.cdim)) + cur_h = numpy.empty(ncells) + for i, c in enumerate(range(cs, ce)): + vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] + centroids[i] = numpy.asarray(cen)[: self.cdim] + cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) + + # Metric M = 1/h_target² at the cell centroids (parent field). + # Parallel: swarm-migration global_evaluate (partition-agnostic); + # serial: the cheaper local evaluate. + if uw.mpi.size > 1: + M = numpy.asarray( + uw.function.global_evaluate(metric_field.sym, centroids) + ).reshape(-1) + else: + M = numpy.asarray( + uw.function.evaluate(metric_field.sym, centroids) + ).reshape(-1) + M = numpy.clip(M, 1e-30, None) + h_target = 1.0 / numpy.sqrt(M) + + refine = numpy.where(cur_h > h_target)[0] + if refine.size == 0: + if verbose: + uw.pprint(0, f"[adapt] level {level}: no cells need refinement") + break - if node_budget is not None and refine.size > node_budget: - # keep the highest-metric (finest-demand) cells first - order = numpy.argsort(M[refine])[::-1] - refine = refine[order[:node_budget]] + if node_budget is not None and refine.size > node_budget: + # keep the highest-metric (finest-demand) cells first + order = numpy.argsort(M[refine])[::-1] + refine = refine[order[:node_budget]] - cell_ids = [int(cs + j) for j in refine] - markers_per_level.append(cell_ids) - if verbose: - uw.pprint(0, f"[adapt] level {level}: refining {len(cell_ids)} " - f"of {ncells} cells") - current_dm = custom_mg.sbr_refine(current_dm, cell_ids) - sbr_level_dms.append(current_dm) + cell_ids = [int(cs + j) for j in refine] + markers_per_level.append(cell_ids) + if verbose: + uw.pprint(0, f"[adapt] level {level}: refining {len(cell_ids)} " + f"of {ncells} cells") + current_dm = custom_mg.sbr_refine(current_dm, cell_ids) + level_dms.append(current_dm) if verbose: base_n = self.dm_hierarchy[-1].getHeightStratum(0) fin_n = current_dm.getHeightStratum(0) uw.pprint(0, f"[adapt] base finest {base_n[1]-base_n[0]} -> " f"child {fin_n[1]-fin_n[0]} cells " - f"({len(markers_per_level)} SBR level(s))") + f"({len(markers_per_level)} {engine} level(s))") # Wrap the refined finest as the child mesh (on-rank; no redistribute). child = Mesh( @@ -6190,20 +6263,23 @@ def _adapt_sbr(self, metric_field, max_levels=2, node_budget=None, child._relationship_kind = "refinement" child.regions = self.regions child._parent_mesh_version = self._mesh_version - # Markers per SBR level (in each level's cell numbering) — the + # Markers per refinement level (in each level's cell numbering) — the # checkpoint-by-marker payload (design only; storage is a follow-up). child._adapt_markers = markers_per_level - # Mesh-owned custom-P geometric-MG tail. EVERY SBR level is its own MG - # level (one custom-P transfer per refinement step), not a single + child._adapt_engine = engine + # Mesh-owned custom-P geometric-MG tail. EVERY refinement level is its own + # MG level (one custom-P transfer per refinement step), not a single # base-finest -> child jump: the tail is - # [base L0 … base finest] + [SBR level 1 … SBR level n-1] - # and the solver appends its own mesh (the finest SBR level = child). - # Each intermediate SBR level is wrapped here (transient, lives on the - # child); the static base levels reuse the parent's cached wraps. - intermediate_sbr = [ - self._wrap_coarse_level(d) for d in sbr_level_dms[:-1] + # [base L0 … base finest] + [refine level 1 … refine level n-1] + # and the solver appends its own mesh (the finest level = child). Each + # intermediate level is wrapped here (transient, lives on the child); the + # static base levels reuse the parent's cached wraps. (NVB snapshots and + # SBR levels are both just coordinate sets — the custom-P transfers are + # coordinate-based, so the hierarchy is engine-agnostic.) + intermediate = [ + self._wrap_coarse_level(d) for d in level_dms[:-1] ] - child._custom_mg_coarse_meshes = self._coarse_level_meshes() + intermediate_sbr + child._custom_mg_coarse_meshes = self._coarse_level_meshes() + intermediate child._custom_mg_builder = builder self._registered_children.add(child) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 6a2407bc..7245b1b5 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -42,7 +42,8 @@ from petsc4py import PETSc __all__ = ["barycentric_prolongation", "rbf_prolongation", "inject_custom_mg", - "CustomMGHierarchy", "set_custom_fmg", "sbr_refine", "sbr_refine_where"] + "CustomMGHierarchy", "set_custom_fmg", "sbr_refine", "sbr_refine_where", + "nvb_refine"] # --------------------------------------------------------------------------- # @@ -168,6 +169,32 @@ def mark(d, lab): return _sbr_apply(dm, mark) +# --------------------------------------------------------------------------- # +# Newest-vertex bisection (NVB) — GRADED refinement (serial Route A) +# --------------------------------------------------------------------------- # +def nvb_refine(dm, cells, boundaries=(), regions=()): + """NVB-refine an explicit list of ``cells`` (serial, single level), returning a + fresh interpolated ``DMPlex`` with boundary/region labels transferred. + + Counterpart to :func:`sbr_refine` but with a **bounded conforming closure**, so + a marked cell deep in a uniform patch adds O(1) cells locally instead of + draining the longest-edge path to the patch edge — the property that lets + successive levels *grade* (see :mod:`underworld3.utilities.nvb`). + + Single-shot: builds an :class:`~underworld3.utilities.nvb.NVBMesh` from ``dm`` + (longest-edge seed), refines, and emits the DM. For a **multi-level** graded + adapt, drive a *persistent* ``NVBMesh`` across levels instead (so the + refinement-edge labelling — hence the similarity-class / shape-regularity bound + — propagates parent→child); ``Mesh._adapt_nested(engine="nvb")`` does this. + + ``boundaries`` / ``regions`` are ``(name, value)`` iterables of labels to carry. + """ + from underworld3.utilities.nvb import NVBMesh + nvb = NVBMesh.from_dm(dm, boundaries=boundaries, regions=regions) + nvb.refine(set(int(c) for c in cells)) + return nvb.to_dm(boundaries=boundaries, regions=regions, comm=dm.comm) + + # --------------------------------------------------------------------------- # # Coordinate helpers # --------------------------------------------------------------------------- # diff --git a/src/underworld3/utilities/nvb.py b/src/underworld3/utilities/nvb.py new file mode 100644 index 00000000..da614f2d --- /dev/null +++ b/src/underworld3/utilities/nvb.py @@ -0,0 +1,341 @@ +"""Newest-vertex bisection (NVB) — a *graded* simplicial refinement engine. + +PETSc's ``refine_sbr`` (longest-edge bisection) has an **unbounded** conforming +closure for region marking: a single marked cell drains the longest-edge +propagation path clear to the nearest mesh-size contrast, so it can only ever +produce a *uniform-finest patch*, never a graded mesh (a level+1 ring around a +level+2 sub-ring around a finer core). Newest-vertex bisection has a *provably +bounded* closure (Binev–Dahmen–DeVore 2004; Stevenson 2008) given a compatible +initial edge labelling, and yields conforming, 2:1-balanced, shape-regular graded +meshes with a finite number of similarity classes — exactly the staircase we +want, with the simplices UW3 is built on (no p4est/DMForest quad/hex). + +This module is the **serial Route-A** implementation (design note +``docs/developer/design/NVB_GRADED_ADAPT.md``): the triangulation is maintained as +pure-Python/numpy arrays with a per-cell refinement-edge labelling; after a +refinement pass the PETSc ``DMPlex`` is rebuilt from the cell list +(``createFromCellList``) and the boundary / region labels are transferred onto the +children. It plugs in where ``custom_mg.sbr_refine`` does today and feeds the same +coordinate-based custom-P geometric-MG hierarchy. + +Parallel (np>1) is **out of scope** for this implementation: ``createFromCellList`` +builds a fresh serial DM whose point star-forest does not preserve the parent's +decomposition (rank *r* owning the refinements of rank *r*'s base cells), which the +custom-P parallel path requires. That needs a native ``DMPlexTransform`` (Route B); +see the design note. ``mesh.adapt(engine="nvb")`` raises ``NotImplementedError`` at +np>1 rather than silently corrupting the decomposition. + +Data model (newest-vertex convention) +-------------------------------------- +``cell = (peak, b0, b1)`` — ``peak`` is the newest vertex; the **refinement edge** +is ``{b0, b1}`` (opposite the peak). Bisecting splits ``{b0, b1}`` at its midpoint +``m`` and makes ``m`` the peak of both children: ``(m, peak, b0)`` and +``(m, peak, b1)`` — so each child's refinement edge is a parent edge incident to +the apex. Cycling the refinement edge this way (rather than re-picking the +geometric longest edge every step, as LEB does) is what bounds the recursion to a +finite number of similarity classes. +""" + +import numpy as np +from petsc4py import PETSc + +__all__ = ["NVBMesh"] + + +def _fs(a, b): + """A sorted (frozen) vertex pair — the canonical edge key.""" + return (a, b) if a < b else (b, a) + + +class NVBMesh: + """A 2D newest-vertex-bisection triangulation with boundary-edge and + region-cell labels carried across bisections. + + Construct from a DMPlex with :meth:`from_dm`, refine marked cells with + :meth:`refine` (the bounded conforming closure runs automatically), and emit a + fresh labelled DMPlex with :meth:`to_dm`. The instance is **stateful** — the + refinement-edge labelling propagates parent→child, which is what preserves the + similarity-class (shape-regularity) bound across successive refinement passes; + re-seeding from scratch each level would not. + + 2D only for now (the design generalises to tets — Bänsch/Maubach/Stevenson — + but that is a follow-up). + """ + + def __init__(self, coords, tris): + self.coords = [np.asarray(p, float) for p in coords] + self.cells = {} # cid -> (peak, b0, b1) + self.depth = {} # cid -> bisection depth from its base cell + self.region = {} # cid -> region value (or None) + self.edge2cells = {} # (a,b) -> set(cid) + self.edge2mid = {} # (a,b) -> midpoint vertex id + self.edge_label = {} # (a,b) -> boundary value (propagated) + self._next = 0 + # Initial labelling: refinement edge = LONGEST edge (a robust seed that + # always conforming-terminates in 2D; NVB's marked-edge propagation takes + # over from here — this is NOT the same as running longest-edge bisection, + # which re-picks the longest edge at every step). + for t in tris: + self._add_cell(self._longest_edge_order(t)) + + # ---- construction from a DMPlex --------------------------------------- + @classmethod + def from_dm(cls, dm, boundaries=(), regions=()): + """Build an :class:`NVBMesh` from a 2D simplex ``DMPlex``. + + ``boundaries`` / ``regions`` are iterables of ``(name, value)`` (e.g. from a + mesh's boundary / region enum). Boundary-face (edge) labels seed + :attr:`edge_label` keyed by the edge's vertex pair — this is the *only* + thing that lets an interior/fault interface survive refinement, since + geometry alone cannot distinguish which boundary an edge belongs to. Region + labels seed :attr:`region` per base cell and propagate parent→child. + """ + if dm.getDimension() != 2: + raise NotImplementedError("NVBMesh is 2D only (tets are a follow-up).") + vS, vE = dm.getDepthStratum(0) + eS, eE = dm.getDepthStratum(1) + cS, cE = dm.getHeightStratum(0) + coords = dm.getCoordinatesLocal().array.reshape(-1, 2) + if coords.shape[0] != vE - vS: + raise RuntimeError( + f"NVBMesh.from_dm: {coords.shape[0]} coords vs {vE - vS} vertices " + f"(degree-1 coordinate DM expected).") + + tris, cell_pts = [], [] + for c in range(cS, cE): + clos = dm.getTransitiveClosure(c)[0] + verts = [p - vS for p in clos if vS <= p < vE] + if len(verts) != 3: + raise RuntimeError(f"cell {c} has {len(verts)} vertices (not a tri)") + tris.append(tuple(verts)) + cell_pts.append(c) + + self = cls(coords, tris) + + for name, value in boundaries: + if not dm.hasLabel(name): + continue + iset = dm.getStratumIS(name, value) + if iset is None: + continue + for p in iset.getIndices(): + if eS <= p < eE: # a boundary edge + a, b = (int(v - vS) for v in dm.getCone(p)) + self.edge_label[_fs(a, b)] = int(value) + + if regions: + cell_of_pt = {cell_pts[i]: i for i in range(len(cell_pts))} + for name, value in regions: + if not dm.hasLabel(name): + continue + iset = dm.getStratumIS(name, value) + if iset is None: + continue + for p in iset.getIndices(): + if p in cell_of_pt: + self.region[cell_of_pt[p]] = int(value) + return self + + # ---- vertex / cell bookkeeping ---------------------------------------- + def _longest_edge_order(self, t): + a, b, c = t + P = self.coords + keys = (_fs(a, b), _fs(b, c), _fs(c, a)) + b0, b1 = max(keys, key=lambda k: np.linalg.norm(P[k[0]] - P[k[1]])) + peak = ({a, b, c} - {b0, b1}).pop() + return (peak, b0, b1) + + def _add_cell(self, pbr, depth=0, region=None): + cid = self._next + self._next += 1 + self.cells[cid] = pbr + self.depth[cid] = depth + self.region[cid] = region + p, b0, b1 = pbr + for e in (_fs(p, b0), _fs(b0, b1), _fs(b1, p)): + self.edge2cells.setdefault(e, set()).add(cid) + return cid + + def _del_cell(self, cid): + p, b0, b1 = self.cells.pop(cid) + self.depth.pop(cid, None) + self.region.pop(cid, None) + for e in (_fs(p, b0), _fs(b0, b1), _fs(b1, p)): + self.edge2cells[e].discard(cid) + if not self.edge2cells[e]: + del self.edge2cells[e] + + def _midpoint(self, e): + """Midpoint vertex of edge ``e`` (created once). Propagates any boundary + label on ``e`` onto its two halves — midpoints are only ever created on an + edge being bisected, so this is exactly where a boundary edge subdivides.""" + if e not in self.edge2mid: + a, b = e + self.coords.append(0.5 * (self.coords[a] + self.coords[b])) + m = len(self.coords) - 1 + self.edge2mid[e] = m + lab = self.edge_label.pop(e, None) + if lab is not None: + self.edge_label[_fs(a, m)] = lab + self.edge_label[_fs(m, b)] = lab + return self.edge2mid[e] + + def _ref_edge(self, cid): + _, b0, b1 = self.cells[cid] + return _fs(b0, b1) + + def _neighbour(self, cid, e): + others = self.edge2cells.get(e, set()) - {cid} + return next(iter(others)) if others else None + + # ---- the bisection ----------------------------------------------------- + def _split(self, cid, m): + p, b0, b1 = self.cells[cid] + d = self.depth[cid] + 1 + r = self.region[cid] + self._del_cell(cid) + self._add_cell((m, p, b0), d, r) # newest vertex m, ref edge {p,b0} + self._add_cell((m, p, b1), d, r) # newest vertex m, ref edge {p,b1} + + def bisect(self, cid, _depth=0): + """Bisect cell ``cid`` with the bounded conforming completion: if the + refinement edge is not shared as the neighbour's refinement edge, bisect the + neighbour first (recursively) until the edge is *compatible*, then split the + pair at the common midpoint. Terminates (bounded closure) — unlike + longest-edge bisection's drain-to-interface chain.""" + if _depth > 100000: + raise RuntimeError("NVB closure did not terminate (LEPP-like loop)") + e = self._ref_edge(cid) + nb = self._neighbour(cid, e) + while nb is not None and self._ref_edge(nb) != e: + self.bisect(nb, _depth + 1) + nb = self._neighbour(cid, e) # a child of the old neighbour + m = self._midpoint(e) + self._split(cid, m) + if nb is not None: + self._split(nb, m) + + def refine(self, marked): + """Refine each still-live cell id in ``marked`` (a set/iterable).""" + for cid in list(marked): + if cid in self.cells: + self.bisect(cid) + + # ---- export / diagnostics --------------------------------------------- + def arrays(self): + """``(coords (V,2), tris (N,3), regions (N,), cids (N,))`` for live cells, + in a stable cell order. ``regions`` is ``-1`` where unset.""" + coords = np.array(self.coords) + cids = list(self.cells) + tris = np.array([self.cells[c] for c in cids], dtype=np.int64) + regions = np.array( + [self.region[c] if self.region.get(c) is not None else -1 for c in cids], + dtype=np.int64) + return coords, tris, regions, np.array(cids, dtype=np.int64) + + def centroids_h(self): + """``(centroids (N,2), h (N,), cids (N,))`` for live cells, where + ``h = (2·area)**0.5`` is the simplex characteristic size + (``h ≈ (dim!·vol)^(1/dim)`` with ``dim=2``) used by the metric marking.""" + coords, tris, _, cids = self.arrays() + a, b, c = coords[tris[:, 0]], coords[tris[:, 1]], coords[tris[:, 2]] + cen = (a + b + c) / 3.0 + area = 0.5 * np.abs((b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1]) - + (c[:, 0] - a[:, 0]) * (b[:, 1] - a[:, 1])) + return cen, np.sqrt(2.0 * area), cids + + def check_conforming(self): + """``(hanging, overshared)``: ``hanging`` = live cells carrying an edge that + has been bisected (a hanging node); ``overshared`` = edges shared by >2 + cells. Both must be 0 for a conforming mesh.""" + hanging = 0 + for (p, b0, b1) in self.cells.values(): + for e in (_fs(p, b0), _fs(b0, b1), _fs(b1, p)): + if e in self.edge2mid: + hanging += 1 + overshared = sum(1 for cs in self.edge2cells.values() if len(cs) > 2) + return hanging, overshared + + def similarity_classes(self, ndigits=6): + """Number of distinct triangle *shapes* (similarity classes) among live + cells — sorted, scale-normalised edge-length triples, rounded. NVB bounds + this by a small constant (≤4 per base triangle in 2D); a blow-up signals + degenerating elements. Diagnostic only.""" + coords = np.array(self.coords) + classes = set() + for (p, b0, b1) in self.cells.values(): + P = coords[[p, b0, b1]] + ls = np.sort([np.linalg.norm(P[0] - P[1]), + np.linalg.norm(P[1] - P[2]), + np.linalg.norm(P[2] - P[0])]) + ls = ls / ls[-1] + classes.add(tuple(np.round(ls, ndigits))) + return len(classes) + + # ---- DMPlex build + label transfer ------------------------------------ + def to_dm(self, boundaries=(), regions=(), comm=None): + """Build a fresh interpolated ``DMPlex`` from the current triangulation and + transfer boundary / region labels onto it. + + ``boundaries`` / ``regions`` are ``(name, value)`` iterables naming the + labels to create. Boundary edges are matched to their carried label by + **vertex pair** (vertices identified by coordinate, robust to PETSc's + renumbering — never by array index); each labelled edge also labels its two + vertices, matching UW's boundary-label convention so ``Mesh()`` derives + ``UW_Boundaries`` / ``Null_Boundary`` from them. ``All_Boundaries`` is the + geometric outer boundary (``markBoundaryFaces``), independent of our labels. + """ + from scipy.spatial import cKDTree + + coords, tris, region_of, _ = self.arrays() + # createFromCellList needs consistent (CCW) winding; the internal + # (peak,b0,b1) ordering is geometry-agnostic, so reorient the EXPORTED cell + # list only (swap the last two verts where the signed area is negative). + a, b, c = coords[tris[:, 0]], coords[tris[:, 1]], coords[tris[:, 2]] + sa = (b[:, 0] - a[:, 0]) * (c[:, 1] - a[:, 1]) - \ + (c[:, 0] - a[:, 0]) * (b[:, 1] - a[:, 1]) + neg = sa < 0 + tris[neg, 1], tris[neg, 2] = tris[neg, 2], tris[neg, 1].copy() + + dm = PETSc.DMPlex().createFromCellList( + 2, tris.astype(np.int32), coords, interpolate=True, + comm=comm or PETSc.COMM_WORLD) + + vS, vE = dm.getDepthStratum(0) + eS, eE = dm.getDepthStratum(1) + cS, cE = dm.getHeightStratum(0) + + # DM vertex point -> NVB vertex id by coordinate (midpoints are exact float + # averages, computed identically here and in the DM, so the match is exact). + dm_vcoords = dm.getCoordinatesLocal().array.reshape(-1, 2) + _, nearest = cKDTree(np.array(self.coords)).query(dm_vcoords) + nvb_of_dmvert = {vS + i: int(nearest[i]) for i in range(vE - vS)} + + # outer geometric boundary (UW convention) + dm.markBoundaryFaces("All_Boundaries", 1001) + + # named boundary labels: each carried edge + its two vertices + wanted = {value for _, value in boundaries} + if wanted: + for name, value in boundaries: + dm.createLabel(name) + labels = {value: dm.getLabel(name) for name, value in boundaries} + for p in range(eS, eE): + va, vb = (nvb_of_dmvert[v] for v in dm.getCone(p)) + lab = self.edge_label.get(_fs(va, vb)) + if lab in wanted: + labels[lab].setValue(p, lab) + for v in dm.getCone(p): + labels[lab].setValue(v, lab) + + # region cell labels (cells are points [cS, cE) in input order) + if regions: + for name, value in regions: + dm.createLabel(name) + rlabels = {value: dm.getLabel(name) for name, value in regions} + for i in range(cE - cS): + rv = int(region_of[i]) + if rv in rlabels: + rlabels[rv].setValue(cS + i, rv) + + return dm From 01d1cc00392196bb810306238fcccfd693cb222a Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 17:40:53 +1000 Subject: [PATCH 28/52] test+docs(layer2): NVB graded adapt tests + design-note status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/test_0836_nvb_graded_adapt.py (tier_b, 12 tests): engine-level properties (conformity 0 hanging/0 over-shared every generation; bounded closure — single cell local +O(1) and #added ≤ C·#marked; shape-regularity via bounded similarity classes; graded bullseye + fault funnel confine the finest generation near the feature with coarse cells surviving) and the integrated path (graded child link/ engine tag, fewer DOFs than the SBR patch, non-cumulative re-adapt, Poisson FMG matches GAMG, SolCx Stokes η-jump-1e6 velocity-block FMG matches GAMG, 3D guard). Update NVB_GRADED_ADAPT.md: phases 2 (DMPlex wrap) + 3 (Layer-2 wiring) DONE with the resolved gotchas (CCW winding, vertex-pair label match, edge+vertex labels); Route B (parallel native transform) is next and the seam is ready. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 45 +++- tests/test_0836_nvb_graded_adapt.py | 297 ++++++++++++++++++++++ 2 files changed, 332 insertions(+), 10 deletions(-) create mode 100644 tests/test_0836_nvb_graded_adapt.py diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 9da15dad..1e52db6a 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -1,6 +1,10 @@ # Newest-Vertex Bisection (NVB) for graded adapt-on-top -Status: design (2026-06-30). Follows the Layer-2 investigation in +Status: **implemented (serial Route A), 2026-06-30**. Engine in +`src/underworld3/utilities/nvb.py` + `custom_mg.nvb_refine`; wired into +`Mesh.adapt(engine="nvb")`; validated in `tests/test_0836_nvb_graded_adapt.py`. +Parallel (Route B, native transform) is the next step. Follows the Layer-2 +investigation in [`LAYER2_SBR_ADAPT_ON_TOP.md`](LAYER2_SBR_ADAPT_ON_TOP.md). Goal: replace the refinement *engine* under `mesh.adapt(adapter="sbr")` so that successive levels produce a **graded** mesh (a level+1 ring, a level+2 sub-ring dividing *some* @@ -289,15 +293,36 @@ is the DMPlex wrap + label transfer (engineering, not algorithm risk). NVB bisec 1. **Serial NVB core** (numpy) — **DONE** (`nvb_prototype_2d.py`): marked-edge model, `refine`, recursive compatible-closure; conformity (no hanging nodes, 0 over-shared edges) and bounded-closure (1 cell → +2, local) verified; graded - bullseye rendered. TODO before src: similarity-class/shape-regularity assertion - and a #added ≤ C·#marked stress test. -2. **DMPlex wrap**: build-from-cells + label transfer; `nvb_refine` mirroring - `sbr_refine`; render the graded bullseye + fault funnel coloured by level - (the acceptance picture: distributed rings, *not* a uniform core). -3. **Layer-2 wiring**: `engine="nvb"` in `_adapt_nested`; custom-P FMG on the - graded child (Poisson + SolCx Stokes), serial. -4. **Decide on Route B** (native transform) for parallel + incremental, informed - by 1–3. + bullseye rendered. +2. **DMPlex wrap** — **DONE** (`src/underworld3/utilities/nvb.py`, + `custom_mg.nvb_refine`): `NVBMesh.from_dm` / `to_dm` build the interpolated + DMPlex from the cell list and transfer boundary-edge + region-cell labels. + Gotchas resolved: `createFromCellList` needs consistent **CCW winding** (the + `(peak,b0,b1)` refinement-edge order is geometry-agnostic, so reorient the + *exported* cell list only); labels matched by **vertex pair** (vertex identified + by coordinate — midpoints are exact float averages, so the match is exact — + never by array index); each boundary edge also labels its two vertices so + `Mesh()` derives `UW_Boundaries`/`Null_Boundary`, and `All_Boundaries` is the + geometric outer boundary. De-risk: a Dirichlet Poisson on the injected DM solves + to 4e-10. Graded bullseye / fault funnel coloured by generation rendered + (`~/+Simulations/layer2_adapt_on_top_study/nvb_src_{bullseye,fault_funnel}.png`, + `nvb_vs_sbr.png` — concentric rings, **not** a uniform core; 1800 vs 3616 cells). +3. **Layer-2 wiring** — **DONE**: `Mesh._adapt_sbr` renamed `_adapt_nested` with + `engine="sbr"|"nvb"` (default sbr, unchanged); the NVB path drives a *persistent* + `NVBMesh` across `2·max_levels` generations (1→2 area halving) so the + refinement-edge labelling — hence the similarity-class bound — propagates + parent→child, snapshotting one DM per generation into the engine-agnostic + coordinate-based custom-P FMG tail. Validated serial (`tests/test_0836`, + tier_b): conformity, bounded closure (single-cell local + ≤C·#marked), + shape-regularity (similarity classes bounded; =1 on a right-isoceles structured + base — the ideal), graded bullseye + fault funnel, **Poisson FMG** (5 levels) + matches GAMG bit-identically, **SolCx Stokes** (η jump 1e6) velocity-block FMG + matches GAMG iter-for-iter (8). `engine="nvb"` raises `NotImplementedError` at + np>1 (verified clean under `mpirun -n 2`; SBR unaffected) and for dim≠2. +4. **Decide on Route B** (native transform) for parallel + incremental — **NEXT**, + informed by 1–3. The seam is ready: `engine="nvb"` would dispatch to a native + `DMPlexTransform` at np>1, and the custom-P tail is already coordinate-based + (engine-agnostic), so Route B needs no change to the hierarchy or the wiring. ## Alternatives considered diff --git a/tests/test_0836_nvb_graded_adapt.py b/tests/test_0836_nvb_graded_adapt.py new file mode 100644 index 00000000..fb3e40dc --- /dev/null +++ b/tests/test_0836_nvb_graded_adapt.py @@ -0,0 +1,297 @@ +"""Layer-2 NVB graded adapt-on-top: ``mesh.adapt(engine="nvb")``. + +Newest-vertex bisection is a *graded* simplicial refinement engine with a +*bounded* conforming closure — the property PETSc's longest-edge ``refine_sbr`` +lacks (it can only build a uniform-finest patch). These tests pin both the engine +(``underworld3.utilities.nvb.NVBMesh``) and the integrated path: + + - conformity: 0 hanging nodes / 0 over-shared edges at every generation; + - bounded closure: one cell deep in a uniform patch adds O(1) cells locally + (not a drain-to-edge cascade), and #added ≤ C·#marked for a marked region; + - shape-regularity: the number of triangle similarity classes stays bounded + under deep refinement (no slivers); + - grading: a bullseye and a fault funnel confine the FINEST generation near the + feature (NVB grades where SBR refills) — and the NVB child has fewer DOFs than + the SBR uniform patch for the same metric; + - the graded NVB child drives the custom-P geometric-MG FMG: Poisson and SolCx + Stokes (η jump 1e6) match a GAMG reference. + +Serial only (NVB Route A); the np>1 / 3D guards are asserted to raise. +""" +import numpy as np +import pytest +import sympy +import underworld3 as uw +from underworld3.function import analytic as A +from underworld3.utilities.nvb import NVBMesh + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + +BOUNDS = [("Bottom", 11), ("Top", 12), ("Right", 13), ("Left", 14)] + + +def _ev(fn, coords): + return np.asarray(uw.function.evaluate(fn, np.asarray(coords))).reshape(-1) + + +def _ncell(mesh): + cs, ce = mesh.dm.getHeightStratum(0) + return ce - cs + + +def _base(cellSize=0.25, refinement=2): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=cellSize, regular=True, + refinement=refinement, qdegree=3) + + +def _nvb_from_base(base): + return NVBMesh.from_dm(base.dm_hierarchy[-1], boundaries=BOUNDS) + + +# --------------------------------------------------------------------------- # +# Engine-level properties (NVBMesh in isolation — fast, no solves) +# --------------------------------------------------------------------------- # +def test_conformity_every_generation(): + """No hanging nodes / over-shared edges after each refinement generation.""" + nvb = _nvb_from_base(_base()) + assert nvb.check_conforming() == (0, 0) + for _gen in range(5): + cen, h, cids = nvb.centroids_h() + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + marked = cids[r < 0.25] + if marked.size == 0: + break + nvb.refine(set(int(c) for c in marked)) + assert nvb.check_conforming() == (0, 0), f"hanging nodes at gen {_gen}" + + +def test_bounded_closure_single_cell_is_local(): + """One marked cell deep in a uniform patch adds O(1) cells (bounded local + closure) — the SBR drain-to-interface would add hundreds.""" + nvb = _nvb_from_base(_base()) + cen, h, cids = nvb.centroids_h() + # a cell near the centre, far from any boundary + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + target = int(cids[np.argmin(r)]) + n_before = len(nvb.cells) + moved = cen[np.argmin(r)] + nvb.refine({target}) + added = len(nvb.cells) - n_before + assert added <= 8, f"closure not local: +{added} cells from 1 mark" + # the added cells are confined near the marked cell, not drained to the edge + cen2, _, _ = nvb.centroids_h() + d = np.hypot(cen2[:, 0] - moved[0], cen2[:, 1] - moved[1]) + # everything that changed lives within a couple of base cells of the seed + assert d[d < 0.2].size >= added + + +def test_bounded_closure_proportional_to_marked(): + """#added ≤ C·#marked for a marked region (the BDD/Stevenson bound).""" + nvb = _nvb_from_base(_base()) + cen, h, cids = nvb.centroids_h() + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + marked = cids[r < 0.3] + n_before = len(nvb.cells) + nvb.refine(set(int(c) for c in marked)) + added = len(nvb.cells) - n_before + assert added <= 6 * marked.size, f"closure {added} >> 6·{marked.size} marked" + + +def test_shape_regularity_bounded_similarity_classes(): + """Deep refinement does not degenerate elements: the number of triangle + similarity classes stays small (NVB ≤ a few per base triangle).""" + nvb = _nvb_from_base(_base(cellSize=0.5, refinement=1)) + base_classes = nvb.similarity_classes() + for _gen in range(8): + cen, h, cids = nvb.centroids_h() + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + marked = cids[r < 0.3] + if marked.size == 0: + break + nvb.refine(set(int(c) for c in marked)) + # NVB's hallmark: a small constant number of shapes, regardless of depth + assert nvb.similarity_classes() <= max(8, 4 * base_classes) + + +def test_bullseye_finest_generation_confined(): + """A bullseye (nested shrinking disks) confines the FINEST generation near the + centre — a graded staircase, not a uniform-finest core.""" + nvb = _nvb_from_base(_base()) + radii = [0.30, 0.22, 0.15, 0.09, 0.05] + for rad in radii: + cen, h, cids = nvb.centroids_h() + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + marked = cids[r < rad] + if marked.size: + nvb.refine(set(int(c) for c in marked)) + + cen, h, cids = nvb.centroids_h() + depths = np.array([nvb.depth[int(c)] for c in cids]) + r = np.hypot(cen[:, 0] - 0.5, cen[:, 1] - 0.5) + dmax = depths.max() + # finest cells hug the centre; coarse (unrefined) cells survive at the edge + assert r[depths == dmax].max() < 0.20, "finest generation not confined" + assert depths.min() == 0, "no coarse cells survive — refilled, not graded" + # a genuine staircase: several distinct depth levels coexist + assert np.unique(depths).size >= 4 + + +def test_fault_funnel_finest_hugs_the_line(): + """A dipping-line feature funnels the finest generation to within a thin band + of the line (perpendicular grading).""" + nvb = _nvb_from_base(_base(cellSize=0.2, refinement=2)) + # line through (0.2,0.85)->(0.85,0.2): n·x = c with unit normal + n = np.array([1.0, 1.0]) / np.sqrt(2) + c0 = n @ np.array([0.2, 0.85]) + + def dist(cen): + return np.abs(cen @ n - c0) + + for band in [0.18, 0.12, 0.08, 0.05, 0.03]: + cen, h, cids = nvb.centroids_h() + marked = cids[dist(cen) < band] + if marked.size: + nvb.refine(set(int(x) for x in marked)) + + cen, h, cids = nvb.centroids_h() + depths = np.array([nvb.depth[int(x)] for x in cids]) + dmax = depths.max() + assert dist(cen)[depths == dmax].max() < 0.08, "finest band not hugging the line" + assert depths.min() == 0 + assert nvb.check_conforming() == (0, 0) + + +# --------------------------------------------------------------------------- # +# Integrated path: mesh.adapt(engine="nvb") +# --------------------------------------------------------------------------- # +def _band_metric(base, center=0.5, h_fine=1 / 100, width=0.08): + M = uw.discretisation.MeshVariable("Mnvb", base, 1, degree=1) + band = sympy.exp(-(((base.N.x - center) / width) ** 2)) + M.data[:, 0] = _ev(1.0 / (0.07 + (h_fine - 0.07) * band) ** 2, M.coords) + return M + + +def test_adapt_nvb_returns_graded_child(): + base = _base() + n0 = _ncell(base) + M = _band_metric(base) + child = base.adapt(M, max_levels=1, engine="nvb") + + assert child is not base + assert child.parent is base + assert child._relationship_kind == "refinement" + assert child._adapt_engine == "nvb" + assert _ncell(child) > n0 + assert _ncell(base) == n0 # base untouched + # 2·max_levels NVB generations -> base levels + (generations-1) intermediate + assert len(child._custom_mg_coarse_meshes) + 1 == len(base.dm_hierarchy) + 2 + + +def test_nvb_child_fewer_dofs_than_sbr_patch(): + """For the same metric, NVB grades (fewer DOFs) where SBR refills a uniform + patch. Compare at matched isotropic-equivalent resolution (NVB 2 gens ≈ SBR 1 + pass).""" + base = _base() + M = _band_metric(base, h_fine=1 / 90, width=0.06) + sbr = base.adapt(M, max_levels=1, engine="sbr") + nvb = base.adapt(M, max_levels=1, engine="nvb") + assert _ncell(nvb) < _ncell(sbr) + + +def test_nvb_readapt_is_non_cumulative(): + base = _base() + n0 = _ncell(base) + c1 = base.adapt(_band_metric(base, center=0.7), max_levels=1, engine="nvb") + c2 = base.adapt(_band_metric(base, center=0.3), max_levels=1, engine="nvb") + assert _ncell(base) == n0 + assert c2 is not c1 + + +def _poisson(mesh): + p = uw.systems.Poisson(mesh) + p.constitutive_model = uw.constitutive_models.DiffusionModel + p.constitutive_model.Parameters.diffusivity = 1 + p.f = 0.0 + p.add_dirichlet_bc(0.0, "Bottom") + p.add_dirichlet_bc(1.0, "Top") + p.petsc_options["ksp_rtol"] = 1e-8 + p.petsc_options["ksp_type"] = "cg" + return p + + +def test_poisson_fmg_on_nvb_child_matches_gamg(): + base = _base() + child = base.adapt(_band_metric(base), max_levels=1, engine="nvb") + + s = _poisson(child) + s.solve() # NO set_custom_fmg + assert s.snes.getKSP().getPC().getType() == "mg" + assert s.snes.getKSP().getPC().getMGLevels() == len(base.dm_hierarchy) + 2 + assert s.snes.getConvergedReason() > 0 + + g = _poisson(child) + g.preconditioner = "gamg" + g.solve() + rel = np.linalg.norm(s.Unknowns.u.data - g.Unknowns.u.data) / ( + np.linalg.norm(g.Unknowns.u.data) + 1e-30) + assert rel < 1e-4 + # exact linear field T = y + err = np.linalg.norm(s.Unknowns.u.data[:, 0] - s.Unknowns.u.coords[:, 1]) / ( + np.linalg.norm(s.Unknowns.u.coords[:, 1]) + 1e-30) + assert err < 1e-8, f"Dirichlet labels wrong on NVB child: err {err}" + + +def test_solcx_stokes_velocity_fmg_on_nvb_child(): + """SolCx Stokes (η jump 1e6) on a graded NVB child: the velocity block must + auto-pick-up the mesh-owned custom-P FMG and match a GAMG reference.""" + base = _base() + M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) + band = sympy.exp(-(((base.N.x - 0.5) / 0.08) ** 2)) + M.data[:, 0] = _ev(1.0 / (0.07 + (1.0 / 80 - 0.07) * band) ** 2, M.coords) + child = base.adapt(M, max_levels=1, engine="nvb") + + def _solcx(mesh): + sol = A.SolCx(mesh, eta_A=1.0, eta_B=1.0e6, x_c=0.5, n=1) + s = uw.systems.Stokes(mesh) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.saddle_preconditioner = 1.0 / sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_dirichlet_bc((0.0, None), "Left") + s.add_dirichlet_bc((0.0, None), "Right") + s.add_dirichlet_bc((None, 0.0), "Bottom") + s.add_dirichlet_bc((None, 0.0), "Top") + s.petsc_use_pressure_nullspace = True + s.petsc_options["snes_type"] = "ksponly" + s.tolerance = 1e-8 + return s, sol + + sg, solg = _solcx(child) + sg.preconditioner = "gamg" + sg.solve() + it_g = sg.snes.getKSP().getPC().getFieldSplitSubKSP()[0].getIterationNumber() + + s, sol = _solcx(child) + assert s._custom_mg is None + s.solve() + vksp = s.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" + assert vksp.getPC().getMGType() == 2 # PC.MGType.FULL == FMG + assert vksp.getType() == "fgmres" + assert vksp.getIterationNumber() <= it_g # FMG matches/beats GAMG + rel = np.linalg.norm(s.u.data - sg.u.data) / (np.linalg.norm(sg.u.data) + 1e-30) + assert rel < 1e-4 + assert sol.velocity_error(s.u) < 2.0 * solg.velocity_error(sg.u) + 1e-6 + + +def test_nvb_3d_guard_raises(): + """NVB is 2D only this pass — a 3D base must raise NotImplementedError.""" + base3 = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0, 0), maxCoords=(1, 1, 1), cellSize=0.4, regular=False, + refinement=1, qdegree=2) + H = uw.discretisation.MeshVariable("H3", base3, 1, degree=1) + H.data[:, 0] = 1.0 / 0.1**2 + with pytest.raises(NotImplementedError): + base3.adapt(H, max_levels=1, engine="nvb") From 0287b0b4922b5d6552fb81fd4b6a6b4ce6b8069c Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 21:17:39 +1000 Subject: [PATCH 29/52] =?UTF-8?q?docs(layer2):=20NVB=20parallel-readiness?= =?UTF-8?q?=20review=20=E2=80=94=20serial=20Route=20A=20is=20non-blocking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit against the 'must not block parallel' requirement: - co-partitioning invariant is structurally preserved by NVB (bisection replaces a cell with same-owner children; closure only triggers same-owner bisections), so any in-place transform keeps the child co-partitioned with the coarse tail — the custom-P parallel transfer's load-bearing assumption. Only Route A's rank-0 cell-list rebuild breaks it. Measured bar: custom-P FMG already solves np=2 on a locally-refined SBR adapt-on-top child (4-level mg, exact err 2e-11). - nothing in the serial code blocks Route B: engine= dispatch seam, engine-agnostic coordinate custom-P tail (parallel path already built+proven), lineage/copy_into/ snapshots all shared; NVBMesh cell-list to_dm + label transfer are simply not on the parallel path. - irreducible hard kernel = the parallel conforming-closure fixpoint; NVB's labelling is cross-rank-consistent for free (geometric longest-edge seed + deterministic midpoint rule). No PETSc transform bisects a labelled edge, so Route B needs a new C transform type (reusing PETSc bisection-closure infra). - options: manual-SF pure-Python (fragile) vs native C transform (robust); recommend prototyping the parallel closure-fixpoint in Python first to de-risk. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 1e52db6a..c0cede10 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -220,6 +220,78 @@ Conclusion — the serial/parallel split is therefore: in a *locally-owned* base cell, and the coarse-tail partition is bit-identical before/after — the same invariant the SBR path checks. +### Parallel-readiness review (2026-06-30) — serial Route A is non-blocking + +A deliberate audit of the shipped serial engine against the parallel path, with +the requirement that *the serial implementation must not block parallel*. + +**Is the co-partitioning invariant real, and does NVB preserve it?** Yes, and the +custom-P parallel transfer (`custom_mg._build_parallel_transfer` / +`_level_dof_layout`) is what *consumes* it: a fine **owned** DOF's coarse +contributions must come from coarse coords that are **local (incl. ghosts) on the +same rank** — i.e. the fine node must fall in a coarse simplex owned/ghosted by its +rank. That holds iff rank *r*'s fine cells are refinements of rank *r*'s base +cells. NVB preserves this *structurally*: bisecting a cell replaces it with two +children **of the same owner**, and the conforming closure only ever triggers more +*same-owner* bisections on a neighbour's rank — so any **in-place** NVB transform +on the distributed DM keeps the child co-partitioned with the coarse tail, exactly +as `dm.refine()` and `refine_sbr`'s `adaptLabel` do. The *only* thing that breaks +it is Route A's rank-0 `createFromCellList` rebuild — a property of the +*construction route*, not of NVB. **Measured bar:** the custom-P FMG already solves +on a locally-refined SBR adapt-on-top child at np=2 (pc=mg, 4 levels, converged, +exact err 2e-11) — the exact behaviour the NVB parallel path must reproduce, with +the parallel transfer machinery already built and proven. + +**Does anything in the serial code block Route B?** No. The integration is a +parallel-alongside layer: +- `engine="nvb"` dispatch in `_adapt_nested` — Route B slots in here (at np>1, + dispatch to the native transform instead of raising `NotImplementedError`). +- the custom-P tail is **coordinate-based and engine-agnostic**, and its parallel + path (`_build_parallel_transfer`) already exists and is proven — a Route-B child + feeds it unchanged. +- parent/child lineage, `copy_into`, the per-generation intermediate-DM snapshots, + and the tail assembly (`_wrap_coarse_level`) are all identical for A and B. +- the only Route-A-specific pieces — `NVBMesh`'s cell-list `to_dm` and the + coordinate/vertex-pair label transfer — are simply *not on* the parallel path + (an in-place transform preserves numbering + SF, so neither is needed). `NVBMesh` + remains useful as the serial engine and as a per-rank/oracle reference. + +**The irreducible hard kernel** (what Route B must actually build) is the +**parallel conforming-closure fixpoint**: computing, consistently across ranks, the +final set of edges to bisect so the result is conforming with bounded closure, then +bisecting those edges in-place with correct SF. PETSc already does this for +*longest-edge* (`refine_sbr` `adaptLabel`); NVB needs the same closure machinery +with the **newest-vertex rule + marked-edge labelling**. Reassuringly, NVB's +labelling is **cross-rank-consistent for free**: the initial longest-edge seed is +geometric (both ranks compute the same edge for any shared cell — and each cell is +owned by exactly one rank), and the propagation rule (newest vertex = the geometric +midpoint) is deterministic and local, so no labelling-reconciliation protocol is +needed — only the closure *exchange*. + +**No free PETSc transform.** petsc4py exposes `DMPlexTransform` generically but no +Python hook for a custom cell-subdivision rule, and PETSc ships only +`refine_regular` / `refine_alfeld` / `refine_boundary_layer` / `refine_tobox` +(plus `refine_sbr`) — **none** bisects a *labelled/specified* edge. So the +attractive "Python computes the closure, an existing PETSc transform does the +SF-preserving surgery" split is **not** available out of the box: Route B requires +registering a new transform type in C (ideally reusing PETSc's bisection-closure +infrastructure, swapping the longest-edge pick for the newest-vertex rule). + +**Options for Route B**, in increasing cost / decreasing fragility: +1. *Manual-SF Route A-parallel* (pure Python): per-rank `NVBMesh` on local cells + + a ghost layer, cross-rank closure by iterative halo exchange to a fixpoint, then + per-rank `createFromCellList(comm=SELF)` + a **hand-built point-SF** matching + shared points by coordinate. No PETSc rebuild, but the manual SF + parallel + fixpoint are the "fragile" parts the original note flagged. +2. *Native C transform* (`DMPLEXTRANSFORMNVB`): the robust path — reuses PETSc's + tested SF propagation and parallel closure; the cost is PETSc-C and a build. + +**Recommended de-risk before committing to C:** prototype the parallel +closure-fixpoint algorithm (option 1's hard kernel) in Python on a partitioned +mesh — prove the cross-rank closure converges and stays bounded — *separately* from +the SF construction. That validates the genuinely novel part cheaply and tells us +whether option 1 is viable or we go straight to option 2. + ## Checkpointing NVB is deterministic given the **initial labelling + the marked-cell set per From a214de2d3eec84d314ca51132881da8c8890c196 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 21:28:44 +1000 Subject: [PATCH 30/52] docs(layer2): de-risk the parallel NVB closure-fixpoint (Route B kernel) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nvb_parallel_closure_prototype.py: distributed NVB conforming-closure under a faithful discipline (geometric state shared = the point-SF; control flow distributed = owner-only bisection + an explicit cross-rank work queue, no rank touches another's cells). Proves the genuinely novel part of parallel NVB, separately from SF construction: - globally conforming (0 hanging nodes incl. across the partition), every case; - identical to the serial mesh (confluence), every partition (2/3/4-way, diagonal + vertical features straddling boundaries); - converges in <=3 communication rounds, FLAT as the mesh grows n=8->32 (8740 cells) — bounded and mesh-size-independent. What remains for a real parallel engine is only the SF/point construction, which Route B's native DMPlexTransform inherits from PETSc. Design note updated with the result table; this is the milestone that says Route B is worth the C investment. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 27 ++- .../design/nvb_parallel_closure_prototype.py | 176 ++++++++++++++++++ 2 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 docs/developer/design/nvb_parallel_closure_prototype.py diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index c0cede10..2068a947 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -286,11 +286,28 @@ infrastructure, swapping the longest-edge pick for the newest-vertex rule). 2. *Native C transform* (`DMPLEXTRANSFORMNVB`): the robust path — reuses PETSc's tested SF propagation and parallel closure; the cost is PETSc-C and a build. -**Recommended de-risk before committing to C:** prototype the parallel -closure-fixpoint algorithm (option 1's hard kernel) in Python on a partitioned -mesh — prove the cross-rank closure converges and stays bounded — *separately* from -the SF construction. That validates the genuinely novel part cheaply and tells us -whether option 1 is viable or we go straight to option 2. +**Parallel closure-fixpoint — DE-RISKED (2026-06-30).** The genuinely novel part +(the cross-rank conforming closure, the hard kernel shared by both options) is now +validated in Python *separately from* the SF construction: +[`nvb_parallel_closure_prototype.py`](nvb_parallel_closure_prototype.py) runs the +distributed NVB closure under a faithful discipline — geometric state shared (a real +run's point-SF, modelled by a deterministic shared midpoint), but **control flow +distributed**: a cell is bisected only by its owner, and the only cross-rank +coupling is an explicit work queue (no rank touches another's cells). Measured on a +diagonal/vertical feature straddling 2-, 3-, and 4-way partitions: + +| property | result | +|---|---| +| globally conforming (incl. across the partition) | **0 hanging nodes**, every case | +| identical to the serial mesh (confluence) | **EQUALS_SERIAL = True**, every partition | +| communication rounds to converge | **≤ 3**, and **flat as the mesh grows** n=8→32 (8740 cells) | + +So the cross-rank closure converges in a *bounded, mesh-size-independent* number of +rounds, stays conforming, and is partition-independent — exactly the guarantees a +parallel NVB needs. **What remains for a real parallel engine is only the SF / point +construction**, which Route B's native `DMPlexTransform` inherits from PETSc (and on +which option 1's manual-SF path could now be built, the algorithm being proven). +This is the milestone that says Route B is worth the C investment. ## Checkpointing diff --git a/docs/developer/design/nvb_parallel_closure_prototype.py b/docs/developer/design/nvb_parallel_closure_prototype.py new file mode 100644 index 00000000..33771d10 --- /dev/null +++ b/docs/developer/design/nvb_parallel_closure_prototype.py @@ -0,0 +1,176 @@ +"""Design-validation prototype: the PARALLEL NVB conforming-closure FIXPOINT. + +Companion to ``NVB_GRADED_ADAPT.md`` (Parallel-readiness review). This de-risks the +genuinely novel part of the parallel (Route B) NVB engine — completing the bounded +conforming closure *across a partition boundary* — separately from the DMPlex / SF +construction. It proves, on a partitioned mesh, that the cross-rank closure: + + (a) CONVERGES in a bounded number of communication rounds (single-digit, + independent of mesh size); + (b) is globally CONFORMING (0 hanging nodes, including across the partition); + (c) is IDENTICAL to the serial NVB mesh — partition-independent (confluence). + +Faithful distributed model (single process, G "ranks"): geometric/topological +state is legitimately SHARED (a real run's point-SF keeps shared vertices / edges / +midpoints consistent — modelled here by one ``NVBMesh`` + a deterministic shared +midpoint), but CONTROL FLOW obeys the distributed discipline — a cell is only ever +bisected by its OWNER, and the sole cross-rank coupling is an explicit work queue of +cells to (attempt to) bisect; no rank reads/writes another rank's cells. + +If the closure converges + matches serial under this discipline, the only remaining +work for a real parallel engine is the SF / point construction, which a native +``DMPlexTransform`` (Route B) inherits from PETSc. + +Run: ``python docs/developer/design/nvb_parallel_closure_prototype.py`` (in the uw +env). Imports the *shipped* engine ``underworld3.utilities.nvb.NVBMesh``. +""" +import numpy as np +from underworld3.utilities.nvb import NVBMesh, _fs as fs + + +# ---- a structured triangulated unit square (pure geometry, no PETSc) -------- +def structured_square(n): + xs = np.linspace(0, 1, n + 1) + coords = [(x, y) for y in xs for x in xs] + vid = lambda i, j: j * (n + 1) + i + tris = [] + for j in range(n): + for i in range(n): + a, b, c, d = vid(i, j), vid(i + 1, j), vid(i + 1, j + 1), vid(i, j + 1) + tris += [(a, b, d), (b, c, d)] + return coords, tris + + +# ---- distributed NVB closure (owner-only bisection + work-queue coupling) --- +class DistributedNVB: + def __init__(self, coords, tris, owner_of_centroid): + self.g = NVBMesh(coords, tris) + self.owner = {} + C = np.array(self.g.coords) + for cid, (p, b0, b1) in self.g.cells.items(): + self.owner[cid] = owner_of_centroid(C[[p, b0, b1]].mean(0)) + self.max_round_worklist = 0 + # children inherit their parent's owner — wrap _split to record it + orig_split = self.g._split + + def split(cid, m): + rank = self.owner.get(cid) + before = set(self.g.cells) + orig_split(cid, m) + self.owner.pop(cid, None) + for newcid in set(self.g.cells) - before: + self.owner[newcid] = rank + self.g._split = split + + def _try_bisect(self, rank, cid, requeue): + """Owner ``rank`` attempts to bisect owned cell ``cid``. Touches only owned + cells; defers (enqueues) when the closure needs a non-owned neighbour; a + shared-edge split enqueues the across-edge neighbour to split at the same + deterministic midpoint. Returns True iff ``cid`` was split.""" + while True: + if cid not in self.g.cells: + return False + e = self.g._ref_edge(cid) + nb = self.g._neighbour(cid, e) + if nb is None or self.g._ref_edge(nb) == e: + break # boundary / compatible + if self.owner[nb] == rank: + if not self._try_bisect(rank, nb, requeue): + requeue.add(cid) # nb deferred -> I defer too + return False + continue # nb split -> re-eval + requeue.add(nb) # non-owned: its owner refines it + requeue.add(cid) + return False + m = self.g._midpoint(e) + self.g._split(cid, m) + if nb is not None: + if self.owner[nb] == rank: + self.g._split(nb, m) + else: + requeue.add(nb) # cross: split at same m + return True + + def refine(self, marked): + work = set(int(c) for c in marked) + rounds = 0 + while work: + rounds += 1 + self.max_round_worklist = max(self.max_round_worklist, len(work)) + nxt = set() + for cid in list(work): # one communication round + if cid in self.g.cells: + self._try_bisect(self.owner[cid], cid, nxt) + work = nxt + if rounds > 1000: + raise RuntimeError("closure did not converge") + return rounds + + +# ---- confluence + conformity oracle ----------------------------------------- +def signature(nvb): + coords, tris, _, _ = nvb.arrays() + verts = np.array(sorted(map(tuple, np.round(coords, 9)))) + cents = np.array(sorted(map(tuple, np.round(coords[tris].mean(1), 9)))) + return verts, cents + + +def equal(a, b): + (va, ca), (vb, cb) = a, b + return (va.shape == vb.shape and ca.shape == cb.shape + and np.allclose(va, vb, atol=1e-8) and np.allclose(ca, cb, atol=1e-8)) + + +def serial_ref(coords, tris, pred, levels): + nvb = NVBMesh(coords, tris) + for _ in range(levels): + cen, h, cids = nvb.centroids_h() + m = cids[np.array([pred(c) for c in cen])] + if m.size == 0: + break + nvb.refine(set(int(c) for c in m)) + return nvb + + +def run_case(n, G, pred, owner_fn, levels, label): + ser = serial_ref(*structured_square(n), pred, levels) + dist = DistributedNVB(*structured_square(n), owner_fn) + rlog = [] + for _ in range(levels): + cen, h, cids = dist.g.centroids_h() + m = cids[np.array([pred(c) for c in cen])] + if m.size == 0: + break + rlog.append(dist.refine(set(int(c) for c in m))) + hang, over = dist.g.check_conforming() + ok_c = (hang, over) == (0, 0) + ok_e = equal(signature(ser), signature(dist.g)) + print(f"[{label}] n={n} G={G} lvl={levels}: serial {len(ser.cells)} | " + f"dist {len(dist.g.cells)} (conf {hang},{over}) | rounds {rlog} | " + f"CONFORMING={ok_c} EQUALS_SERIAL={ok_e}") + assert ok_c and ok_e + return max(rlog) if rlog else 0 + + +def main(): + diag = lambda c: abs((c[0] + c[1]) - 1.0) < 0.18 + diag_t = lambda c: abs((c[0] + c[1]) - 1.0) < 0.10 + band = lambda c: abs(c[0] - 0.5) < 0.16 + LR = lambda c: 0 if c[0] < 0.5 else 1 + quad = lambda c: (0 if c[0] < 0.5 else 1) + (0 if c[1] < 0.5 else 2) + stripes = lambda c: int(c[0] * 4) % 3 + + print("=== convergence / conformity / confluence across partitions ===") + mx = [run_case(8, 2, band, LR, 3, "band|LR2"), + run_case(8, 4, diag, quad, 3, "diag|quad4"), + run_case(8, 3, diag, stripes, 3, "diag|stripes3")] + print("\n=== bounded rounds vs mesh size (feature fixed, base refined) ===") + for n in (8, 16, 24, 32): + mx.append(run_case(n, 4, diag_t, quad, 4, f"diag|quad4|n{n}")) + print(f"\nMAX communication rounds over ALL cases: {max(mx)} " + f"(bounded, single-digit, independent of mesh size)") + print("PARALLEL-CLOSURE DE-RISK PASS.") + + +if __name__ == "__main__": + main() From 4e8772e59210ffcead2af696c8342d0bca19e077 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 21:39:24 +1000 Subject: [PATCH 31/52] =?UTF-8?q?docs(layer2):=20Route=20B=20implementatio?= =?UTF-8?q?n=20plan=20=E2=80=94=20feasibility=20established?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symbol audit (dyld_info on installed libpetsc 3.25) + reading PETSc's SBR transform establish that the native parallel NVB transform is a SELF-CONTAINED UW extension (no PETSc rebuild, no fork) and a small delta from SBR: - the parallel-hard machinery is all EXPORTED: DMPlexTransformRegister, DMPlexPointQueue*, DMLabelPropagate* (the cross-rank closure), generic plex/ label API. Only 5 small 2D geometry/orientation helpers are hidden, all short + locally reimplementable (SBR's triangle-split cone tables are static fns in the .c; barycentre placement ~12 lines). - NVB = SBR with the edge pick changed from longest-edge to newest-vertex; the newest vertex is identified by a per-vertex AGE label (base 0, midpoint=gen), cross-rank-consistent for free. All split geometry/closure/SF reused. - build wiring mirrors the existing _function extension (.pyx + .c vs libpetsc + private headers). Plan of record is validation-gated: infra gate (SBR-clone transform built+parallel) -> NVB edge rule (confluence vs serial) -> _adapt_nested np>1 dispatch -> acceptance (np>1 graded child, Poisson/SolCx FMG, co-partitioning). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 2068a947..9ee85d7a 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -309,6 +309,83 @@ construction**, which Route B's native `DMPlexTransform` inherits from PETSc (an which option 1's manual-SF path could now be built, the algorithm being proven). This is the milestone that says Route B is worth the C investment. +## Route B implementation plan (native transform) — feasibility established 2026-06-30 + +A symbol/feasibility audit against the installed custom PETSc (`petsc-3.25`, +`libpetsc.3.25.0.dylib`) shows Route B is a **self-contained UW extension — no PETSc +rebuild, no PETSc fork** — and is a *small delta* from the shipped SBR transform. + +**The key insight (NVB ≈ SBR + a different edge pick).** Reading PETSc's SBR +transform (`src/dm/impls/plex/transform/impls/refine/sbr/plexrefsbr.c`): the +topological surgery (`CellTransform`, `GetSubcellOrientation`, the `RT_*` refinement +types, `SBRGetTriangleSplit{Single,Double}`, barycentre coordinate placement) is +**pure simplex-bisection geometry — identical for NVB**. The parallel conforming +closure is **already generic and SF-correct**: `DMLabelPropagate{Begin,Push,End}` +over the point-SF (the loop that exchanges the `splitPoints` edge label between +ranks) *is* the cross-rank closure fixpoint the Python prototype validated. The +**only** NVB-specific change is the *edge selected to split* in `SetUp`: SBR picks +the geometric **longest** edge; NVB picks the **newest-vertex refinement edge**. + +**The newest-vertex rule with no stored bisection tree.** The newest vertex of a +triangle is identifiable by a per-vertex **age** label (base vertices = 0; each +midpoint = its creation generation). The refinement edge is the edge *opposite* the +unique highest-age vertex; when all three tie (a base cell) fall back to the +longest-edge seed. This is local, label-only, and **cross-rank-consistent for free** +(age = generation count is deterministic, so a shared midpoint gets the same age on +both ranks). The transform updates the age label on the midpoints it creates; the +Python driver carries it across generations. This reproduces the serial `NVBMesh` +exactly (longest-edge seed on base cells, newest-vertex thereafter). + +**Symbol audit — what an external extension can link (no rebuild).** Checked with +`dyld_info -exports` on the installed `libpetsc`: + +| needed | exported? | +|---|---| +| `DMPlexTransformRegister` | **yes** | +| `DMPlexPointQueue{Create,Enqueue,Dequeue,Empty,EmptyCollective,Destroy}` | **yes** | +| `DMLabelPropagate{Begin,Push,End}` (the parallel closure) | **yes** | +| `DMPlexTransformCellTransformIdentity`, `…GetSubcellOrientationIdentity` | **yes** | +| all generic plex/label API (`DMPlexGetCone/Support/TransitiveClosure`, `DMLabel*`, `DMGetPointSF`, `DMPlexTransformGet{DM,Active}`, …) | **yes** | +| `DMPlexTransformCellRefine_Regular` | hidden | +| `DMPlexTransformGetSubcellOrientation_Regular` | hidden | +| `DMPlexTransformMapCoordinatesBarycenter_Internal` (~12 lines) | hidden | +| `DMPlexTransformSetDimensions_Internal` (~20 lines) | hidden | +| `DMPolytopeTypeGetArrangement` (triangle: a fixed 6-orientation table) | hidden | + +So **the entire parallel-hard machinery (closure + SF + registration) is linkable**; +the only hidden symbols are five small **2D geometry/orientation** helpers, all +short and reimplementable locally (the `SBRGetTriangleSplit{Single,Double}` cone +tables are *static functions in the SBR `.c`* — copy them; the regular triangle +(1→4) / segment (1→2) cone tables are small fixed arrays; barycentre placement and +set-dimensions are a few lines each). + +**Build wiring.** UW already compiles Cython+C extensions against PETSc with a +shared `conf` (PETSc + petsc4py includes, `libpetsc` link) — e.g. the +`underworld3.function._function` extension pairs a `.pyx` with a hand-written +`petsc_tools.c`. Add `underworld3.utilities._nvb_transform` the same way: +`_nvb_transform.pyx` (imports register the `nvb` transform via +`DMPlexTransformRegister` and exposes a thin `nvb_refine`) + `nvb_transform.c` (the +`SetUp`/`CellTransform`/helpers, including `#include `). +`$PETSC_DIR/include` carries the private headers, so the compile needs no rebuilt +PETSc. + +**Plan of record (validation-gated):** +1. *Infra gate* — scaffold `_nvb_transform` that registers `nvb` and reproduces SBR + serially (copy SBR's `SetUp`/`CellTransform` + reimplement the 5 helpers), built + into UW with no PETSc rebuild; confirm `setType("nvb")` + `apply` works and + preserves the SF in parallel (np=2). Proves the whole build/registration/private- + header/parallel-SF stack. +2. *NVB rule* — replace the longest-edge pick in `SetUp` with the age-based + newest-vertex refinement edge (seed cells split their ref edge, closure + propagates ref edges); maintain the vertex-age label. Validate against the serial + `NVBMesh` (confluence: identical mesh) serially and at np>1. +3. *Integration* — `_adapt_nested(engine="nvb")` at np>1 dispatches to the transform + (via `adaptLabel` with `dm_plex_transform_type=nvb`, the same in-place path SBR + uses) instead of raising; the child stays co-partitioned (in-place transform), so + the existing parallel custom-P tail consumes it unchanged. +4. *Acceptance* — np>1 graded child: Poisson + SolCx FMG match GAMG; co-partitioning + invariant holds; result matches the serial NVB mesh. + ## Checkpointing NVB is deterministic given the **initial labelling + the marked-cell set per From 05c96611611fd305448a96bb6e68940b5b7d6475 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Tue, 30 Jun 2026 21:43:03 +1000 Subject: [PATCH 32/52] =?UTF-8?q?feat(layer2):=20Route=20B=20infra=20gate?= =?UTF-8?q?=20=E2=80=94=20native=20uwnvb=20DMPlexTransform=20registered=20?= =?UTF-8?q?+=20parallel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of the native parallel NVB transform: prove the whole Route B stack with no PETSc rebuild. New self-contained UW extension underworld3.utilities._nvb_transform (_nvb_transform.pyx registers the type on import + nvb_transform.c implements the transform), wired into setup.py like the existing _function extension (links libpetsc, uses PETSc private transform headers from $PETSC_DIR/include). nvb_transform.c registers 'uwnvb' via the exported DMPlexTransformRegister and sets the transform ops, reimplementing the two short non-exported helpers it needs (SetDimensions ~4 lines, MapCoordinatesBarycenter ~6 lines); celltransform is currently Identity (Stage 2 swaps in the newest-vertex bisection SetUp + cell transform, reusing the exported DMLabelPropagate cross-rank closure). VALIDATED: builds clean (no PETSc rebuild); import registers 'uwnvb'; setType('uwnvb')+setUp()+apply() work serially (8->8 identity); and at np=2 on a DISTRIBUTED DM the apply produces a valid distributed DM with a populated point-SF (9 shared points rank0) + global-vec roundtrip. Confirms registration-from-extension, private-header compile, petsc4py routing, and parallel SF preservation all work — the previously-unknown infrastructure for Route B. Underworld development team with AI support from Claude Code --- setup.py | 12 +++ src/underworld3/utilities/_nvb_transform.pyx | 18 ++++ src/underworld3/utilities/nvb_transform.c | 105 +++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 src/underworld3/utilities/_nvb_transform.pyx create mode 100644 src/underworld3/utilities/nvb_transform.c diff --git a/setup.py b/setup.py index 890a9f4d..a2827dcc 100644 --- a/setup.py +++ b/setup.py @@ -249,6 +249,18 @@ def configure(): extra_compile_args=extra_compile_args, **conf, ), + # Native NVB DMPlexTransform (Route B parallel graded refinement). Registers + # the "uwnvb" transform type into PETSc on import; uses PETSc private transform + # headers (in $PETSC_DIR/include) and links libpetsc — no PETSc rebuild. + Extension( + "underworld3.utilities._nvb_transform", + sources=[ + "src/underworld3/utilities/_nvb_transform.pyx", + "src/underworld3/utilities/nvb_transform.c", + ], + extra_compile_args=extra_compile_args, + **conf, + ), ] diff --git a/src/underworld3/utilities/_nvb_transform.pyx b/src/underworld3/utilities/_nvb_transform.pyx new file mode 100644 index 00000000..7afeaad0 --- /dev/null +++ b/src/underworld3/utilities/_nvb_transform.pyx @@ -0,0 +1,18 @@ +# cython: language_level=3 +"""Registers the native NVB DMPlexTransform ("uwnvb") into PETSc on import. + +Route B of docs/developer/design/NVB_GRADED_ADAPT.md. The C transform lives in +nvb_transform.c; this thin module just calls its registration entry once PETSc is +initialised (importing petsc4py.PETSc guarantees that). After import, +``PETSc.DMPlexTransform().setType("uwnvb")`` and the ``dm_plex_transform_type=uwnvb`` +option resolve to the NVB transform. +""" +from petsc4py import PETSc as _PETSc # ensures PetscInitialize has run + +cdef extern int UWNVBTransformRegister() + +cdef int _ierr = UWNVBTransformRegister() +if _ierr != 0: + raise RuntimeError("UWNVBTransformRegister failed with PETSc error %d" % _ierr) + +registered = True diff --git a/src/underworld3/utilities/nvb_transform.c b/src/underworld3/utilities/nvb_transform.c new file mode 100644 index 00000000..3f6eff00 --- /dev/null +++ b/src/underworld3/utilities/nvb_transform.c @@ -0,0 +1,105 @@ +/* + Newest-vertex bisection (NVB) as a native DMPlexTransform — UW extension. + + Route B of docs/developer/design/NVB_GRADED_ADAPT.md: a graded, parallel, + SF-preserving simplicial refinement transform registered into PETSc from UW + (no PETSc rebuild). NVB is a small delta from PETSc's SBR transform + (src/dm/impls/plex/transform/impls/refine/sbr/plexrefsbr.c): identical + bisection geometry + the same DMLabelPropagate cross-rank closure; the only + NVB-specific change is the edge chosen to split (newest-vertex refinement edge + via a per-vertex "age" label, vs SBR's geometric longest edge). + + STAGE 1 (this file, current): the INFRA GATE — register "uwnvb" and prove the + build / registration / private-header / parallel-SF stack with an identity + transform. The two PETSc helpers the real transform needs that are NOT exported + from libpetsc (SetDimensions, MapCoordinatesBarycenter) are reimplemented here + (both are a few lines); the closure (DMLabelPropagate*) and registration are + exported and linked directly. STAGE 2 will replace SetUp_UWNVB / the cell + transform with the newest-vertex bisection logic. +*/ +#include /*I "petscdmplextransform.h" I*/ +#include + +typedef struct { + DMLabel splitPoints; /* STAGE 2: edges (1) / triangles (2) to split */ +} DMPlexTransform_UWNVB; + +/* --- reimplemented (non-exported) helpers ------------------------------- */ +static PetscErrorCode DMPlexTransformSetDimensions_UWNVB(DMPlexTransform tr, DM dm, DM tdm) +{ + PetscInt dim, cdim; + + PetscFunctionBegin; + PetscCall(DMGetDimension(dm, &dim)); + PetscCall(DMSetDimension(tdm, dim)); + PetscCall(DMGetCoordinateDim(dm, &cdim)); + PetscCall(DMSetCoordinateDim(tdm, cdim)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* New vertices land at the barycentre of their source point's vertices — for an + edge that is exactly the midpoint (NVB's bisection vertex). */ +static PetscErrorCode DMPlexTransformMapCoordinates_UWNVB(DMPlexTransform tr, DMPolytopeType pct, DMPolytopeType ct, PetscInt p, PetscInt r, PetscInt Nv, PetscInt dE, const PetscScalar in[], PetscScalar out[]) +{ + PetscInt v, d; + + PetscFunctionBeginHot; + PetscCheck(ct == DM_POLYTOPE_POINT, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not for refined point type %s", DMPolytopeTypes[ct]); + for (d = 0; d < dE; ++d) out[d] = 0.0; + for (v = 0; v < Nv; ++v) + for (d = 0; d < dE; ++d) out[d] += in[v * dE + d]; + for (d = 0; d < dE; ++d) out[d] /= Nv; + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* --- transform ops ------------------------------------------------------ */ +static PetscErrorCode DMPlexTransformSetUp_UWNVB(DMPlexTransform tr) +{ + /* STAGE 1: identity — no split points. STAGE 2: build the newest-vertex + splitPoints label + run the DMLabelPropagate cross-rank closure (see the + SBR template + NVB_GRADED_ADAPT.md). */ + PetscFunctionBegin; + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformDestroy_UWNVB(DMPlexTransform tr) +{ + DMPlexTransform_UWNVB *nvb = (DMPlexTransform_UWNVB *)tr->data; + + PetscFunctionBegin; + PetscCall(DMLabelDestroy(&nvb->splitPoints)); + PetscCall(PetscFree(tr->data)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformInitialize_UWNVB(DMPlexTransform tr) +{ + PetscFunctionBegin; + tr->ops->setup = DMPlexTransformSetUp_UWNVB; + tr->ops->destroy = DMPlexTransformDestroy_UWNVB; + tr->ops->setdimensions = DMPlexTransformSetDimensions_UWNVB; + tr->ops->celltransform = DMPlexTransformCellTransformIdentity; /* STAGE 2: NVB bisection */ + tr->ops->getsubcellorientation = DMPlexTransformGetSubcellOrientationIdentity; + tr->ops->mapcoordinates = DMPlexTransformMapCoordinates_UWNVB; + PetscFunctionReturn(PETSC_SUCCESS); +} + +PETSC_EXTERN PetscErrorCode DMPlexTransformCreate_UWNVB(DMPlexTransform tr) +{ + DMPlexTransform_UWNVB *nvb; + + PetscFunctionBegin; + PetscValidHeaderSpecific(tr, DMPLEXTRANSFORM_CLASSID, 1); + PetscCall(PetscNew(&nvb)); + tr->data = nvb; + PetscCall(DMPlexTransformInitialize_UWNVB(tr)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Idempotent registration entry called from the Cython module on import. */ +PetscErrorCode UWNVBTransformRegister(void) +{ + PetscFunctionBegin; + PetscCall(DMPlexTransformRegister("uwnvb", DMPlexTransformCreate_UWNVB)); + PetscFunctionReturn(PETSC_SUCCESS); +} From 5cd9cebb6f757fd929225fc19186163de24b93e4 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 09:00:54 +1000 Subject: [PATCH 33/52] docs(layer2): settle + prove the Route B build model (headers + binary, no rebuild) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answer the build question definitively: the uwnvb transform extension compiles against PETSc headers (public + private petsc/private/dmplextransformimpl.h from $PETSC_DIR/include) and links the PETSc BINARY libpetsc — no PETSc source compile, no fork. Three tiers: (1) all parallel-critical API exported + linked (DMPlexTransformRegister, DMPlexPointQueue*, DMLabelPropagate* closure); (2) private header for the ops struct (same internal API PETSc's own transforms use, from outside the tree); (3) the few non-exported helpers reimplemented locally (tiny: SetDimensions, MapCoordinatesBarycenter + the regular tri-1to4/seg-1to2 cone tables). DMPolytopeTypeGetArrangement is a static inline in public petscdm.h -> usable by inclusion, not a reimplement. Proven by the Stage-1 gate (compiled, registered, np=2). Caveat recorded: tiers 2/3 couple to PETSc's pinned-3.25 internal ABI -> assert version at build, re-verify on upgrade. Exact source locations to copy/reimplement listed for a turnkey Stage-2 pass. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 9ee85d7a..36c405d7 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -369,6 +369,41 @@ shared `conf` (PETSc + petsc4py includes, `libpetsc` link) — e.g. the `$PETSC_DIR/include` carries the private headers, so the compile needs no rebuilt PETSc. +### Build model (settled + proven) — headers + binary, no PETSc rebuild + +The extension **compiles against PETSc headers and links the PETSc binary**; it does +not compile/rebuild any PETSc `.c` and does not fork PETSc. Three tiers: + +1. **Exported symbols → linked from `libpetsc`.** All parallel-critical API is + exported (verified `dyld_info -exports`): `DMPlexTransformRegister`, + `DMPlexPointQueue*`, `DMLabelPropagate{Begin,Push,End}` (the cross-rank closure), + `DMPlexTransformCellTransformIdentity`, `…GetSubcellOrientationIdentity`, and all + generic plex/label API. +2. **Private headers → `#include `** for the + `_p_DMPlexTransform` struct (`tr->ops->…`, `tr->data`, `tr->trType`). Ships in + `$PETSC_DIR/include`. This is how PETSc's own in-tree transforms are written — the + supported internal API, used from outside the tree. +3. **Non-exported helpers → reimplemented locally** (short, 2D): `SetDimensions` + (~4 lines), `MapCoordinatesBarycenter` (~6 lines) — both already in the Stage-1 + file — plus the regular **triangle 1→4** and **segment 1→2** cone/orientation + tables (from `plexrefregular.c`'s `DMPlexTransformCellRefine_Regular` / + `…GetSubcellOrientation_Regular`). NOTE: `DMPolytopeTypeGetArrangement` is a + **`static inline` in the public `petscdm.h`** — usable by inclusion, *not* a + reimplement (so the copied `SBRGetTriangleSplit{Single,Double}` work as-is). + +**Proven:** the Stage-1 infra gate did exactly this (private header + reimplemented +helpers + linked `libpetsc`) and compiled, registered `uwnvb`, and applied on a +distributed DM at np=2 with a valid point-SF. **Caveat to record + assert at build:** +tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build +(3.25); re-verify on any PETSc upgrade. + +**Source to copy/reimplement (exact locations):** +- `SetUp` + `CellTransform` + `GetSubcellOrientation` + `SBRGetTriangleSplit{Single,Double}` + + the `RT_*` enum: `…/transform/impls/refine/sbr/plexrefsbr.c` (already read in full). +- `CellRefine_Regular` tri(1→4)/seg(1→2) cone tables + `GetSubcellOrientation_Regular`: + `…/transform/impls/refine/regular/plexrefregular.c` (`DMPlexTransformCellRefine_Regular`, ~line 645+). +- `DMPolytopeTypeGetArrangement`: inline in `include/petscdm.h:609` (include, don't copy). + **Plan of record (validation-gated):** 1. *Infra gate* — scaffold `_nvb_transform` that registers `nvb` and reproduces SBR serially (copy SBR's `SetUp`/`CellTransform` + reimplement the 5 helpers), built From 8d7544a6b4e8dc1a6ad12fae699af4ecd66e1682 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 15:12:04 +1000 Subject: [PATCH 34/52] feat(layer2/nvb): native uwnvb DMPlexTransform reproduces refine_sbr (Route B Stage 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fills in the Stage-1 "uwnvb" transform (previously an identity gate) with a self-contained clone of PETSc's SBR longest-edge bisection, registered from UW with no PETSc rebuild. It reproduces refine_sbr byte-for-byte — identical triangulation (per-cell vertex geometry, not just counts) for every marking pattern including full uniform refinement — which exercises the entire native transform stack on the pinned PETSc 3.25: registration, the SetUp closure, DMLabelPropagate cross-rank propagation, cell transform, subcell orientation, and coordinate mapping. Depends only on: exported libpetsc symbols (DMPlexPointQueue*, DMLabelPropagate*, the Identity helpers, registration); static-inline header helpers (DMPolytopeTypeGetArrangement, ComposeOrientation, DMPlex_DistD_Internal); and four non-exported PETSc helpers reimplemented verbatim (SetDimensions, MapCoordinatesBarycenter, and the SEGMENT/TRIANGLE cases of the regular cell-refine and subcell-orientation) — all verified byte-identical to the 3.25.0 source. No PETSc rebuild, no fork. This is the SBR-equivalent base; the graded newest-vertex edge choice layers on top (a single-bisection, multi-pass transform — see the design note's Stage 2b finding for why a one-pass double/triple-split scheme cannot grade). - tests/test_0837_nvb_native_transform.py: uwnvb == refine_sbr equivalence (tier_b, skips cleanly without the custom-PETSc extension). - docs/developer/design/NVB_GRADED_ADAPT.md: Stage 2a done; Stage 2b finding and the single-bisection multi-pass redesign; FMG-hierarchy-index label reuse for checkpointing (to investigate next phase). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 76 ++- src/underworld3/utilities/nvb_transform.c | 621 +++++++++++++++++++++- tests/test_0837_nvb_native_transform.py | 95 ++++ 3 files changed, 751 insertions(+), 41 deletions(-) create mode 100644 tests/test_0837_nvb_native_transform.py diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 36c405d7..fd8bc096 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -405,21 +405,54 @@ tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build - `DMPolytopeTypeGetArrangement`: inline in `include/petscdm.h:609` (include, don't copy). **Plan of record (validation-gated):** -1. *Infra gate* — scaffold `_nvb_transform` that registers `nvb` and reproduces SBR - serially (copy SBR's `SetUp`/`CellTransform` + reimplement the 5 helpers), built - into UW with no PETSc rebuild; confirm `setType("nvb")` + `apply` works and - preserves the SF in parallel (np=2). Proves the whole build/registration/private- - header/parallel-SF stack. -2. *NVB rule* — replace the longest-edge pick in `SetUp` with the age-based - newest-vertex refinement edge (seed cells split their ref edge, closure - propagates ref edges); maintain the vertex-age label. Validate against the serial - `NVBMesh` (confluence: identical mesh) serially and at np>1. -3. *Integration* — `_adapt_nested(engine="nvb")` at np>1 dispatches to the transform - (via `adaptLabel` with `dm_plex_transform_type=nvb`, the same in-place path SBR - uses) instead of raising; the child stays co-partitioned (in-place transform), so - the existing parallel custom-P tail consumes it unchanged. -4. *Acceptance* — np>1 graded child: Poisson + SolCx FMG match GAMG; co-partitioning - invariant holds; result matches the serial NVB mesh. +1. *Infra gate* — **DONE (2026-06-30).** scaffold `_nvb_transform` that registers + `uwnvb`; identity transform; proved build/registration/private-header/parallel-SF + stack (np=2, valid point-SF). +2. *SBR-equivalent base (Stage 2a)* — **DONE (2026-07-01).** self-contained clone of + SBR (SBR's `SetUp`/`CellTransform`/`GetSubcellOrientation`/`SBRGetTriangleSplit{,}` + + the 4 reimplemented helpers) reproduces `refine_sbr` **byte-for-byte** — identical + triangulation for every marking pattern incl. full uniform refine. `tests/ + test_0837_nvb_native_transform.py` (tier_b). Longest-edge edge pick; the newest- + vertex choice layers on top. +3. *Newest-vertex grading (Stage 2b)* — **IN PROGRESS; redesign required.** See + "Stage 2b finding" below. +4. *Integration + acceptance* — `_adapt_nested(engine="nvb")` at np>1 via + `adaptLabel(dm_plex_transform_type=uwnvb)` (in-place, co-partitioned); Poisson + + SolCx FMG match GAMG; result matches the serial NVB mesh (properties). + +### Stage 2b finding (2026-07-01): grading needs single-bisection, multi-pass + +Two attempts to add the newest-vertex edge choice on top of the Stage-2a base both +**failed to grade** (a mark deep in a refined patch drained the whole patch, exactly +like SBR): + +- **Age-derived edge (opposite highest-age vertex).** `age = max(endpoint ages)+1` + produces **ties** — every vertex born in the same pass shares an age — so after any + uniform refinement "the newest vertex" is ambiguous and the longest-edge tie-break + silently reverts to longest-edge bisection. Pure vertex-age cannot reconstruct the + true NVB refinement edge; this is why the serial `NVBMesh` tracks `(peak,b0,b1)` + **explicitly** per cell. +- **Explicit per-cell refinement-edge slot** (a cone-slot 0/1/2 label, stored at child + creation, read verbatim thereafter). Mechanically sound (slot populated, survives + clone, read by `SetUp`; Stage 2a still exact) but **still drains.** The deeper + obstacle: a single `DMPlexTransform` pass must stay **conforming**, so a cell caught + in the closure across a *non*-refinement edge is forced into a **double/triple + (green/blue) split** in the same pass. These one-pass multi-edge splits do **not** + reproduce the compatible refinement-edge structure that `NVBMesh` builds via + **sequential single bisections** (bisect the ref edge, *replace* the cell, recurse) — + so the ref edges come out incompatible and the closure cascades. + +**Redesign (the sound path): single-bisection-only, multi-pass.** Each transform pass +splits **only compatible refinement edges** — an edge that is the refinement edge of +*every* cell in its support. That is always a clean conforming bisection (both cells +single-split, share the midpoint), with a **trivial** child slot (each child's peak is +the new midpoint, so its ref edge is the edge opposite the midpoint). A driver loop +iterates passes, pulling in the neighbour across an incompatible ref edge (it must +refine its own ref edge first), until the marked set is satisfied — exactly +`NVBMesh`'s recursion, batched over the DM. This keeps every intermediate mesh +conforming, needs no green/blue child tables, and preserves co-partitioning (each pass +is an in-place transform). The Stage-2a clone stays the base; Stage 2b changes the +`SetUp` to mark only compatible ref edges + adds the driver loop. ## Checkpointing @@ -429,6 +462,19 @@ consistent with the marker-sidecar scheme already designed for SBR. The coordinate-built custom-P sidesteps the canonical-numbering fragility (same as SBR). +**FMG-hierarchy-index label — reuse for checkpointing (L.M., 2026-07-01, TODO next +phase).** The custom-P / FMG round-trip through checkpoints already carries a +**mesh-hierarchy-index label** (which coarse level each point belongs to) to +reconstruct the transfer hierarchy on reload. When implementing the single- +bisection multi-pass Stage 2b, investigate: **(a)** whether that existing hierarchy +label can be *reused* to encode the NVB level/marker replay (one label serving both +FMG-transfer reconstruction and adapt replay), and **(b)** whether the *new* NVB +structures (per-vertex generation + the compatible-ref-edge state, or the +per-pass marked-edge sets) are the natural thing to *checkpoint the FMG itself* +against — i.e. store the NVB replay data and rebuild the whole FMG tail from it, +rather than storing meshes. Decide before finalising the checkpoint format so the +two mechanisms share one label rather than duplicating. + ## 3D (tetrahedra) NVB is the right engine in 3D too, and the design generalises with no change of diff --git a/src/underworld3/utilities/nvb_transform.c b/src/underworld3/utilities/nvb_transform.c index 3f6eff00..4af7131e 100644 --- a/src/underworld3/utilities/nvb_transform.c +++ b/src/underworld3/utilities/nvb_transform.c @@ -7,25 +7,47 @@ (src/dm/impls/plex/transform/impls/refine/sbr/plexrefsbr.c): identical bisection geometry + the same DMLabelPropagate cross-rank closure; the only NVB-specific change is the edge chosen to split (newest-vertex refinement edge - via a per-vertex "age" label, vs SBR's geometric longest edge). - - STAGE 1 (this file, current): the INFRA GATE — register "uwnvb" and prove the - build / registration / private-header / parallel-SF stack with an identity - transform. The two PETSc helpers the real transform needs that are NOT exported - from libpetsc (SetDimensions, MapCoordinatesBarycenter) are reimplemented here - (both are a few lines); the closure (DMLabelPropagate*) and registration are - exported and linked directly. STAGE 2 will replace SetUp_UWNVB / the cell - transform with the newest-vertex bisection logic. + vs SBR's geometric longest edge). + + STAGE 2a (this file): a SELF-CONTAINED clone of SBR, registered as "uwnvb". It + reproduces `refine_sbr` byte-for-byte on a serial mesh (validated: identical + triangulation for every marking pattern incl. full uniform refine) and matches + it under the parallel closure. It depends only on: + - exported libpetsc symbols (DMPlexPointQueue*, DMLabelPropagate*, the + Identity cell-transform / orientation helpers, registration, plus the + ordinary DMPlex / DMLabel / PetscSection API); + - static-inline header helpers (DMPolytopeTypeGetArrangement, + DMPolytopeTypeComposeOrientation, DMPlex_DistD_Internal); + - four PETSc helpers that are NOT exported from libpetsc and are therefore + reimplemented verbatim below (SetDimensions, MapCoordinatesBarycenter, and + the SEGMENT+TRIANGLE cases of the regular cell-refine / subcell-orientation + — the only polytopes SBR uses in 2D). + + The newest-vertex edge choice (Stage 2b) is a graded, multi-pass single- + bisection transform layered on this SBR-equivalent base; see the design note. + + Reference (read-only) copy of the PETSc 3.25.0 sources this clones: + src/dm/impls/plex/transform/impls/refine/sbr/plexrefsbr.c + src/dm/impls/plex/transform/impls/refine/regular/plexrefregular.c + src/dm/impls/plex/transform/interface/plextransform.c (the two _Internal helpers) */ #include /*I "petscdmplextransform.h" I*/ #include +/* Same payload as PETSc's DMPlexRefine_SBR (private, so we declare our own). */ typedef struct { - DMLabel splitPoints; /* STAGE 2: edges (1) / triangles (2) to split */ -} DMPlexTransform_UWNVB; + DMLabel splitPoints; /* edges to bisect (1) and triangles to divide (2) */ + PetscSection secEdgeLen; /* section for the edge-length field */ + PetscReal *edgeLen; /* lazily-computed edge lengths */ +} DMPlexRefine_UWNVB; -/* --- reimplemented (non-exported) helpers ------------------------------- */ -static PetscErrorCode DMPlexTransformSetDimensions_UWNVB(DMPlexTransform tr, DM dm, DM tdm) +/* ======================================================================== */ +/* Reimplemented PETSc helpers that libpetsc does not export. */ +/* Byte-identical to the 3.25.0 originals (verified against source). */ +/* ======================================================================== */ + +/* == DMPlexTransformSetDimensions_Internal == */ +static PetscErrorCode UWNVBSetDimensions(DMPlexTransform tr, DM dm, DM tdm) { PetscInt dim, cdim; @@ -37,9 +59,8 @@ static PetscErrorCode DMPlexTransformSetDimensions_UWNVB(DMPlexTransform tr, DM PetscFunctionReturn(PETSC_SUCCESS); } -/* New vertices land at the barycentre of their source point's vertices — for an - edge that is exactly the midpoint (NVB's bisection vertex). */ -static PetscErrorCode DMPlexTransformMapCoordinates_UWNVB(DMPlexTransform tr, DMPolytopeType pct, DMPolytopeType ct, PetscInt p, PetscInt r, PetscInt Nv, PetscInt dE, const PetscScalar in[], PetscScalar out[]) +/* == DMPlexTransformMapCoordinatesBarycenter_Internal == */ +static PetscErrorCode UWNVBMapCoordinatesBarycenter(DMPlexTransform tr, DMPolytopeType pct, DMPolytopeType ct, PetscInt p, PetscInt r, PetscInt Nv, PetscInt dE, const PetscScalar in[], PetscScalar out[]) { PetscInt v, d; @@ -52,21 +73,567 @@ static PetscErrorCode DMPlexTransformMapCoordinates_UWNVB(DMPlexTransform tr, DM PetscFunctionReturn(PETSC_SUCCESS); } -/* --- transform ops ------------------------------------------------------ */ +/* == DMPlexTransformCellRefine_Regular, SEGMENT + TRIANGLE cases only == + (regular 1->2 edge split and 1->4 triangle split — the uniform refinement + SBR delegates to for fully-split cells and edges). */ +static PetscErrorCode UWNVBRegularCellRefine(DMPlexTransform tr, DMPolytopeType source, PetscInt p, PetscInt *rt, PetscInt *Nt, DMPolytopeType *target[], PetscInt *size[], PetscInt *cone[], PetscInt *ornt[]) +{ + /* Split an edge with a new midpoint vertex, making two new edges: + 0--0--0--1--1 */ + static DMPolytopeType segT[] = {DM_POLYTOPE_POINT, DM_POLYTOPE_SEGMENT}; + static PetscInt segS[] = {1, 2}; + static PetscInt segC[] = {DM_POLYTOPE_POINT, 1, 0, 0, DM_POLYTOPE_POINT, 0, 0, DM_POLYTOPE_POINT, 0, 0, DM_POLYTOPE_POINT, 1, 1, 0}; + static PetscInt segO[] = {0, 0, 0, 0}; + /* Add 3 edges inside every triangle, making 4 new triangles. */ + static DMPolytopeType triT[] = {DM_POLYTOPE_SEGMENT, DM_POLYTOPE_TRIANGLE}; + static PetscInt triS[] = {3, 4}; + static PetscInt triC[] = {DM_POLYTOPE_POINT, 1, 0, 0, DM_POLYTOPE_POINT, 1, 1, 0, DM_POLYTOPE_POINT, 1, 1, 0, DM_POLYTOPE_POINT, 1, 2, 0, DM_POLYTOPE_POINT, 1, 2, 0, DM_POLYTOPE_POINT, 1, 0, 0, DM_POLYTOPE_SEGMENT, 1, 0, 0, DM_POLYTOPE_SEGMENT, 0, 2, DM_POLYTOPE_SEGMENT, 1, 2, 1, DM_POLYTOPE_SEGMENT, 1, 0, 1, DM_POLYTOPE_SEGMENT, 1, 1, 0, DM_POLYTOPE_SEGMENT, 0, 0, DM_POLYTOPE_SEGMENT, 0, 1, DM_POLYTOPE_SEGMENT, 1, 1, 1, DM_POLYTOPE_SEGMENT, 1, 2, 0, DM_POLYTOPE_SEGMENT, 0, 0, DM_POLYTOPE_SEGMENT, 0, 1, DM_POLYTOPE_SEGMENT, 0, 2}; + static PetscInt triO[] = {0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, -1, 0, 0, 0, 0, 0}; + + PetscFunctionBegin; + if (rt) *rt = 0; + switch (source) { + case DM_POLYTOPE_SEGMENT: + *Nt = 2; + *target = segT; + *size = segS; + *cone = segC; + *ornt = segO; + break; + case DM_POLYTOPE_TRIANGLE: + *Nt = 2; + *target = triT; + *size = triS; + *cone = triC; + *ornt = triO; + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "UWNVB regular refine only handles SEGMENT/TRIANGLE, not %s", DMPolytopeTypes[source]); + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* == DMPlexTransformGetSubcellOrientation_Regular, SEGMENT + TRIANGLE cases == */ +static PetscErrorCode UWNVBRegularGetSubcellOrientation(DMPlexTransform tr, DMPolytopeType sct, PetscInt sp, PetscInt so, DMPolytopeType tct, PetscInt r, PetscInt o, PetscInt *rnew, PetscInt *onew) +{ + static PetscInt seg_seg[] = {1, -1, 0, -1, 0, 0, 1, 0}; + static PetscInt tri_seg[] = {2, -1, 1, -1, 0, -1, 1, -1, 0, -1, 2, -1, 0, -1, 2, -1, 1, -1, 0, 0, 1, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 1, 0}; + static PetscInt tri_tri[] = {1, -3, 0, -3, 2, -3, 3, -2, 0, -2, 2, -2, 1, -2, 3, -1, 2, -1, 1, -1, 0, -1, 3, -3, 0, 0, 1, 0, 2, 0, 3, 0, 1, 1, 2, 1, 0, 1, 3, 1, 2, 2, 0, 2, 1, 2, 3, 2}; + + PetscFunctionBeginHot; + *rnew = r; + *onew = o; + if (!so) PetscFunctionReturn(PETSC_SUCCESS); + switch (sct) { + case DM_POLYTOPE_POINT: + break; + case DM_POLYTOPE_SEGMENT: + switch (tct) { + case DM_POLYTOPE_POINT: + break; + case DM_POLYTOPE_SEGMENT: + *rnew = seg_seg[(so + 1) * 4 + r * 2]; + *onew = DMPolytopeTypeComposeOrientation(tct, o, seg_seg[(so + 1) * 4 + r * 2 + 1]); + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell type %s is not produced by %s", DMPolytopeTypes[tct], DMPolytopeTypes[sct]); + } + break; + case DM_POLYTOPE_TRIANGLE: + switch (tct) { + case DM_POLYTOPE_SEGMENT: + *rnew = tri_seg[(so + 3) * 6 + r * 2]; + *onew = DMPolytopeTypeComposeOrientation(tct, o, tri_seg[(so + 3) * 6 + r * 2 + 1]); + break; + case DM_POLYTOPE_TRIANGLE: + *rnew = tri_tri[(so + 3) * 8 + r * 2]; + *onew = DMPolytopeTypeComposeOrientation(tct, o, tri_tri[(so + 3) * 8 + r * 2 + 1]); + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell type %s is not produced by %s", DMPolytopeTypes[tct], DMPolytopeTypes[sct]); + } + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "UWNVB regular orientation only handles SEGMENT/TRIANGLE, not %s", DMPolytopeTypes[sct]); + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* ======================================================================== */ +/* SBR algorithm, cloned verbatim and renamed UWNVB. */ +/* ======================================================================== */ + +static PetscErrorCode UWNVBGetEdgeLen(DMPlexTransform tr, PetscInt edge, PetscReal *len) +{ + DMPlexRefine_UWNVB *nvb = (DMPlexRefine_UWNVB *)tr->data; + DM dm; + PetscInt off; + + PetscFunctionBeginHot; + PetscCall(DMPlexTransformGetDM(tr, &dm)); + PetscCall(PetscSectionGetOffset(nvb->secEdgeLen, edge, &off)); + if (nvb->edgeLen[off] <= 0.0) { + DM cdm; + Vec coordsLocal; + const PetscScalar *coords; + const PetscInt *cone; + PetscScalar *cA, *cB; + PetscInt coneSize, cdim; + + PetscCall(DMGetCoordinateDM(dm, &cdm)); + PetscCall(DMPlexGetCone(dm, edge, &cone)); + PetscCall(DMPlexGetConeSize(dm, edge, &coneSize)); + PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Edge %" PetscInt_FMT " cone size must be 2, not %" PetscInt_FMT, edge, coneSize); + PetscCall(DMGetCoordinateDim(dm, &cdim)); + PetscCall(DMGetCoordinatesLocalNoncollective(dm, &coordsLocal)); + PetscCall(VecGetArrayRead(coordsLocal, &coords)); + PetscCall(DMPlexPointLocalRead(cdm, cone[0], coords, &cA)); + PetscCall(DMPlexPointLocalRead(cdm, cone[1], coords, &cB)); + nvb->edgeLen[off] = DMPlex_DistD_Internal(cdim, cA, cB); + PetscCall(VecRestoreArrayRead(coordsLocal, &coords)); + } + *len = nvb->edgeLen[off]; + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Mark local edges that should be split. + STAGE 2a: pick the longest edge of each marked cell (== SBR). STAGE 2b picks + the newest-vertex refinement edge instead. */ +static PetscErrorCode UWNVBSplitLocalEdges(DMPlexTransform tr, DMPlexPointQueue queue) +{ + DMPlexRefine_UWNVB *nvb = (DMPlexRefine_UWNVB *)tr->data; + DM dm; + + PetscFunctionBegin; + PetscCall(DMPlexTransformGetDM(tr, &dm)); + while (!DMPlexPointQueueEmpty(queue)) { + PetscInt p = -1; + const PetscInt *support; + PetscInt supportSize, s; + + PetscCall(DMPlexPointQueueDequeue(queue, &p)); + PetscCall(DMPlexGetSupport(dm, p, &support)); + PetscCall(DMPlexGetSupportSize(dm, p, &supportSize)); + for (s = 0; s < supportSize; ++s) { + const PetscInt cell = support[s]; + const PetscInt *cone; + PetscInt coneSize, c; + PetscInt cval, eval, maxedge; + PetscReal len, maxlen; + + PetscCall(DMLabelGetValue(nvb->splitPoints, cell, &cval)); + if (cval == 2) continue; + PetscCall(DMPlexGetCone(dm, cell, &cone)); + PetscCall(DMPlexGetConeSize(dm, cell, &coneSize)); + PetscCall(UWNVBGetEdgeLen(tr, cone[0], &maxlen)); + maxedge = cone[0]; + for (c = 1; c < coneSize; ++c) { + PetscCall(UWNVBGetEdgeLen(tr, cone[c], &len)); + if (len > maxlen) { + maxlen = len; + maxedge = cone[c]; + } + } + PetscCall(DMLabelGetValue(nvb->splitPoints, maxedge, &eval)); + if (eval != 1) { + PetscCall(DMLabelSetValue(nvb->splitPoints, maxedge, 1)); + PetscCall(DMPlexPointQueueEnqueue(queue, maxedge)); + } + PetscCall(DMLabelSetValue(nvb->splitPoints, cell, 2)); + } + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode UWNVBSplitPoint(PETSC_UNUSED DMLabel label, PetscInt p, PETSC_UNUSED PetscInt val, void *ctx) +{ + DMPlexPointQueue queue = (DMPlexPointQueue)ctx; + + PetscFunctionBegin; + PetscCall(DMPlexPointQueueEnqueue(queue, p)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* + The 'splitPoints' label marks mesh points to be divided: edges with 1, + triangles with 2. The refinement type is then: + + vertex: 0 (RT_VERTEX) + edge unsplit: 1 (RT_EDGE) + edge split: 2 (RT_EDGE_SPLIT) + triangle unsplit: 3 (RT_TRIANGLE) + triangle split all edges: 4 (RT_TRIANGLE_SPLIT) + triangle split edges 0 1: 5 (RT_TRIANGLE_SPLIT_01) ... etc +*/ +typedef enum { + RT_VERTEX, + RT_EDGE, + RT_EDGE_SPLIT, + RT_TRIANGLE, + RT_TRIANGLE_SPLIT, + RT_TRIANGLE_SPLIT_01, + RT_TRIANGLE_SPLIT_10, + RT_TRIANGLE_SPLIT_12, + RT_TRIANGLE_SPLIT_21, + RT_TRIANGLE_SPLIT_20, + RT_TRIANGLE_SPLIT_02, + RT_TRIANGLE_SPLIT_0, + RT_TRIANGLE_SPLIT_1, + RT_TRIANGLE_SPLIT_2 +} RefinementType; + static PetscErrorCode DMPlexTransformSetUp_UWNVB(DMPlexTransform tr) { - /* STAGE 1: identity — no split points. STAGE 2: build the newest-vertex - splitPoints label + run the DMLabelPropagate cross-rank closure (see the - SBR template + NVB_GRADED_ADAPT.md). */ + DMPlexRefine_UWNVB *nvb = (DMPlexRefine_UWNVB *)tr->data; + DM dm; + DMLabel active; + PetscSF pointSF; + DMPlexPointQueue queue = NULL; + IS refineIS; + const PetscInt *refineCells; + PetscInt pStart, pEnd, p, eStart, eEnd, e, edgeLenSize, Nc, c; + PetscBool empty; + PetscFunctionBegin; + PetscCall(DMPlexTransformGetDM(tr, &dm)); + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Split Points", &nvb->splitPoints)); + /* Create edge lengths */ + PetscCall(DMGetCoordinatesLocalSetUp(dm)); + PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd)); + PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &nvb->secEdgeLen)); + PetscCall(PetscSectionSetChart(nvb->secEdgeLen, eStart, eEnd)); + for (e = eStart; e < eEnd; ++e) PetscCall(PetscSectionSetDof(nvb->secEdgeLen, e, 1)); + PetscCall(PetscSectionSetUp(nvb->secEdgeLen)); + PetscCall(PetscSectionGetStorageSize(nvb->secEdgeLen, &edgeLenSize)); + PetscCall(PetscCalloc1(edgeLenSize, &nvb->edgeLen)); + /* Add edges of cells that are marked for refinement to edge queue */ + PetscCall(DMPlexTransformGetActive(tr, &active)); + PetscCheck(active, PetscObjectComm((PetscObject)tr), PETSC_ERR_ARG_WRONGSTATE, "DMPlexTransform must have an adaptation label in order to use UWNVB algorithm"); + PetscCall(DMPlexPointQueueCreate(1024, &queue)); + PetscCall(DMLabelGetStratumIS(active, DM_ADAPT_REFINE, &refineIS)); + PetscCall(DMLabelGetStratumSize(active, DM_ADAPT_REFINE, &Nc)); + if (refineIS) PetscCall(ISGetIndices(refineIS, &refineCells)); + for (c = 0; c < Nc; ++c) { + const PetscInt cell = refineCells[c]; + PetscInt depth; + + PetscCall(DMPlexGetPointDepth(dm, cell, &depth)); + if (depth == 1) { + PetscCall(DMLabelSetValue(nvb->splitPoints, cell, 1)); + PetscCall(DMPlexPointQueueEnqueue(queue, cell)); + } else { + PetscInt *closure = NULL; + PetscInt Ncl, cl; + + PetscCall(DMLabelSetValue(nvb->splitPoints, cell, depth)); + PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &Ncl, &closure)); + for (cl = 0; cl < Ncl; cl += 2) { + const PetscInt edge = closure[cl]; + + if (edge >= eStart && edge < eEnd) { + PetscCall(DMLabelSetValue(nvb->splitPoints, edge, 1)); + PetscCall(DMPlexPointQueueEnqueue(queue, edge)); + } + } + PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &Ncl, &closure)); + } + } + if (refineIS) PetscCall(ISRestoreIndices(refineIS, &refineCells)); + PetscCall(ISDestroy(&refineIS)); + /* Setup communication */ + PetscCall(DMGetPointSF(dm, &pointSF)); + PetscCall(DMLabelPropagateBegin(nvb->splitPoints, pointSF)); + /* While edge queue is not empty (collective): split locally, then push the + newly-marked edges across the point SF, repeat until globally empty. */ + PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty)); + while (!empty) { + PetscCall(UWNVBSplitLocalEdges(tr, queue)); + PetscCall(DMLabelPropagatePush(nvb->splitPoints, pointSF, UWNVBSplitPoint, queue)); + PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty)); + } + PetscCall(DMLabelPropagateEnd(nvb->splitPoints, pointSF)); + /* Calculate refineType for each cell */ + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Refine Type", &tr->trType)); + PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); + for (p = pStart; p < pEnd; ++p) { + DMLabel trType = tr->trType; + DMPolytopeType ct; + PetscInt val; + + PetscCall(DMPlexGetCellType(dm, p, &ct)); + switch (ct) { + case DM_POLYTOPE_POINT: + PetscCall(DMLabelSetValue(trType, p, RT_VERTEX)); + break; + case DM_POLYTOPE_SEGMENT: + PetscCall(DMLabelGetValue(nvb->splitPoints, p, &val)); + if (val == 1) PetscCall(DMLabelSetValue(trType, p, RT_EDGE_SPLIT)); + else PetscCall(DMLabelSetValue(trType, p, RT_EDGE)); + break; + case DM_POLYTOPE_TRIANGLE: + PetscCall(DMLabelGetValue(nvb->splitPoints, p, &val)); + if (val == 2) { + const PetscInt *cone; + PetscReal lens[3]; + PetscInt vals[3], i; + + PetscCall(DMPlexGetCone(dm, p, &cone)); + for (i = 0; i < 3; ++i) { + PetscCall(DMLabelGetValue(nvb->splitPoints, cone[i], &vals[i])); + vals[i] = vals[i] < 0 ? 0 : vals[i]; + PetscCall(UWNVBGetEdgeLen(tr, cone[i], &lens[i])); + } + if (vals[0] && vals[1] && vals[2]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT)); + else if (vals[0] && vals[1]) PetscCall(DMLabelSetValue(trType, p, lens[0] > lens[1] ? RT_TRIANGLE_SPLIT_01 : RT_TRIANGLE_SPLIT_10)); + else if (vals[1] && vals[2]) PetscCall(DMLabelSetValue(trType, p, lens[1] > lens[2] ? RT_TRIANGLE_SPLIT_12 : RT_TRIANGLE_SPLIT_21)); + else if (vals[2] && vals[0]) PetscCall(DMLabelSetValue(trType, p, lens[2] > lens[0] ? RT_TRIANGLE_SPLIT_20 : RT_TRIANGLE_SPLIT_02)); + else if (vals[0]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_0)); + else if (vals[1]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_1)); + else if (vals[2]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_2)); + else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " does not fit any refinement type (%" PetscInt_FMT ", %" PetscInt_FMT ", %" PetscInt_FMT ")", p, vals[0], vals[1], vals[2]); + } else PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE)); + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot handle points of type %s", DMPolytopeTypes[ct]); + } + PetscCall(DMLabelGetValue(nvb->splitPoints, p, &val)); + } + /* Cleanup */ + PetscCall(DMPlexPointQueueDestroy(&queue)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformGetSubcellOrientation_UWNVB(DMPlexTransform tr, DMPolytopeType sct, PetscInt sp, PetscInt so, DMPolytopeType tct, PetscInt r, PetscInt o, PetscInt *rnew, PetscInt *onew) +{ + PetscInt rt; + + PetscFunctionBeginHot; + PetscCall(DMLabelGetValue(tr->trType, sp, &rt)); + *rnew = r; + *onew = o; + switch (rt) { + case RT_TRIANGLE_SPLIT_01: + case RT_TRIANGLE_SPLIT_10: + case RT_TRIANGLE_SPLIT_12: + case RT_TRIANGLE_SPLIT_21: + case RT_TRIANGLE_SPLIT_20: + case RT_TRIANGLE_SPLIT_02: + break; + case RT_TRIANGLE_SPLIT_0: + case RT_TRIANGLE_SPLIT_1: + case RT_TRIANGLE_SPLIT_2: + switch (tct) { + case DM_POLYTOPE_SEGMENT: + break; + case DM_POLYTOPE_TRIANGLE: + *onew = so < 0 ? -(o + 1) : o; + *rnew = so < 0 ? (r + 1) % 2 : r; + break; + default: + break; + } + break; + case RT_EDGE_SPLIT: + case RT_TRIANGLE_SPLIT: + PetscCall(UWNVBRegularGetSubcellOrientation(tr, sct, sp, so, tct, r, o, rnew, onew)); + break; + default: + PetscCall(DMPlexTransformGetSubcellOrientationIdentity(tr, sct, sp, so, tct, r, o, rnew, onew)); + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Add 1 edge inside this triangle, making 2 new triangles (single bisection). */ +static PetscErrorCode UWNVBGetTriangleSplitSingle(PetscInt o, PetscInt *Nt, DMPolytopeType *target[], PetscInt *size[], PetscInt *cone[], PetscInt *ornt[]) +{ + const PetscInt *arr = DMPolytopeTypeGetArrangement(DM_POLYTOPE_TRIANGLE, o); + static DMPolytopeType triT1[] = {DM_POLYTOPE_SEGMENT, DM_POLYTOPE_TRIANGLE}; + static PetscInt triS1[] = {1, 2}; + static PetscInt triC1[] = {DM_POLYTOPE_POINT, 2, 0, 0, 0, DM_POLYTOPE_POINT, 1, 1, 0, DM_POLYTOPE_SEGMENT, 1, 0, 0, DM_POLYTOPE_SEGMENT, 1, 1, 0, DM_POLYTOPE_SEGMENT, 0, 0, DM_POLYTOPE_SEGMENT, 1, 1, 1, DM_POLYTOPE_SEGMENT, 1, 2, 0, + DM_POLYTOPE_SEGMENT, 0, 0}; + static PetscInt triO1[] = {0, 0, 0, 0, -1, 0, 0, 0}; + + PetscFunctionBeginHot; + /* To get the other divisions, we reorient the triangle */ + triC1[2] = arr[0 * 2]; + triC1[7] = arr[1 * 2]; + triC1[11] = arr[0 * 2]; + triC1[15] = arr[1 * 2]; + triC1[22] = arr[1 * 2]; + triC1[26] = arr[2 * 2]; + *Nt = 2; + *target = triT1; + *size = triS1; + *cone = triC1; + *ornt = triO1; + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Add 2 edges inside this triangle, making 3 new triangles (double bisection). */ +static PetscErrorCode UWNVBGetTriangleSplitDouble(PetscInt o, PetscInt *Nt, DMPolytopeType *target[], PetscInt *size[], PetscInt *cone[], PetscInt *ornt[]) +{ + PetscInt e0, e1; + const PetscInt *arr = DMPolytopeTypeGetArrangement(DM_POLYTOPE_TRIANGLE, o); + static DMPolytopeType triT2[] = {DM_POLYTOPE_SEGMENT, DM_POLYTOPE_TRIANGLE}; + static PetscInt triS2[] = {2, 3}; + static PetscInt triC2[] = {DM_POLYTOPE_POINT, 2, 0, 0, 0, DM_POLYTOPE_POINT, 1, 1, 0, DM_POLYTOPE_POINT, 1, 1, 0, DM_POLYTOPE_POINT, 1, 2, 0, DM_POLYTOPE_SEGMENT, 1, 0, 0, DM_POLYTOPE_SEGMENT, 1, 1, 0, DM_POLYTOPE_SEGMENT, 0, 0, DM_POLYTOPE_SEGMENT, 1, 1, 1, DM_POLYTOPE_SEGMENT, 1, 2, 0, DM_POLYTOPE_SEGMENT, 0, 1, DM_POLYTOPE_SEGMENT, 1, 2, 1, DM_POLYTOPE_SEGMENT, 0, 0, DM_POLYTOPE_SEGMENT, 0, 1}; + static PetscInt triO2[] = {0, 0, 0, 0, 0, 0, -1, 0, 0, -1, 0, 0, 0}; + + PetscFunctionBeginHot; + /* To get the other divisions, we reorient the triangle */ + triC2[2] = arr[0 * 2]; + triC2[3] = arr[0 * 2 + 1] ? 1 : 0; + triC2[7] = arr[1 * 2]; + triC2[11] = arr[1 * 2]; + triC2[15] = arr[2 * 2]; + /* Swap the first two edges if the triangle is reversed */ + e0 = o < 0 ? 23 : 19; + e1 = o < 0 ? 19 : 23; + triC2[e0] = arr[0 * 2]; + triC2[e0 + 1] = 0; + triC2[e1] = arr[1 * 2]; + triC2[e1 + 1] = o < 0 ? 1 : 0; + triO2[6] = DMPolytopeTypeComposeOrientation(DM_POLYTOPE_SEGMENT, -1, arr[2 * 2 + 1]); + /* Swap the first two edges if the triangle is reversed */ + e0 = o < 0 ? 34 : 30; + e1 = o < 0 ? 30 : 34; + triC2[e0] = arr[1 * 2]; + triC2[e0 + 1] = o < 0 ? 0 : 1; + triC2[e1] = arr[2 * 2]; + triC2[e1 + 1] = o < 0 ? 1 : 0; + triO2[9] = DMPolytopeTypeComposeOrientation(DM_POLYTOPE_SEGMENT, -1, arr[2 * 2 + 1]); + /* Swap the last two edges if the triangle is reversed */ + triC2[41] = arr[2 * 2]; + triC2[42] = o < 0 ? 0 : 1; + triC2[45] = o < 0 ? 1 : 0; + triC2[48] = o < 0 ? 0 : 1; + triO2[11] = DMPolytopeTypeComposeOrientation(DM_POLYTOPE_SEGMENT, 0, arr[1 * 2 + 1]); + triO2[12] = DMPolytopeTypeComposeOrientation(DM_POLYTOPE_SEGMENT, 0, arr[2 * 2 + 1]); + *Nt = 2; + *target = triT2; + *size = triS2; + *cone = triC2; + *ornt = triO2; + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformCellTransform_UWNVB(DMPlexTransform tr, DMPolytopeType source, PetscInt p, PetscInt *rt, PetscInt *Nt, DMPolytopeType *target[], PetscInt *size[], PetscInt *cone[], PetscInt *ornt[]) +{ + DMLabel trType = tr->trType; + PetscInt val; + + PetscFunctionBeginHot; + PetscCheck(p >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point argument is invalid"); + PetscCall(DMLabelGetValue(trType, p, &val)); + if (rt) *rt = val; + switch (source) { + case DM_POLYTOPE_POINT: + case DM_POLYTOPE_POINT_PRISM_TENSOR: + case DM_POLYTOPE_QUADRILATERAL: + case DM_POLYTOPE_SEG_PRISM_TENSOR: + case DM_POLYTOPE_TETRAHEDRON: + case DM_POLYTOPE_HEXAHEDRON: + case DM_POLYTOPE_TRI_PRISM: + case DM_POLYTOPE_TRI_PRISM_TENSOR: + case DM_POLYTOPE_QUAD_PRISM_TENSOR: + case DM_POLYTOPE_PYRAMID: + PetscCall(DMPlexTransformCellTransformIdentity(tr, source, p, NULL, Nt, target, size, cone, ornt)); + break; + case DM_POLYTOPE_SEGMENT: + if (val == RT_EDGE) PetscCall(DMPlexTransformCellTransformIdentity(tr, source, p, NULL, Nt, target, size, cone, ornt)); + else PetscCall(UWNVBRegularCellRefine(tr, source, p, NULL, Nt, target, size, cone, ornt)); + break; + case DM_POLYTOPE_TRIANGLE: + switch (val) { + case RT_TRIANGLE_SPLIT_0: + PetscCall(UWNVBGetTriangleSplitSingle(2, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_1: + PetscCall(UWNVBGetTriangleSplitSingle(0, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_2: + PetscCall(UWNVBGetTriangleSplitSingle(1, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_21: + PetscCall(UWNVBGetTriangleSplitDouble(-3, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_10: + PetscCall(UWNVBGetTriangleSplitDouble(-2, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_02: + PetscCall(UWNVBGetTriangleSplitDouble(-1, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_12: + PetscCall(UWNVBGetTriangleSplitDouble(0, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_20: + PetscCall(UWNVBGetTriangleSplitDouble(1, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT_01: + PetscCall(UWNVBGetTriangleSplitDouble(2, Nt, target, size, cone, ornt)); + break; + case RT_TRIANGLE_SPLIT: + PetscCall(UWNVBRegularCellRefine(tr, source, p, NULL, Nt, target, size, cone, ornt)); + break; + default: + PetscCall(DMPlexTransformCellTransformIdentity(tr, source, p, NULL, Nt, target, size, cone, ornt)); + } + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No refinement strategy for %s", DMPolytopeTypes[source]); + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformSetFromOptions_UWNVB(DMPlexTransform tr, PetscOptionItems PetscOptionsObject) +{ + PetscInt cells[256], n = 256, i; + PetscBool flg; + + PetscFunctionBegin; + PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options"); + PetscCall(PetscOptionsIntArray("-dm_plex_transform_uwnvb_ref_cell", "Mark cells for refinement", "", cells, &n, &flg)); + if (flg) { + DMLabel active; + + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Adaptation Label", &active)); + for (i = 0; i < n; ++i) PetscCall(DMLabelSetValue(active, cells[i], DM_ADAPT_REFINE)); + PetscCall(DMPlexTransformSetActive(tr, active)); + PetscCall(DMLabelDestroy(&active)); + } + PetscOptionsHeadEnd(); + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformView_UWNVB(DMPlexTransform tr, PetscViewer viewer) +{ + PetscBool isascii; + + PetscFunctionBegin; + PetscValidHeaderSpecific(tr, DMPLEXTRANSFORM_CLASSID, 1); + PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2); + PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &isascii)); + if (isascii) { + PetscViewerFormat format; + const char *name; + + PetscCall(PetscObjectGetName((PetscObject)tr, &name)); + PetscCall(PetscViewerASCIIPrintf(viewer, "UWNVB refinement %s\n", name ? name : "")); + PetscCall(PetscViewerGetFormat(viewer, &format)); + if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) PetscCall(DMLabelView(tr->trType, viewer)); + } else { + SETERRQ(PetscObjectComm((PetscObject)tr), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlexTransform writing", ((PetscObject)viewer)->type_name); + } PetscFunctionReturn(PETSC_SUCCESS); } static PetscErrorCode DMPlexTransformDestroy_UWNVB(DMPlexTransform tr) { - DMPlexTransform_UWNVB *nvb = (DMPlexTransform_UWNVB *)tr->data; + DMPlexRefine_UWNVB *nvb = (DMPlexRefine_UWNVB *)tr->data; PetscFunctionBegin; + PetscCall(PetscFree(nvb->edgeLen)); + PetscCall(PetscSectionDestroy(&nvb->secEdgeLen)); PetscCall(DMLabelDestroy(&nvb->splitPoints)); PetscCall(PetscFree(tr->data)); PetscFunctionReturn(PETSC_SUCCESS); @@ -75,18 +642,20 @@ static PetscErrorCode DMPlexTransformDestroy_UWNVB(DMPlexTransform tr) static PetscErrorCode DMPlexTransformInitialize_UWNVB(DMPlexTransform tr) { PetscFunctionBegin; + tr->ops->view = DMPlexTransformView_UWNVB; + tr->ops->setfromoptions = DMPlexTransformSetFromOptions_UWNVB; tr->ops->setup = DMPlexTransformSetUp_UWNVB; tr->ops->destroy = DMPlexTransformDestroy_UWNVB; - tr->ops->setdimensions = DMPlexTransformSetDimensions_UWNVB; - tr->ops->celltransform = DMPlexTransformCellTransformIdentity; /* STAGE 2: NVB bisection */ - tr->ops->getsubcellorientation = DMPlexTransformGetSubcellOrientationIdentity; - tr->ops->mapcoordinates = DMPlexTransformMapCoordinates_UWNVB; + tr->ops->setdimensions = UWNVBSetDimensions; + tr->ops->celltransform = DMPlexTransformCellTransform_UWNVB; + tr->ops->getsubcellorientation = DMPlexTransformGetSubcellOrientation_UWNVB; + tr->ops->mapcoordinates = UWNVBMapCoordinatesBarycenter; PetscFunctionReturn(PETSC_SUCCESS); } PETSC_EXTERN PetscErrorCode DMPlexTransformCreate_UWNVB(DMPlexTransform tr) { - DMPlexTransform_UWNVB *nvb; + DMPlexRefine_UWNVB *nvb; PetscFunctionBegin; PetscValidHeaderSpecific(tr, DMPLEXTRANSFORM_CLASSID, 1); diff --git a/tests/test_0837_nvb_native_transform.py b/tests/test_0837_nvb_native_transform.py new file mode 100644 index 00000000..a02357ee --- /dev/null +++ b/tests/test_0837_nvb_native_transform.py @@ -0,0 +1,95 @@ +"""Layer-2 Route B, Stage 2a: the native ``uwnvb`` DMPlexTransform. + +``underworld3.utilities._nvb_transform`` registers a self-contained C +``DMPlexTransform`` named ``uwnvb`` into PETSc on import (no PETSc rebuild). This +stage is the *SBR-equivalent* base of the native newest-vertex transform: it must +reproduce PETSc's ``refine_sbr`` (longest-edge bisection) byte-for-byte, which +exercises the whole native stack end-to-end — registration, the ``SetUp`` +closure, ``DMLabelPropagate`` cross-rank propagation, the cell transform, subcell +orientation, and coordinate mapping — on the pinned PETSc. + +The graded newest-vertex edge choice is layered on this base in a later stage; +here we only pin the equivalence so the transform infrastructure can't regress. + +Skips cleanly where the extension isn't built (e.g. a non-custom-PETSc env). +""" +import numpy as np +import pytest +import underworld3 as uw +from petsc4py import PETSc + +pytestmark = [pytest.mark.level_1, pytest.mark.tier_b] + +_nvb = pytest.importorskip( + "underworld3.utilities._nvb_transform", + reason="native uwnvb transform not built (needs the custom-PETSc/amr env)", +) + +DM_ADAPT_REFINE = 1 + + +def _base_dm(cellSize=0.2): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=cellSize, regular=False, + qdegree=2, + ).dm.clone() + + +def _refine(dm, ttype, marked): + """Refine ``dm`` with transform ``ttype`` on the given marked cell ids, via + the same ``adaptLabel`` path production code uses.""" + opts = PETSc.Options() + had = opts.hasName("dm_plex_transform_type") + prev = opts.getString("dm_plex_transform_type") if had else None + opts.setValue("dm_plex_transform_type", ttype) + try: + d = dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + for c in marked: + lab.setValue(int(c), DM_ADAPT_REFINE) + return d.adaptLabel("adapt") + finally: + if had: + opts.setValue("dm_plex_transform_type", prev) + else: + opts.delValue("dm_plex_transform_type") + + +def _triangulation(dm): + """The set of triangles as sorted rounded vertex-coordinate triples — a + geometry-level fingerprint independent of point numbering.""" + coords = dm.getCoordinatesLocal().array.reshape(-1, 2) + vs, ve = dm.getDepthStratum(0) + cs, ce = dm.getHeightStratum(0) + tris = [] + for c in range(cs, ce): + cl = dm.getTransitiveClosure(c)[0] + verts = [p for p in cl if vs <= p < ve] + tris.append(tuple(sorted(tuple(np.round(coords[p - vs], 10)) for p in verts))) + return sorted(tris) + + +def test_registered(): + assert _nvb.registered is True + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_uwnvb_matches_refine_sbr(seed): + """Native ``uwnvb`` produces an identical triangulation to ``refine_sbr`` for + a variety of marked sets, including full uniform refinement.""" + rng = np.random.default_rng(seed) + base = _base_dm() + ncell = base.getHeightStratum(0)[1] + if seed == 0: + marked = [0] + elif seed == 1: + marked = sorted(rng.choice(ncell, size=min(5, ncell), replace=False).tolist()) + else: + marked = list(range(ncell)) # full uniform refine + + tri_sbr = _triangulation(_refine(_base_dm(), "refine_sbr", marked)) + tri_nvb = _triangulation(_refine(_base_dm(), "uwnvb", marked)) + assert len(tri_nvb) == len(tri_sbr) + assert tri_nvb == tri_sbr From fc94499a95be6964ed1a208696fa956a4cf67db2 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 16:19:45 +1000 Subject: [PATCH 35/52] feat(layer2/nvb): graded NVB via single-bisection multi-pass native transform (Route B Stage 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds graded newest-vertex refinement on top of the Stage-2a native transform, and fixes the grading failure of the earlier one-pass approach. A single conforming DMPlexTransform pass is forced into green/blue multi-edge splits that corrupt the refinement-edge structure; instead we refine as repeated CONFORMING single bisections of *compatible* refinement edges, which keeps every child's slot trivially correct and the closure bounded. As built (all in C, so the SF-sensitive parallel logic stays in one place): - each triangle carries a uwnvb_refedge cone-slot label (refedge = cone[slot]; longest-edge seed on the base). No generation label is needed — single bisection makes each child's newest vertex the new midpoint, so its refinement edge is the edge opposite that midpoint (geometric). - uwnvb_bisect: a dumb transform that single-splits exactly the edges named by a label, reusing the Stage-2a cell-transform/orientation/coordinate ops. - UWNVBRefine(dm, want_label): closure (grow the bisect set by LEPP blocking neighbours) + drain (each sub-pass single-splits the compatible refinement edges — an edge that is the refinement edge of all its incident cells, reconciled across ranks by an MPI_LAND PetscSFReduce — then rebuilds the slot + set). Exposed as underworld3.utilities._nvb_transform.refine(dm, want_label). Validated (tests/test_0838, tier_b): refinement edges match the serial NVBMesh reference EXACTLY over four uniform refines (0 mismatch, 64/128/256/512); a mark deep in a 2x-refined patch adds +2 cells (== NVBMesh) versus SBR's +4824; a shrinking bullseye grades to 805 cells versus SBR's 102868 uniform-patch cells; conforming (0 hanging / 0 over-shared) and deterministic. Stage-2a (uwnvb == refine_sbr, test_0837) still passes. Runs conforming at np=1/2/3. Remaining for full parallel confluence: the closure is on-rank only, so a closure crossing a partition under-refines slightly there (still conforming — the SF-reconciled compatibility test prevents cross-partition hanging nodes). The SF-reconciled cross-rank closure (DMLabelPropagate of requested refinement edges, as SBR's SetUp does) is the next step, followed by wiring _adapt_nested(engine=nvb) at np>1. See docs/developer/design/NVB_GRADED_ADAPT.md. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 86 +++- src/underworld3/utilities/_nvb_transform.pyx | 24 + src/underworld3/utilities/nvb_transform.c | 479 +++++++++++++++++++ tests/test_0838_nvb_graded_native.py | 165 +++++++ 4 files changed, 737 insertions(+), 17 deletions(-) create mode 100644 tests/test_0838_nvb_graded_native.py diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index fd8bc096..dff9ba55 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -414,11 +414,49 @@ tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build triangulation for every marking pattern incl. full uniform refine. `tests/ test_0837_nvb_native_transform.py` (tier_b). Longest-edge edge pick; the newest- vertex choice layers on top. -3. *Newest-vertex grading (Stage 2b)* — **IN PROGRESS; redesign required.** See - "Stage 2b finding" below. -4. *Integration + acceptance* — `_adapt_nested(engine="nvb")` at np>1 via - `adaptLabel(dm_plex_transform_type=uwnvb)` (in-place, co-partitioned); Poisson + - SolCx FMG match GAMG; result matches the serial NVB mesh (properties). +3. *Newest-vertex grading (Stage 2b)* — **DONE serially (2026-07-01)** via the + single-bisection multi-pass driver (see below). Matches serial `NVBMesh` + refinement edges exactly over repeated uniform refine; a deep mark is bounded + (+2 vs SBR's +4824); conforming, deterministic, graded bullseye (805 vs SBR's + 102 868 cells). `tests/test_0838_nvb_graded_native.py`. Runs conforming at + np=1/2/3; full parallel *confluence* needs the cross-rank closure (below). +4. *Integration + acceptance* — `_adapt_nested(engine="nvb")` at np>1 via the + driver; Poisson + SolCx FMG match GAMG; result matches the serial NVB mesh. + +### Stage 2b as built — single-bisection multi-pass driver (WORKS) + +`_nvb_transform.refine(dm, want_label)` → C entry `UWNVBRefine`: +- each triangle carries a **`uwnvb_refedge`** cone-slot (0/1/2) label; `refedge = + cone[slot]`, longest-edge seed on the base. No generation label is needed — + single bisection makes each child's peak the new midpoint, so its refinement + edge is just the edge opposite that midpoint (purely geometric). +- **closure**: grow the set `C` of cells that must bisect by adding, for each + `C`-cell whose refinement edge is blocked by a neighbour with a different + refinement edge, that neighbour (LEPP). +- **drain**: repeat — mark the *compatible* refinement edges (an edge that is the + refinement edge of **all** its incident cells; reconciled across ranks by an + `MPI_LAND` `PetscSFReduce`), single-split exactly those via a dumb + **`uwnvb_bisect`** transform (reuses the Stage-2a cell-transform/orientation/ + coordinate ops), then rebuild the slot label (split child → edge opposite the + midpoint; unsplit cell → map the parent refinement *edge* to its child edge — + copying the slot *index* is wrong because the identity transform may renumber + the cone) and the `C` set (bisected cells leave; blocked cells stay). Loop until + `C` is globally empty. Every sub-pass touches each cell at most once ⇒ pure + single-splits, always conforming, child slots trivial. + +Gotchas fixed: the transform copies the *adaptation* label onto its output, so it +accumulates stale marks across calls (filter `want` to triangle cells + strip the +label from the output); and the drain loop condition must use the **global** `|C|` +(`MPIU_Allreduce`) or a rank with no local marks skips the collective loop and +deadlocks. + +**Remaining for full parallel confluence:** the closure is currently on-rank only, +so a closure that would cross a partition under-refines slightly there (np=2 gives +161 cells where serial gives 163 — still conforming, since the `SF`-reconciled +compatibility test prevents cross-partition hanging nodes). The fix is the +SF-reconciled cross-rank closure: propagate *requested* refinement edges across +the point SF with `DMLabelPropagate` (exactly as SBR's `SetUp` does), so a +`C`-cell blocked by an off-rank neighbour marks that neighbour on its owning rank. ### Stage 2b finding (2026-07-01): grading needs single-bisection, multi-pass @@ -462,18 +500,32 @@ consistent with the marker-sidecar scheme already designed for SBR. The coordinate-built custom-P sidesteps the canonical-numbering fragility (same as SBR). -**FMG-hierarchy-index label — reuse for checkpointing (L.M., 2026-07-01, TODO next -phase).** The custom-P / FMG round-trip through checkpoints already carries a -**mesh-hierarchy-index label** (which coarse level each point belongs to) to -reconstruct the transfer hierarchy on reload. When implementing the single- -bisection multi-pass Stage 2b, investigate: **(a)** whether that existing hierarchy -label can be *reused* to encode the NVB level/marker replay (one label serving both -FMG-transfer reconstruction and adapt replay), and **(b)** whether the *new* NVB -structures (per-vertex generation + the compatible-ref-edge state, or the -per-pass marked-edge sets) are the natural thing to *checkpoint the FMG itself* -against — i.e. store the NVB replay data and rebuild the whole FMG tail from it, -rather than storing meshes. Decide before finalising the checkpoint format so the -two mechanisms share one label rather than duplicating. +**FMG-hierarchy checkpoint — reuse vs new structures (L.M. question, investigated +2026-07-01).** Finding: the *existing* FMG-hierarchy checkpoint +(`fmg-checkpoint-hierarchy.md`) does **not** carry a reusable per-node level label. +It stores the **coarsest** level as a one-DM sidecar (`mymesh.hierarchy.L0.h5`) plus +a refine count (`hierarchy_coarse_levels`) and `setCoarseDM`-links on reload; the +per-node "level" label reconstruction was **explicitly rejected** there because a +`createFromCellList` coarse DM loses the canonical `refine()` numbering PETSc's +*nested* interpolator requires (err 77). So there is no shared label to reuse for +option (a). + +**That err-77 objection does not apply to NVB/adapt-on-top:** the Layer-2 custom-P +transfers are built from **coordinates**, not parent-child numbering — so +label-/replay-based reconstruction *is* viable here (the same reason SBR +adapt-on-top uses a marker-sidecar). ⇒ **Option (b) is the clean path:** checkpoint +the NVB portion of the FMG tail by persisting its **replay data** — the +per-generation marked-cell sets (deterministic ⇒ replay reproduces every level +bit-identically) — and carry the `uwnvb_refedge` slot + per-vertex generation as +labels so a reload can resume/extend adaptation without recomputing. The base +uniform hierarchy keeps its existing coarsest-sidecar mechanism; the NVB top levels +add a marker-sidecar. **Implication for this implementation:** keep the slot + +generation as ordinary DMLabels (they already survive `DMClone`/label-transfer and +checkpoint via the normal label I/O), and have the driver accept/emit the +per-generation marked-cell set as the replay record. No new bespoke label format; +the two mechanisms stay decoupled (base = sidecar+count, NVB top = marker replay), +which is correct since they reconstruct fundamentally differently (nested-numbering +vs coordinate-built). ## 3D (tetrahedra) diff --git a/src/underworld3/utilities/_nvb_transform.pyx b/src/underworld3/utilities/_nvb_transform.pyx index 7afeaad0..31ac473e 100644 --- a/src/underworld3/utilities/_nvb_transform.pyx +++ b/src/underworld3/utilities/_nvb_transform.pyx @@ -8,11 +8,35 @@ initialised (importing petsc4py.PETSc guarantees that). After import, option resolve to the NVB transform. """ from petsc4py import PETSc as _PETSc # ensures PetscInitialize has run +from petsc4py.PETSc cimport DM, PetscDM cdef extern int UWNVBTransformRegister() +cdef extern int UWNVBRefine(PetscDM dm, const char *wantName, PetscDM *rdm) cdef int _ierr = UWNVBTransformRegister() if _ierr != 0: raise RuntimeError("UWNVBTransformRegister failed with PETSc error %d" % _ierr) registered = True + + +def refine(dm, want_label="adapt"): + """Graded NVB refinement of ``dm``. + + Every cell flagged in the ``want_label`` adaptation label (value + ``DM_ADAPT_REFINE = 1``) is bisected once, with the bounded newest-vertex + conforming closure, returning a fresh ``PETSc.DMPlex`` that carries an updated + per-triangle ``uwnvb_refedge`` slot label. Refinement runs as repeated + conforming single-bisection sub-passes (the ``uwnvb_bisect`` transform) so the + output stays co-partitioned with the input — the custom-P FMG tail consumes it + unchanged. Call repeatedly (re-marking) for multi-level grading. + + The whole refine (closure + drain + slot maintenance) runs in C. + """ + cdef DM c_dm = dm + cdef DM out = _PETSc.DMPlex() + cdef bytes name = want_label.encode("utf8") + cdef int ierr = UWNVBRefine(c_dm.dm, name, &out.dm) + if ierr != 0: + raise RuntimeError("UWNVBRefine failed with PETSc error %d" % ierr) + return out diff --git a/src/underworld3/utilities/nvb_transform.c b/src/underworld3/utilities/nvb_transform.c index 4af7131e..66272cf2 100644 --- a/src/underworld3/utilities/nvb_transform.c +++ b/src/underworld3/utilities/nvb_transform.c @@ -665,10 +665,489 @@ PETSC_EXTERN PetscErrorCode DMPlexTransformCreate_UWNVB(DMPlexTransform tr) PetscFunctionReturn(PETSC_SUCCESS); } +/* ======================================================================== */ +/* Stage 2b — graded NVB by single-bisection multi-pass. */ +/* */ +/* The `uwnvb` transform above is the SBR-equivalent (longest-edge, one-pass */ +/* with green/blue closure) used as a validated native-SBR. Grading needs a */ +/* different execution: each triangle carries an explicit refinement-edge */ +/* SLOT (cone index 0/1/2), and refinement proceeds as repeated CONFORMING */ +/* single bisections of *compatible* refinement edges (an edge that is the */ +/* refinement edge of every incident cell). A one-pass conforming transform */ +/* is forced into green/blue multi-edge splits, which corrupt the refedge */ +/* structure and defeat grading (see NVB_GRADED_ADAPT.md, Stage 2b finding); */ +/* single bisections keep child slots trivially correct (opposite the new */ +/* midpoint) so the closure stays bounded. */ +/* */ +/* `uwnvb_bisect` is the per-sub-pass engine: given an edge label marking a */ +/* set of pairwise-independent-per-cell edges (each the refedge of its cells) */ +/* it single-splits exactly those edges. The C driver UWNVBRefine() below */ +/* computes the closure + drains it over uwnvb_bisect sub-passes, maintaining */ +/* the slot label — all in C so the SF-sensitive parallel logic stays in one */ +/* place. */ +/* ======================================================================== */ +#define UWNVB_SLOT_LABEL "uwnvb_refedge" /* per-triangle refinement-edge cone slot 0/1/2 */ +#define UWNVB_BISECT_LABEL "uwnvb_bisect_edges" /* per-edge: 1 => bisect this sub-pass */ + +/* The vertex of a triangle opposite edge slot `i`, given the three edges' + endpoint pairs (the vertex shared by the other two edges). */ +static inline PetscInt UWNVBOppVertex(const PetscInt everts[3][2], PetscInt i) +{ + PetscInt j = (i + 1) % 3, k = (i + 2) % 3, a, b; + for (a = 0; a < 2; ++a) + for (b = 0; b < 2; ++b) + if (everts[j][a] == everts[k][b]) return everts[j][a]; + return -1; +} + +/* SetUp for uwnvb_bisect: mark the edges named by UWNVB_BISECT_LABEL and set the + refinement type of each incident triangle from how many of its edges are marked + (1 => single bisection on that edge; 2/3 => the green/blue split types, kept as + a safety net — the driver only ever feeds pairwise-independent single edges). */ +static PetscErrorCode DMPlexTransformSetUp_UWNVBBisect(DMPlexTransform tr) +{ + DMPlexRefine_UWNVB *nvb = (DMPlexRefine_UWNVB *)tr->data; + DM dm; + DMLabel bisect; + IS bisectIS; + const PetscInt *bisectEdges; + PetscInt pStart, pEnd, p, eStart, eEnd, e, edgeLenSize, Nb, i; + + PetscFunctionBegin; + PetscCall(DMPlexTransformGetDM(tr, &dm)); + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Split Points", &nvb->splitPoints)); + /* edge lengths (for the 2/3-marked disambiguation only) */ + PetscCall(DMGetCoordinatesLocalSetUp(dm)); + PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd)); + PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &nvb->secEdgeLen)); + PetscCall(PetscSectionSetChart(nvb->secEdgeLen, eStart, eEnd)); + for (e = eStart; e < eEnd; ++e) PetscCall(PetscSectionSetDof(nvb->secEdgeLen, e, 1)); + PetscCall(PetscSectionSetUp(nvb->secEdgeLen)); + PetscCall(PetscSectionGetStorageSize(nvb->secEdgeLen, &edgeLenSize)); + PetscCall(PetscCalloc1(edgeLenSize, &nvb->edgeLen)); + /* mark the requested edges + their incident triangles */ + PetscCall(DMGetLabel(dm, UWNVB_BISECT_LABEL, &bisect)); + PetscCheck(bisect, PetscObjectComm((PetscObject)tr), PETSC_ERR_ARG_WRONGSTATE, "uwnvb_bisect needs the '%s' edge label", UWNVB_BISECT_LABEL); + PetscCall(DMLabelGetStratumIS(bisect, 1, &bisectIS)); + PetscCall(DMLabelGetStratumSize(bisect, 1, &Nb)); + if (bisectIS) { + PetscCall(ISGetIndices(bisectIS, &bisectEdges)); + for (i = 0; i < Nb; ++i) { + const PetscInt edge = bisectEdges[i]; + const PetscInt *supp; + PetscInt suppSize, s; + + PetscCall(DMLabelSetValue(nvb->splitPoints, edge, 1)); + PetscCall(DMPlexGetSupport(dm, edge, &supp)); + PetscCall(DMPlexGetSupportSize(dm, edge, &suppSize)); + for (s = 0; s < suppSize; ++s) PetscCall(DMLabelSetValue(nvb->splitPoints, supp[s], 2)); + } + PetscCall(ISRestoreIndices(bisectIS, &bisectEdges)); + } + PetscCall(ISDestroy(&bisectIS)); + /* refine type per point (no closure: exactly the marked edges are split) */ + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Refine Type", &tr->trType)); + PetscCall(DMPlexGetChart(dm, &pStart, &pEnd)); + for (p = pStart; p < pEnd; ++p) { + DMLabel trType = tr->trType; + DMPolytopeType ct; + PetscInt val; + + PetscCall(DMPlexGetCellType(dm, p, &ct)); + switch (ct) { + case DM_POLYTOPE_POINT: + PetscCall(DMLabelSetValue(trType, p, RT_VERTEX)); + break; + case DM_POLYTOPE_SEGMENT: + PetscCall(DMLabelGetValue(nvb->splitPoints, p, &val)); + PetscCall(DMLabelSetValue(trType, p, val == 1 ? RT_EDGE_SPLIT : RT_EDGE)); + break; + case DM_POLYTOPE_TRIANGLE: + PetscCall(DMLabelGetValue(nvb->splitPoints, p, &val)); + if (val == 2) { + const PetscInt *cone; + PetscReal lens[3]; + PetscInt vals[3], k; + + PetscCall(DMPlexGetCone(dm, p, &cone)); + for (k = 0; k < 3; ++k) { + PetscCall(DMLabelGetValue(nvb->splitPoints, cone[k], &vals[k])); + vals[k] = vals[k] == 1 ? 1 : 0; + PetscCall(UWNVBGetEdgeLen(tr, cone[k], &lens[k])); + } + if (vals[0] && vals[1] && vals[2]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT)); + else if (vals[0] && vals[1]) PetscCall(DMLabelSetValue(trType, p, lens[0] > lens[1] ? RT_TRIANGLE_SPLIT_01 : RT_TRIANGLE_SPLIT_10)); + else if (vals[1] && vals[2]) PetscCall(DMLabelSetValue(trType, p, lens[1] > lens[2] ? RT_TRIANGLE_SPLIT_12 : RT_TRIANGLE_SPLIT_21)); + else if (vals[2] && vals[0]) PetscCall(DMLabelSetValue(trType, p, lens[2] > lens[0] ? RT_TRIANGLE_SPLIT_20 : RT_TRIANGLE_SPLIT_02)); + else if (vals[0]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_0)); + else if (vals[1]) PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_1)); + else PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE_SPLIT_2)); + } else PetscCall(DMLabelSetValue(trType, p, RT_TRIANGLE)); + break; + default: + SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot handle points of type %s", DMPolytopeTypes[ct]); + } + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +static PetscErrorCode DMPlexTransformInitialize_UWNVBBisect(DMPlexTransform tr) +{ + PetscFunctionBegin; + tr->ops->view = DMPlexTransformView_UWNVB; + tr->ops->setup = DMPlexTransformSetUp_UWNVBBisect; + tr->ops->destroy = DMPlexTransformDestroy_UWNVB; + tr->ops->setdimensions = UWNVBSetDimensions; + tr->ops->celltransform = DMPlexTransformCellTransform_UWNVB; + tr->ops->getsubcellorientation = DMPlexTransformGetSubcellOrientation_UWNVB; + tr->ops->mapcoordinates = UWNVBMapCoordinatesBarycenter; + PetscFunctionReturn(PETSC_SUCCESS); +} + +PETSC_EXTERN PetscErrorCode DMPlexTransformCreate_UWNVBBisect(DMPlexTransform tr) +{ + DMPlexRefine_UWNVB *nvb; + + PetscFunctionBegin; + PetscValidHeaderSpecific(tr, DMPLEXTRANSFORM_CLASSID, 1); + PetscCall(PetscNew(&nvb)); + tr->data = nvb; + PetscCall(DMPlexTransformInitialize_UWNVBBisect(tr)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* ---- driver helpers ---------------------------------------------------- */ + +/* Length of edge `e` from the local coordinates (standalone, no transform cache). */ +static PetscErrorCode uwnvb_edge_len(DM dm, PetscInt e, PetscReal *len) +{ + DM cdm; + Vec coordsLocal; + const PetscScalar *coords; + const PetscInt *cone; + PetscScalar *cA, *cB; + PetscInt cdim; + + PetscFunctionBegin; + PetscCall(DMGetCoordinateDM(dm, &cdm)); + PetscCall(DMGetCoordinateDim(dm, &cdim)); + PetscCall(DMPlexGetCone(dm, e, &cone)); + PetscCall(DMGetCoordinatesLocalNoncollective(dm, &coordsLocal)); + PetscCall(VecGetArrayRead(coordsLocal, &coords)); + PetscCall(DMPlexPointLocalRead(cdm, cone[0], coords, &cA)); + PetscCall(DMPlexPointLocalRead(cdm, cone[1], coords, &cB)); + *len = DMPlex_DistD_Internal(cdim, cA, cB); + PetscCall(VecRestoreArrayRead(coordsLocal, &coords)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Refinement edge of triangle `cell`: the stored cone slot, or the longest-edge + seed when no slot is set (base mesh / uninitialised cell). */ +static PetscErrorCode uwnvb_refedge(DM dm, DMLabel slot, PetscInt cell, PetscInt *refedge, PetscInt *refslot) +{ + const PetscInt *cone; + PetscInt s = -1, k; + + PetscFunctionBegin; + PetscCall(DMPlexGetCone(dm, cell, &cone)); + if (slot) PetscCall(DMLabelGetValue(slot, cell, &s)); + if (s >= 0 && s < 3) { + *refslot = s; + *refedge = cone[s]; + } else { + PetscReal len, maxlen = -1.0; + PetscInt best = 0; + for (k = 0; k < 3; ++k) { + PetscCall(uwnvb_edge_len(dm, cone[k], &len)); + if (len > maxlen) { maxlen = len; best = k; } + } + *refslot = best; + *refedge = cone[best]; + } + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* Make a point-indexed 0/1 array globally consistent under logical-AND across + the point SF (serial: no-op). After this, a shared point's value is the AND of + every rank's local value — used so an edge is "agreed" only if all incident + cells (on any rank) accept it as their refinement edge. */ +static PetscErrorCode uwnvb_sf_land(DM dm, PetscInt *val) +{ + PetscSF sf; + PetscInt nroots; + + PetscFunctionBegin; + PetscCall(DMGetPointSF(dm, &sf)); + PetscCall(PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL)); + if (nroots < 0) PetscFunctionReturn(PETSC_SUCCESS); /* no SF (serial) */ + PetscCall(PetscSFReduceBegin(sf, MPIU_INT, val, val, MPI_LAND)); + PetscCall(PetscSFReduceEnd(sf, MPIU_INT, val, val, MPI_LAND)); + PetscCall(PetscSFBcastBegin(sf, MPIU_INT, val, val, MPI_REPLACE)); + PetscCall(PetscSFBcastEnd(sf, MPIU_INT, val, val, MPI_REPLACE)); + PetscFunctionReturn(PETSC_SUCCESS); +} + +/* ---- driver: graded NVB refinement of `dm` ----------------------------- */ +/* + Refine `dm` so that every cell flagged in the `wantName` adaptation label + (values DM_ADAPT_REFINE) is bisected once, with the bounded NVB conforming + closure, and return the refined DM carrying an updated `uwnvb_refedge` slot. + + Two phases (see NVB_GRADED_ADAPT.md): + 1. CLOSURE — grow C (cells that must bisect) by adding, for each C cell whose + refinement edge is blocked by a neighbour with a different refinement edge, + that neighbour (LEPP; bounded). + 2. DRAIN — repeatedly single-split the *compatible* refinement edges (refedge + of all incident cells) via the uwnvb_bisect transform, until C is empty. + Each pass is conforming and touches each cell at most once (single split), + so child slots are trivial (edge opposite the new midpoint). + + NOTE: the CLOSURE currently walks on-rank neighbours only; the SF-reconciled + cross-rank closure (DMLabelPropagate of requested edges) is the remaining + parallel step. Compatibility (DRAIN) is already SF-reconciled via uwnvb_sf_land. +*/ +PETSC_EXTERN PetscErrorCode UWNVBRefine(DM dm, const char *wantName, DM *rdm) +{ + MPI_Comm comm; + DM work; + DMLabel want, C, slot = NULL; + PetscInt cStart, cEnd, c, guard, nC; + + PetscFunctionBegin; + PetscValidHeaderSpecific(dm, DM_CLASSID, 1); + PetscAssertPointer(rdm, 3); + PetscCall(PetscObjectGetComm((PetscObject)dm, &comm)); + PetscCall(DMGetLabel(dm, wantName, &want)); + PetscCheck(want, comm, PETSC_ERR_ARG_WRONGSTATE, "Adaptation label '%s' not found on DM", wantName); + + work = dm; + PetscCall(PetscObjectReference((PetscObject)work)); + PetscCall(DMGetLabel(work, UWNVB_SLOT_LABEL, &slot)); /* NULL on the base */ + + /* --- 1. closure: C = want, grown by blocking neighbours (on-rank) --- */ + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "uwnvb_C", &C)); + { + IS wantIS; + const PetscInt *wantCells; + PetscInt nWant, i, *stack, top = 0, cap; + + PetscCall(DMLabelGetStratumIS(want, DM_ADAPT_REFINE, &wantIS)); + PetscCall(DMLabelGetStratumSize(want, DM_ADAPT_REFINE, &nWant)); + PetscCall(DMPlexGetHeightStratum(work, 0, &cStart, &cEnd)); + cap = cEnd - cStart; /* every C member is a triangle; ≤ #cells, never overflows */ + PetscCall(PetscMalloc1(cap > 0 ? cap : 1, &stack)); + if (wantIS) { + PetscCall(ISGetIndices(wantIS, &wantCells)); + for (i = 0; i < nWant; ++i) { + DMPolytopeType ct; + PetscInt v; + /* Ignore any non-triangle marks — the adaptation label can carry stale + entries transferred from a previous refine onto non-cell points. */ + if (wantCells[i] < cStart || wantCells[i] >= cEnd) continue; + PetscCall(DMPlexGetCellType(work, wantCells[i], &ct)); + if (ct != DM_POLYTOPE_TRIANGLE) continue; + PetscCall(DMLabelGetValue(C, wantCells[i], &v)); + if (v != 1) { + PetscCall(DMLabelSetValue(C, wantCells[i], 1)); + stack[top++] = wantCells[i]; + } + } + PetscCall(ISRestoreIndices(wantIS, &wantCells)); + } + PetscCall(ISDestroy(&wantIS)); + while (top > 0) { + PetscInt cell = stack[--top], refedge, refslot, ss, s; + const PetscInt *supp; + PetscCall(uwnvb_refedge(work, slot, cell, &refedge, &refslot)); + PetscCall(DMPlexGetSupport(work, refedge, &supp)); + PetscCall(DMPlexGetSupportSize(work, refedge, &ss)); + for (s = 0; s < ss; ++s) { + PetscInt d = supp[s], de, ds, v; + if (d == cell) continue; + PetscCall(uwnvb_refedge(work, slot, d, &de, &ds)); + if (de != refedge) { /* d blocks cell: d must refine its own refedge first */ + PetscCall(DMLabelGetValue(C, d, &v)); + if (v != 1) { + PetscCall(DMLabelSetValue(C, d, 1)); + if (top < cap) stack[top++] = d; + } + } + } + } + PetscCall(PetscFree(stack)); + } + + /* --- 2. drain: single-split compatible refedges of C until C empty --- */ + PetscCall(DMLabelGetStratumSize(C, 1, &nC)); + { /* GLOBAL count: ranks with no local C must still enter the collective loop */ + PetscInt nCG = nC; + PetscCallMPI(MPIU_Allreduce(&nC, &nCG, 1, MPIU_INT, MPI_SUM, comm)); + nC = nCG; + } + guard = 0; + while (nC > 0) { + DMPlexTransform tr; + DM out; + DMLabel bisect, slotOut, Cout; + PetscInt *agree, pStart, pEnd, p, e, eStart, eEnd, nready = 0; + + PetscCheck(++guard < 100000, comm, PETSC_ERR_PLIB, "UWNVB drain did not converge"); + + /* agree[e] = 1 iff e is the refinement edge of all its incident cells */ + PetscCall(DMPlexGetChart(work, &pStart, &pEnd)); + PetscCall(PetscMalloc1(pEnd - pStart, &agree)); + for (p = 0; p < pEnd - pStart; ++p) agree[p] = 1; + PetscCall(DMPlexGetHeightStratum(work, 0, &cStart, &cEnd)); + for (c = cStart; c < cEnd; ++c) { + DMPolytopeType ct; + const PetscInt *cone; + PetscInt refedge, refslot, k; + PetscCall(DMPlexGetCellType(work, c, &ct)); + if (ct != DM_POLYTOPE_TRIANGLE) continue; + PetscCall(uwnvb_refedge(work, slot, c, &refedge, &refslot)); + PetscCall(DMPlexGetCone(work, c, &cone)); + for (k = 0; k < 3; ++k) + if (cone[k] != refedge) agree[cone[k] - pStart] = 0; /* c vetoes non-refedges */ + } + PetscCall(uwnvb_sf_land(work, agree)); + + /* ready edges = refedge of a C cell that is agreed; mark them for bisection */ + PetscCall(DMLabelCreate(PETSC_COMM_SELF, UWNVB_BISECT_LABEL, &bisect)); + { + IS cis; + const PetscInt *ccells; + PetscInt n, i; + PetscCall(DMLabelGetStratumIS(C, 1, &cis)); + PetscCall(DMLabelGetStratumSize(C, 1, &n)); + if (cis) { + PetscCall(ISGetIndices(cis, &ccells)); + for (i = 0; i < n; ++i) { + PetscInt refedge, refslot, bv; + PetscCall(uwnvb_refedge(work, slot, ccells[i], &refedge, &refslot)); + if (agree[refedge - pStart]) { + PetscCall(DMLabelGetValue(bisect, refedge, &bv)); + if (bv != 1) { PetscCall(DMLabelSetValue(bisect, refedge, 1)); ++nready; } + } + } + PetscCall(ISRestoreIndices(cis, &ccells)); + } + PetscCall(ISDestroy(&cis)); + } + PetscCall(PetscFree(agree)); + { + PetscInt nreadyG = nready; + PetscCallMPI(MPIU_Allreduce(&nready, &nreadyG, 1, MPIU_INT, MPI_SUM, comm)); + PetscCheck(nreadyG > 0, comm, PETSC_ERR_PLIB, "UWNVB drain stalled: C non-empty but no compatible edge"); + } + + /* attach the bisect label to `work` and single-split those edges. `work` may + be the caller's dm on the first pass, so the label is removed again below. */ + PetscCall(DMAddLabel(work, bisect)); + PetscCall(DMLabelDestroy(&bisect)); /* work holds the ref now */ + PetscCall(DMPlexTransformCreate(comm, &tr)); + PetscCall(DMPlexTransformSetType(tr, "uwnvb_bisect")); + PetscCall(DMPlexTransformSetDM(tr, work)); + PetscCall(DMPlexTransformSetUp(tr)); + PetscCall(DMPlexTransformApply(tr, work, &out)); + { /* clean the transient bisect label off both work (esp. if == caller dm) and out */ + DMLabel tmp = NULL; + PetscCall(DMRemoveLabel(work, UWNVB_BISECT_LABEL, &tmp)); + PetscCall(DMLabelDestroy(&tmp)); + tmp = NULL; + PetscCall(DMRemoveLabel(out, UWNVB_BISECT_LABEL, &tmp)); + PetscCall(DMLabelDestroy(&tmp)); + } + + /* rebuild the slot label on `out` (single split => child refedge is the edge + opposite the new midpoint; unsplit cells copy their slot) */ + { + DMLabel staleS = NULL; + PetscCall(DMGetLabel(out, UWNVB_SLOT_LABEL, &staleS)); + if (staleS) { PetscCall(DMRemoveLabel(out, UWNVB_SLOT_LABEL, &staleS)); PetscCall(DMLabelDestroy(&staleS)); } + } + PetscCall(DMCreateLabel(out, UWNVB_SLOT_LABEL)); + PetscCall(DMGetLabel(out, UWNVB_SLOT_LABEL, &slotOut)); + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "uwnvb_C", &Cout)); + + PetscCall(DMPlexGetHeightStratum(work, 0, &cStart, &cEnd)); + for (c = cStart; c < cEnd; ++c) { + DMPolytopeType ct; + PetscInt rt, refedge, refslot, cval, pNew; + PetscCall(DMPlexGetCellType(work, c, &ct)); + if (ct != DM_POLYTOPE_TRIANGLE) continue; + PetscCall(DMLabelGetValue(tr->trType, c, &rt)); + PetscCall(uwnvb_refedge(work, slot, c, &refedge, &refslot)); + PetscCall(DMLabelGetValue(C, c, &cval)); + + if (rt == RT_TRIANGLE) { + /* Unsplit: the refinement edge is unchanged, but the identity transform may + renumber the cone — so map the parent refinement EDGE to its child edge + and record THAT slot (copying the index would corrupt the labelling). */ + const PetscInt *ccone; + PetscInt childRefEdge, j, slotval = 0; + PetscCall(DMPlexTransformGetTargetPoint(tr, DM_POLYTOPE_TRIANGLE, DM_POLYTOPE_TRIANGLE, c, 0, &pNew)); + PetscCall(DMPlexTransformGetTargetPoint(tr, DM_POLYTOPE_SEGMENT, DM_POLYTOPE_SEGMENT, refedge, 0, &childRefEdge)); + PetscCall(DMPlexGetCone(out, pNew, &ccone)); + for (j = 0; j < 3; ++j) + if (ccone[j] == childRefEdge) { slotval = j; break; } + PetscCall(DMLabelSetValue(slotOut, pNew, slotval)); + if (cval == 1) PetscCall(DMLabelSetValue(Cout, pNew, 1)); + } else { + /* single split on refedge: two children, peak = midpoint m */ + PetscInt m, r, j; + PetscCheck(rt == RT_TRIANGLE_SPLIT_0 || rt == RT_TRIANGLE_SPLIT_1 || rt == RT_TRIANGLE_SPLIT_2, + comm, PETSC_ERR_PLIB, "UWNVB drain produced a non-single split (rt=%" PetscInt_FMT ")", rt); + PetscCall(DMPlexTransformGetTargetPoint(tr, DM_POLYTOPE_SEGMENT, DM_POLYTOPE_POINT, refedge, 0, &m)); + for (r = 0; r < 2; ++r) { + const PetscInt *ccone; + PetscInt cev[3][2], slotval = 0; + PetscCall(DMPlexTransformGetTargetPoint(tr, DM_POLYTOPE_TRIANGLE, DM_POLYTOPE_TRIANGLE, c, r, &pNew)); + PetscCall(DMPlexGetCone(out, pNew, &ccone)); + for (j = 0; j < 3; ++j) { + const PetscInt *ec; + PetscCall(DMPlexGetCone(out, ccone[j], &ec)); + cev[j][0] = ec[0]; + cev[j][1] = ec[1]; + } + for (j = 0; j < 3; ++j) + if (UWNVBOppVertex(cev, j) == m) { slotval = j; break; } + PetscCall(DMLabelSetValue(slotOut, pNew, slotval)); + /* children of a bisected cell are done -> NOT added to Cout */ + } + } + } + + PetscCall(DMPlexTransformDestroy(&tr)); + PetscCall(DMDestroy(&work)); + work = out; + slot = slotOut; + PetscCall(DMLabelDestroy(&C)); + C = Cout; + PetscCall(DMLabelGetStratumSize(C, 1, &nC)); + { + PetscInt nCG = nC; + PetscCallMPI(MPIU_Allreduce(&nC, &nCG, 1, MPIU_INT, MPI_SUM, comm)); + nC = nCG; + } + } + + PetscCall(DMLabelDestroy(&C)); + /* Drop the transferred adaptation label so repeated refines don't accumulate + stale marks (the transform copies user labels onto every output). */ + { + DMLabel staleW = NULL; + PetscCall(DMGetLabel(work, wantName, &staleW)); + if (staleW) { PetscCall(DMRemoveLabel(work, wantName, &staleW)); PetscCall(DMLabelDestroy(&staleW)); } + } + *rdm = work; + PetscFunctionReturn(PETSC_SUCCESS); +} + /* Idempotent registration entry called from the Cython module on import. */ PetscErrorCode UWNVBTransformRegister(void) { PetscFunctionBegin; PetscCall(DMPlexTransformRegister("uwnvb", DMPlexTransformCreate_UWNVB)); + PetscCall(DMPlexTransformRegister("uwnvb_bisect", DMPlexTransformCreate_UWNVBBisect)); PetscFunctionReturn(PETSC_SUCCESS); } diff --git a/tests/test_0838_nvb_graded_native.py b/tests/test_0838_nvb_graded_native.py new file mode 100644 index 00000000..00908f42 --- /dev/null +++ b/tests/test_0838_nvb_graded_native.py @@ -0,0 +1,165 @@ +"""Layer-2 Route B, Stage 2b: graded NVB via the native single-bisection driver. + +``underworld3.utilities._nvb_transform.refine(dm, want_label)`` refines every cell +flagged in the adaptation label once, with the bounded newest-vertex conforming +closure, running as repeated conforming single-bisection sub-passes (the +``uwnvb_bisect`` transform) driven from C. Unlike PETSc's longest-edge +``refine_sbr`` (unbounded closure — a uniform-finest patch only), NVB *grades*: a +mark deep in a refined patch adds O(1) cells locally. + +These tests pin the graded behaviour against the validated serial reference +``underworld3.utilities.nvb.NVBMesh``: + - the refinement edges match ``NVBMesh`` exactly under repeated uniform refine; + - a mark deep in a refined patch is bounded/local (NVB) not a drain (SBR); + - the mesh stays conforming (0 hanging / 0 over-shared) and refinement is + deterministic; + - a shrinking bullseye grades (many levels coexist), with far fewer cells than + the SBR uniform patch. + +Serial (the SF-reconciled cross-rank closure for full parallel confluence is a +follow-up; the driver already runs conforming at np>1). +""" +import numpy as np +import pytest +import underworld3 as uw +from petsc4py import PETSc + +pytestmark = [pytest.mark.level_1, pytest.mark.tier_b] + +_nvb = pytest.importorskip( + "underworld3.utilities._nvb_transform", + reason="native uwnvb transform not built (needs the custom-PETSc/amr env)", +) +from underworld3.utilities.nvb import NVBMesh # noqa: E402 + +DM_ADAPT_REFINE = 1 +CENTER = np.array([0.5, 0.5]) + + +def _base(cellSize=0.2): + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=cellSize, regular=False, qdegree=2, + ).dm.clone() + + +def _cents(dm): + cs, ce = dm.getHeightStratum(0) + return np.array([dm.computeCellGeometryFVM(c)[1] for c in range(cs, ce)]), cs, ce + + +def _ncells(dm): + cs, ce = dm.getHeightStratum(0) + return ce - cs + + +def _refine(dm, pred): + d = dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + C, cs, ce = _cents(d) + for c in range(cs, ce): + if pred(C[c - cs]): + lab.setValue(c, DM_ADAPT_REFINE) + return _nvb.refine(d, "adapt") + + +def _conforming(dm): + es, ee = dm.getDepthStratum(1) + over = sum(1 for e in range(es, ee) if dm.getSupportSize(e) > 2) + coords = dm.getCoordinatesLocal().array.reshape(-1, 2) + vs, ve = dm.getDepthStratum(0) + vset = {tuple(np.round(coords[v - vs], 9)) for v in range(vs, ve)} + hang = 0 + for e in range(es, ee): + c = dm.getCone(e) + mp = tuple(np.round(0.5 * (coords[c[0] - vs] + coords[c[1] - vs]), 9)) + if mp in vset: + hang += 1 + return over, hang + + +def _refmap_dm(dm): + """Each cell's refinement edge, keyed by centroid, as a frozenset of endpoint coords.""" + slot = dm.getLabel("uwnvb_refedge") + co = dm.getCoordinatesLocal().array.reshape(-1, 2) + vs, ve = dm.getDepthStratum(0) + cs, ce = dm.getHeightStratum(0) + ref = {} + for c in range(cs, ce): + cone = dm.getCone(c) + s = slot.getValue(c) + verts = set() + for e in cone: + a, b = dm.getCone(e) + verts |= {a, b} + cen = tuple(np.round(np.array([co[v - vs] for v in verts]).mean(0), 8)) + a, b = dm.getCone(cone[s]) + ref[cen] = frozenset({tuple(np.round(co[a - vs], 8)), tuple(np.round(co[b - vs], 8))}) + return ref + + +def _refmap_nvbmesh(m): + C = np.array(m.coords) + ref = {} + for _, (p, b0, b1) in m.cells.items(): + cen = tuple(np.round(C[[p, b0, b1]].mean(0), 8)) + ref[cen] = frozenset({tuple(np.round(C[b0], 8)), tuple(np.round(C[b1], 8))}) + return ref + + +def test_refedges_match_nvbmesh_under_uniform_refine(): + """The native driver reproduces NVBMesh's refinement-edge structure exactly + over repeated uniform refinement (the invariant that makes closure bounded).""" + base = _base(0.35) + m = NVBMesh.from_dm(base) + dm = base + for _ in range(4): + m.refine(set(m.cells.keys())) + d = dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + cs, ce = d.getHeightStratum(0) + for c in range(cs, ce): + lab.setValue(c, DM_ADAPT_REFINE) + dm = _nvb.refine(d, "adapt") + assert _ncells(dm) == len(m.cells) + rd, rn = _refmap_dm(dm), _refmap_nvbmesh(m) + assert len(rd) == len(rn) + assert all(cen in rn and rn[cen] == rd[cen] for cen in rd) + + +def test_deep_mark_is_bounded_not_a_drain(): + """A single cell marked deep in a 2x-refined patch adds O(1) cells (NVB), not + the whole patch (the SBR longest-edge drain).""" + dm = _base(0.1) + for _ in range(2): + dm = _refine(dm, lambda x: np.linalg.norm(x - CENTER) < 0.35) + before = _ncells(dm) + C, cs, ce = _cents(dm) + tgt = C[np.argmin(np.linalg.norm(C - CENTER, axis=1))] + dm2 = _refine(dm, lambda x, t=tgt: np.allclose(x, t)) + added = _ncells(dm2) - before + assert 0 < added <= 12, f"deep mark added {added} cells (should be a small local closure)" + assert _conforming(dm2) == (0, 0) + + +def test_bullseye_grades_and_is_conforming(): + """A shrinking bullseye keeps many refinement levels (grades) and stays + conforming — far fewer cells than an SBR uniform patch would give.""" + dm = _base(0.12) + for R in (0.4, 0.28, 0.18, 0.11, 0.06): + dm = _refine(dm, lambda x, R=R: np.linalg.norm(x - CENTER) < R) + assert _conforming(dm) == (0, 0) + # multiple distinct cell areas => graded (not a single uniform-fine patch) + a = np.array([dm.computeCellGeometryFVM(c)[0] for c in range(*dm.getHeightStratum(0))]) + a = a[a > 0] + levels = np.unique(np.round(np.log2(a.max() / a)).astype(int)) + assert len(levels) >= 4, f"expected a graded spread of levels, got {levels}" + + +def test_deterministic(): + a = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) + b = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) + assert _ncells(a) == _ncells(b) From be9f41c99228d046caf35b041be313c229a01558 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 17:03:14 +1000 Subject: [PATCH 36/52] feat(layer2/nvb): SF-propagated cross-rank closure for parallel NVB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the on-rank-only closure with the SF-reconciled version: each C-cell's refinement edge is marked in a `req` edge label and propagated across the point SF with DMLabelPropagate (the same mechanism SBR's SetUp uses — edges are shared SF points, cells are not), so a cell blocked by an off-rank neighbour marks that neighbour on its owning rank. Each rank derives its own C-cells locally from the requested edges incident to them. This makes the LEPP closure cross-rank complete. Serial behaviour is unchanged (tests 0837 + 0838 still pass); the driver runs at np=1/2/3 producing conforming, graded output (locally 0 over-shared; 0 hanging nodes at np=1; SF-reconciled compatibility prevents one-sided edge bisection). Known limitation (documented in NVB_GRADED_ADAPT.md): the parallel result is valid but not yet bit-confluent — global cell counts differ by a few cells across np (uniform refine 159/352/760 at np=1 vs 157/346/743 at np=2). This is a drain sub-pass sequencing issue, NOT the closure (it affects uniform refine, whose closure is trivial), NOT the longest-edge seed (no exact ties → geometrically deterministic), and NOT the SF-LAND compatibility indexing (verified point-SF ilocal = point numbers). Confluence is a focused follow-up (per-sub-pass serial-vs-parallel comparison), then Stage 2c wires _adapt_nested(engine=nvb) np>1. Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 26 +++++-- src/underworld3/utilities/nvb_transform.c | 82 +++++++++++++++-------- 2 files changed, 73 insertions(+), 35 deletions(-) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index dff9ba55..534765c8 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -450,14 +450,28 @@ label from the output); and the drain loop condition must use the **global** `|C (`MPIU_Allreduce`) or a rank with no local marks skips the collective loop and deadlocks. -**Remaining for full parallel confluence:** the closure is currently on-rank only, -so a closure that would cross a partition under-refines slightly there (np=2 gives -161 cells where serial gives 163 — still conforming, since the `SF`-reconciled -compatibility test prevents cross-partition hanging nodes). The fix is the -SF-reconciled cross-rank closure: propagate *requested* refinement edges across -the point SF with `DMLabelPropagate` (exactly as SBR's `SetUp` does), so a +The **cross-rank closure** is implemented: `C`'s refinement edges are marked in a +`req` edge label and propagated across the point SF with `DMLabelPropagate` +(exactly as SBR's `SetUp` does — edges are shared SF points, cells are not), so a `C`-cell blocked by an off-rank neighbour marks that neighbour on its owning rank. +**Remaining — parallel confluence (drain sequencing):** the parallel result is +**valid** (locally conforming; 0 hanging nodes at np=1; the SF-reconciled +compatibility reduce prevents one-sided edge bisection, so no cross-partition +hanging nodes) but **not bit-confluent** — global cell counts differ by a few +cells across `np` (e.g. uniform refine: 159/352/760 at np=1 vs 157/346/743 at +np=2). This is *not* the closure (it affects uniform refine, whose closure is +trivial), *not* the longest-edge seed (no exact ties in the test meshes → the seed +is geometrically deterministic), and *not* the `uwnvb_sf_land` indexing (verified: +the point-SF `ilocal` holds point numbers, so `PetscSFReduce(val,val,MPI_LAND)` +indexes `val[point]` correctly). The remaining suspect is the **drain sub-pass +sequencing**: with the same refinement-edge seed and the same cell set, the batches +of compatible edges bisected per sub-pass — and the conformity-closure bisections +that make NVB uniform refine super-linear (26→64) — appear to resolve to a +slightly different (still valid) mesh depending on the partition. Pinning it needs +a per-sub-pass comparison of the compatible-edge set serial-vs-parallel to find the +divergence. Then Stage 2c: wire `_adapt_nested(engine="nvb")` at np>1 to the driver. + ### Stage 2b finding (2026-07-01): grading needs single-bisection, multi-pass Two attempts to add the newest-vertex edge choice on top of the Stage-2a base both diff --git a/src/underworld3/utilities/nvb_transform.c b/src/underworld3/utilities/nvb_transform.c index 66272cf2..8fe724b1 100644 --- a/src/underworld3/utilities/nvb_transform.c +++ b/src/underworld3/utilities/nvb_transform.c @@ -924,57 +924,81 @@ PETSC_EXTERN PetscErrorCode UWNVBRefine(DM dm, const char *wantName, DM *rdm) PetscCall(PetscObjectReference((PetscObject)work)); PetscCall(DMGetLabel(work, UWNVB_SLOT_LABEL, &slot)); /* NULL on the base */ - /* --- 1. closure: C = want, grown by blocking neighbours (on-rank) --- */ + /* --- 1. closure: C = want, grown by blocking neighbours across refinement + edges. Blockers are discovered by propagating REQUESTED refinement edges over + the point SF (edges are shared SF points; cells are not) — the same + DMLabelPropagate mechanism SBR uses, so the closure is cross-rank complete and + bounded (LEPP; Rivara/Stevenson). Each rank derives its own C cells locally + from the requested edges incident to them. --- */ PetscCall(DMLabelCreate(PETSC_COMM_SELF, "uwnvb_C", &C)); { - IS wantIS; - const PetscInt *wantCells; - PetscInt nWant, i, *stack, top = 0, cap; + IS wantIS; + const PetscInt *wantCells; + DMLabel req; + DMPlexPointQueue queue; + PetscSF pointSF; + PetscInt nWant, i; + PetscBool empty; + PetscCall(DMPlexGetHeightStratum(work, 0, &cStart, &cEnd)); + PetscCall(DMGetPointSF(work, &pointSF)); + PetscCall(DMLabelCreate(PETSC_COMM_SELF, "uwnvb_req", &req)); + PetscCall(DMPlexPointQueueCreate(1024, &queue)); + /* seed: each want triangle joins C and requests its refinement edge */ PetscCall(DMLabelGetStratumIS(want, DM_ADAPT_REFINE, &wantIS)); PetscCall(DMLabelGetStratumSize(want, DM_ADAPT_REFINE, &nWant)); - PetscCall(DMPlexGetHeightStratum(work, 0, &cStart, &cEnd)); - cap = cEnd - cStart; /* every C member is a triangle; ≤ #cells, never overflows */ - PetscCall(PetscMalloc1(cap > 0 ? cap : 1, &stack)); if (wantIS) { PetscCall(ISGetIndices(wantIS, &wantCells)); for (i = 0; i < nWant; ++i) { DMPolytopeType ct; - PetscInt v; - /* Ignore any non-triangle marks — the adaptation label can carry stale + PetscInt refedge, refslot, rv; + /* Ignore non-triangle marks — the adaptation label can carry stale entries transferred from a previous refine onto non-cell points. */ if (wantCells[i] < cStart || wantCells[i] >= cEnd) continue; PetscCall(DMPlexGetCellType(work, wantCells[i], &ct)); if (ct != DM_POLYTOPE_TRIANGLE) continue; - PetscCall(DMLabelGetValue(C, wantCells[i], &v)); - if (v != 1) { - PetscCall(DMLabelSetValue(C, wantCells[i], 1)); - stack[top++] = wantCells[i]; + PetscCall(DMLabelSetValue(C, wantCells[i], 1)); + PetscCall(uwnvb_refedge(work, slot, wantCells[i], &refedge, &refslot)); + PetscCall(DMLabelGetValue(req, refedge, &rv)); + if (rv != 1) { + PetscCall(DMLabelSetValue(req, refedge, 1)); + PetscCall(DMPlexPointQueueEnqueue(queue, refedge)); } } PetscCall(ISRestoreIndices(wantIS, &wantCells)); } PetscCall(ISDestroy(&wantIS)); - while (top > 0) { - PetscInt cell = stack[--top], refedge, refslot, ss, s; - const PetscInt *supp; - PetscCall(uwnvb_refedge(work, slot, cell, &refedge, &refslot)); - PetscCall(DMPlexGetSupport(work, refedge, &supp)); - PetscCall(DMPlexGetSupportSize(work, refedge, &ss)); - for (s = 0; s < ss; ++s) { - PetscInt d = supp[s], de, ds, v; - if (d == cell) continue; - PetscCall(uwnvb_refedge(work, slot, d, &de, &ds)); - if (de != refedge) { /* d blocks cell: d must refine its own refedge first */ - PetscCall(DMLabelGetValue(C, d, &v)); - if (v != 1) { - PetscCall(DMLabelSetValue(C, d, 1)); - if (top < cap) stack[top++] = d; + PetscCall(DMLabelPropagateBegin(req, pointSF)); + PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)work, queue, &empty)); + while (!empty) { + while (!DMPlexPointQueueEmpty(queue)) { + PetscInt e = -1, ss, s; + const PetscInt *supp; + PetscCall(DMPlexPointQueueDequeue(queue, &e)); + PetscCall(DMPlexGetSupport(work, e, &supp)); + PetscCall(DMPlexGetSupportSize(work, e, &ss)); + for (s = 0; s < ss; ++s) { + PetscInt d = supp[s], de, ds, cv, rv; + PetscCall(uwnvb_refedge(work, slot, d, &de, &ds)); + if (de != e) { /* d blocks a request on e: it must bisect its own refedge first */ + PetscCall(DMLabelGetValue(C, d, &cv)); + if (cv != 1) PetscCall(DMLabelSetValue(C, d, 1)); + PetscCall(DMLabelGetValue(req, de, &rv)); + if (rv != 1) { + PetscCall(DMLabelSetValue(req, de, 1)); + PetscCall(DMPlexPointQueueEnqueue(queue, de)); + } } } } + /* exchange newly-requested edges across ranks; the callback enqueues any + that arrive here so their incident cells are processed next round */ + PetscCall(DMLabelPropagatePush(req, pointSF, UWNVBSplitPoint, queue)); + PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)work, queue, &empty)); } - PetscCall(PetscFree(stack)); + PetscCall(DMLabelPropagateEnd(req, pointSF)); + PetscCall(DMLabelDestroy(&req)); + PetscCall(DMPlexPointQueueDestroy(&queue)); } /* --- 2. drain: single-split compatible refedges of C until C empty --- */ From 31091e572e188c10799c159d6015b54ba9478498 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 17:48:52 +1000 Subject: [PATCH 37/52] =?UTF-8?q?fix(layer2/nvb):=20parallel=20confluence?= =?UTF-8?q?=20=E2=80=94=20OR-reconcile=20the=20per-pass=20bisection=20set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nested-distributed-refine SEGFAULT and the parallel non-confluence were the same single bug: the drain marked edges to bisect purely from local C cells, so a shared edge could be split on one rank and kept whole on its neighbour. That makes DMPlexTransformCreateSF map a split-edge child on one rank onto an unsplit vertex on the other — a stratum-crossing point SF. On the next refine, DMLabelPropagate reduces an edge's request onto that vertex (whose support is edges, not cells) and reads out of range → SEGV; and the divergent split sets produced slightly different (each valid) meshes per partition. Fix: uwnvb_sf_lor — an MPI_LOR PetscSFReduce+Bcast of the bisect-mark array over the point SF, so an edge chosen on any rank is split on every rank owning a copy. Since agree[] is already MPI_LAND-reconciled (the edge is the refinement edge of the incident cell on all sharers), splitting it everywhere is exactly the conforming closure — the OR fixes the SF and completes the cross-partition closure in one step. Serial is unchanged (no SF graph → no-op). Now bit-confluent with serial at any communicator size: uniform 159/352/760, graded bullseye 215, deep 5-level nesting 96/180/288/390/472 all equal at np=1/2/3/4, 0 hanging nodes, stratum-consistent output SF. New regression test test_parallel_confluence pins it (passes np=1/2/3). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 68 ++++++++++++++--------- src/underworld3/utilities/nvb_transform.c | 44 ++++++++++++--- tests/test_0838_nvb_graded_native.py | 52 ++++++++++++++++- 3 files changed, 128 insertions(+), 36 deletions(-) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 534765c8..491933d0 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -1,9 +1,13 @@ # Newest-Vertex Bisection (NVB) for graded adapt-on-top -Status: **implemented (serial Route A), 2026-06-30**. Engine in +Status: **implemented — serial Route A (2026-06-30) + parallel Route B native +transform, bit-confluent (2026-07-01)**. Serial engine in `src/underworld3/utilities/nvb.py` + `custom_mg.nvb_refine`; wired into `Mesh.adapt(engine="nvb")`; validated in `tests/test_0836_nvb_graded_adapt.py`. -Parallel (Route B, native transform) is the next step. Follows the Layer-2 +The native `uwnvb` `DMPlexTransform` (`src/underworld3/utilities/nvb_transform.c`, +`tests/test_083{7,8}`) grades in parallel and produces a mesh identical to the +serial one at any communicator size. Remaining: Stage 2c integration +(`_adapt_nested(engine="nvb")` at np>1) + FMG tail. Follows the Layer-2 investigation in [`LAYER2_SBR_ADAPT_ON_TOP.md`](LAYER2_SBR_ADAPT_ON_TOP.md). Goal: replace the refinement *engine* under `mesh.adapt(adapter="sbr")` so that successive levels @@ -414,12 +418,14 @@ tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build triangulation for every marking pattern incl. full uniform refine. `tests/ test_0837_nvb_native_transform.py` (tier_b). Longest-edge edge pick; the newest- vertex choice layers on top. -3. *Newest-vertex grading (Stage 2b)* — **DONE serially (2026-07-01)** via the - single-bisection multi-pass driver (see below). Matches serial `NVBMesh` +3. *Newest-vertex grading (Stage 2b)* — **DONE, serial and parallel (2026-07-01)** + via the single-bisection multi-pass driver (see below). Matches serial `NVBMesh` refinement edges exactly over repeated uniform refine; a deep mark is bounded (+2 vs SBR's +4824); conforming, deterministic, graded bullseye (805 vs SBR's - 102 868 cells). `tests/test_0838_nvb_graded_native.py`. Runs conforming at - np=1/2/3; full parallel *confluence* needs the cross-rank closure (below). + 102 868 cells). **Bit-confluent in parallel**: the refined mesh is identical to + the serial one at any communicator size — uniform refine 159/352/760 and graded + bullseye 215 cells agree exactly at np=1/2/3/4, 0 hanging nodes. + `tests/test_0838_nvb_graded_native.py` (`test_parallel_confluence`). 4. *Integration + acceptance* — `_adapt_nested(engine="nvb")` at np>1 via the driver; Poisson + SolCx FMG match GAMG; result matches the serial NVB mesh. @@ -450,27 +456,35 @@ label from the output); and the drain loop condition must use the **global** `|C (`MPIU_Allreduce`) or a rank with no local marks skips the collective loop and deadlocks. -The **cross-rank closure** is implemented: `C`'s refinement edges are marked in a -`req` edge label and propagated across the point SF with `DMLabelPropagate` -(exactly as SBR's `SetUp` does — edges are shared SF points, cells are not), so a -`C`-cell blocked by an off-rank neighbour marks that neighbour on its owning rank. - -**Remaining — parallel confluence (drain sequencing):** the parallel result is -**valid** (locally conforming; 0 hanging nodes at np=1; the SF-reconciled -compatibility reduce prevents one-sided edge bisection, so no cross-partition -hanging nodes) but **not bit-confluent** — global cell counts differ by a few -cells across `np` (e.g. uniform refine: 159/352/760 at np=1 vs 157/346/743 at -np=2). This is *not* the closure (it affects uniform refine, whose closure is -trivial), *not* the longest-edge seed (no exact ties in the test meshes → the seed -is geometrically deterministic), and *not* the `uwnvb_sf_land` indexing (verified: -the point-SF `ilocal` holds point numbers, so `PetscSFReduce(val,val,MPI_LAND)` -indexes `val[point]` correctly). The remaining suspect is the **drain sub-pass -sequencing**: with the same refinement-edge seed and the same cell set, the batches -of compatible edges bisected per sub-pass — and the conformity-closure bisections -that make NVB uniform refine super-linear (26→64) — appear to resolve to a -slightly different (still valid) mesh depending on the partition. Pinning it needs -a per-sub-pass comparison of the compatible-edge set serial-vs-parallel to find the -divergence. Then Stage 2c: wire `_adapt_nested(engine="nvb")` at np>1 to the driver. +The parallel path needs **two** SF reconciliations, and both are now in place: + +1. **Cross-rank closure.** `C`'s refinement edges are marked in a `req` edge label + and propagated across the point SF with `DMLabelPropagate` (exactly as SBR's + `SetUp` does — edges are shared SF points, cells are not), so a `C`-cell blocked + by an off-rank neighbour marks that neighbour on its owning rank. +2. **Bisection-set consistency (`uwnvb_sf_lor`).** The per-pass set of edges to + split must be *identical across ranks* for every shared edge. Marking it purely + from the local `C` cells is not enough: a shared edge can be the agreed + refinement edge of a `C`-cell on rank A while rank B's copy of the incident cell + is not (yet) in its local `C`. Rank A then splits the shared edge (midpoint + two + child edges) while rank B keeps it whole, so `DMPlexTransformCreateSF` maps rank + A's child *edge* onto rank B's *vertex* — a **stratum-crossing point SF**. On the + next refine, `DMLabelPropagate` reduces an edge's request onto that vertex, whose + support is edges (not cells) → out-of-range read → **segfault**. The fix is an + `MPI_LOR` `PetscSFReduce`+`Bcast` of the bisect-mark array over the point SF: an + edge chosen on *any* rank is split on *every* rank owning a copy. Because the + `agree` set (already `MPI_LAND`-reconciled) guarantees such an edge is the + refinement edge of the incident cell on **all** sharers, splitting it everywhere + is exactly the conforming closure — so the OR both fixes the SF and completes the + cross-partition closure. + +**Confluence (2026-07-01): achieved.** With both reconciliations, the refined mesh +is **bit-identical to the serial mesh** at any communicator size (uniform +159/352/760, graded bullseye 215, deep 5-level nesting 96/180/288/390/472 — all +equal at np=1/2/3/4; 0 hanging nodes; the output point SF is stratum-consistent). +The root cause of the earlier non-confluence *and* the nested-distributed segfault +was the single missing `uwnvb_sf_lor` — one bug, both symptoms. Next: Stage 2c — +wire `_adapt_nested(engine="nvb")` at np>1 to the driver. ### Stage 2b finding (2026-07-01): grading needs single-bisection, multi-pass diff --git a/src/underworld3/utilities/nvb_transform.c b/src/underworld3/utilities/nvb_transform.c index 8fe724b1..c9b63d19 100644 --- a/src/underworld3/utilities/nvb_transform.c +++ b/src/underworld3/utilities/nvb_transform.c @@ -887,6 +887,26 @@ static PetscErrorCode uwnvb_sf_land(DM dm, PetscInt *val) PetscFunctionReturn(PETSC_SUCCESS); } +/* Point-indexed 0/1 array made globally consistent under logical-OR across the + point SF (serial: no-op). After this, a shared point's value is 1 if ANY rank + marked it — used so a shared edge chosen for bisection on one rank is split on + every rank that owns a copy, keeping the child point SF conforming. */ +static PetscErrorCode uwnvb_sf_lor(DM dm, PetscInt *val) +{ + PetscSF sf; + PetscInt nroots; + + PetscFunctionBegin; + PetscCall(DMGetPointSF(dm, &sf)); + PetscCall(PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL)); + if (nroots < 0) PetscFunctionReturn(PETSC_SUCCESS); /* no SF (serial) */ + PetscCall(PetscSFReduceBegin(sf, MPIU_INT, val, val, MPI_LOR)); + PetscCall(PetscSFReduceEnd(sf, MPIU_INT, val, val, MPI_LOR)); + PetscCall(PetscSFBcastBegin(sf, MPIU_INT, val, val, MPI_REPLACE)); + PetscCall(PetscSFBcastEnd(sf, MPIU_INT, val, val, MPI_REPLACE)); + PetscFunctionReturn(PETSC_SUCCESS); +} + /* ---- driver: graded NVB refinement of `dm` ----------------------------- */ /* Refine `dm` so that every cell flagged in the `wantName` adaptation label @@ -1035,27 +1055,37 @@ PETSC_EXTERN PetscErrorCode UWNVBRefine(DM dm, const char *wantName, DM *rdm) } PetscCall(uwnvb_sf_land(work, agree)); - /* ready edges = refedge of a C cell that is agreed; mark them for bisection */ - PetscCall(DMLabelCreate(PETSC_COMM_SELF, UWNVB_BISECT_LABEL, &bisect)); + /* ready edges = refedge of a C cell that is agreed. Collect them in a + point-indexed mark array, then OR-reconcile across the point SF so a shared + edge chosen on any rank is split on every rank owning a copy — otherwise the + child point SF crosses strata (a split-edge child on one rank maps to an + unsplit vertex on its neighbour) and DMLabelPropagate corrupts. */ { + PetscInt *bmark; IS cis; const PetscInt *ccells; PetscInt n, i; + PetscCall(PetscCalloc1(pEnd - pStart, &bmark)); PetscCall(DMLabelGetStratumIS(C, 1, &cis)); PetscCall(DMLabelGetStratumSize(C, 1, &n)); if (cis) { PetscCall(ISGetIndices(cis, &ccells)); for (i = 0; i < n; ++i) { - PetscInt refedge, refslot, bv; + PetscInt refedge, refslot; PetscCall(uwnvb_refedge(work, slot, ccells[i], &refedge, &refslot)); - if (agree[refedge - pStart]) { - PetscCall(DMLabelGetValue(bisect, refedge, &bv)); - if (bv != 1) { PetscCall(DMLabelSetValue(bisect, refedge, 1)); ++nready; } - } + if (agree[refedge - pStart]) bmark[refedge - pStart] = 1; } PetscCall(ISRestoreIndices(cis, &ccells)); } PetscCall(ISDestroy(&cis)); + PetscCall(uwnvb_sf_lor(work, bmark)); + + PetscCall(DMPlexGetDepthStratum(work, 1, &eStart, &eEnd)); + PetscCall(DMLabelCreate(PETSC_COMM_SELF, UWNVB_BISECT_LABEL, &bisect)); + for (e = eStart; e < eEnd; ++e) { + if (bmark[e - pStart]) { PetscCall(DMLabelSetValue(bisect, e, 1)); ++nready; } + } + PetscCall(PetscFree(bmark)); } PetscCall(PetscFree(agree)); { diff --git a/tests/test_0838_nvb_graded_native.py b/tests/test_0838_nvb_graded_native.py index 00908f42..a01ad57a 100644 --- a/tests/test_0838_nvb_graded_native.py +++ b/tests/test_0838_nvb_graded_native.py @@ -16,8 +16,11 @@ - a shrinking bullseye grades (many levels coexist), with far fewer cells than the SBR uniform patch. -Serial (the SF-reconciled cross-rank closure for full parallel confluence is a -follow-up; the driver already runs conforming at np>1). +Parallel confluence is covered by ``test_parallel_confluence`` below: the driver +SF-reconciles both the conforming closure (``req`` via ``DMLabelPropagate``) and +the per-pass bisection set (``uwnvb_sf_lor`` — a shared edge chosen on any rank is +split on every rank owning a copy), so the refined mesh is bit-identical to the +serial one at any communicator size. """ import numpy as np import pytest @@ -163,3 +166,48 @@ def test_deterministic(): a = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) b = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) assert _ncells(a) == _ncells(b) + + +def _global_owned_cells(dm): + """Total cells across all ranks, counting each cell only on its owner (a + partition-independent size, so it must match the serial value under confluence).""" + from mpi4py import MPI + + cs, ce = dm.getHeightStratum(0) + sf = dm.getPointSF() + try: + _, ilocal, _ = sf.getGraph() + except (ValueError, TypeError): + ilocal = None # serial: no SF graph -> every local cell is owned + leaves = set() if ilocal is None else {int(x) for x in ilocal} + owned = sum(1 for c in range(cs, ce) if c not in leaves) + return dm.comm.tompi4py().allreduce(owned, op=MPI.SUM) + + +def test_parallel_confluence(): + """The refined mesh is partition-independent: the global owned-cell count and + the (serially-validated) reference values agree at any communicator size, and + no edge is over-shared. Runs on 1..N ranks; the assert is the confluence pin.""" + base = _base(0.2) + + # uniform: serial reference sizes for cellSize=0.2 (see the confluence sweep) + dm = base + for expected in (159, 352, 760): + d = dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + cs, ce = d.getHeightStratum(0) + for c in range(cs, ce): + lab.setValue(c, DM_ADAPT_REFINE) + dm = _nvb.refine(d, "adapt") + assert _global_owned_cells(dm) == expected + + # graded bullseye: serial reference size + dm = base + for R in (0.4, 0.28, 0.18): + dm = _refine(dm, lambda x, R=R: np.linalg.norm(x - CENTER) < R) + assert _global_owned_cells(dm) == 215 + # no locally over-shared edge (each rank stays conforming) + es, ee = dm.getDepthStratum(1) + assert all(dm.getSupportSize(e) <= 2 for e in range(es, ee)) From d37819768006fb7c268c826408b471f0974f345e Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 17:51:47 +1000 Subject: [PATCH 38/52] test(layer2/nvb): skip serial NVBMesh cross-checks under np>1 The four NVBMesh-reference tests compare per-rank local cells to the serial reference engine, so they only make sense at np=1. Mark them serial_only; test_parallel_confluence (partition-independent global counts) is the parallel pin. np=2/3: 1 passed + 4 skipped; serial: all 9 pass. Underworld development team with AI support from Claude Code --- tests/test_0838_nvb_graded_native.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_0838_nvb_graded_native.py b/tests/test_0838_nvb_graded_native.py index a01ad57a..fbc2eb71 100644 --- a/tests/test_0838_nvb_graded_native.py +++ b/tests/test_0838_nvb_graded_native.py @@ -38,6 +38,14 @@ DM_ADAPT_REFINE = 1 CENTER = np.array([0.5, 0.5]) +# The NVBMesh cross-checks below use the serial reference engine and compare +# per-rank local cells; they are meaningful only serially. Parallel behaviour is +# pinned by test_parallel_confluence (partition-independent global counts). +serial_only = pytest.mark.skipif( + PETSc.COMM_WORLD.getSize() > 1, + reason="serial NVBMesh reference; parallel confluence covered by test_parallel_confluence", +) + def _base(cellSize=0.2): return uw.meshing.UnstructuredSimplexBox( @@ -111,6 +119,7 @@ def _refmap_nvbmesh(m): return ref +@serial_only def test_refedges_match_nvbmesh_under_uniform_refine(): """The native driver reproduces NVBMesh's refinement-edge structure exactly over repeated uniform refinement (the invariant that makes closure bounded).""" @@ -133,6 +142,7 @@ def test_refedges_match_nvbmesh_under_uniform_refine(): assert all(cen in rn and rn[cen] == rd[cen] for cen in rd) +@serial_only def test_deep_mark_is_bounded_not_a_drain(): """A single cell marked deep in a 2x-refined patch adds O(1) cells (NVB), not the whole patch (the SBR longest-edge drain).""" @@ -148,6 +158,7 @@ def test_deep_mark_is_bounded_not_a_drain(): assert _conforming(dm2) == (0, 0) +@serial_only def test_bullseye_grades_and_is_conforming(): """A shrinking bullseye keeps many refinement levels (grades) and stays conforming — far fewer cells than an SBR uniform patch would give.""" @@ -162,6 +173,7 @@ def test_bullseye_grades_and_is_conforming(): assert len(levels) >= 4, f"expected a graded spread of levels, got {levels}" +@serial_only def test_deterministic(): a = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) b = _refine(_base(0.12), lambda x: np.linalg.norm(x - CENTER) < 0.2) From a5f45df882b61e8764aad9d2e915529b90a08bc7 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 18:07:06 +1000 Subject: [PATCH 39/52] =?UTF-8?q?feat(layer2/nvb):=20Stage=202c=20?= =?UTF-8?q?=E2=80=94=20wire=20native=20uwnvb=20transform=20into=20mesh.ada?= =?UTF-8?q?pt=20(parallel)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh.adapt(engine="nvb") now dispatches to the native uwnvb DMPlexTransform when the compiled extension is present, giving graded NVB refinement in parallel (was serial-only via the NVBMesh cell-list engine, which raised at np>1). The transform is in-place — the child stays co-partitioned with the parent, so the mesh-owned custom-P geometric-MG tail is valid at np>1 — and carries the boundary/region labels forward automatically. _adapt_nested: for engine="nvb" with the extension available, mark cells whose current size exceeds the metric target and refine once per pass via _nvb_transform.refine (up to 2*max_levels passes; a bisection halves the area). Collective stop via allreduce(sel.size) so a rank with no local marks still joins the refine when a neighbour's edge forces a closure bisection. metric evaluated with global_evaluate at np>1. NVBMesh remains the serial fallback (raises at np>1 only when the extension is absent). Verified: adapt(engine="nvb") is bit-confluent (child=366 cells at np=1/2/4), boundaries + 4-level MG tail preserved; Poisson on the graded child drives custom-P FMG converging in 3 iters == GAMG at np=1/2/4. New test_0839_nvb_parallel_adapt.py (confluence + FMG acceptance, passes np=1/2/3); existing SBR + serial-NVB adapt suites (22 tests) still green. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 112 +++++++++++++---- tests/test_0836_nvb_graded_adapt.py | 5 +- tests/test_0839_nvb_parallel_adapt.py | 115 ++++++++++++++++++ 3 files changed, 210 insertions(+), 22 deletions(-) create mode 100644 tests/test_0839_nvb_parallel_adapt.py diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index 21681123..c0709d7c 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6030,14 +6030,14 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, *unbounded for region marking*, so it produces a **uniform-finest patch**, not a graded mesh (a marked cell drains the longest-edge path to the patch edge). Robust and fine for the MG hierarchy. - * ``"nvb"`` — newest-vertex bisection (:mod:`underworld3.utilities.nvb`), - a **graded** engine with a *bounded* conforming closure: a marked cell - adds O(1) cells locally, so successive levels grade (a level+1 ring around - a finer core) and DOFs concentrate near the feature. Serial only this pass - (``NotImplementedError`` at np>1 — the parallel decomposition cannot be - preserved through the cell-list DM rebuild; a native transform is the - parallel path). Bisects 1→2, so one isotropic-equivalent ``max_levels`` is - run as **two** NVB generations. + * ``"nvb"`` — newest-vertex bisection, a **graded** engine with a *bounded* + conforming closure: a marked cell adds O(1) cells locally, so successive + levels grade (a level+1 ring around a finer core) and DOFs concentrate near + the feature. Runs **in parallel** via the native ``uwnvb`` ``DMPlexTransform`` + (in-place, co-partitioned with the parent, bit-confluent serial↔parallel); + when that compiled extension is absent it falls back to the serial + ``NVBMesh`` cell-list engine (``NotImplementedError`` at np>1). Bisects 1→2, + so one isotropic-equivalent ``max_levels`` is run as **two** NVB passes. The child owns a custom-P geometric-MG hierarchy with **one level per SBR refinement step** — ``[base L0 … base finest, SBR-1, …, SBR-n(child)]`` — @@ -6137,31 +6137,101 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, "dm_hierarchy of coarse levels supplies the geometric-MG tail). " "Build the mesh with e.g. refinement=2." ) + # The native uwnvb DMPlexTransform (Route B) is the parallel NVB engine: + # in-place (co-partitioned with the parent), graded, and bit-confluent + # serial<->parallel. Prefer it whenever the compiled extension is present; + # fall back to the serial NVBMesh cell-list engine only when it is not + # (which then restricts engine='nvb' to np=1). + _nvbx = None if engine == "nvb": - if uw.mpi.size > 1: - raise NotImplementedError( - "adapt(engine='nvb') is serial-only: the cell-list DMPlex " - "rebuild does not preserve the parent's parallel decomposition " - "(required by the custom-P parallel path). Use engine='sbr' at " - "np>1, or the native-transform (Route B) NVB when available." - ) if self.dim != 2: raise NotImplementedError( "adapt(engine='nvb') is 2D only this pass (tets are a follow-up)." ) + try: + from underworld3.utilities import _nvb_transform as _nvbx + except ImportError: + _nvbx = None + if _nvbx is None and uw.mpi.size > 1: + raise NotImplementedError( + "adapt(engine='nvb') at np>1 needs the native uwnvb transform " + "(underworld3.utilities._nvb_transform), which is not built in " + "this environment. Build the custom-PETSc/amr env, or use " + "engine='sbr' at np>1." + ) dim = self.dim edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex + DM_ADAPT_REFINE = 1 # PETSc DMAdaptFlag: refine this cell markers_per_level = [] level_dms = [] # one DM per refinement level - if engine == "nvb": - # Persistent NVBMesh: the refinement-edge labelling propagates - # parent→child across generations, which preserves the similarity-class - # (shape-regularity) bound. Each generation bisects 1→2 (h /√2), so we - # run up to 2·max_levels generations to reach the same isotropic- - # equivalent target as the SBR path's max_levels passes (1→4 each). + if engine == "nvb" and _nvbx is not None: + # Native uwnvb DMPlexTransform. Each pass marks the cells whose current + # size still exceeds the metric target and bisects them once (with the + # bounded newest-vertex conforming closure); the transform is in-place so + # the output stays co-partitioned with the parent and carries the + # boundary/region labels forward automatically. A single bisection halves + # the area (h → h/√2), so we allow up to 2·max_levels passes to reach the + # same isotropic target as the SBR path's max_levels quad-splits. + current_dm = self.dm_hierarchy[-1] # static base finest (distributed) + n_gen = 2 * max_levels + for level in range(n_gen): + cs, ce = current_dm.getHeightStratum(0) + ncells = ce - cs + if ncells: + centroids = numpy.empty((ncells, self.cdim)) + cur_h = numpy.empty(ncells) + for i, c in enumerate(range(cs, ce)): + vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] + centroids[i] = numpy.asarray(cen)[: self.cdim] + cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) + if uw.mpi.size > 1: + M = numpy.asarray( + uw.function.global_evaluate(metric_field.sym, centroids) + ).reshape(-1) + else: + M = numpy.asarray( + uw.function.evaluate(metric_field.sym, centroids) + ).reshape(-1) + M = numpy.clip(M, 1e-30, None) + h_target = 1.0 / numpy.sqrt(M) + sel = numpy.where(cur_h > h_target)[0] + if node_budget is not None and sel.size > node_budget: + order = numpy.argsort(M[sel])[::-1] + sel = sel[order[:node_budget]] + else: + sel = numpy.empty(0, dtype=int) # rank owns no cells this level + + # Collective stop: refine while ANY rank still has cells to split + # (a rank with none may still bisect via the cross-rank closure). + # mpi4py allreduce defaults to MPI.SUM. + if uw.mpi.comm.allreduce(int(sel.size)) == 0: + if verbose: + uw.pprint(0, f"[adapt] nvb pass {level}: nothing to refine") + break + + marked = [int(cs + j) for j in sel] + markers_per_level.append(marked) + d = current_dm.clone() + d.createLabel("adapt") + lab = d.getLabel("adapt") + lab.setDefaultValue(0) + for cidx in marked: + lab.setValue(cidx, DM_ADAPT_REFINE) + current_dm = _nvbx.refine(d, "adapt") + level_dms.append(current_dm) + if verbose: + fs, fe = current_dm.getHeightStratum(0) + uw.pprint(0, f"[adapt] nvb pass {level}: marked {len(marked)} " + f"-> {fe - fs} cells (rank-local)") + if not level_dms: + current_dm = self.dm_hierarchy[-1].clone() + elif engine == "nvb": + # Serial fallback (no native transform): persistent NVBMesh cell-list + # engine. The refinement-edge labelling propagates parent→child across + # generations, preserving the similarity-class (shape-regularity) bound. from underworld3.utilities.nvb import NVBMesh carry = [(b.name, b.value) for b in self.boundaries if b.name not in ("Null_Boundary", "All_Boundaries")] diff --git a/tests/test_0836_nvb_graded_adapt.py b/tests/test_0836_nvb_graded_adapt.py index fb3e40dc..816ef164 100644 --- a/tests/test_0836_nvb_graded_adapt.py +++ b/tests/test_0836_nvb_graded_adapt.py @@ -16,7 +16,10 @@ - the graded NVB child drives the custom-P geometric-MG FMG: Poisson and SolCx Stokes (η jump 1e6) match a GAMG reference. -Serial only (NVB Route A); the np>1 / 3D guards are asserted to raise. +These tests exercise the serial NVBMesh reference engine (Route A) and the +integrated path; the 3D guard is asserted to raise. Parallel NVB (the native +uwnvb transform, Route B) and its confluence + FMG acceptance live in +``test_0839_nvb_parallel_adapt.py``. """ import numpy as np import pytest diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py new file mode 100644 index 00000000..8e08dc1d --- /dev/null +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -0,0 +1,115 @@ +"""Layer-2 Route B, Stage 2c: parallel ``mesh.adapt(engine="nvb")``. + +The native ``uwnvb`` ``DMPlexTransform`` is the parallel NVB engine — in-place +(co-partitioned with the parent, so the custom-P geometric-MG tail stays valid), +graded, and bit-confluent serial↔parallel. ``discretisation_mesh._adapt_nested`` +dispatches to it whenever the compiled extension is present. + +These tests pin the integrated parallel path: + - **confluence**: the adapted child's global (owner-counted) cell total is + partition-independent — identical at any communicator size, so the same + assert holds serially and under ``mpirun``; + - **labels carried**: the child keeps the parent's boundary labels through the + transform, and owns the ``[base … child]`` custom-P MG tail; + - **FMG acceptance**: a Poisson solve on the graded child drives the mesh-owned + custom-P geometric multigrid (``preconditioner="auto"``) and converges in the + same iteration count as a GAMG reference. + +Runs on 1..N ranks; the asserts are partition-independent, so ``mpirun -np K`` +validates confluence directly. +""" +import numpy as np +import pytest +import sympy +import underworld3 as uw +from petsc4py import PETSc + +pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] + +# Parallel NVB needs the native transform; skip cleanly where it is not built. +pytest.importorskip( + "underworld3.utilities._nvb_transform", + reason="native uwnvb transform not built (needs the custom-PETSc/amr env)", +) + + +def _base(): + """A base mesh with a one-level MG tail (refinement=1) so nested adapt has a + coarse hierarchy to extend.""" + return uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.25, regular=False, + qdegree=2, refinement=1, + ) + + +def _bullseye_metric(mesh, h_near=0.05, h_far=0.4, radius=0.25): + M = uw.discretisation.MeshVariable("Mbe", mesh, 1, degree=1) + with mesh.access(M): + c = M.coords + d = np.sqrt((c[:, 0] - 0.5) ** 2 + (c[:, 1] - 0.5) ** 2) + h = np.where(d < radius, h_near, h_far) + M.data[:, 0] = 1.0 / h ** 2 + return M + + +def _global_owned_cells(dm): + from mpi4py import MPI + + cs, ce = dm.getHeightStratum(0) + sf = dm.getPointSF() + try: + _, ilocal, _ = sf.getGraph() + except (ValueError, TypeError): + ilocal = None + leaves = set() if ilocal is None else {int(x) for x in ilocal} + owned = sum(1 for c in range(cs, ce) if c not in leaves) + return dm.comm.tompi4py().allreduce(owned, op=MPI.SUM) + + +def test_nvb_adapt_is_confluent_and_carries_labels(): + """The graded child is partition-independent and keeps its lineage/labels.""" + mesh = _base() + child = mesh.adapt(_bullseye_metric(mesh), max_levels=2, engine="nvb") + + # confluence: same global cell count at any communicator size (serial value) + assert _global_owned_cells(child.dm) == 366 + + # lineage + boundary labels carried through the transform + assert child.parent is mesh + assert child._relationship_kind == "refinement" + names = {b.name for b in child.boundaries} + assert {"Top", "Bottom", "Left", "Right"} <= names + + # mesh-owned custom-P tail: [base L0, base finest, nvb-1, nvb-2] under child + assert len(child._custom_mg_coarse_meshes) >= 2 + + +def test_poisson_fmg_on_nvb_child_matches_gamg(): + """Custom-P geometric MG on the graded NVB child converges like GAMG.""" + mesh = _base() + child = mesh.adapt(_bullseye_metric(mesh), max_levels=2, engine="nvb") + + x, y = child.X + exact = sympy.sin(sympy.pi * x) * sympy.sin(sympy.pi * y) + + def solve(pc): + u = uw.discretisation.MeshVariable(f"u_{pc}", child, 1, degree=1) + poisson = uw.systems.Poisson(child, u_Field=u) + poisson.constitutive_model = uw.constitutive_models.DiffusionModel + poisson.constitutive_model.Parameters.diffusivity = 1.0 + poisson.f = 2 * sympy.pi ** 2 * exact + for b in ("Top", "Bottom", "Left", "Right"): + poisson.add_dirichlet_bc(0.0, b) + poisson.preconditioner = "auto" if pc == "fmg" else "gamg" + if pc == "gamg": + poisson.petsc_options["pc_type"] = "gamg" + poisson.petsc_options["ksp_rtol"] = 1e-8 + poisson.solve() + return poisson.snes.getKSP().getIterationNumber() + + fmg_its = solve("fmg") + gamg_its = solve("gamg") + # both are effective preconditioners on this small graded mesh; the custom-P + # FMG must not need materially more iterations than the AMG reference. + assert fmg_its <= gamg_its + 2, f"fmg {fmg_its} vs gamg {gamg_its}" + assert fmg_its <= 10 From 05d4d77ef9ff2baab06ee2d36d3a46305ceca764 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 18:07:38 +1000 Subject: [PATCH 40/52] docs(layer2/nvb): mark Stage 2c (parallel adapt wiring) done Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index 491933d0..b21eec9a 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -426,8 +426,19 @@ tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build the serial one at any communicator size — uniform refine 159/352/760 and graded bullseye 215 cells agree exactly at np=1/2/3/4, 0 hanging nodes. `tests/test_0838_nvb_graded_native.py` (`test_parallel_confluence`). -4. *Integration + acceptance* — `_adapt_nested(engine="nvb")` at np>1 via the - driver; Poisson + SolCx FMG match GAMG; result matches the serial NVB mesh. +4. *Integration + acceptance (Stage 2c)* — **DONE (2026-07-01).** + `_adapt_nested` dispatches `engine="nvb"` to the native `uwnvb` transform when + the compiled extension is present (serial `NVBMesh` is the fallback, raising at + np>1 only if the extension is absent). Each pass marks cells whose size exceeds + the metric target and refines once (`_nvb_transform.refine`), up to + `2·max_levels` passes, with a collective `allreduce` stop so a rank with no local + marks still joins the closure bisection; the metric is `global_evaluate`d at + np>1. Verified: `mesh.adapt(engine="nvb")` is bit-confluent (child = 366 cells at + np=1/2/4), carries boundary labels + the `[base … child]` custom-P MG tail, and a + Poisson solve on the graded child drives custom-P FMG converging in **3 iters == + GAMG** at np=1/2/4. `tests/test_0839_nvb_parallel_adapt.py`. Still open: SolCx + Stokes FMG on the parallel child (serial covered in `test_0836`), and the + marker-replay checkpoint. ### Stage 2b as built — single-bisection multi-pass driver (WORKS) @@ -483,8 +494,8 @@ is **bit-identical to the serial mesh** at any communicator size (uniform 159/352/760, graded bullseye 215, deep 5-level nesting 96/180/288/390/472 — all equal at np=1/2/3/4; 0 hanging nodes; the output point SF is stratum-consistent). The root cause of the earlier non-confluence *and* the nested-distributed segfault -was the single missing `uwnvb_sf_lor` — one bug, both symptoms. Next: Stage 2c — -wire `_adapt_nested(engine="nvb")` at np>1 to the driver. +was the single missing `uwnvb_sf_lor` — one bug, both symptoms. **Stage 2c (wiring +`_adapt_nested(engine="nvb")` to this transform) is now done — see item 4 above.** ### Stage 2b finding (2026-07-01): grading needs single-bisection, multi-pass From cbfd64e9f9ba75e000e1ebdec129ab7cc20aaef0 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 18:20:36 +1000 Subject: [PATCH 41/52] test(layer2/nvb): parallel SolCx Stokes FMG acceptance on the NVB child MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds test_solcx_stokes_velocity_fmg_on_nvb_child to test_0839: SolCx (η jump 1e6) on a graded NVB child, velocity block auto-picks-up the mesh-owned custom-P FMG (geometric mg, FULL/FMG cycle) and matches the GAMG reference. Confluent child (800 cells) and a partition-independent solution. Verified np=1/2/4: FMG vel-iters == GAMG (8/9/7), rel|u| ~0. Underworld development team with AI support from Claude Code --- tests/test_0839_nvb_parallel_adapt.py | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py index 8e08dc1d..30b65e06 100644 --- a/tests/test_0839_nvb_parallel_adapt.py +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -22,6 +22,7 @@ import pytest import sympy import underworld3 as uw +from underworld3.function import analytic as A from petsc4py import PETSc pytestmark = [pytest.mark.level_2, pytest.mark.tier_b] @@ -113,3 +114,60 @@ def solve(pc): # FMG must not need materially more iterations than the AMG reference. assert fmg_its <= gamg_its + 2, f"fmg {fmg_its} vs gamg {gamg_its}" assert fmg_its <= 10 + + +def _solcx_metric(base): + """A viscosity-jump band metric around x=0.5 (the SolCx interface).""" + M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) + with base.access(M): + c = M.coords + band = np.exp(-(((c[:, 0] - 0.5) / 0.08) ** 2)) + M.data[:, 0] = 1.0 / (0.07 + (1.0 / 80 - 0.07) * band) ** 2 + return M + + +def test_solcx_stokes_velocity_fmg_on_nvb_child(): + """SolCx Stokes (η jump 1e6) on a graded NVB child, in parallel: the velocity + block auto-picks-up the mesh-owned custom-P FMG and matches a GAMG reference + iteration-for-iteration, with a partition-independent solution.""" + base = uw.meshing.UnstructuredSimplexBox( + minCoords=(0, 0), maxCoords=(1, 1), cellSize=0.25, regular=True, + refinement=2, qdegree=3, + ) + child = base.adapt(_solcx_metric(base), max_levels=1, engine="nvb") + assert _global_owned_cells(child.dm) == 800 # confluent at any np + + def solcx(pc): + sol = A.SolCx(child, eta_A=1.0, eta_B=1.0e6, x_c=0.5, n=1) + s = uw.systems.Stokes(child) + s.constitutive_model = uw.constitutive_models.ViscousFlowModel + s.constitutive_model.Parameters.shear_viscosity_0 = sol.fn_viscosity + s.saddle_preconditioner = 1.0 / sol.fn_viscosity + s.bodyforce = sol.fn_bodyforce + s.add_dirichlet_bc((0.0, None), "Left") + s.add_dirichlet_bc((0.0, None), "Right") + s.add_dirichlet_bc((None, 0.0), "Bottom") + s.add_dirichlet_bc((None, 0.0), "Top") + s.petsc_use_pressure_nullspace = True + s.petsc_options["snes_type"] = "ksponly" + s.tolerance = 1e-8 + if pc == "gamg": + s.preconditioner = "gamg" + s.solve() + vksp = s.snes.getKSP().getPC().getFieldSplitSubKSP()[0] + return s, vksp + + sg, vg = solcx("gamg") + it_g = vg.getIterationNumber() + + s, vksp = solcx("fmg") + assert s.snes.getConvergedReason() > 0 + assert vksp.getPC().getType() == "mg" # geometric MG (from the mesh tail), not AMG + assert vksp.getPC().getMGType() == 2 # PC.MGType.FULL == FMG + assert vksp.getIterationNumber() <= it_g + 1 # FMG matches the AMG reference + + from mpi4py import MPI + comm = child.dm.comm.tompi4py() + num = comm.allreduce(float(np.sum((s.u.data - sg.u.data) ** 2)), op=MPI.SUM) + den = comm.allreduce(float(np.sum(sg.u.data ** 2)), op=MPI.SUM) + assert np.sqrt(num / (den + 1e-30)) < 1e-4 # same solution as GAMG From c16e60d8c4f1f2caa215a3e9588c53b657fa9ce0 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 20:04:41 +1000 Subject: [PATCH 42/52] feat(layer2/adapt): accept an analytic metric in _adapt_nested (fixes patchy grading) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit adapt(metric, engine=...) now accepts any evaluatable metric (sympy / UWexpression), not only a MeshVariable: metric_sym = getattr(metric_field, "sym", metric_field), used in all three refinement branches. Motivation: a baked P1 refinement_metric field stores M = 1/h², which for a typical h_far/h_near ratio swings ~80x across one base cell. P1-interpolating that peaked quantity on the coarse base aliases badly, so a thin finite-width feature gets *patchy* refinement levels along its length (measured: along-fault h_target varied 3.9x even though the distance is ~0 all along the fault). Passing the analytic wedge instead samples the metric exactly at each centroid → uniform along-feature resolution (residual 1.3x is only the P1 distance field's own error; exact distance gives 1.0x). MeshVariable metrics are unchanged (metric_sym == metric_field.sym). New test_adapt_accepts_analytic_metric (confluent, conforming, np1/2). Existing SBR + NVB adapt suites (22) still pass. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 18 +++++++++---- tests/test_0839_nvb_parallel_adapt.py | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index c0709d7c..c70db97d 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6164,6 +6164,14 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex DM_ADAPT_REFINE = 1 # PETSc DMAdaptFlag: refine this cell + # The metric may be a MeshVariable (use its .sym) OR any evaluatable + # expression (sympy / UWexpression). Passing the analytic metric avoids + # baking a sharply-peaked M = 1/h² into a P1 field on the coarse base — + # that P1 interpolation aliases badly (a width-w feature ≈ one base cell), + # giving *patchy* refinement levels along a thin feature even though the + # metric is smooth. An analytic metric is sampled exactly at each centroid. + metric_sym = getattr(metric_field, "sym", metric_field) + markers_per_level = [] level_dms = [] # one DM per refinement level @@ -6189,11 +6197,11 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) if uw.mpi.size > 1: M = numpy.asarray( - uw.function.global_evaluate(metric_field.sym, centroids) + uw.function.global_evaluate(metric_sym, centroids) ).reshape(-1) else: M = numpy.asarray( - uw.function.evaluate(metric_field.sym, centroids) + uw.function.evaluate(metric_sym, centroids) ).reshape(-1) M = numpy.clip(M, 1e-30, None) h_target = 1.0 / numpy.sqrt(M) @@ -6244,7 +6252,7 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, centroids, cur_h, cids = nvb.centroids_h() M = numpy.clip( numpy.asarray( - uw.function.evaluate(metric_field.sym, centroids) + uw.function.evaluate(metric_sym, centroids) ).reshape(-1), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) sel = numpy.where(cur_h > h_target)[0] @@ -6283,11 +6291,11 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, # serial: the cheaper local evaluate. if uw.mpi.size > 1: M = numpy.asarray( - uw.function.global_evaluate(metric_field.sym, centroids) + uw.function.global_evaluate(metric_sym, centroids) ).reshape(-1) else: M = numpy.asarray( - uw.function.evaluate(metric_field.sym, centroids) + uw.function.evaluate(metric_sym, centroids) ).reshape(-1) M = numpy.clip(M, 1e-30, None) h_target = 1.0 / numpy.sqrt(M) diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py index 30b65e06..0d40af94 100644 --- a/tests/test_0839_nvb_parallel_adapt.py +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -116,6 +116,31 @@ def solve(pc): assert fmg_its <= 10 +def test_adapt_accepts_analytic_metric(): + """adapt(engine="nvb") accepts an analytic metric expression (not only a + MeshVariable). Evaluating the metric analytically at each centroid avoids + P1-interpolating a peaked M=1/h² on the coarse base — the cause of *patchy* + refinement along a thin feature. The child stays confluent.""" + mesh = _base() + x, y = mesh.X + # bullseye wedge as a pure expression: h_near at centre, h_far away + d = sympy.sqrt((x - 0.5) ** 2 + (y - 0.5) ** 2) + h_near, h_far, w = 0.05, 0.4, 0.25 + h = h_near + (h_far - h_near) * sympy.Min(d / w, 1) + metric_expr = 1.0 / h ** 2 + + child = mesh.adapt(metric_expr, max_levels=2, engine="nvb") + assert child.parent is mesh + # refined (more cells than the base finest) and partition-independent count + base_finest = mesh.dm_hierarchy[-1] + bs, be = base_finest.getHeightStratum(0) + assert _global_owned_cells(child.dm) > (be - bs) + assert _global_owned_cells(child.dm) == 174 # confluent (same at any np) + # conforming: no locally over-shared edge + es, ee = child.dm.getDepthStratum(1) + assert all(child.dm.getSupportSize(e) <= 2 for e in range(es, ee)) + + def _solcx_metric(base): """A viscosity-jump band metric around x=0.5 (the SolCx interface).""" M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) From 1d5b7c0ac2f9470a1d0f03987754e66c2afa08a2 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 21:01:45 +1000 Subject: [PATCH 43/52] feat(layer2/adapt): callable exact-distance metric, resolved on each refined level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapt metric is re-evaluated on the centroids of every refined NVB/SBR level, but a distance-driven fault metric supplied as a P1 MeshVariable (or an expression referencing Surface.distance.sym) is interpolated from the *base* mesh — so it aliases: baking M = 1/h² into a P1 field swings ~80x across one base cell (patchy along-fault levels, 3.9x h_target variation), and even an analytic wedge still rides the P1 *distance* field (1.3x). Let mesh.adapt() accept a plain callable metric(centroids) -> M, evaluated directly at each level's centroids, and add Surface.refinement_metric_function(...) that returns one built on the surface's EXACT signed distance. Surface.signed_distance / unsigned_distance expose the exact point-to-polyline (2D) / implicit-distance (3D) primitive at arbitrary query points; _compute_distance_field is refactored to share it (_signed_distance_at) so the P1 field and the callable use one code path. The distance now resolves itself at the new resolution as levels appear (on-fault h_target uniform to ~1e-9, 1.0x). A coordinate-driven callable is also partition-independent, so it needs no global_evaluate/swarm migration and the child stays bit-confluent (206 cells at np=1/2/4 in the test; 1590 in the weak-fault demo). tests/test_0839_nvb_parallel_adapt.py: test_adapt_accepts_callable_metric_exact_surface_distance (np1/np2). Underworld development team with AI support from Claude Code --- docs/developer/design/NVB_GRADED_ADAPT.md | 36 +++- .../discretisation/discretisation_mesh.py | 78 +++++--- src/underworld3/meshing/surfaces.py | 176 +++++++++++++++--- tests/test_0839_nvb_parallel_adapt.py | 37 ++++ 4 files changed, 268 insertions(+), 59 deletions(-) diff --git a/docs/developer/design/NVB_GRADED_ADAPT.md b/docs/developer/design/NVB_GRADED_ADAPT.md index b21eec9a..1fd641e9 100644 --- a/docs/developer/design/NVB_GRADED_ADAPT.md +++ b/docs/developer/design/NVB_GRADED_ADAPT.md @@ -436,9 +436,39 @@ tiers 2/3 couple us to PETSc's internal struct/cone ABI for the pinned build np>1. Verified: `mesh.adapt(engine="nvb")` is bit-confluent (child = 366 cells at np=1/2/4), carries boundary labels + the `[base … child]` custom-P MG tail, and a Poisson solve on the graded child drives custom-P FMG converging in **3 iters == - GAMG** at np=1/2/4. `tests/test_0839_nvb_parallel_adapt.py`. Still open: SolCx - Stokes FMG on the parallel child (serial covered in `test_0836`), and the - marker-replay checkpoint. + GAMG** at np=1/2/4. SolCx Stokes FMG on the parallel child is also covered (the + velocity block picks up the mesh-owned custom-P FMG and matches GAMG + iteration-for-iteration; child = 800 cells at np=1/2/4). + `tests/test_0839_nvb_parallel_adapt.py`. Still open: the marker-replay checkpoint. + +### Metric resolution on the refined levels — callable, exact-distance metric + +The metric is re-evaluated on the centroids of **each refined level** (not once on +the base). Three metric kinds, in increasing "self-resolving" order: + +1. a **MeshVariable** (`.sym`) — a P1 field on the *base* nodes; +2. any **sympy / UWexpression**; +3. a plain **callable** `metric(centroids) -> M`. + +(1) and (2) go through `uw.function.evaluate`, so anything they reference is +interpolated from the base mesh. For a distance-driven fault metric this aliases +twice: baking `M = 1/h²` into a P1 field swings ~80× across one base cell (→ *patchy* +levels, 3.9× along-fault `h_target` variation where it should be uniform `h_near`); +and even an analytic wedge built from `Surface.distance.sym` still interpolates the +P1 *distance* field (residual 1.3×). + +A **callable** is evaluated directly at each level's centroids. +`Surface.refinement_metric_function(h_near, h_far, width, profile)` returns one built +on the surface's **exact** signed distance — `Surface.signed_distance(coords)` / +`unsigned_distance(coords)` run the exact point-to-polyline (2D) / implicit-distance +(3D) primitive at *arbitrary* query points (the same primitive that backs the P1 +`distance` field, factored out as `_signed_distance_at`). The distance therefore +resolves itself at the new resolution as levels appear — on-fault `h_target` uniform +to ~1e-9 (1.0×). Because it is coordinate-driven it is **partition-independent** (each +rank evaluates its own centroids; no `global_evaluate` / swarm migration) and the +child stays bit-confluent. `test_adapt_accepts_callable_metric_exact_surface_distance` +(child = 206 cells at any np); weak-fault demo in +`~/+Simulations/nvb_parallel_fault_study` (1590 cells, 4.1× graded band, np=1/2/4). ### Stage 2b as built — single-bisection multi-pass driver (WORKS) diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index c70db97d..c185fba0 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6048,8 +6048,15 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, Parameters ---------- - metric_field : MeshVariable + metric_field : MeshVariable, sympy expression, or callable Scalar metric ``M = 1/h²`` (target edge length ``h``); larger ⇒ finer. + A **MeshVariable** or **sympy/UWexpression** is sampled through + ``uw.function.evaluate`` (so anything it references is interpolated + from the *base* mesh). A **callable** ``metric(centroids) -> M`` is + evaluated directly at each refined level's centroids — use this for a + metric built from exact geometry (e.g. + :meth:`Surface.refinement_metric_function`) so a thin feature refines + to a clean, uniform-width band instead of a P1-aliased *patchy* one. Same interface as :meth:`remesh` / ``adaptivity.create_metric``. max_levels : int Maximum SBR depth applied on top of the base finest (bounds the @@ -6164,14 +6171,42 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex DM_ADAPT_REFINE = 1 # PETSc DMAdaptFlag: refine this cell - # The metric may be a MeshVariable (use its .sym) OR any evaluatable - # expression (sympy / UWexpression). Passing the analytic metric avoids - # baking a sharply-peaked M = 1/h² into a P1 field on the coarse base — - # that P1 interpolation aliases badly (a width-w feature ≈ one base cell), - # giving *patchy* refinement levels along a thin feature even though the - # metric is smooth. An analytic metric is sampled exactly at each centroid. + # The metric may be one of three kinds, in increasing "self-resolving" + # order: + # 1. a MeshVariable (use its .sym) — a P1 field on the base nodes; + # 2. any evaluatable expression (sympy / UWexpression); + # 3. a plain CALLABLE metric(centroids) -> M array. + # (1) and (2) are sampled through uw.function.evaluate, so anything they + # reference (e.g. a Surface.distance P1 field) is interpolated from the + # *base* mesh — a sharply-peaked M = 1/h² aliases across a base cell and + # gives *patchy* refinement levels along a thin feature. A callable is + # evaluated directly at each refined level's centroids, so a metric built + # from EXACT geometry (Surface.refinement_metric_function) resolves itself + # at the new resolution — a clean, uniform-width band. A callable driven + # purely by coordinates is also partition-independent, so it needs no + # swarm-migration global_evaluate in parallel (each rank evaluates its + # own centroids). + import sympy as _sympy + metric_is_callable = ( + callable(metric_field) + and not hasattr(metric_field, "sym") + and not isinstance(metric_field, _sympy.Basic) + ) metric_sym = getattr(metric_field, "sym", metric_field) + def _eval_metric(centroids): + if metric_is_callable: + return numpy.asarray( + metric_field(centroids), dtype=float + ).reshape(-1) + if uw.mpi.size > 1: + return numpy.asarray( + uw.function.global_evaluate(metric_sym, centroids) + ).reshape(-1) + return numpy.asarray( + uw.function.evaluate(metric_sym, centroids) + ).reshape(-1) + markers_per_level = [] level_dms = [] # one DM per refinement level @@ -6195,15 +6230,7 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] centroids[i] = numpy.asarray(cen)[: self.cdim] cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) - if uw.mpi.size > 1: - M = numpy.asarray( - uw.function.global_evaluate(metric_sym, centroids) - ).reshape(-1) - else: - M = numpy.asarray( - uw.function.evaluate(metric_sym, centroids) - ).reshape(-1) - M = numpy.clip(M, 1e-30, None) + M = numpy.clip(_eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) sel = numpy.where(cur_h > h_target)[0] if node_budget is not None and sel.size > node_budget: @@ -6250,10 +6277,7 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, n_gen = 2 * max_levels for level in range(n_gen): centroids, cur_h, cids = nvb.centroids_h() - M = numpy.clip( - numpy.asarray( - uw.function.evaluate(metric_sym, centroids) - ).reshape(-1), 1e-30, None) + M = numpy.clip(_eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) sel = numpy.where(cur_h > h_target)[0] if sel.size == 0: @@ -6287,17 +6311,9 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) # Metric M = 1/h_target² at the cell centroids (parent field). - # Parallel: swarm-migration global_evaluate (partition-agnostic); - # serial: the cheaper local evaluate. - if uw.mpi.size > 1: - M = numpy.asarray( - uw.function.global_evaluate(metric_sym, centroids) - ).reshape(-1) - else: - M = numpy.asarray( - uw.function.evaluate(metric_sym, centroids) - ).reshape(-1) - M = numpy.clip(M, 1e-30, None) + # A callable is evaluated directly on the centroids; a field/expr + # goes through global_evaluate (parallel) or evaluate (serial). + M = numpy.clip(_eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) refine = numpy.where(cur_h > h_target)[0] diff --git a/src/underworld3/meshing/surfaces.py b/src/underworld3/meshing/surfaces.py index 21191979..b3bbfbd4 100644 --- a/src/underworld3/meshing/surfaces.py +++ b/src/underworld3/meshing/surfaces.py @@ -1169,6 +1169,90 @@ def abs_distance(self) -> "uw.discretisation.MeshVariable": return self._abs_distance_var + def _signed_distance_at(self, coords: np.ndarray) -> np.ndarray: + """Exact signed distance from arbitrary query points to this surface. + + This is the geometric primitive underneath the ``distance`` / + ``abs_distance`` fields, but evaluated at *any* points rather than only + the mesh nodes. It does **not** touch or interpolate the P1 distance + field — it re-runs the exact point-to-polyline (2D) / implicit-distance + (3D) computation directly. That makes it safe to call on the centroids + of a *refined* mesh during adaptation: the distance resolves itself at + the new resolution instead of aliasing a coarse P1 field. + + Parameters + ---------- + coords : ndarray, shape (N, 2) or (N, 3) + Query points in model (internal) coordinate space — the same space + the surface control points live in (cf. ``mesh._coords``). + + Returns + ------- + ndarray, shape (N,) + Signed distance at each query point (edge-clamped: beyond a finite + edge it is the radial distance to the nearest endpoint). + """ + self._ensure_discretized() + coords = np.asarray(coords, dtype=float) + + if self.is_2d: + from underworld3.utilities.geometry_tools import ( + signed_distance_pointcloud_polyline_2d + ) + + coords_2d = coords[:, :2] + if hasattr(self, "_vertices_2d") and self._vertices_2d is not None: + vertices_2d = self._vertices_2d + else: + vertices_2d = self._pv_mesh.points[:, :2] + return signed_distance_pointcloud_polyline_2d(coords_2d, vertices_2d) + + # 3D: pyvista implicit distance to the surface polydata + pv = _require_pyvista() + pv_pts = pv.PolyData(np.ascontiguousarray(coords[:, :3])) + dist_result = pv_pts.compute_implicit_distance(self._pv_mesh) + return np.asarray(dist_result.point_data["implicit_distance"]) + + def signed_distance(self, coords: np.ndarray) -> np.ndarray: + """Exact **signed** distance from arbitrary points to the surface. + + Unlike ``distance`` (a P1 :class:`MeshVariable` sampled at the mesh + nodes), this evaluates the exact geometry at *whatever* points you pass + — so it stays accurate on a refined mesh. Positive on one side of the + surface, negative on the other. + + Parameters + ---------- + coords : ndarray, shape (N, 2) or (N, 3) + Query points in model coordinate space. + + Returns + ------- + ndarray, shape (N,) + Signed distance at each point. + """ + return self._signed_distance_at(coords) + + def unsigned_distance(self, coords: np.ndarray) -> np.ndarray: + """Exact **unsigned**, edge-clamped distance from arbitrary points. + + ``abs()`` of :meth:`signed_distance`. Because the underlying distance is + edge-clamped (it falls back to the nearest endpoint beyond a finite + edge), this is the true distance to the *finite* surface — the right + quantity for a distance-driven refinement metric. + + Parameters + ---------- + coords : ndarray, shape (N, 2) or (N, 3) + Query points in model coordinate space. + + Returns + ------- + ndarray, shape (N,) + Unsigned distance at each point (``>= 0``). + """ + return np.abs(self._signed_distance_at(coords)) + def _compute_distance_field(self) -> None: """Compute signed distance field from mesh nodes to surface. @@ -1197,31 +1281,9 @@ def _compute_distance_field(self) -> None: # coordinate space used to create them — typically model coordinates. coords = np.asarray(self.mesh._coords) - if self.is_2d: - # 2D: Use geometry_tools for signed distance to polyline - from underworld3.utilities.geometry_tools import ( - signed_distance_pointcloud_polyline_2d - ) - - # Get 2D coordinates - coords_2d = coords[:, :2] - - # Use stored 2D vertices for distance computation - if hasattr(self, '_vertices_2d') and self._vertices_2d is not None: - vertices_2d = self._vertices_2d - else: - # Fall back to pyvista mesh points - vertices_2d = self._pv_mesh.points[:, :2] - - distances = signed_distance_pointcloud_polyline_2d(coords_2d, vertices_2d) - - else: - # 3D: Use pyvista's compute_implicit_distance - pv = _require_pyvista() - pv_mesh = pv.PolyData(coords) - dist_result = pv_mesh.compute_implicit_distance(self._pv_mesh) - # Keep signed distance - helpers use sympy.Abs() when needed - distances = dist_result.point_data["implicit_distance"] + # Exact signed distance at the mesh nodes (same primitive that + # signed_distance()/unsigned_distance() expose for arbitrary points). + distances = self._signed_distance_at(coords) # Companion UNSIGNED distance field. The per-node distance is already # edge-clamped (the segment/surface distance falls back to the nearest @@ -1574,6 +1636,70 @@ def refinement_metric( return metric + def refinement_metric_function( + self, + h_near, + h_far, + width=None, + profile: str = "linear", + ): + r"""Return a **callable** refinement metric based on exact distance. + + This is the self-resolving companion to :meth:`refinement_metric`. Where + ``refinement_metric`` bakes ``M = 1/h²`` into a P1 :class:`MeshVariable` + sampled at the *base* mesh nodes, this returns a plain function + ``metric(coords) -> M`` that computes the **exact** distance + (:meth:`unsigned_distance`) at whatever points it is handed and maps it + through the same profile. + + Pass it straight to :meth:`Mesh.adapt`, which re-evaluates the callable + at the cell centroids of **each refined level**. The metric therefore + resolves itself at the new resolution as mesh levels appear — it never + interpolates a coarse P1 field, so a thin feature refines to a clean, + uniform-width band instead of the *patchy* levels a P1-interpolated + ``1/h²`` produces (that peaked quantity aliases across a base cell). + + Parameters + ---------- + h_near, h_far : float or quantity + Target edge length near / far from the surface (same unit handling + as :meth:`refinement_metric`). + width : float or quantity, optional + Transition distance; defaults to ``2 * h_far``. + profile : {"linear", "smoothstep", "gaussian"} + Distance-to-size profile. Default ``"linear"``. + + Returns + ------- + callable + ``metric(coords: ndarray[N, dim]) -> ndarray[N]`` giving ``M = 1/h²`` + at each query point. + + Examples + -------- + >>> fault = uw.meshing.Surface("fault", mesh, fault_points) + >>> fault.discretize() + >>> metric = fault.refinement_metric_function(h_near=0.005, h_far=0.05, + ... width=0.02) + >>> child = mesh.adapt(metric, max_levels=3, engine="nvb") + """ + if self.mesh is None: + raise RuntimeError( + f"Surface '{self.name}' must be attached to a mesh to create " + "a refinement metric" + ) + + h_near = _to_nd_length(h_near) + h_far = _to_nd_length(h_far) + width = _to_nd_length(width) if width is not None else 2.0 * h_far + + def metric(coords: np.ndarray) -> np.ndarray: + d = self.unsigned_distance(coords) + h = _profile_to_edge_lengths(d, h_near, h_far, width, profile) + return 1.0 / (h ** 2) + + return metric + # --- Factory methods --- @classmethod diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py index 0d40af94..8b0306ef 100644 --- a/tests/test_0839_nvb_parallel_adapt.py +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -141,6 +141,43 @@ def test_adapt_accepts_analytic_metric(): assert all(child.dm.getSupportSize(e) <= 2 for e in range(es, ee)) +def test_adapt_accepts_callable_metric_exact_surface_distance(): + """adapt(engine="nvb") accepts a CALLABLE metric evaluated at each refined + level's centroids. A callable built from a Surface's EXACT signed distance + (``Surface.refinement_metric_function``) resolves itself at the new + resolution — no P1 field is interpolated — so a thin feature refines to a + clean band. Because it is coordinate-driven it is partition-independent + (needs no global_evaluate) and the child stays confluent.""" + mesh = _base() + # a thin diagonal "fault" polyline + s = np.linspace(0.0, 1.0, 21) + trace = np.column_stack([0.2 + 0.6 * s, 0.15 + 0.7 * s, np.zeros_like(s)]) + fault = uw.meshing.Surface("f839", mesh, trace, symbol="F") + fault.discretize() + + # exact distance is a real geometric primitive at arbitrary points + d_on = fault.unsigned_distance(trace[:, :2]) + assert float(np.max(d_on)) < 1e-6 # exactly zero on the trace + # step exactly perpendicular to the trace direction (0.6, 0.7) + perp = np.array([0.7, -0.6]) / np.hypot(0.6, 0.7) + # use interior points only (endpoints clamp to the finite edge) + off = trace[5:-5, :2] + 0.1 * perp + assert abs(float(np.mean(fault.unsigned_distance(off))) - 0.1) < 1e-6 + + metric = fault.refinement_metric_function(h_near=0.03, h_far=0.4, width=0.08) + assert callable(metric) + + child = mesh.adapt(metric, max_levels=2, engine="nvb") + assert child.parent is mesh + base_finest = mesh.dm_hierarchy[-1] + bs, be = base_finest.getHeightStratum(0) + assert _global_owned_cells(child.dm) > (be - bs) # refined + # conforming (no over-shared edge) and confluent at any communicator size + es, ee = child.dm.getDepthStratum(1) + assert all(child.dm.getSupportSize(e) <= 2 for e in range(es, ee)) + assert _global_owned_cells(child.dm) == 206 # confluent (same at any np) + + def _solcx_metric(base): """A viscosity-jump band metric around x=0.5 (the SolCx interface).""" M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) From 6fcbed4aca4b083a7afcdb36ecf63ffc1c3a5049 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 1 Jul 2026 21:47:07 +1000 Subject: [PATCH 44/52] refactor(layer2/adapt): normalise the metric to one callable; fn.evaluate is the field adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _adapt_nested had two implicit paths (callable vs field/expr) branched inside a per-call helper. Collapse to a single eval_metric(centroids) -> M callable built once: a user callable is used as-is; a MeshVariable/expression is wrapped in an uw.function.(global_)evaluate adapter. So fn.evaluate is not a separate path — it IS a callable in the framework, just the default one for a field/expression. One code path, evaluated at each refined level's centroids. This also documents the composition the wrapper makes obvious: a user callable may call global_evaluate itself when the metric genuinely depends on a base field (e.g. |grad T| for convection adaptivity) — with the caveat that a field is sampled from the base-mesh interpolant (base-resolution, no self-sharpening, unlike exact geometry) and that np>1 needs global_evaluate, not evaluate. Behaviour unchanged: test_0839 (5) passes np1/np2. Underworld development team with AI support from Claude Code --- .../discretisation/discretisation_mesh.py | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index c185fba0..5f86f7d9 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -6171,41 +6171,44 @@ def _adapt_nested(self, metric_field, max_levels=2, node_budget=None, edge_factor = math.factorial(dim) # h ≈ (dim! · vol)**(1/dim) for a simplex DM_ADAPT_REFINE = 1 # PETSc DMAdaptFlag: refine this cell - # The metric may be one of three kinds, in increasing "self-resolving" - # order: - # 1. a MeshVariable (use its .sym) — a P1 field on the base nodes; - # 2. any evaluatable expression (sympy / UWexpression); - # 3. a plain CALLABLE metric(centroids) -> M array. - # (1) and (2) are sampled through uw.function.evaluate, so anything they - # reference (e.g. a Surface.distance P1 field) is interpolated from the - # *base* mesh — a sharply-peaked M = 1/h² aliases across a base cell and - # gives *patchy* refinement levels along a thin feature. A callable is - # evaluated directly at each refined level's centroids, so a metric built - # from EXACT geometry (Surface.refinement_metric_function) resolves itself - # at the new resolution — a clean, uniform-width band. A callable driven - # purely by coordinates is also partition-independent, so it needs no - # swarm-migration global_evaluate in parallel (each rank evaluates its - # own centroids). + # The metric is normalised to a single callable `eval_metric(centroids) + # -> M`, re-evaluated at each refined level's centroids. There is only one + # code path; the metric *kind* just decides which callable we build: + # + # * a plain CALLABLE metric(centroids) -> M is used as-is; + # * a MeshVariable (.sym) or sympy/UWexpression is wrapped in an + # `uw.function.(global_)evaluate` adapter — i.e. fn.evaluate IS a + # callable in this framework, just the default one for a field/expr. + # + # The distinction is about *where the metric resolves*. The evaluate + # adapter samples the base-mesh interpolant, so anything the field/expr + # references (e.g. a Surface.distance P1 field, or a peaked M = 1/h²) + # aliases across a base cell → *patchy* levels along a thin feature. A + # callable built from EXACT geometry (Surface.refinement_metric_function) + # instead resolves itself at the refined resolution — a clean band — and, + # being coordinate-driven, is partition-independent (no swarm-migration + # global_evaluate). A user callable is free to call global_evaluate itself + # when the metric genuinely depends on a base field (e.g. |∇T| for + # convection); use global_evaluate, not evaluate, at np>1. import sympy as _sympy metric_is_callable = ( callable(metric_field) and not hasattr(metric_field, "sym") and not isinstance(metric_field, _sympy.Basic) ) - metric_sym = getattr(metric_field, "sym", metric_field) - def _eval_metric(centroids): - if metric_is_callable: - return numpy.asarray( - metric_field(centroids), dtype=float - ).reshape(-1) - if uw.mpi.size > 1: - return numpy.asarray( - uw.function.global_evaluate(metric_sym, centroids) - ).reshape(-1) - return numpy.asarray( - uw.function.evaluate(metric_sym, centroids) - ).reshape(-1) + if metric_is_callable: + def eval_metric(centroids): + return numpy.asarray(metric_field(centroids), dtype=float).reshape(-1) + else: + # Wrap the field/expression as a callable over the (global_)evaluate + # sampler — the same framework, with fn.evaluate as the adapter. + metric_sym = getattr(metric_field, "sym", metric_field) + _sampler = (uw.function.global_evaluate if uw.mpi.size > 1 + else uw.function.evaluate) + + def eval_metric(centroids): + return numpy.asarray(_sampler(metric_sym, centroids)).reshape(-1) markers_per_level = [] level_dms = [] # one DM per refinement level @@ -6230,7 +6233,7 @@ def _eval_metric(centroids): vol, cen = current_dm.computeCellGeometryFVM(c)[0:2] centroids[i] = numpy.asarray(cen)[: self.cdim] cur_h[i] = (edge_factor * abs(float(vol))) ** (1.0 / dim) - M = numpy.clip(_eval_metric(centroids), 1e-30, None) + M = numpy.clip(eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) sel = numpy.where(cur_h > h_target)[0] if node_budget is not None and sel.size > node_budget: @@ -6277,7 +6280,7 @@ def _eval_metric(centroids): n_gen = 2 * max_levels for level in range(n_gen): centroids, cur_h, cids = nvb.centroids_h() - M = numpy.clip(_eval_metric(centroids), 1e-30, None) + M = numpy.clip(eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) sel = numpy.where(cur_h > h_target)[0] if sel.size == 0: @@ -6313,7 +6316,7 @@ def _eval_metric(centroids): # Metric M = 1/h_target² at the cell centroids (parent field). # A callable is evaluated directly on the centroids; a field/expr # goes through global_evaluate (parallel) or evaluate (serial). - M = numpy.clip(_eval_metric(centroids), 1e-30, None) + M = numpy.clip(eval_metric(centroids), 1e-30, None) h_target = 1.0 / numpy.sqrt(M) refine = numpy.where(cur_h > h_target)[0] From 29ab5524f229da3ade6fa9f215e9eaf823c126f0 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Thu, 2 Jul 2026 16:13:02 +1000 Subject: [PATCH 45/52] feat(surfaces): Surface.remap_to(mesh) + director property for adapt-on-top faults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remap_to(new_mesh): re-home a Surface onto an adapt() child (drops cached distance fields bound to the old mesh; geometry unchanged; re-registers). Lets ONE fault Surface drive both the geometry-only refinement metric (evaluated before the child exists) and the child-side constitutive model (distance/normal live on the child) — the nested SBR/NVB adapt returns a child and does not auto-notify surfaces the way in-place remesh does. director: unit surface-normal as a symbolic column vector = normalised grad of the signed-distance field. The natural TI weak-plane director; exact for a planar fault. Underworld development team with AI support from Claude Code --- src/underworld3/meshing/surfaces.py | 90 +++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/underworld3/meshing/surfaces.py b/src/underworld3/meshing/surfaces.py index b3bbfbd4..661f7560 100644 --- a/src/underworld3/meshing/surfaces.py +++ b/src/underworld3/meshing/surfaces.py @@ -1253,6 +1253,40 @@ def unsigned_distance(self, coords: np.ndarray) -> np.ndarray: """ return np.abs(self._signed_distance_at(coords)) + @property + def director(self) -> sympy.Matrix: + r"""Unit surface-normal as a symbolic column vector — the TI director n̂. + + The gradient of a signed-distance field is its unit normal, so this is + ``∇d / |∇d|`` built from the :attr:`distance` field. It is the natural + **director** for :class:`~underworld3.constitutive_models.TransverseIsotropicFlowModel` + (the weak-plane orientation): the same fault object that drives the + refinement metric and the weak-zone viscosity also supplies the normal:: + + ti = uw.constitutive_models.TransverseIsotropicFlowModel + stokes.constitutive_model = ti + stokes.constitutive_model.Parameters.director = fault.director + + For a **planar** surface the signed distance is linear, so ``∇d`` is the + exact constant unit normal everywhere. Near a curved surface or a finite + edge it is the local unit normal wherever the field is smooth. The + normalisation makes it robust where ``|∇d|`` drifts from 1 (interpolated + distance, edge clamping). + + Returns + ------- + sympy.Matrix + ``(dim, 1)`` unit normal expression, evaluatable on this surface's + mesh (call :meth:`remap_to` first if the surface was built on a + different mesh, e.g. an ``adapt`` parent). + """ + d = self.distance.sym[0] + X = self.mesh.X + dim = self.mesh.dim + grad = [sympy.diff(d, X[i]) for i in range(dim)] + norm = sympy.sqrt(sum(g ** 2 for g in grad)) + sympy.sympify(1e-30) + return sympy.Matrix([g / norm for g in grad]) + def _compute_distance_field(self) -> None: """Compute signed distance field from mesh nodes to surface. @@ -1510,6 +1544,62 @@ def _on_mesh_adapted(self, adapted_mesh: "Mesh") -> None: # Mark all variable proxies as stale (they project to mesh nodes) self._mark_all_proxies_stale() + def remap_to(self, new_mesh: "Mesh") -> "Surface": + """Re-home this surface onto a different mesh and return ``self``. + + The surface **geometry** (control points, polyline/polydata vertices) is + unchanged — only the mesh its *fields* are computed against changes. + Cached distance fields (which were bound to the old mesh) are dropped so + they recompute, at **exact** distance, on the new mesh's nodes on next + access. + + This is the companion to :meth:`Mesh.adapt` for the nested (child- + returning) engines (``engine="sbr"``/``"nvb"``): ``adapt`` leaves the + base mesh untouched and returns a refined *child*, so — unlike the + in-place :meth:`Mesh.remesh` — registered surfaces are not auto-notified. + Re-homing lets **one** fault object drive both the geometry-only + refinement metric (:meth:`refinement_metric_function`, evaluated before + the child exists) and the child-side constitutive model (its + ``distance`` / normal live on the child):: + + fault = uw.meshing.Surface("fault", base, trace) + metric = fault.refinement_metric_function(h_near, h_far, width) + child = base.adapt(metric, engine="nvb") + fault.remap_to(child) # distance now on the child + eta = fault.influence_function(width, value_near=1e-3, value_far=1.0) + + Parameters + ---------- + new_mesh : Mesh + The mesh to re-home onto (e.g. an ``adapt`` child). + + Returns + ------- + Surface + ``self`` (for chaining). + """ + if new_mesh is self.mesh: + return self + + old_mesh = self.mesh + if old_mesh is not None and hasattr(old_mesh, "unregister_surface"): + old_mesh.unregister_surface(self) + + self.mesh = new_mesh + self._dim = None # re-detect from the new mesh + if new_mesh is not None and hasattr(new_mesh, "register_surface"): + new_mesh.register_surface(self) + + # Drop cached fields bound to the OLD mesh; the geometry (pyvista mesh / + # 2D vertices) is mesh-independent and stays as-is. .distance / .abs_distance + # rebuild lazily on the new mesh's nodes (exact) on next access. + self._distance_var = None + self._abs_distance_var = None + self._distance_stale = True + self._mark_all_proxies_stale() + + return self + def refinement_metric( self, h_near, From a4add4b7cbaf697d6399b6ad3b1755ce72e07192 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Thu, 2 Jul 2026 22:38:32 +1000 Subject: [PATCH 46/52] fix(custom_mg): don't auto-inject velocity-block FMG onto scalar solvers on adapt children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adapt() child owns a custom-P FMG coarse tail that solve() auto-picks-up via maybe_inject_custom_mg. That tail is built to accelerate the Stokes VELOCITY block, but the hook also fired for scalar solvers (field_id=None). For a semi-Lagrangian advection-diffusion solver the assembled operator is boundary-reduced in a way the coarse DS-copy reduced map does not reproduce, so the per-level BC reductions disagree and the transfers don't chain — PCMG's Galerkin PtAP gets a rectangular A (e.g. 12725x12486) and dies with PETSc error 60. This bit any AdvDiffusionSLCN on an adapted mesh. Fix, in maybe_inject_custom_mg (mesh-owned/opportunistic path only): - skip the auto-pickup when the solver carries a DuDt trace-back operator (semi-Lagrangian advection-diffusion) — its operator structure is incompatible and a scalar AD solve is cheap (GAMG is fine); an explicit set_custom_fmg() still works if wanted; - plus a dimensional guard (finest transfer must match the assembled operator and actually coarsen) as defense-in-depth for other field_id=None cases. Restructured so the solver-set (set_custom_fmg) path is unchanged and errors there still surface. Velocity-block (field_id=0) and P1-scalar-Poisson auto-pickup are unaffected. test_0839: test_advdiff_scalar_on_nvb_child_no_custom_mg_crash (raised PETSc 60 before; passes now). Full NVB suite 11/11. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 70 +++++++++++++++++++++----- tests/test_0839_nvb_parallel_adapt.py | 26 ++++++++++ 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 70b7d64a..2770fe9d 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -660,18 +660,64 @@ def maybe_inject_custom_mg(solver, field_id=None): the solver — so every solver on an adapted mesh drives geometric MG with no per-solver call. A solver-set hierarchy (if present) always wins. """ - if solver._custom_mg is None: - coarse = getattr(solver.mesh, "_custom_mg_coarse_meshes", None) - if coarse is None: - return # nothing to inject - builder = getattr(solver.mesh, "_custom_mg_builder", "barycentric") - solver._custom_mg = { - "mode": "hierarchy", - "hierarchy": CustomMGHierarchy(list(coarse) + [solver.mesh], - builder=builder, field_id=field_id), - "verbose": False, - } - inject_custom_mg(solver) + # Solver-set hierarchy (set_custom_fmg): the user asked for it explicitly — + # build + install directly and let any error surface. + if solver._custom_mg is not None: + inject_custom_mg(solver) + return + + # Mesh-owned hierarchy (adapt() child): OPPORTUNISTIC auto-pickup. It must never + # crash a solve, so build the transfers and verify the finest one matches this + # solver's assembled operator before installing. It does NOT for a scalar solver + # whose DM carries auxiliary fields (e.g. semi-Lagrangian advection-diffusion): + # _reduced_map then counts the full unconstrained DOFs, not the reduced operator + # size, and the PtAP in PCMG setup fails. In that case skip and fall back to the + # solver's own preconditioner. (The Stokes velocity block, field_id=0, and P1 + # scalar Poisson match and are unaffected.) + coarse = getattr(solver.mesh, "_custom_mg_coarse_meshes", None) + if coarse is None: + return # nothing to inject + + # Semi-Lagrangian advection-diffusion (carries a DuDt trace-back operator): its + # assembled operator is boundary-reduced in a way the coarse DS-copy does NOT + # reproduce, so the per-level BC reductions disagree and the custom-P transfers + # don't chain (rectangular PtAP -> PETSc error 60). A scalar AD solve is cheap and + # doesn't need geometric FMG, so skip the OPPORTUNISTIC mesh-owned auto-pickup and + # let it use its default preconditioner. An explicit set_custom_fmg() still works. + if getattr(solver, "DuDt", None) is not None: + return + + builder = getattr(solver.mesh, "_custom_mg_builder", "barycentric") + h = CustomMGHierarchy(list(coarse) + [solver.mesh], builder=builder, + field_id=field_id) + try: + Ps = h.build(solver) + except Exception as exc: # pragma: no cover - defensive + import warnings + warnings.warn(f"custom_mg: mesh-owned FMG build failed ({exc}); using the " + "solver's default preconditioner.") + return + + # Dimensional guard (checkable for the monolithic operator, field_id is None): + # the finest transfer must chain to the operator PCMG will Galerkin against. + if field_id is None and len(Ps): + try: + solver.snes.setUp() + op_n = int(solver.snes.getJacobian()[0].getSize()[0]) + pr, pc = (int(v) for v in Ps[-1].getSize()) + if op_n > 0 and (pr != op_n or pc >= pr): # rows!=op or no coarsening + import warnings + warnings.warn( + "custom_mg: mesh-owned adapt-mesh FMG transfer is incompatible " + f"with this solver's operator (transfer {pr}x{pc}, operator {op_n}); " + "skipping the auto-pickup (using the default preconditioner). " + "set_custom_fmg() an explicit hierarchy to override.") + return + except Exception: + pass # can't check -> don't block working cases + + h.install(solver, verbose=False) + solver._custom_mg = {"mode": "hierarchy", "hierarchy": h, "verbose": False} def inject_custom_mg(solver): diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py index 8b0306ef..21ee80c5 100644 --- a/tests/test_0839_nvb_parallel_adapt.py +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -178,6 +178,32 @@ def test_adapt_accepts_callable_metric_exact_surface_distance(): assert _global_owned_cells(child.dm) == 206 # confluent (same at any np) +def test_advdiff_scalar_on_nvb_child_no_custom_mg_crash(): + """A semi-Lagrangian advection-diffusion solve on an NVB adapt child must not + choke on the mesh-owned custom-P FMG auto-pickup. That tail is the Stokes + VELOCITY-block preconditioner; auto-injecting it on a scalar SLCN operator built + a transfer that mismatched the (boundary-reduced) operator -> rectangular PtAP, + PETSc error 60. maybe_inject_custom_mg now skips it (the solver has a DuDt + trace-back operator) and the scalar solve uses its default PC and completes.""" + mesh = _base() + child = mesh.adapt(_bullseye_metric(mesh), max_levels=2, engine="nvb") + assert getattr(child, "_custom_mg_coarse_meshes", None) is not None # tail present + + T = uw.discretisation.MeshVariable("Tad", child, 1, degree=2) + x, y = child.X + T.data[:, 0] = np.asarray(uw.function.evaluate( + sympy.exp(-(((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.02)), T.coords)).reshape(-1) + v = sympy.Matrix([[-(y - 0.5), (x - 0.5)]]) # solid-body rotation + adv = uw.systems.AdvDiffusionSLCN(child, u_Field=T, V_fn=v, order=1, + monotone_mode="clamp") + adv.constitutive_model = uw.constitutive_models.DiffusionModel + adv.constitutive_model.Parameters.diffusivity = 1.0e-3 + adv.f = 0.0 + adv.solve(timestep=0.01) # raised PETSc err 60 (PtAP) before the fix + assert np.isfinite(np.asarray(T.data)).all() + assert float(T.data[:, 0].max()) <= 1.01 # bounded (monotone clamp) + + def _solcx_metric(base): """A viscosity-jump band metric around x=0.5 (the SolCx interface).""" M = uw.discretisation.MeshVariable("Mjump", base, 1, degree=1) From 1e1fabdd66f115d4949817e314e48760540b3ecc Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 3 Jul 2026 07:36:34 +1000 Subject: [PATCH 47/52] refactor(custom_mg): rename maybe_inject_custom_mg -> auto_inject_custom_mg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "maybe_" is a hedging, non-descriptive prefix. The function is the solve-time hook that AUTO-injects a custom-P MG hierarchy when one is available (solver-set via set_custom_fmg, or mesh-owned on an adapt() child) and no-ops otherwise — so name it for what it does. Pure rename across the solver pyx (4 call sites) + custom_mg.py + the test comment; behaviour unchanged. NVB suite 11/11. Underworld development team with AI support from Claude Code --- .../cython/petsc_generic_snes_solvers.pyx | 16 ++++++++-------- src/underworld3/utilities/custom_mg.py | 2 +- tests/test_0839_nvb_parallel_adapt.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/underworld3/cython/petsc_generic_snes_solvers.pyx b/src/underworld3/cython/petsc_generic_snes_solvers.pyx index 6e332486..bc541d79 100644 --- a/src/underworld3/cython/petsc_generic_snes_solvers.pyx +++ b/src/underworld3/cython/petsc_generic_snes_solvers.pyx @@ -2820,8 +2820,8 @@ class SNES_Scalar(SolverBaseClass): # first PCSetUp (so the Galerkin coarse operators are built from it). # Picks up a solver-set (set_custom_mg) OR a mesh-owned (adapt child) # hierarchy. No-op unless one is present. - from underworld3.utilities.custom_mg import maybe_inject_custom_mg - maybe_inject_custom_mg(self, field_id=None) + from underworld3.utilities.custom_mg import auto_inject_custom_mg + auto_inject_custom_mg(self, field_id=None) # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) @@ -3842,8 +3842,8 @@ class SNES_Vector(SolverBaseClass): # Custom geometric-MG prolongation on the (top-level vector) PC, if # registered via set_custom_fmg or owned by an adapt() mesh. Mirrors the # SNES_Scalar hook. - from underworld3.utilities.custom_mg import maybe_inject_custom_mg - maybe_inject_custom_mg(self, field_id=None) + from underworld3.utilities.custom_mg import auto_inject_custom_mg + auto_inject_custom_mg(self, field_id=None) # solve self._snes_solve_with_retries(gvec, divergence_retries, verbose) @@ -7701,8 +7701,8 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): # once the monolithic Jacobian is assembled (see custom_mg). Picks up # a solver-set (set_custom_fmg field_id=0) or mesh-owned (adapt child) # hierarchy on the velocity block. - from underworld3.utilities.custom_mg import maybe_inject_custom_mg - maybe_inject_custom_mg(self, field_id=0) + from underworld3.utilities.custom_mg import auto_inject_custom_mg + auto_inject_custom_mg(self, field_id=0) self._snes_solve_with_retries(gvec, divergence_retries, verbose) else: @@ -7719,8 +7719,8 @@ class SNES_Stokes_SaddlePt(SolverBaseClass): # once the monolithic Jacobian is assembled (see custom_mg). Picks up # a solver-set (set_custom_fmg field_id=0) or mesh-owned (adapt child) # hierarchy on the velocity block. - from underworld3.utilities.custom_mg import maybe_inject_custom_mg - maybe_inject_custom_mg(self, field_id=0) + from underworld3.utilities.custom_mg import auto_inject_custom_mg + auto_inject_custom_mg(self, field_id=0) self._snes_solve_with_retries(gvec, divergence_retries, verbose) # Project the rigid-body rotation gauge out of the converged solution. diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 2770fe9d..10289794 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -649,7 +649,7 @@ def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", solver.is_setup = False -def maybe_inject_custom_mg(solver, field_id=None): +def auto_inject_custom_mg(solver, field_id=None): """Solve-hook entry: inject custom-P FMG from either a solver-set hierarchy (``set_custom_fmg``) or a **mesh-owned** one (``mesh.adapt`` refinement child). diff --git a/tests/test_0839_nvb_parallel_adapt.py b/tests/test_0839_nvb_parallel_adapt.py index 21ee80c5..3d19cfd2 100644 --- a/tests/test_0839_nvb_parallel_adapt.py +++ b/tests/test_0839_nvb_parallel_adapt.py @@ -183,7 +183,7 @@ def test_advdiff_scalar_on_nvb_child_no_custom_mg_crash(): choke on the mesh-owned custom-P FMG auto-pickup. That tail is the Stokes VELOCITY-block preconditioner; auto-injecting it on a scalar SLCN operator built a transfer that mismatched the (boundary-reduced) operator -> rectangular PtAP, - PETSc error 60. maybe_inject_custom_mg now skips it (the solver has a DuDt + PETSc error 60. auto_inject_custom_mg now skips it (the solver has a DuDt trace-back operator) and the scalar solve uses its default PC and completes.""" mesh = _base() child = mesh.adapt(_bullseye_metric(mesh), max_levels=2, engine="nvb") From 5ac985efcf26ec1f567f5e824684800897d5566e Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 3 Jul 2026 10:39:48 +1000 Subject: [PATCH 48/52] fix(custom_mg): operator-faithful finest reduced map on adapt children The custom-P finest reduced map was read from the DM global section via _reduced_map(solver.dm, field_id). On an adapt() child the section can be read before the SNES finalizes it, so it could disagree with the assembled operator the PCMG Galerkins against -> a rectangular finest transfer and a bare PETSc error 60 in the PtAP. auto_inject_custom_mg worked around this by SKIPPING custom-P for any semi-Lagrangian AdvDiffusion (DuDt present), falling back to GAMG. Fix, in CustomMGHierarchy.build: - call solver.snes.setUp() before reading the finest reduced map, so the DM global section is the finalized space the operator lives on; - validate the finest reduced-map size against the assembled operator (_assert_finest_matches_operator) and raise an actionable error instead of a cryptic error 60 if a genuine adapt-child section inconsistency ever arises. The mesh-owned auto-pickup wraps build() in try/except, so this degrades gracefully to the default preconditioner. Remove the vestigial DuDt skip-guard: a semi-Lagrangian AdvDiffusion on an NVB adapt child now installs custom-P geometric MG (PC 'mg') and solves correctly, matching a default-preconditioner solve to iterative tolerance. Tests: test_1016 gains test_finest_map_operator_mismatch_raises (guard fires) and test_advdiff_on_nvb_adapt_child_gets_custom_mg (real auto-pickup path, no skip, correct result). test_0838, test_0839, and the full custom_mg suite (1014/1015/1016/1017, 47 tests) pass. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 72 +++++++++++++++++------ tests/test_1016_custom_mg_hierarchy.py | 80 ++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 16 deletions(-) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 10289794..8d082be1 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -576,6 +576,19 @@ def build(self, solver): nlev = len(self.level_meshes) parallel = mpi.size > 1 + # Operator-faithful finest level: finalize the DM section and assemble the + # operator BEFORE reading the finest reduced map. The finest transfer's row + # space must be the space the operator's PCMG will Galerkin against; the DM + # global section is that space only once the SNES is set up (an adapt() + # child can otherwise carry a not-yet-finalized / auxiliary section that + # disagrees with the assembled operator -> rectangular finest transfer -> + # cryptic PETSc error 60 in the PtAP). setUp is idempotent (the install + # paths call it again). + try: + solver.snes.setUp() + except Exception: + pass + coords, maps, ncomp = [], [], [] for k, mesh in enumerate(self.level_meshes): c = np.asarray(mesh._get_coords_for_basis(degree, continuous)) @@ -599,6 +612,14 @@ def build(self, solver): raise RuntimeError(f"inconsistent component counts across levels: {ncomp}") nc = ncomp[0] + # Operator-faithful check: the finest reduced map must span exactly the + # assembled operator's rows. Checkable directly for the monolithic + # single-field operator (field_id is None — scalar / single-field vector, + # e.g. Poisson, Projection, semi-Lagrangian AdvDiffusion on an adapt child). + # Fail here with an actionable message rather than deep inside PETSc's PtAP. + if self.field_id is None: + self._assert_finest_matches_operator(solver, maps[-1], parallel) + Ps = [] for l in range(1, nlev): if parallel: @@ -620,6 +641,35 @@ def build(self, solver): self.transfers = Ps return Ps + @staticmethod + def _assert_finest_matches_operator(solver, finest_map, parallel): + """Guarantee the finest reduced map spans the assembled operator's rows. + + The finest transfer is Galerkin-multiplied against the solver's real + operator (``PtAP``); if the row space disagrees the product is rectangular + and PETSc aborts with a bare error 60. On a plain mesh the DM global section + and the operator always agree; the guard matters for adapt() children whose + DM section could be stale relative to the freshly assembled operator.""" + try: + op_n = int(solver.snes.getJacobian()[0].getSize()[0]) + except Exception: + return # can't read operator -> skip + if op_n <= 0: + return + if parallel: + _l2g, rstart, rend, _n = finest_map + red_n = int(solver.dm.comm.tompi4py().allreduce(int(rend - rstart))) + else: + red_n = int(len(finest_map)) # r2f length = reduced global size + if red_n != op_n: + raise RuntimeError( + f"custom_mg: finest reduced-map size {red_n} != assembled operator " + f"size {op_n}. The DM global section disagrees with the operator " + f"(an adapt-child section inconsistency); the finest transfer would " + f"be rectangular and the Galerkin PtAP would abort (PETSc error 60). " + f"Rebuild the solver so its DM section matches the operator before " + f"installing custom-P.") + def install(self, solver, verbose=False): if self.transfers is None: raise RuntimeError("call build() before install()") @@ -667,26 +717,16 @@ def auto_inject_custom_mg(solver, field_id=None): return # Mesh-owned hierarchy (adapt() child): OPPORTUNISTIC auto-pickup. It must never - # crash a solve, so build the transfers and verify the finest one matches this - # solver's assembled operator before installing. It does NOT for a scalar solver - # whose DM carries auxiliary fields (e.g. semi-Lagrangian advection-diffusion): - # _reduced_map then counts the full unconstrained DOFs, not the reduced operator - # size, and the PtAP in PCMG setup fails. In that case skip and fall back to the - # solver's own preconditioner. (The Stokes velocity block, field_id=0, and P1 - # scalar Poisson match and are unaffected.) + # crash a solve, so build the transfers (which now validate the finest reduced + # map against the assembled operator — see CustomMGHierarchy.build) inside a + # try/except and fall back to the solver's default preconditioner on any failure. + # The finest map is derived from the DM section AFTER snes.setUp() finalizes it, + # so it is faithful to the operator on adapt children too — including scalar + # semi-Lagrangian advection-diffusion (which earlier had to be skipped). coarse = getattr(solver.mesh, "_custom_mg_coarse_meshes", None) if coarse is None: return # nothing to inject - # Semi-Lagrangian advection-diffusion (carries a DuDt trace-back operator): its - # assembled operator is boundary-reduced in a way the coarse DS-copy does NOT - # reproduce, so the per-level BC reductions disagree and the custom-P transfers - # don't chain (rectangular PtAP -> PETSc error 60). A scalar AD solve is cheap and - # doesn't need geometric FMG, so skip the OPPORTUNISTIC mesh-owned auto-pickup and - # let it use its default preconditioner. An explicit set_custom_fmg() still works. - if getattr(solver, "DuDt", None) is not None: - return - builder = getattr(solver.mesh, "_custom_mg_builder", "barycentric") h = CustomMGHierarchy(list(coarse) + [solver.mesh], builder=builder, field_id=field_id) diff --git a/tests/test_1016_custom_mg_hierarchy.py b/tests/test_1016_custom_mg_hierarchy.py index 402ee9c4..fdc6c674 100644 --- a/tests/test_1016_custom_mg_hierarchy.py +++ b/tests/test_1016_custom_mg_hierarchy.py @@ -58,3 +58,83 @@ def test_rbf_builder_also_works(): s.solve() assert s.snes.getKSP().getPC().getType()=="mg" assert s.snes.getConvergedReason()>0 + + +# --------------------------------------------------------------------------- # +# Operator-faithful finest reduced map (adapt-child regression) +# --------------------------------------------------------------------------- # +def test_finest_map_operator_mismatch_raises(): + """The finest reduced map must span exactly the assembled operator's rows. + A stale/oversized finest map (adapt-child section inconsistency) must fail + with an actionable error, not a bare PETSc PtAP error 60.""" + m0, coarse, fine = _hierarchy() + s=_poisson(fine); s._build(False,False,None); s.snes.setUp() + op_n = int(s.snes.getJacobian()[0].getSize()[0]) + # A correct finest map (length == op_n) passes; a doctored oversized one raises. + good = np.arange(op_n) + custom_mg.CustomMGHierarchy._assert_finest_matches_operator(s, good, parallel=False) + bad = np.arange(op_n + 17) + with pytest.raises(RuntimeError, match="finest reduced-map size"): + custom_mg.CustomMGHierarchy._assert_finest_matches_operator(s, bad, parallel=False) + + +def test_advdiff_on_nvb_adapt_child_gets_custom_mg(): + """TASK C: a semi-Lagrangian AdvDiffusion on an NVB adapt() child now installs + custom-P geometric MG via the mesh-owned auto-pickup (previously skipped by a + DuDt guard because the finest reduced map read from the DM section could + disagree with the assembled operator). The finest map is now read after the + SNES section is finalized and validated against the operator, so the transfer + is faithful on adapt children -> the PC becomes 'mg', the solve converges, and + the result matches a default-preconditioner solve.""" + pytest.importorskip( + "underworld3.utilities._nvb_transform", + reason="native uwnvb transform not built (needs the custom-PETSc/amr env)") + if PETSc.COMM_WORLD.getSize() > 1: + pytest.skip("serial regression; parallel adapt covered elsewhere") + + base = uw.meshing.UnstructuredSimplexBox( + minCoords=(0,0), maxCoords=(1,1), cellSize=0.15, regular=False, + qdegree=3, refinement=1) + x, y = base.CoordinateSystem.X + + def metric(coords): + d = np.abs(coords[:,0]-0.5) + h = np.where(d<0.1, 0.03, np.minimum(0.03+(0.12-0.03)*(d-0.1)/0.2, 0.12)) + return 1.0/h**2 + + child = base.adapt(metric, max_levels=3, engine="nvb") + assert child._custom_mg_coarse_meshes is not None # adapt child carries a tail + + def make(name): + T = uw.discretisation.MeshVariable(name, child, 1, degree=2) + T.data[:,0] = np.asarray(uw.function.evaluate( + sympy.exp(-(((x-0.3)**2+(y-0.7)**2)/(2*0.05**2))), T.coords)).reshape(-1) + V = uw.discretisation.MeshVariable(name+"V", child, child.dim, degree=2) + V.data[:,0] = 0.4 + adv = uw.systems.AdvDiffusionSLCN(child, u_Field=T, V_fn=V.sym, order=1, + monotone_mode="clamp") + adv.constitutive_model = uw.constitutive_models.DiffusionModel + adv.constitutive_model.Parameters.diffusivity = 2.0e-4 + adv.f = 0.0 + adv.add_dirichlet_bc(0.0, "Top") + return adv, T + + # reference: mesh-owned pickup disabled (clear the coarse tail) + advR, TR = make("Tc_ref") + saved = child._custom_mg_coarse_meshes + child._custom_mg_coarse_meshes = None + advR.solve(timestep=0.01) + ref = np.asarray(TR.data[:,0]).copy() + child._custom_mg_coarse_meshes = saved + + # custom-P via the real auto-pickup path (no monkeypatch) + advC, TC = make("Tc_cmg") + advC.solve(timestep=0.01) + cust = np.asarray(TC.data[:,0]) + + assert advC.snes.getKSP().getPC().getType() == "mg" # custom-P installed + assert advC._custom_mg is not None # registered, not skipped + assert advC.snes.getConvergedReason() > 0 + # same solution up to iterative tolerance (different preconditioners) + rel = np.linalg.norm(cust-ref)/max(np.linalg.norm(ref), 1e-30) + assert rel < 1e-4 From 7311fe6d6074fee8c79e1d906fb863778bb826a4 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 3 Jul 2026 10:53:20 +1000 Subject: [PATCH 49/52] feat(custom_mg): cross-partition transfer for non-nested coarse tails at np>1 The parallel custom-P path assembled each rank's prolongation block from that rank's LOCAL coarse coords, which is correct only for CO-PARTITIONED NESTED levels (refine() / on-rank adapt). For a genuinely non-nested coarse mesh (independent gmsh mesh at a different cellSize) at np>1 the coarse and fine partitions differ: a fine leaf on rank r can sit in a coarse cell owned by rank s, so rank-local point location misses it -> zero columns (the guard fired: 23 at np2, 48 at np4) or a wrong nearest-DOF fallback. So set_custom_fmg([independent coarse meshes]) was serial-only. Add a cross-partition transfer (_build_crosspart_transfer / _gather_coarse_cloud): a coarse MG level is small, so all-gather the coarse node cloud (coords + each node/component's GLOBAL reduced column index, -1 for constrained) deduplicated by coordinate; every rank then locates its OWNED fine nodes against the FULL coarse mesh. Columns are coarse global reduced indices (MPIAIJ handles off-rank); constrained coarse DOFs stay as barycentric vertices but drop from the columns (reduced->reduced). Fine rows stay rank-local. Routing via cross_partition on CustomMGHierarchy / set_custom_fmg: - "auto" (default): rank-local co-partitioned build first, rebuild a level cross-partition only if it has zero columns -> validated nested/adapt path stays bit-identical on the fast builder, non-nested tails fixed automatically; - True: force cross-partition; False: force rank-local. Validated: an independent (non-co-partitioned) coarse box tail converges in the same iteration count as serial (6 iters) and matches a GAMG reference to ~1e-8 at np1/np2/np4. New tests/parallel test_parallel_custom_fmg_nonnested (np2, np4). Nested co-partitioned parallel tests (scalar, Stokes velocity block, vector) and the serial suite unchanged. Design note updated. Underworld development team with AI support from Claude Code --- .../GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md | 38 +++++- src/underworld3/utilities/custom_mg.py | 122 ++++++++++++++++-- .../test_1017_custom_mg_parallel_mpi.py | 20 +++ 3 files changed, 165 insertions(+), 15 deletions(-) diff --git a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md index 4380f1d0..ef2d53b4 100644 --- a/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md +++ b/docs/developer/design/GENERALIZED_FMG_HIERARCHY_AND_ADAPT.md @@ -76,13 +76,45 @@ The supported parallel path is the **nested, co-partitioned** hierarchy: (`localToGlobal` gives the reduced global ordering across ranks). - `P` is assembled as an MPIAIJ matrix (fine local rows; coarse local + ghost columns). -**Non-nested / independent-mesh custom-P** (cross-rank point-location) is **serial-only / -experimental** — it is not on the parallel-supported path. The production case -(targeted refinement = nested) needs only the rank-local path. +**Non-nested / independent-mesh custom-P** in parallel (cross-rank point location) is now +supported via a **cross-partition transfer** (`_build_crosspart_transfer`, +`_gather_coarse_cloud`). When the coarse and fine meshes are partitioned independently, a +fine leaf on rank *r* can sit in a coarse cell owned by rank *s*, so the rank-local builder +either misses it (nearest-DOF fallback — wrong) or leaves a coarse DOF with no fine image +(zero column). The fix exploits the fact that a coarse MG level is, by definition, **small**: + +- **All-gather the coarse node cloud** (coords + each node/component's GLOBAL reduced column + index, `-1` for a BC-constrained DOF), deduplicated by rounded coordinate (ghost copies are + bit-identical). Every rank then holds the *full* coarse mesh. +- Each rank locates its **owned** fine nodes against that full cloud → point location spans + partitions. Columns are the coarse global reduced indices (off-rank columns are fine for + MPIAIJ); constrained coarse DOFs stay as barycentric vertices but drop from the columns + (reduced→reduced). Fine rows stay rank-local. + +`CustomMGHierarchy(..., cross_partition=...)` / `set_custom_fmg(..., cross_partition=...)`: +`"auto"` (default) builds the rank-local co-partitioned transfer first and rebuilds a level +cross-partition **only if it has zero columns** (the signature of a cross-partition miss) — +so the validated nested/adapt path stays on the fast rank-local builder bit-for-bit, while +non-nested tails are fixed automatically. `True` forces cross-partition (use when a coarse +level is known non-nested and might mis-locate without producing a zero column); `False` +forces the rank-local path. Validated: an independent (non-co-partitioned) coarse box tail +converges in the same iteration count as serial and matches a GAMG reference to ~1e-8 at +np2 and np4. Non-load-balancing SBR → bound the added refinement depth (configurable cap); document the imbalance/level trade-off. +### Operator-faithful finest reduced map +The finest transfer's row space **must** equal the assembled operator's space (PCMG Galerkins +`PᵀAP` against the real operator). The finest reduced map is read from the DM global section, +which is that space only **after** `snes.setUp()` finalizes it — critical on an `adapt()` +child, whose section can otherwise be read before finalization and disagree with the operator +(a rectangular finest transfer → bare PETSc error 60 in the PtAP). `CustomMGHierarchy.build` +calls `snes.setUp()` before reading the finest map and asserts its size against the assembled +operator (`_assert_finest_matches_operator`), failing with an actionable message instead. This +removed the earlier defensive skip of semi-Lagrangian advection-diffusion on adapt children — +such solves now install custom-P geometric MG and match a default-preconditioner solve. + ## Correctness invariants (must hold, all levels) 1. BCs applied at every level; transfers reduced→reduced (no zero columns). 2. Each prolongation reproduces constants (row-sums = 1 — partition of unity). diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 8d082be1..39a827f4 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -386,16 +386,86 @@ def _build_parallel_transfer(cc, fc, lay_c, lay_f, ncomp, builder, comm): return P -def _assert_no_zero_columns_parallel(P, comm): - """Parallel zero-column guard: a coarse DOF with no fine image -> singular - Galerkin coarse operator. Column sums via P^T·1 (weights are positive, so a - zero sum means an empty column).""" +def _gather_coarse_cloud(cc, lay_c, ncomp, comm): + """All-gather the (small) coarse node cloud across ranks and deduplicate. + + Returns ``(coords_u, cols_u)``: ``coords_u`` is the FULL coarse node cloud + (Nu, dim) — every coarse node in the mesh, on every rank — and ``cols_u`` is + (Nu, ncomp) the GLOBAL reduced column index of each node/component (``-1`` for a + BC-constrained DOF). Ghost copies are bit-identical and dedup by rounded + coordinate; constrained nodes stay in the cloud as barycentric vertices but + carry ``-1`` so they are dropped from the transfer columns.""" + m4 = comm.tompi4py() + ncn = cc.shape[0] + cols_local = np.asarray(lay_c[0], dtype=np.int64).reshape(ncn, ncomp) + cc_all = np.vstack(m4.allgather(np.ascontiguousarray(cc, dtype=float))) + cols_all = np.vstack(m4.allgather(np.ascontiguousarray(cols_local))) + _key, uidx = np.unique(np.round(cc_all, 9), axis=0, return_index=True) + uidx = np.sort(uidx) + return cc_all[uidx], cols_all[uidx] + + +def _build_crosspart_transfer(cc, fc, lay_c, lay_f, ncomp, builder, comm): + """One reduced->reduced prolongation for a NON-NESTED (independently + partitioned) coarse level. + + The co-partitioned builder locates each rank's fine nodes only in that rank's + LOCAL coarse coords; when the coarse and fine partitions differ a fine leaf on + rank ``r`` can sit in a coarse cell owned by rank ``s`` -> missed (nearest-DOF + fallback, wrong) or a coarse DOF with no fine image (zero column). Here every + rank locates its OWNED fine nodes against the FULL coarse cloud (all-gathered, + small — that is what makes a coarse MG level coarse), so point location spans + partitions. Columns are the coarse GLOBAL reduced indices (off-rank columns are + fine for MPIAIJ); constrained coarse DOFs (col < 0) drop out.""" + _l2g_c, cstart, cend, _ = lay_c + l2g_f, fstart, fend, _ = lay_f + coords_u, cols_u = _gather_coarse_cloud(cc, lay_c, ncomp, comm) + + Pn = builder(coords_u, fc).tocsr() # (n_f_nodes_local, Nu) + nloc_f = fend - fstart + nloc_c = cend - cstart + + P = PETSc.Mat().create(comm=comm) + P.setSizes(((nloc_f, None), (nloc_c, None))) + P.setType("aij") + P.setUp() + for i in range(Pn.shape[0]): # fine local node + js = Pn.indices[Pn.indptr[i]:Pn.indptr[i + 1]] + ws = Pn.data[Pn.indptr[i]:Pn.indptr[i + 1]] + for c in range(ncomp): + grow = int(l2g_f[i * ncomp + c]) + if grow < fstart or grow >= fend: # set OWNED rows only + continue + gcols, vals = [], [] + for jj, w in zip(js.tolist(), ws.tolist()): + gcol = int(cols_u[jj, c]) + if gcol >= 0: # skip constrained coarse DOFs + gcols.append(gcol) + vals.append(w) + if gcols: + P.setValues([grow], gcols, vals, addv=PETSc.InsertMode.INSERT_VALUES) + P.assemble() + return P + + +def _count_zero_columns_parallel(P, comm): + """Number of empty columns of ``P`` across all ranks (coarse DOFs with no fine + image). Column sums via P^T·1 (barycentric/RBF weights are positive, so a zero + sum is an empty column). Used to auto-detect a cross-partition point-location + miss (non-nested coarse level) and, separately, as the hard guard below.""" ones_f = P.createVecLeft(); ones_f.set(1.0) colsum = P.createVecRight() P.multTranspose(ones_f, colsum) nzero_local = int((colsum.array == 0.0).sum()) nzero = comm.tompi4py().allreduce(nzero_local) ones_f.destroy(); colsum.destroy() + return nzero + + +def _assert_no_zero_columns_parallel(P, comm): + """Parallel zero-column guard: a coarse DOF with no fine image -> singular + Galerkin coarse operator.""" + nzero = _count_zero_columns_parallel(P, comm) if nzero: raise RuntimeError( f"parallel transfer has {nzero} zero columns (coarse DOFs with no fine " @@ -547,17 +617,30 @@ class CustomMGHierarchy: Per-level node prolongation builder. field_id : int or None Field index for multi-field solvers (e.g. 0 = velocity); None = single field. + cross_partition : {"auto", True, False} + Parallel (np>1) transfer strategy. ``False`` = rank-local point location + (the co-partitioned nested / adapt-child fast path; each rank uses only its + LOCAL coarse coords). ``True`` = all-gather the coarse cloud so every rank + locates its fine nodes against the FULL coarse mesh (required when coarse + and fine are partitioned independently — non-nested coarse tails). ``"auto"`` + (default) uses the fast path and, if it produces a zero-column transfer + (the signature of a cross-partition point-location miss), rebuilds that + level cross-partition. Serial builds ignore this. """ - def __init__(self, level_meshes, builder="barycentric", field_id=None): + def __init__(self, level_meshes, builder="barycentric", field_id=None, + cross_partition="auto"): if builder not in _BUILDERS: raise ValueError("builder must be 'barycentric' or 'rbf'") if len(level_meshes) < 2: raise ValueError("need at least 2 levels (>=1 coarse + finest)") + if cross_partition not in ("auto", True, False): + raise ValueError("cross_partition must be 'auto', True or False") self.level_meshes = list(level_meshes) self.builder = _BUILDERS[builder] self.builder_name = builder self.field_id = field_id + self.cross_partition = cross_partition self.transfers = None def build(self, solver): @@ -621,12 +704,22 @@ def build(self, solver): self._assert_finest_matches_operator(solver, maps[-1], parallel) Ps = [] + comm = solver.dm.comm for l in range(1, nlev): if parallel: - P = _build_parallel_transfer(coords[l - 1], coords[l], - maps[l - 1], maps[l], nc, - self.builder, solver.dm.comm) - _assert_no_zero_columns_parallel(P, solver.dm.comm) + args = (coords[l - 1], coords[l], maps[l - 1], maps[l], nc, + self.builder, comm) + if self.cross_partition is True: + P = _build_crosspart_transfer(*args) + else: + P = _build_parallel_transfer(*args) + # "auto": a zero-column transfer means the coarse level is NOT + # co-partitioned with the fine level (a fine leaf sits in an + # off-rank coarse cell). Rebuild it spanning partitions. + if (self.cross_partition == "auto" + and _count_zero_columns_parallel(P, comm) > 0): + P = _build_crosspart_transfer(*args) + _assert_no_zero_columns_parallel(P, comm) Ps.append(P) else: Pr = _reduced_transfer(coords[l - 1], coords[l], maps[l - 1], @@ -680,7 +773,7 @@ def install(self, solver, verbose=False): # Entry points # --------------------------------------------------------------------------- # def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", - field_id=None, verbose=False): + field_id=None, cross_partition="auto", verbose=False): """Generalized custom-P FMG with BC-per-level reduction (the correct path). Registers a :class:`CustomMGHierarchy` on the solver so that the next @@ -689,11 +782,16 @@ def set_custom_fmg(solver, coarse_meshes, *, builder="barycentric", map is derived directly from its DM by copying the solver's fields + DS (``_coarse_reduced_map``), so ``coarse_meshes`` need only carry the same boundary labels as the solver's mesh. For a saddle-point (Stokes) solver pass - ``field_id=0`` to target the velocity sub-block.""" + ``field_id=0`` to target the velocity sub-block. + + ``cross_partition`` selects the parallel (np>1) transfer strategy (see + :class:`CustomMGHierarchy`); the default ``"auto"`` handles both nested and + non-nested coarse tails.""" solver._custom_mg = { "mode": "hierarchy", "hierarchy": CustomMGHierarchy(list(coarse_meshes) + [solver.mesh], - builder=builder, field_id=field_id), + builder=builder, field_id=field_id, + cross_partition=cross_partition), "verbose": verbose, } solver.is_setup = False diff --git a/tests/parallel/test_1017_custom_mg_parallel_mpi.py b/tests/parallel/test_1017_custom_mg_parallel_mpi.py index 3a3e29a5..d73046a3 100644 --- a/tests/parallel/test_1017_custom_mg_parallel_mpi.py +++ b/tests/parallel/test_1017_custom_mg_parallel_mpi.py @@ -68,6 +68,26 @@ def test_parallel_custom_fmg_scalar(): assert _rel_l2(c.Unknowns.u.data, g.Unknowns.u.data) < 1e-4 +@pytest.mark.mpi(min_size=2) +def test_parallel_custom_fmg_nonnested(): + """NON-NESTED coarse tail in parallel: the coarse mesh is an INDEPENDENT box at + a different cellSize (partitioned independently of the fine mesh), so a fine + leaf on rank r can sit in a coarse cell owned by rank s. The co-partitioned + rank-local builder misses those (zero columns); the "auto" cross-partition path + all-gathers the coarse cloud so every rank locates its fine nodes against the + full coarse mesh. Result must converge and match a GAMG reference.""" + assert uw.mpi.size > 1 + fine = _box(0.07) # NO refine hierarchy + coarse = _box(0.25) # independent, non-nested + g = _poisson(fine); g.preconditioner = "gamg"; g.solve() + c = _poisson(fine) + custom_mg.set_custom_fmg(c, [coarse], builder="barycentric") # cross_partition="auto" + c.solve() + assert c.snes.getKSP().getPC().getType() == "mg" + assert c.snes.getConvergedReason() > 0 + assert _rel_l2(c.Unknowns.u.data, g.Unknowns.u.data) < 1e-4 + + @pytest.mark.mpi(min_size=2) def test_parallel_custom_fmg_stokes_velocity_block(): """Custom-P on the SolCx velocity block converges in few MG iters in parallel From e79324d36e530724fd241676452eb7662880ec44 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 15 Jul 2026 17:57:15 -1000 Subject: [PATCH 50/52] style: state the sanctioned failure mode on the custom_mg setUp swallow The style gate (Charter S4 except-pass rule, #364) flags a bare 'except Exception: pass' introduced on the stranded branch pre-Charter. Minimal fix: document the sanctioned failure mode; no behaviour change. Underworld development team with AI support from Claude Code --- src/underworld3/utilities/custom_mg.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/underworld3/utilities/custom_mg.py b/src/underworld3/utilities/custom_mg.py index 39a827f4..9d4a252d 100644 --- a/src/underworld3/utilities/custom_mg.py +++ b/src/underworld3/utilities/custom_mg.py @@ -670,6 +670,9 @@ def build(self, solver): try: solver.snes.setUp() except Exception: + # Sanctioned swallow: setUp can fail on a not-yet-fully-configured + # SNES (pre-solve injection). The install paths call setUp again; + # the finest map then reads the DM's current global section. pass coords, maps, ncomp = [], [], [] From 4bb80add10224bfa761b27e88e68ded2b6a60da7 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 15 Jul 2026 18:01:42 -1000 Subject: [PATCH 51/52] docs: add the two new adapt design docs to the Design Documents toctree LAYER2_SBR_ADAPT_ON_TOP and NVB_GRADED_ADAPT arrive with the custom-mg landing; wiring them into docs/developer/index.md keeps the docs build at (below) the development warning baseline. Underworld development team with AI support from Claude Code --- docs/developer/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer/index.md b/docs/developer/index.md index 7f3965e9..a720ba89 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -147,6 +147,8 @@ design/WHY_UNITS_NOT_DIMENSIONALITY design/SYMBOL_DISAMBIGUATION_2025-12 design/ADAPTIVE_MESHING_DESIGN design/mesh-adaptation-formulation +design/LAYER2_SBR_ADAPT_ON_TOP +design/NVB_GRADED_ADAPT design/ARCHITECTURE_ANALYSIS design/MATHEMATICAL_MIXIN_DESIGN design/COORDINATE_MIGRATION_GUIDE From 01e9972250402fd28c4824cd7417462586f9852c Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 15 Jul 2026 18:33:31 -1000 Subject: [PATCH 52/52] fix(adapt): guard the legacy in-place mesh.adapt(metric) call shape Post-review hardening for the #365 landing. The branch renamed the in-place MMG remesher adapt() -> remesh() and gave adapt() new child-returning semantics, but a legacy caller got no signal: on a plain mesh a bare adapt(metric) raised a confusing "needs refinement>=1" RuntimeError, and on a refined mesh it silently returned an ignored child, leaving the base unmodified. - mesh.adapt(metric) with no new-API keyword now raises a TypeError directing the caller to mesh.remesh(metric) for in-place adaptation, or to pass an explicit keyword (max_levels=, engine=, ...) to opt in to the child-returning nested path. No silent redirection: the return semantics differ. All existing callers (tests, skills) already pass explicit keywords. - tests/test_0834_adapt_legacy_call_guard.py (level_1, tier_a): the guard fires for bare and verbose-only calls, the message names both paths, and an explicit keyword opts in. - Stale in-place docstring/doc surfaces updated to remesh: docs/advanced/mesh-adaptation.md, adaptivity.py docstrings, the _set_coords guidance message, _replace_from_adapted_mesh and Surface._on_mesh_adapted / refinement-metric docstrings. - nvb.py: TODO(#360) comments on the two coordinate-row == vertex-order assumptions (serial-only reachable; holds for fresh createFromCellList / undistributed base DMs; np>1 uses the native transform). Underworld development team with AI support from Claude Code --- docs/advanced/mesh-adaptation.md | 14 ++--- src/underworld3/adaptivity.py | 14 ++--- .../discretisation/discretisation_mesh.py | 41 +++++++++++-- .../discretisation_mesh_variables.py | 4 +- src/underworld3/meshing/surfaces.py | 14 ++--- src/underworld3/utilities/nvb.py | 11 ++++ tests/test_0834_adapt_legacy_call_guard.py | 60 +++++++++++++++++++ 7 files changed, 131 insertions(+), 27 deletions(-) create mode 100644 tests/test_0834_adapt_legacy_call_guard.py diff --git a/docs/advanced/mesh-adaptation.md b/docs/advanced/mesh-adaptation.md index 48942edf..0eb94178 100644 --- a/docs/advanced/mesh-adaptation.md +++ b/docs/advanced/mesh-adaptation.md @@ -30,7 +30,7 @@ metric = uw.adaptivity.metric_from_gradient( ) # Adapt the mesh -mesh.adapt(metric) +mesh.remesh(metric) ``` ## Core Concepts @@ -65,7 +65,7 @@ for each edge vector $\mathbf{e}$. Edges that are too long get subdivided; regio UW3 offers **two complementary** ways to put resolution where it is needed: -| | `mesh.adapt(...)` (this page) | `smooth_mesh_interior(method="anisotropic")` | +| | `mesh.remesh(...)` (this page) | `smooth_mesh_interior(method="anisotropic")` | |---|---|---| | Mechanism | **Re-mesh** (MMG): insert/remove/retriangulate | **Redistribute** the existing nodes (move only) | | Node budget | *Changes* — targets an **absolute** edge length `h` | **Fixed** — relative redistribution to a target *density* | @@ -75,7 +75,7 @@ needed: | Cost | Re-mesh + full variable transfer | A few cheap SPD elliptic solves (no re-mesh) | | Parallel | MMG re-partition | O(N), GAMG-parallelisable, no transfer | -Use `mesh.adapt` when you need a genuinely finer mesh (more +Use `mesh.remesh` when you need a genuinely finer mesh (more elements) and can afford to rebuild the problem. Use the **node-snuggling** redistribution when you want to *reshape* the existing mesh toward a feature every timestep cheaply, keeping the @@ -241,7 +241,7 @@ metric = fault.refinement_metric( ) # Adapt mesh -mesh.adapt(metric) +mesh.remesh(metric) # Check result print(f"Adapted mesh: {mesh.X.coords.shape[0]} nodes") @@ -258,7 +258,7 @@ p = uw.discretisation.MeshVariable("p", mesh, 1, degree=1) ### Variables Are Reset -After `mesh.adapt()`, all variables on the old mesh become invalid. Variables on the new mesh start uninitialized. +After `mesh.remesh()`, all variables on the old mesh become invalid. Variables on the new mesh start uninitialized. **For analytical initialization:** ```python @@ -347,7 +347,7 @@ pixi run -e amr python -c "import underworld3; print('AMR ready')" | Function | Purpose | |----------|---------| -| `mesh.adapt(metric)` | Adapt mesh using metric field | +| `mesh.remesh(metric)` | Re-mesh (in place) using metric field | | `uw.adaptivity.create_metric(mesh, h)` | Convert h-field to metric | | `uw.adaptivity.metric_from_gradient(field, ...)` | Metric from field gradient | | `uw.adaptivity.metric_from_field(indicator, ...)` | Metric from indicator field | @@ -380,7 +380,7 @@ For the mathematically inclined, see the [Developer Design Document](../develope When you want to concentrate resolution on an evolving feature **every timestep** without re-meshing — keeping the topology and all field data intact — use `smooth_mesh_interior` (the node-moving -mover) instead of `mesh.adapt`: +mover) instead of `mesh.remesh`: ```python import underworld3 as uw diff --git a/src/underworld3/adaptivity.py b/src/underworld3/adaptivity.py index c22a1fb8..589a01fe 100644 --- a/src/underworld3/adaptivity.py +++ b/src/underworld3/adaptivity.py @@ -108,7 +108,7 @@ def create_metric( Returns ------- MeshVariable - Scalar MeshVariable containing metric values ready for mesh.adapt(). + Scalar MeshVariable containing metric values ready for mesh.remesh(). Notes ----- @@ -132,7 +132,7 @@ def create_metric( >>> # Create metric from h-field computed elsewhere >>> h_field = compute_error_based_h(solution) # User function >>> metric = uw.adaptivity.create_metric(mesh, h_field) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) See Also -------- @@ -197,7 +197,7 @@ def metric_from_gradient( Returns ------- MeshVariable - Scalar MeshVariable containing metric values ready for mesh.adapt(). + Scalar MeshVariable containing metric values ready for mesh.remesh(). Notes ----- @@ -243,14 +243,14 @@ def metric_from_gradient( >>> metric = uw.adaptivity.metric_from_gradient( ... T, h_min=0.005, h_max=0.05, profile="smoothstep" ... ) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) >>> # Refine based on strain rate >>> # First compute strain rate magnitude as scalar field >>> SR = uw.discretisation.MeshVariable("SR", mesh, 1) >>> # ... populate SR with strain rate second invariant ... >>> metric = uw.adaptivity.metric_from_gradient(SR, h_min=0.01, h_max=0.1) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) See Also -------- @@ -359,7 +359,7 @@ def metric_from_field( Returns ------- MeshVariable - Scalar MeshVariable containing metric values ready for mesh.adapt(). + Scalar MeshVariable containing metric values ready for mesh.remesh(). Notes ----- @@ -380,7 +380,7 @@ def metric_from_field( >>> # Refine based on error estimate >>> error = compute_error_estimate(solution) # User function >>> metric = uw.adaptivity.metric_from_field(error, h_min=0.005, h_max=0.05) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) >>> # Refine at phase boundaries (phi transitions from 0 to 1) >>> # Want fine mesh where phi is near 0.5 diff --git a/src/underworld3/discretisation/discretisation_mesh.py b/src/underworld3/discretisation/discretisation_mesh.py index 139acead..5317095a 100644 --- a/src/underworld3/discretisation/discretisation_mesh.py +++ b/src/underworld3/discretisation/discretisation_mesh.py @@ -3338,7 +3338,7 @@ def _assert_coord_mutation_allowed(self): " Use instead:\n" " • mesh.deform(new_coords, dt=…) — impose an arbitrary " "node displacement (free surface / prescribed motion)\n" - " • mesh.adapt(metric) / mesh.OT_adapt(field)\n" + " • mesh.remesh(metric) / mesh.OT_adapt(field)\n" " • uw.meshing.smooth_mesh_interior(…) / uw.meshing.follow_metric(…)\n" " These route the field + SL/DDt-history transfer " "(remesh_with_field_transfer). For trusted scheme-internal trial " @@ -6224,8 +6224,8 @@ def _coarse_level_meshes(self): self._coarse_level_meshes_cache = cached return cached - def adapt(self, metric_field, max_levels=2, node_budget=None, - builder="barycentric", adapter="sbr", engine="sbr", verbose=False): + def adapt(self, metric_field, max_levels=None, node_budget=None, + builder=None, adapter=None, engine=None, verbose=False): r""" Nested **adapt-on-top**: return a refined **child** mesh. @@ -6259,6 +6259,13 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, own MG level; the transfers between consecutive levels each span a single refinement.) + .. note:: + ``mesh.adapt(metric)`` with **no other keyword** raises a + ``TypeError``: that call shape used to be the in-place MMG remesher + (now :meth:`remesh`), and a legacy caller would silently discard + the returned child. Pass any keyword — e.g. + ``adapt(metric, max_levels=2)`` — to opt in to the new semantics. + Parameters ---------- metric_field : MeshVariable, sympy expression, or callable @@ -6271,7 +6278,7 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, :meth:`Surface.refinement_metric_function`) so a thin feature refines to a clean, uniform-width band instead of a P1-aliased *patchy* one. Same interface as :meth:`remesh` / ``adaptivity.create_metric``. - max_levels : int + max_levels : int, default 2 Maximum SBR depth applied on top of the base finest (bounds the on-rank imbalance). Each level re-marks against the metric. node_budget : int or None @@ -6322,6 +6329,32 @@ def adapt(self, metric_field, max_levels=2, node_budget=None, """ import warnings + # Legacy-call guard. Before 2026-07 ``mesh.adapt(metric)`` was the + # IN-PLACE MMG remesher (now :meth:`remesh`); the nested adapt-on-top + # returns a NEW child mesh instead, which a legacy caller would silently + # discard. A bare call with no new-API keyword is therefore ambiguous + # and refused rather than redirected (the return semantics differ). + if (max_levels is None and node_budget is None and builder is None + and adapter is None and engine is None): + raise TypeError( + "mesh.adapt(metric) has changed meaning: the in-place metric " + "adaptation this call shape used to perform is now " + "mesh.remesh(metric). mesh.adapt(...) performs nested " + "adapt-on-top refinement and RETURNS A NEW child mesh (the " + "base mesh is not modified) — opt in explicitly by passing " + "any of its keywords, e.g. mesh.adapt(metric, max_levels=2) " + "or mesh.adapt(metric, engine='nvb'), and keep the returned " + "child." + ) + if max_levels is None: + max_levels = 2 + if builder is None: + builder = "barycentric" + if adapter is None: + adapter = "sbr" + if engine is None: + engine = "sbr" + if adapter == "mmg": warnings.warn( "mesh.adapt(adapter='mmg') is deprecated; the in-place MMG " diff --git a/src/underworld3/discretisation/discretisation_mesh_variables.py b/src/underworld3/discretisation/discretisation_mesh_variables.py index aadc1e40..fff0f357 100644 --- a/src/underworld3/discretisation/discretisation_mesh_variables.py +++ b/src/underworld3/discretisation/discretisation_mesh_variables.py @@ -1809,7 +1809,7 @@ def _replace_from_adapted_mesh(self, temp_var, adapted_mesh): """ Replace internal storage after mesh adaptation. - Called by mesh.adapt() to update this variable's internal PETSc + Called by mesh.remesh() to update this variable's internal PETSc structures after the mesh's discretization has changed. The data has already been interpolated to temp_var; this method copies that data into this variable's updated storage. @@ -1824,7 +1824,7 @@ def _replace_from_adapted_mesh(self, temp_var, adapted_mesh): Notes ----- - This is an internal method called by mesh.adapt(). Users should + This is an internal method called by mesh.remesh(). Users should not need to call this directly. After this method returns, all user references to this variable diff --git a/src/underworld3/meshing/surfaces.py b/src/underworld3/meshing/surfaces.py index 75c1434b..dbbf6a61 100644 --- a/src/underworld3/meshing/surfaces.py +++ b/src/underworld3/meshing/surfaces.py @@ -1524,13 +1524,13 @@ def save(self, filename: str) -> None: # --- Mesh Adaptation Support --- def _on_mesh_adapted(self, adapted_mesh: "Mesh") -> None: - """Called by mesh.adapt() to update after mesh adaptation. + """Called by mesh.remesh() to update after mesh adaptation. Marks the distance field as stale so it will be recomputed on next access. The surface geometry (control points, pyvista mesh) is unchanged - only the cached distance values need updating. - The distance MeshVariable itself is reinitialized by mesh.adapt() along + The distance MeshVariable itself is reinitialized by mesh.remesh() along with all other MeshVariables - we just need to mark the data as stale. Args: @@ -1611,7 +1611,7 @@ def refinement_metric( r"""Create a metric field for mesh adaptation based on distance from this surface. Returns a MeshVariable containing refinement metric values that can - be passed directly to mesh.adapt(). Higher metric values produce finer + be passed directly to mesh.remesh(). Higher metric values produce finer mesh (smaller elements). Parameters @@ -1681,7 +1681,7 @@ def refinement_metric( >>> >>> # With plain floats (nondimensional coordinates) >>> metric = fault.refinement_metric(h_near=0.005, h_far=0.05) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) >>> >>> # With quantities (automatic nondimensionalisation) >>> metric = fault.refinement_metric( @@ -1689,7 +1689,7 @@ def refinement_metric( ... h_far=uw.quantity(30, "km"), ... width=uw.quantity(10, "km"), ... ) - >>> mesh.adapt(metric) + >>> mesh.remesh(metric) """ if self.mesh is None: raise RuntimeError( @@ -2463,7 +2463,7 @@ def refinement_metric( Returns ------- MeshVariable - Scalar metric field suitable for ``mesh.adapt()``. + Scalar metric field suitable for ``mesh.remesh()``. """ h_near = _to_nd_length(h_near) h_far = _to_nd_length(h_far) @@ -3068,7 +3068,7 @@ def fault_metric(mesh, faults, method="ma", *, cell_size, method="ma")`` ``"anisotropic"`` 2×2 tensor (sympy Matrix) ``smooth_mesh_interior( method="anisotropic")`` - ``"adapt"``/``"mmg"`` ``h⁻²`` MeshVariable ``mesh.adapt(...)`` + ``"adapt"``/``"mmg"`` ``h⁻²`` MeshVariable ``mesh.remesh(...)`` =================== ============================ =========================== **``cell_size`` is honoured differently by each** — this is the key diff --git a/src/underworld3/utilities/nvb.py b/src/underworld3/utilities/nvb.py index da614f2d..7887322b 100644 --- a/src/underworld3/utilities/nvb.py +++ b/src/underworld3/utilities/nvb.py @@ -95,6 +95,14 @@ def from_dm(cls, dm, boundaries=(), regions=()): vS, vE = dm.getDepthStratum(0) eS, eE = dm.getDepthStratum(1) cS, cE = dm.getHeightStratum(0) + # TODO(#360): coordinate row i is assumed to be vertex point vS+i — the + # assumption class outlawed for Mesh coordinate arrays by #360. It holds + # HERE because this serial cell-list engine only ever sees fresh + # createFromCellList / undistributed base DMs whose degree-1 coordinate + # section is vertex-ordered (checked below), and the np>1 path uses the + # native transform instead (NotImplementedError before reaching this). + # If NVBMesh ever ingests distributed/permuted DMs, switch to a + # section-offset mapping (cf. Mesh._coord_rows_for_points). coords = dm.getCoordinatesLocal().array.reshape(-1, 2) if coords.shape[0] != vE - vS: raise RuntimeError( @@ -307,6 +315,9 @@ def to_dm(self, boundaries=(), regions=(), comm=None): # DM vertex point -> NVB vertex id by coordinate (midpoints are exact float # averages, computed identically here and in the DM, so the match is exact). + # TODO(#360): row i == vertex point vS+i is assumed, as in from_dm above. + # Safe here: the DM was created two lines up by createFromCellList on this + # rank (serial-only path), so its coordinate section is vertex-ordered. dm_vcoords = dm.getCoordinatesLocal().array.reshape(-1, 2) _, nearest = cKDTree(np.array(self.coords)).query(dm_vcoords) nvb_of_dmvert = {vS + i: int(nearest[i]) for i in range(vE - vS)} diff --git a/tests/test_0834_adapt_legacy_call_guard.py b/tests/test_0834_adapt_legacy_call_guard.py new file mode 100644 index 00000000..2e7f15c7 --- /dev/null +++ b/tests/test_0834_adapt_legacy_call_guard.py @@ -0,0 +1,60 @@ +"""Legacy-call guard for the mesh.adapt() semantics change. + +Before 2026-07 ``mesh.adapt(metric)`` was the IN-PLACE MMG remesher (now +``mesh.remesh``). The nested adapt-on-top ``adapt`` returns a NEW child mesh +instead, which a legacy in-place caller would silently discard (or, on an +unrefined base, hit a confusing "needs refinement>=1" RuntimeError). A bare +call with no new-API keyword must therefore raise a TypeError pointing at +``mesh.remesh(metric)`` and at the opt-in keywords — never silently redirect +(the return semantics differ). +""" + +import pytest + +import underworld3 as uw + +pytestmark = [pytest.mark.level_1, pytest.mark.tier_a] + + +@pytest.fixture(scope="module") +def mesh_and_metric(): + mesh = uw.meshing.UnstructuredSimplexBox( + minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0), cellSize=0.5, qdegree=2 + ) + metric = uw.discretisation.MeshVariable("Hguard", mesh, 1, degree=1) + metric.data[:, 0] = 100.0 + return mesh, metric + + +def test_bare_adapt_call_raises_typeerror(mesh_and_metric): + """mesh.adapt(metric) with no new-API keyword is the legacy in-place call + shape and must refuse with guidance, not run either behaviour.""" + mesh, metric = mesh_and_metric + with pytest.raises(TypeError, match=r"mesh\.remesh\(metric\)"): + mesh.adapt(metric) + + +def test_bare_adapt_with_verbose_still_raises(mesh_and_metric): + """verbose= exists in both signatures, so it does not disambiguate.""" + mesh, metric = mesh_and_metric + with pytest.raises(TypeError, match="child mesh"): + mesh.adapt(metric, verbose=True) + + +def test_error_message_names_both_paths(mesh_and_metric): + """The message must state the in-place replacement AND the opt-in shape.""" + mesh, metric = mesh_and_metric + with pytest.raises(TypeError) as excinfo: + mesh.adapt(metric) + message = str(excinfo.value) + assert "mesh.remesh(metric)" in message + assert "child mesh" in message + assert "max_levels" in message + + +def test_explicit_keyword_opts_in(mesh_and_metric): + """Passing any new-API keyword bypasses the guard and reaches the nested + path (which then reports the refinement>=1 requirement for this base).""" + mesh, metric = mesh_and_metric + with pytest.raises(RuntimeError, match="refinement>=1"): + mesh.adapt(metric, max_levels=1)