From 7957e839922c7c39f90b2935fcdfbb63d3f1e6ce Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Feb 2026 10:33:31 -0800
Subject: [PATCH 01/70] chore(deps): bump sentencepiece from 0.2.0 to 0.2.1
 (#114)

Bumps [sentencepiece](https://github.com/google/sentencepiece) from 0.2.0 to 0.2.1.
- [Release notes](https://github.com/google/sentencepiece/releases)
- [Commits](https://github.com/google/sentencepiece/compare/v0.2.0...v0.2.1)

---
updated-dependencies:
- dependency-name: sentencepiece
  dependency-version: 0.2.1
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b4a6e1e7..c531c681 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ requests==2.32.3
 spacy==3.7.5
 pydantic==2.11.4
 Pillow==11.2.1
-sentencepiece==0.2.0
+sentencepiece==0.2.1
 protobuf==6.30.2
 pytesseract==0.3.13
 aiohttp==3.11.18

From 04091e04dd8f999ada1e3f1cf3625d260ab286f1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Feb 2026 10:33:56 -0800
Subject: [PATCH 02/70] chore(deps): bump cryptography from 44.0.2 to 46.0.5
 (#119)

Bumps [cryptography](https://github.com/pyca/cryptography) from 44.0.2 to 46.0.5.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/44.0.2...46.0.5)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-version: 46.0.5
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c531c681..3fdc78cd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,7 @@ setuptools>=68.0.0
 pydantic-settings==2.3.4
 typer==0.12.3
 sphinx==7.2.6
-cryptography==44.0.2
+cryptography==46.0.5
 
 # Testing dependencies
 pytest==7.4.0

From 1ab0b95d40275107adaa2bacb2cb9dee7c343cd7 Mon Sep 17 00:00:00 2001
From: sid mohan <sidmohan001@gmail.com>
Date: Fri, 13 Feb 2026 10:35:41 -0800
Subject: [PATCH 03/70] chore(deps): bump pillow and protobuf pins

---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3fdc78cd..2078e115 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,9 +3,9 @@ pandas==2.2.3
 requests==2.32.3
 spacy==3.7.5
 pydantic==2.11.4
-Pillow==11.2.1
+Pillow==12.1.1
 sentencepiece==0.2.1
-protobuf==6.30.2
+protobuf==6.33.5
 pytesseract==0.3.13
 aiohttp==3.11.18
 numpy==1.26.4
@@ -20,4 +20,4 @@ cryptography==46.0.5
 # Testing dependencies
 pytest==7.4.0
 pytest-asyncio==0.21.0
-pytest-cov==4.1.0
\ No newline at end of file
+pytest-cov==4.1.0

From 8163e80ab68d14b99fc1f113580b6434cbe86860 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 16 Feb 2026 03:14:35 +0000
Subject: [PATCH 04/70] chore: bump version to 4.3.0a1 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 111dc917..67aab652 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0"
+__version__ = "4.3.0a1"

From 33fae4ed6d82956e39e7a22df474eb2eb7af62d7 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 19 Feb 2026 03:11:15 +0000
Subject: [PATCH 05/70] chore: bump version to 4.3.0b2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 67aab652..7f5ba1d2 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a1"
+__version__ = "4.3.0b2"

From 744370b08358de72772ff22f6efb55e08cac23a8 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 23 Feb 2026 03:16:15 +0000
Subject: [PATCH 06/70] chore: bump version to 4.3.0a2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 7f5ba1d2..9ef0c4a9 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b2"
+__version__ = "4.3.0a2"

From 84d52dbba36d54b13ab42a91240784ff1edfa362 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 26 Feb 2026 03:09:33 +0000
Subject: [PATCH 07/70] chore: bump version to 4.3.0b3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 9ef0c4a9..31595a63 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a2"
+__version__ = "4.3.0b3"

From 4108984ec03fbfa099d56c9fae950de294bd98f8 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 2 Mar 2026 03:12:17 +0000
Subject: [PATCH 08/70] chore: bump version to 4.3.0a3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 31595a63..6a0400b0 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b3"
+__version__ = "4.3.0a3"

From cb7d951fce8f50be816e9c7599604f6ef26b3bbb Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 5 Mar 2026 03:08:13 +0000
Subject: [PATCH 09/70] chore: bump version to 4.3.0b4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 6a0400b0..10d2475c 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a3"
+__version__ = "4.3.0b4"

From 107df2805856e8c3986641b3fe58e0de24608ea8 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 9 Mar 2026 03:13:14 +0000
Subject: [PATCH 10/70] chore: bump version to 4.3.0a4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 10d2475c..1e1c3b3b 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b4"
+__version__ = "4.3.0a4"

From 0788e82dc15249f3613dee9b9ad471c69b23dd9e Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 12 Mar 2026 03:08:47 +0000
Subject: [PATCH 11/70] chore: bump version to 4.3.0b5 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 1e1c3b3b..f74eef4c 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a4"
+__version__ = "4.3.0b5"

From 14ae62e9bec2089111a5ce04c29ce2bb0ff2a255 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 16 Mar 2026 03:17:24 +0000
Subject: [PATCH 12/70] chore: bump version to 4.3.0a5 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index f74eef4c..6a3dae89 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b5"
+__version__ = "4.3.0a5"

From fac558d19083cab0811a4af8356177db3908aee6 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 19 Mar 2026 03:13:18 +0000
Subject: [PATCH 13/70] chore: bump version to 4.3.0b6 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 6a3dae89..8923bd60 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a5"
+__version__ = "4.3.0b6"

From 14347bec43fb76e68ca93fa643f12d45299134ae Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 23 Mar 2026 03:15:54 +0000
Subject: [PATCH 14/70] chore: bump version to 4.3.0a6 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 8923bd60..12f598de 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b6"
+__version__ = "4.3.0a6"

From 189b77de38e3bb8db79e941788d2893cdc134048 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 26 Mar 2026 03:13:33 +0000
Subject: [PATCH 15/70] chore: bump version to 4.3.0b7 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 12f598de..b2380e72 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a6"
+__version__ = "4.3.0b7"

From d4602d0d2b014986f02d4b3083b4e3ca1c3245f8 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 30 Mar 2026 03:18:21 +0000
Subject: [PATCH 16/70] chore: bump version to 4.3.0a7 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index b2380e72..9dcb5920 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b7"
+__version__ = "4.3.0a7"

From 879ae7047cda7daaf4901cca6b50b7a544f2e005 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 2 Apr 2026 03:14:22 +0000
Subject: [PATCH 17/70] chore: bump version to 4.3.0b8 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 9dcb5920..7c19e895 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a7"
+__version__ = "4.3.0b8"

From 3b1a5e8b4149ecd8245b9fe8c54fdb6ef661916d Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 6 Apr 2026 03:17:32 +0000
Subject: [PATCH 18/70] chore: bump version to 4.3.0a8 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 7c19e895..ff818429 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b8"
+__version__ = "4.3.0a8"

From 372d3080b2fd5ac2ee99d8e28dcacb080ded3fef Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 9 Apr 2026 03:14:19 +0000
Subject: [PATCH 19/70] chore: bump version to 4.3.0b9 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index ff818429..f02f7efd 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a8"
+__version__ = "4.3.0b9"

From 936ca144544a640f7049479f85463aef17af36be Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 13 Apr 2026 03:18:37 +0000
Subject: [PATCH 20/70] chore: bump version to 4.3.0a9 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index f02f7efd..5e5421fd 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b9"
+__version__ = "4.3.0a9"

From 22e9820ca638a70df8e0ad5e9b4e9f393ba31677 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 16 Apr 2026 03:19:57 +0000
Subject: [PATCH 21/70] chore: bump version to 4.3.0b10 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 5e5421fd..6a48ac4f 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a9"
+__version__ = "4.3.0b10"

From 0a67ad3d33f0fd4107723e1292e9b403682f88db Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 20 Apr 2026 03:17:54 +0000
Subject: [PATCH 22/70] chore: bump version to 4.3.0a10 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 6a48ac4f..635b146e 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b10"
+__version__ = "4.3.0a10"

From aa72c2e171e84d84a807baf823482d97b90cf868 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 23 Apr 2026 03:15:13 +0000
Subject: [PATCH 23/70] chore: bump version to 4.3.0b11 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 635b146e..bc4f11a2 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0a10"
+__version__ = "4.3.0b11"

From 47520244a44a8dd9ef955af572dd3cff6ce6e12f Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sat, 25 Apr 2026 19:53:12 -0700
Subject: [PATCH 24/70] Add v4.4 bridge release runway (#130)

* feat: add v4.4 bridge release runway

* docs: clarify contributor workflow

* style: apply lint formatting
---
 .github/ISSUE_TEMPLATE/bug_report.yml         |  62 +++++++
 .github/ISSUE_TEMPLATE/config.yml             |   5 +
 .github/ISSUE_TEMPLATE/feature_request.yml    |  41 +++++
 .github/PULL_REQUEST_TEMPLATE.md              |  37 ++++
 .github/workflows/ci.yml                      |  10 +-
 .github/workflows/release.yml                 |  34 +++-
 CONTRIBUTING.md                               | 112 ++++++++++---
 SECURITY.md                                   |  18 ++
 datafog/__init__.py                           | 104 +++++++++++-
 datafog/config.py                             |   8 +-
 datafog/models/anonymizer.py                  |   7 +-
 .../text_processing/spacy_pii_annotator.py    |   7 +-
 docs/conf.py                                  |   4 +
 docs/important-concepts.rst                   |  12 +-
 docs/index.rst                                |   8 +-
 docs/roadmap.rst                              |  65 ++++++-
 docs/v44-bridge-release.rst                   | 145 ++++++++++++++++
 docs/v5-compatibility-matrix.rst              | 158 ++++++++++++++++++
 docs/v5-cut-line.rst                          | 142 ++++++++++++++++
 docs/v5-product-brief.rst                     | 114 +++++++++++++
 setup.py                                      |   3 +-
 tests/test_telemetry.py                       |   9 +-
 tests/test_v44_bridge_api.py                  |  82 +++++++++
 23 files changed, 1141 insertions(+), 46 deletions(-)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml
 create mode 100644 .github/ISSUE_TEMPLATE/config.yml
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml
 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md
 create mode 100644 SECURITY.md
 create mode 100644 docs/v44-bridge-release.rst
 create mode 100644 docs/v5-compatibility-matrix.rst
 create mode 100644 docs/v5-cut-line.rst
 create mode 100644 docs/v5-product-brief.rst
 create mode 100644 tests/test_v44_bridge_api.py

diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 00000000..39ba49ac
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,62 @@
+name: Bug report
+description: Report something that is broken or surprising.
+title: "bug: "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: Thanks for reporting a DataFog issue.
+  - type: textarea
+    id: summary
+    attributes:
+      label: Summary
+      description: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Reproduction
+      description: Minimal code, command, or steps to reproduce.
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: DataFog version
+      placeholder: "4.3.0"
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      placeholder: "3.12.4"
+  - type: dropdown
+    id: profile
+    attributes:
+      label: Install profile
+      options:
+        - core
+        - cli
+        - nlp
+        - nlp-advanced
+        - ocr
+        - distributed
+        - all
+        - not sure
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment details
+      description: OS, package manager, relevant dependency versions, or CI link.
+  - type: textarea
+    id: extra
+    attributes:
+      label: Additional context
+      description: Logs, screenshots, or related issues.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 00000000..3505f60f
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Security report
+    url: https://github.com/DataFog/datafog-python/security/advisories/new
+    about: Please report security vulnerabilities privately.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
new file mode 100644
index 00000000..8d64b92e
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -0,0 +1,41 @@
+name: Feature request
+description: Suggest an improvement or new workflow.
+title: "feat: "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What user problem would this solve?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: What would you like DataFog to do?
+    validations:
+      required: true
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Core scan/redaction
+        - CLI
+        - LLM guardrails
+        - NLP engines
+        - OCR/image processing
+        - Spark/distributed processing
+        - Packaging/install
+        - Documentation
+        - Other
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+  - type: textarea
+    id: extra
+    attributes:
+      label: Additional context
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 00000000..c3a6b2d3
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,37 @@
+## Summary
+
+Describe the change and why it is needed.
+
+## Type
+
+- [ ] Bug fix
+- [ ] Feature
+- [ ] Docs
+- [ ] Tests
+- [ ] Chore
+
+## Target Branch
+
+- [ ] This PR targets `dev`
+- [ ] This PR targets `main` for a release/hotfix
+
+## Validation
+
+Commands run:
+
+```bash
+
+```
+
+Optional profiles tested:
+
+- [ ] core
+- [ ] cli
+- [ ] nlp
+- [ ] nlp-advanced
+- [ ] ocr
+- [ ] distributed
+
+## Notes For Reviewers
+
+Mention API changes, migrations, warnings, or release-note needs.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 03df4cb1..4ffb6fda 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,8 +29,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         install-profile: ["core", "nlp", "nlp-advanced"]
+        exclude:
+          # v4.4.0 claims Python 3.13 support for core + CLI first.
+          # Optional heavyweight profiles remain validated separately before
+          # we advertise Python 3.13 support for them.
+          - python-version: "3.13"
+            install-profile: "nlp"
+          - python-version: "3.13"
+            install-profile: "nlp-advanced"
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 091f9329..f99b874c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -139,8 +139,40 @@ jobs:
           OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py
 
   # ── 3. Build & Publish ────────────────────────────────────────────────
+  python313-core:
+    needs: determine-release
+    if: needs.determine-release.outputs.has_changes == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ needs.determine-release.outputs.target_branch }}
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: "pip"
+
+      - name: Install core + CLI dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-cov coverage
+          pip install -e ".[dev,cli]"
+
+      - name: Run Python 3.13 core + CLI tests
+        run: |
+          pytest tests/ \
+            -m "not slow" \
+            --ignore=tests/test_gliner_annotator.py \
+            --ignore=tests/test_image_service.py \
+            --ignore=tests/test_ocr_integration.py \
+            --ignore=tests/test_spark_integration.py \
+            --ignore=tests/test_text_service_integration.py
+
   publish:
-    needs: [determine-release, test]
+    needs: [determine-release, test, python313-core]
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.version.outputs.version }}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0ec54416..285dd51e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,29 +1,99 @@
-# Contributing guidelines
+# Contributing to DataFog Python
 
-# Contributors
+Thanks for helping improve DataFog. The project welcomes issues, bug reports,
+documentation fixes, tests, and pull requests.
 
-- sroy9675
-- pselvana
-- sidmohan0
+Please follow the [Code of Conduct](CODE_OF_CONDUCT.md) in all project spaces.
+
+## Branch And PR Policy
+
+DataFog uses `dev` as the default development branch and `main` as the stable
+release branch.
+
+Use this workflow for normal contributions:
+
+1. Fork the repository or create a topic branch from `dev`.
+2. Name branches with a GitHub username prefix when practical, for example
+   `sidmohan0/dfpy-v44-bridge` or `yourname/fix-cli-redaction`.
+3. Open pull requests into `dev`.
+4. Keep pull requests focused and include tests or docs when behavior changes.
+
+Use `main` only for stable release promotion or urgent release hotfixes.
+Do not use `dev` or `main` as working branches.
+
+Maintainers should prefer pull requests even for small changes. Protected branch
+rules should prevent branch deletion, require CI before merge, and avoid direct
+pushes except for explicit emergency maintenance.
+
+## Local Development
+
+```bash
+git clone https://github.com/datafog/datafog-python
+cd datafog-python
+python -m venv .venv
+source .venv/bin/activate  # Windows: .venv\Scripts\activate
+python -m pip install --upgrade pip
+pip install -e ".[dev,cli]"
+```
+
+For optional NLP or OCR work, install the relevant extras:
+
+```bash
+pip install -e ".[dev,cli,nlp]"
+pip install -e ".[dev,cli,nlp,nlp-advanced]"
+pip install -e ".[all,dev]"
+```
 
-for their help
+## Tests
 
-The datafog community appreciates your contributions via issues and
-pull requests. Note that the [code of conduct](CODE_OF_CONDUCT.md)
-applies to all interactions with the datafog project, including
-issues and pull requests.
+Run the core test suite before opening a pull request:
 
-When submitting pull requests, please follow the style guidelines of
-the project, ensure that your code is tested and documented, and write
-good commit messages, e.g., following [these
-guidelines](https://chris.beams.io/posts/git-commit/).
+```bash
+pytest tests/ -m "not slow" \
+  --ignore=tests/test_gliner_annotator.py \
+  --ignore=tests/test_image_service.py \
+  --ignore=tests/test_ocr_integration.py \
+  --ignore=tests/test_spark_integration.py \
+  --ignore=tests/test_text_service_integration.py
+```
 
-By submitting a pull request, you are licensing your code under the
-project [license](LICENSE) and affirming that you either own copyright
-(automatic for most individuals) or are authorized to distribute under
-the project license (e.g., in case your employer retains copyright on
-your work).
+Run the focused test file for the area you changed whenever possible. For
+documentation-only changes, build the docs:
 
-### Legal Notice
+```bash
+sphinx-build -b html docs docs/_build/html
+```
 
-When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+## Pull Request Checklist
+
+Before requesting review:
+
+- Rebase or merge the latest `dev`.
+- Add or update tests for behavior changes.
+- Update docs for user-facing changes.
+- Keep public API changes explicit in the PR description.
+- Note any optional dependency profile you tested, such as `core`, `nlp`, or
+  `nlp-advanced`.
+
+## Commit Messages
+
+Use clear, descriptive commit messages. Conventional-style prefixes are welcome
+but not required, for example:
+
+- `fix: handle empty scan input`
+- `docs: clarify branch policy`
+- `test: cover v5 preview redaction wrapper`
+
+## Legal
+
+By submitting a pull request, you license your contribution under the project
+[license](LICENSE). You also affirm that you authored the contribution or have
+the right to submit it under the project license.
+
+## Contributors
+
+Thanks to early contributors including:
+
+- sroy9675
+- pselvana
+- sidmohan0
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 00000000..2821acd0
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,18 @@
+# Security Policy
+
+Please do not report security vulnerabilities in public issues.
+
+Use GitHub private vulnerability reporting when available:
+
+https://github.com/DataFog/datafog-python/security/advisories/new
+
+If private vulnerability reporting is unavailable, contact the maintainers
+directly and include:
+
+- affected versions;
+- a minimal reproduction or proof of concept;
+- expected impact;
+- any known mitigations.
+
+We will acknowledge valid reports as quickly as practical and coordinate fixes
+before public disclosure.
diff --git a/datafog/__init__.py b/datafog/__init__.py
index b3ca498e..e3974ad7 100644
--- a/datafog/__init__.py
+++ b/datafog/__init__.py
@@ -8,11 +8,17 @@
 - pip install datafog[all] - for all features
 """
 
+import warnings
+
 from .__about__ import __version__
 from .agent import create_guardrail, filter_output, sanitize, scan_prompt
 
 # Core API functions - always available (lightweight)
 from .core import anonymize_text, detect_pii, get_supported_entities, scan_text
+from .engine import Entity, RedactResult, ScanResult
+from .engine import redact as _redact_entities
+from .engine import scan as _scan
+from .engine import scan_and_redact as _scan_and_redact
 
 # Essential models - always available
 from .models.common import EntityTypes
@@ -134,6 +140,88 @@ def _missing_dependency(*args, **kwargs):
 )
 
 
+_REDACT_PRESETS = {
+    "default": "token",
+    "llm": "token",
+    "mask": "mask",
+    "hash": "hash",
+    "replace": "pseudonymize",
+    "pseudonymize": "pseudonymize",
+}
+
+
+def _warn_v5_replacement(old_api: str, replacement: str) -> None:
+    warnings.warn(
+        f"datafog.{old_api}() is deprecated for v5. Use {replacement} instead. "
+        "This compatibility shim will remain through the v5.x line.",
+        FutureWarning,
+        stacklevel=3,
+    )
+
+
+def scan(
+    text: str,
+    engine: str = "regex",
+    entity_types: list[str] | None = None,
+) -> ScanResult:
+    """
+    v5-preview scan entrypoint.
+
+    Defaults to the lightweight regex engine so the core install works without
+    optional dependency fallback warnings.
+    """
+    return _scan(text=text, engine=engine, entity_types=entity_types)
+
+
+def redact(
+    text: str,
+    entities: list[Entity] | None = None,
+    engine: str = "regex",
+    entity_types: list[str] | None = None,
+    strategy: str = "token",
+    preset: str | None = None,
+) -> RedactResult:
+    """
+    v5-preview redaction entrypoint.
+
+    If entities are provided, redact those spans. Otherwise, scan text first
+    using the selected engine and redact the detected entities.
+    """
+    if preset is not None:
+        try:
+            strategy = _REDACT_PRESETS[preset]
+        except KeyError as exc:
+            allowed = ", ".join(sorted(_REDACT_PRESETS))
+            raise ValueError(f"preset must be one of: {allowed}") from exc
+
+    if entities is not None:
+        return _redact_entities(text=text, entities=entities, strategy=strategy)
+
+    return _scan_and_redact(
+        text=text,
+        engine=engine,
+        entity_types=entity_types,
+        strategy=strategy,
+    )
+
+
+def protect(
+    entity_types: list[str] | None = None,
+    engine: str = "regex",
+    strategy: str = "token",
+    on_detect: str = "redact",
+):
+    """
+    v5-preview guardrail factory.
+    """
+    return create_guardrail(
+        entity_types=entity_types,
+        engine=engine,
+        strategy=strategy,
+        on_detect=on_detect,
+    )
+
+
 # Simple API for core functionality (backward compatibility)
 def detect(text: str) -> list:
     """
@@ -150,6 +238,12 @@ def detect(text: str) -> list:
         >>> detect("Contact john@example.com")
         [{'type': 'EMAIL', 'value': 'john@example.com', 'start': 8, 'end': 24}]
     """
+    _warn_v5_replacement("detect", "datafog.scan()")
+
+    return _detect_impl(text)
+
+
+def _detect_impl(text: str) -> list:
     import time as _time
 
     _start = _time.monotonic()
@@ -217,11 +311,13 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict:
             'findings': [{'type': 'EMAIL', 'value': 'john@example.com', ...}]
         }
     """
+    _warn_v5_replacement("process", "datafog.scan() or datafog.redact()")
+
     import time as _time
 
     _start = _time.monotonic()
 
-    findings = detect(text)
+    findings = _detect_impl(text)
 
     result = {"original": text, "findings": findings}
 
@@ -268,6 +364,12 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict:
 # Core exports
 __all__ = [
     "__version__",
+    "Entity",
+    "ScanResult",
+    "RedactResult",
+    "scan",
+    "redact",
+    "protect",
     "detect",
     "process",
     "detect_pii",
diff --git a/datafog/config.py b/datafog/config.py
index 36a6b3d8..b195893b 100644
--- a/datafog/config.py
+++ b/datafog/config.py
@@ -9,7 +9,7 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic_settings import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class DataFogConfig(BaseSettings):
@@ -25,6 +25,8 @@ class DataFogConfig(BaseSettings):
     uses Pydantic for data validation and settings management.
     """
 
+    model_config = SettingsConfigDict(env_prefix="DATAFOG_", case_sensitive=False)
+
     # API Keys and Authentication
     api_key: str = os.environ.get("DATAFOG_API_KEY", "")
 
@@ -48,10 +50,6 @@ class DataFogConfig(BaseSettings):
     # Logging
     log_level: str = "INFO"
 
-    class Config:
-        env_prefix = "DATAFOG_"
-        case_sensitive = False
-
     def update(self, **kwargs):
         """Update configuration with new values"""
         for key, value in kwargs.items():
diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py
index 79af53ca..6b81f5be 100644
--- a/datafog/models/anonymizer.py
+++ b/datafog/models/anonymizer.py
@@ -7,7 +7,7 @@
 from enum import Enum
 from typing import List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from .annotator import AnnotationResult
 from .common import EntityTypes
@@ -34,14 +34,13 @@ class AnonymizerRequest(BaseModel):
 
 
 class AnonymizationResult(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
     anonymized_text: str
     anonymized_entities: List[dict] = Field(
         default_factory=list, alias="replaced_entities"
     )
 
-    class Config:
-        populate_by_name = True
-
 
 class Anonymizer(BaseModel):
     anonymizer_type: AnonymizerType = AnonymizerType.REPLACE
diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py
index e7bc5732..e871db8a 100644
--- a/datafog/processing/text_processing/spacy_pii_annotator.py
+++ b/datafog/processing/text_processing/spacy_pii_annotator.py
@@ -1,7 +1,7 @@
 import logging
 from typing import Any, Dict, List
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 PII_ANNOTATION_LABELS = [
     "CARDINAL",
@@ -27,6 +27,8 @@
 
 
 class SpacyPIIAnnotator(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     nlp: Any
 
     @classmethod
@@ -75,6 +77,3 @@ def annotate(self, text: str) -> Dict[str, List[str]]:
             return {
                 label: [] for label in PII_ANNOTATION_LABELS
             }  # Return empty annotations in case of error
-
-    class Config:
-        arbitrary_types_allowed = True
diff --git a/docs/conf.py b/docs/conf.py
index 0c10e751..1cb1c895 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -30,3 +30,7 @@
 napoleon_use_rtype = False
 napoleon_use_ivar = False
 napoleon_use_param = False
+
+# Keep API docs buildable from the lightweight core/dev install. These
+# integrations are documented, but they live behind optional extras.
+autodoc_mock_imports = ["PIL", "pytesseract", "spacy"]
diff --git a/docs/important-concepts.rst b/docs/important-concepts.rst
index 791fa0a0..d08c932c 100644
--- a/docs/important-concepts.rst
+++ b/docs/important-concepts.rst
@@ -1,6 +1,6 @@
-===========
+==================
 Important Concepts
-===========
+==================
 
 Overview
 --------
@@ -20,6 +20,7 @@ Key data models to support PII annotation and OCR analysis.
 Processors
 ^^^^^^^^^^^
 Main processors:
+
 * SpacyAnnotator
     Text annotation with spaCy
 * DonutProcessor
@@ -30,6 +31,7 @@ Main processors:
 Services
 ^^^^^^^^^^^
 Core services:
+
 * ImageService
     Image handling and OCR
 * SparkService
@@ -47,6 +49,7 @@ Data Models
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    AnnotatorRequest
    AnnotationResult
    AnalysisExplanation
@@ -57,6 +60,7 @@ Data Models
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    EntityTypes
    Pattern
    PatternRecognizer
@@ -68,6 +72,7 @@ Data Models
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    SpacyAnnotator
 
 Processors
@@ -98,6 +103,7 @@ Services
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    ImageDownloader
    ImageService
 
@@ -107,6 +113,7 @@ Services
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    SparkService
 
 
@@ -116,4 +123,5 @@ Services
 .. autosummary::
    :toctree: generated/
    :template: class.rst
+
    TextService
diff --git a/docs/index.rst b/docs/index.rst
index e092c758..a22af1c5 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -12,13 +12,17 @@ DataFog is an open-source tool for PII detection and anonymization of unstructur
    python-sdk
    definitions
    roadmap
+   v44-bridge-release
+   v5-product-brief
+   v5-compatibility-matrix
+   v5-cut-line
 
 =====================
 Getting Started
 =====================
 
----------------------
 Installation
+------------
 
 Install DataFog via pip:
 
@@ -102,5 +106,3 @@ Scan image for PII:
    asyncio.run(run_ocr_pipeline_demo())
 
 For detailed information on the Python SDK, see :doc:`python-sdk`.
-
-
diff --git a/docs/roadmap.rst b/docs/roadmap.rst
index 63850b63..acf8b6a0 100644
--- a/docs/roadmap.rst
+++ b/docs/roadmap.rst
@@ -9,6 +9,69 @@ to a lightweight, modular architecture with optional extras.
    :local:
    :depth: 1
 
+v4.4.0 - Python 3.13 and v5 Migration Bridge
+--------------------------------------------
+
+Before v5.0.0, DataFog should ship a focused v4.4.0 bridge release. The
+purpose is to give users a concrete compatibility win while introducing the
+v5 direction gently.
+
+v4.4.0 should focus on:
+
+* Python 3.13 support for the core SDK and CLI.
+* Dependency validation for optional profiles without blocking core/CLI.
+* v5-style preview APIs where they can land safely.
+* Targeted deprecation warnings with no warnings on import.
+* Migration docs and release notes that announce the v5 path.
+
+Scope artifact:
+
+* :doc:`v44-bridge-release`
+
+v5.0.0 - Offline PII Firewall for AI Apps
+-----------------------------------------
+
+The v5.0.0 release is scoped around a sharper adoption wedge:
+
+   DataFog should be the fastest, easiest offline PII firewall for AI apps,
+   logs, and datasets.
+
+The release should prioritize trust and time-to-first-value over broad
+enterprise surface area. The first path should be a core install, a simple
+top-level API, no network surprises, and copy-pasteable workflows for LLM
+prompts/outputs, logs, JSONL datasets, and CI checks.
+
+Scope artifacts:
+
+* :doc:`v5-product-brief`
+* :doc:`v5-compatibility-matrix`
+* :doc:`v5-cut-line`
+
+v5.0.0 must focus on:
+
+* Stable top-level APIs: ``scan``, ``redact``, ``protect``, and ``restore``.
+* Privacy-safe defaults: no default network behavior, no runtime package
+  installation, and no implicit model downloads.
+* Policy-based redaction with presets for LLMs, logs, strict workflows, and
+  datasets.
+* Reversible token sessions that are explicit and opt-in.
+* LLM guardrails, including sync, async, and streaming protection.
+* CLI workflows for stdin, files, directories, CSV, JSONL, machine-readable
+  output, and CI-friendly exit codes.
+* Custom recognizers and stronger structured detection for app/log/secrets
+  data.
+* Modern packaging and release gates for install profiles, no-network
+  behavior, import time, wheel size, accuracy, coverage, and benchmarks.
+
+Deferred to v5.1+:
+
+* OCR overhaul.
+* Spark overhaul.
+* Cloud DLP integrations.
+* Enterprise dashboards and analytics.
+* Broad multilingual model tuning.
+* Large Presidio-style framework expansion.
+
 ✅ 4.1.0 (Released)
 --------------------
 The ``4.1.0`` release represents a major architectural shift to a lightweight
@@ -80,4 +143,4 @@ Version ``4.5.0`` will introduce:
 * **Performance monitoring** and metrics collection
 
 The lightweight core will remain unchanged, ensuring existing
-integrations continue to work without modification.
\ No newline at end of file
+integrations continue to work without modification.
diff --git a/docs/v44-bridge-release.rst b/docs/v44-bridge-release.rst
new file mode 100644
index 00000000..981b517d
--- /dev/null
+++ b/docs/v44-bridge-release.rst
@@ -0,0 +1,145 @@
+=================================
+v4.4 Bridge Release Scope
+=================================
+
+Status
+------
+
+Scope and implementation notes for the v4.4.0 transition release.
+
+Linear project: `DataFog Python v4.4.0: Python 3.13 + v5 Migration Bridge <https://linear.app/threadfork/project/datafog-python-v440-python-313-v5-migration-bridge-0afcd4211a58>`_
+
+Release Thesis
+--------------
+
+v4.4.0 should be a bridge release before v5. It should give users an
+immediate compatibility win with Python 3.13 and introduce the v5 migration
+path without forcing a disruptive API change.
+
+The release should feel positive:
+
+* Python 3.13 support for the core SDK and CLI.
+* A preview of the v5-style API path.
+* Targeted, actionable deprecation warnings.
+* Migration docs that explain what is changing and why.
+
+Required Scope
+--------------
+
+Python 3.13 Core and CLI Support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+v4.4.0 should support Python 3.13 for the lightweight core package and CLI
+workflow.
+
+Required changes:
+
+* Package metadata allows Python 3.13.
+* Package classifiers include Python 3.13.
+* CI runs the core/CLI profile on Python 3.13.
+* Release notes clearly state that Python 3.13 support is guaranteed for
+  core and CLI.
+
+Optional Dependency Stance
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Optional heavy profiles should be validated where practical, but they should
+not block v4.4.0 unless they break the core or CLI install.
+
+.. list-table::
+   :header-rows: 1
+   :widths: 20 30 50
+
+   * - Profile
+     - v4.4.0 support stance
+     - Notes
+   * - ``core``
+     - Required on Python 3.13
+     - Must install and run scan/redaction tests.
+   * - ``cli``
+     - Required on Python 3.13
+     - Must install and run CLI smoke tests.
+   * - ``nlp``
+     - Validate where practical
+     - spaCy advertises Python 3.13 support in current releases, but model
+       install behavior should be tested before claiming full support.
+   * - ``nlp-advanced``
+     - Provisional
+     - GLiNER and PyTorch dependency behavior should be tested separately.
+   * - ``ocr``
+     - Provisional
+     - OCR is not on the v4.4 or v5.0 critical path.
+   * - ``distributed``
+     - Provisional
+     - Spark support should not block the bridge release.
+
+v5 Preview APIs
+~~~~~~~~~~~~~~~
+
+v4.4.0 should make the recommended v5 path available early where it can be
+done without large internal churn.
+
+Preview path:
+
+* ``datafog.scan``
+* ``datafog.redact``
+* ``datafog.protect`` if it can be exposed through existing guardrail helpers
+  without pulling in the full v5 policy engine.
+
+The preview APIs should delegate through current stable internals and should
+not require optional heavy dependencies.
+
+For the bridge release, preview APIs should default to the lightweight regex
+engine so a core-only install does not emit optional dependency fallback
+warnings. Users can still opt into ``engine="smart"`` explicitly.
+
+Deprecation Warning Policy
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+v4.4.0 should introduce migration signals, not warning noise.
+
+Rules:
+
+* No warnings on ``import datafog``.
+* Warnings only fire when selected legacy convenience APIs are called.
+* Warning messages name the v5-style replacement.
+* Warnings use ``stacklevel`` so they point to user code where practical.
+* ``DataFog`` and ``TextService`` should not warn by default in v4.4.0.
+
+Candidate warning targets:
+
+* ``datafog.detect`` -> ``datafog.scan``
+* ``datafog.process`` -> ``datafog.redact``
+
+Deferred warning targets:
+
+* ``detect_pii`` -> ``scan``
+* ``anonymize_text`` -> ``redact``
+* ``scan_text`` -> ``scan``
+* Old CLI commands once v5-style CLI aliases exist.
+
+Non-Goals
+---------
+
+v4.4.0 should not attempt to complete v5.
+
+Out of scope:
+
+* Full v5 policy engine.
+* Reversible token sessions.
+* Streaming guardrails.
+* OCR overhaul.
+* Spark overhaul.
+* Modern packaging migration to ``pyproject.toml``.
+* Broad dependency cleanup beyond what Python 3.13 requires.
+
+Success Criteria
+----------------
+
+v4.4.0 is ready when:
+
+* Python 3.13 core/CLI CI is green.
+* Package metadata advertises Python 3.13 support.
+* v5-style API docs are visible.
+* Deprecation warnings are targeted and tested.
+* Release notes frame the release as a compatibility win and migration runway.
diff --git a/docs/v5-compatibility-matrix.rst b/docs/v5-compatibility-matrix.rst
new file mode 100644
index 00000000..41c9e339
--- /dev/null
+++ b/docs/v5-compatibility-matrix.rst
@@ -0,0 +1,158 @@
+=======================
+v5 Compatibility Matrix
+=======================
+
+Status
+------
+
+Scope lock draft for the v4 to v5 migration contract.
+
+Migration Stance
+----------------
+
+v5 should be adoption-first without being hostile to existing users.
+The default posture is:
+
+* Make the new API obvious and stable.
+* Keep existing public APIs working through compatibility shims where
+  practical.
+* Warn only when a user is on a path that should move to the v5 API.
+* Break behavior only when required for trust, privacy, or install
+  predictability.
+
+Compatibility Matrix
+--------------------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 26 24 20 30
+
+   * - Surface
+     - v4 State
+     - v5 Status
+     - v5 Direction
+   * - ``datafog.scan``
+     - v4.4 bridge preview top-level API.
+     - Stable public API.
+     - Keep as the obvious scan entrypoint returning typed scan results.
+   * - ``datafog.redact``
+     - v4.4 bridge preview top-level API.
+     - Stable public API.
+     - Expand user-facing presets and policy support while preserving typed
+       redaction metadata.
+   * - ``datafog.protect``
+     - v4.4 bridge preview alias around ``create_guardrail``.
+     - Stable public API.
+     - Add decorator/context API for sync, async, and streaming workflows.
+   * - ``datafog.restore``
+     - No primary public API.
+     - Stable public API.
+     - Restore text from explicit reversible token sessions only.
+   * - ``datafog.sanitize``
+     - Top-level LLM helper.
+     - Supported shim.
+     - Keep as alias-like convenience around ``redact``.
+   * - ``datafog.scan_prompt``
+     - Top-level LLM helper.
+     - Supported shim.
+     - Keep for compatibility; docs should prefer ``scan`` or ``protect``.
+   * - ``datafog.filter_output``
+     - Top-level LLM helper.
+     - Supported shim.
+     - Keep for compatibility; docs should prefer ``redact`` or ``protect``.
+   * - ``datafog.create_guardrail``
+     - Top-level guardrail factory.
+     - Supported shim.
+     - Keep, but docs should make ``protect`` the first path.
+   * - ``datafog.detect``
+     - Top-level convenience detection API.
+     - Deprecated shim.
+     - Delegate to ``scan`` and warn with replacement guidance.
+   * - ``datafog.process``
+     - Top-level detect/process helper.
+     - Deprecated shim.
+     - Delegate to ``scan`` / ``redact`` and warn with replacement guidance.
+   * - ``detect_pii``
+     - Core helper returning dicts.
+     - Deprecated shim.
+     - Keep temporarily for v4 users; prefer ``scan`` typed results.
+   * - ``anonymize_text``
+     - Core anonymization helper.
+     - Deprecated shim.
+     - Keep temporarily; prefer ``redact`` with a policy or preset.
+   * - ``scan_text``
+     - Boolean or dict helper.
+     - Deprecated shim.
+     - Keep temporarily; prefer ``scan``.
+   * - ``get_supported_entities``
+     - Returns core regex entity names.
+     - Supported utility.
+     - Keep, but align with recognizer registry and locale packs.
+   * - ``DataFog``
+     - Legacy class-based entrypoint.
+     - Compatibility shim.
+     - Keep through v5 with warnings for new construction patterns.
+   * - ``TextService``
+     - Service class with engine selection and legacy return shapes.
+     - Compatibility shim.
+     - Keep for existing users; new docs should use top-level API.
+   * - ``TextPIIAnnotator``
+     - Lightweight wrapper.
+     - Deprecated shim.
+     - Keep only if needed for compatibility; prefer ``scan``.
+   * - ``RegexAnnotator``
+     - Exposed implementation detail.
+     - Advanced API.
+     - Keep available for advanced users, but docs should prefer registry.
+   * - ``datafog.engine``
+     - Internal boundary introduced before v5.
+     - Internal implementation.
+     - Keep stable enough for wrappers, but public imports should use top-level
+       APIs or public model paths.
+   * - CLI ``scan-text``, ``redact-text``, ``replace-text``, ``hash-text``
+     - Current command set.
+     - Deprecated command shims.
+     - Add v5 ``scan``, ``redact``, and ``audit`` commands; keep old commands
+       with warnings.
+   * - CLI ``scan-image``
+     - OCR-oriented command.
+     - Optional legacy command.
+     - Keep under OCR extra; do not make it a v5.0 adoption path.
+   * - OCR services and processors
+     - Optional extras with heavyweight dependencies.
+     - Deferred for v5.1+ overhaul.
+     - Keep install hints clear; no runtime package installs.
+   * - Spark services and UDFs
+     - Optional distributed path.
+     - Deferred for v5.1+ overhaul.
+     - Keep compatibility where practical; no runtime package installs.
+   * - Telemetry
+     - Default-on opt-out telemetry.
+     - Trust-critical behavior change.
+     - Make opt-in or no-network-by-default.
+   * - ``*_lean`` and ``*_original`` modules
+     - Parallel historical implementations.
+     - Remove or make private after migration path.
+     - Consolidate around the v5 core and delete duplicate runtime surfaces.
+
+Warning Policy
+--------------
+
+Deprecation warnings should be specific and actionable. They should name the
+old API, the v5 replacement, and the planned support window. For example:
+
+.. code-block:: text
+
+   datafog.detect() is deprecated in v5. Use datafog.scan() instead.
+   This compatibility shim will be supported through the v5.x line.
+
+Breaking-Change Criteria
+------------------------
+
+A v5 breaking change is acceptable only when it materially improves one of:
+
+* Privacy or trust defaults.
+* Core install reliability.
+* No-network behavior.
+* Public API clarity.
+* Correctness of redaction metadata.
diff --git a/docs/v5-cut-line.rst b/docs/v5-cut-line.rst
new file mode 100644
index 00000000..71a66a5a
--- /dev/null
+++ b/docs/v5-cut-line.rst
@@ -0,0 +1,142 @@
+=================
+v5.0 Release Line
+=================
+
+Status
+------
+
+Scope lock draft for what belongs in v5.0.0 versus v5.1+.
+
+v5.0 Release Promise
+--------------------
+
+v5.0.0 should ship a sharp, adoption-focused SDK:
+
+   The fastest, easiest offline PII firewall for AI apps, logs, and
+   datasets.
+
+The release is not judged by how much surface area it adds. It is judged by
+whether a new user can trust it quickly, understand it quickly, and protect
+real application data quickly.
+
+Must Ship in v5.0
+-----------------
+
+Core API
+~~~~~~~~
+
+* Stable top-level ``scan``, ``redact``, ``protect``, and ``restore`` APIs.
+* Public typed result objects and entity models.
+* Compatibility shims for important v4 APIs.
+* API snapshot tests and executable quickstart smoke tests.
+
+Trust Defaults
+~~~~~~~~~~~~~~
+
+* Telemetry opt-in or no-network-by-default.
+* No runtime package installation.
+* No implicit model downloads.
+* Safe redaction metadata by default.
+* No-network import and core API tests.
+
+Policy and Redaction
+~~~~~~~~~~~~~~~~~~~~
+
+* ``RedactionPolicy`` with entity filters, thresholds, locale, mapping
+  behavior, and actions per entity type.
+* Built-in presets for ``llm``, ``logs``, ``strict``, and ``dataset``.
+* Reversible ``TokenSession`` only when explicitly requested.
+* HMAC hashing and format-preserving masking strategies.
+
+LLM and App Workflows
+~~~~~~~~~~~~~~~~~~~~~
+
+* ``protect`` decorator for sync and async functions.
+* Streaming output filtering.
+* Guardrail session counters and safe audit metadata.
+* Examples for LLM calls, FastAPI middleware, logging filters, LangChain-style
+  adapters, and LlamaIndex-style adapters.
+
+CLI and Dataset Workflows
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* v5 CLI commands: ``scan``, ``redact``, and ``audit``.
+* Support for direct text, stdin, files, directories, CSV, and JSONL.
+* Machine-readable ``--json`` and ``--jsonl`` output.
+* CI-friendly ``--fail-on-detect`` behavior and exit codes.
+* Dataset audit summaries that avoid raw PII by default.
+
+Detection Quality
+~~~~~~~~~~~~~~~~~
+
+* Custom recognizer registry with regex and validator hooks.
+* Common app/log recognizers for secrets, tokens, API keys, IPv6, IBAN,
+  crypto wallets, and account identifiers.
+* ``us`` and ``global`` locale packs.
+* Deterministic overlap resolution and confidence scoring.
+* Expanded corpus and enforced accuracy thresholds in CI.
+
+Packaging and Release
+~~~~~~~~~~~~~~~~~~~~~
+
+* ``pyproject.toml`` as primary packaging metadata.
+* ``py.typed`` marker.
+* Clean runtime dependencies and optional extras.
+* CI gates for install profiles, import time, wheel size, no-network behavior,
+  coverage, accuracy, and benchmark regressions.
+* v4 to v5 migration guide and launch assets.
+
+Defer to v5.1+
+--------------
+
+OCR Overhaul
+~~~~~~~~~~~~
+
+OCR is useful, but it is not the adoption wedge for v5.0. Keep current OCR
+compatibility where practical and move OCR quality, preprocessing, and
+processor architecture to v5.1+.
+
+Spark and Distributed Processing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Spark support should not block the first v5 release. Keep install errors and
+compatibility sane, then revisit distributed workflows after the core API and
+CLI are stable.
+
+Cloud Integrations
+~~~~~~~~~~~~~~~~~~
+
+AWS, GCP, Azure, hosted APIs, and cloud DLP bridges are deferred. v5.0 should
+win first as an offline local SDK.
+
+Enterprise Analytics
+~~~~~~~~~~~~~~~~~~~~
+
+Dashboards, organization-wide reporting, and advanced analytics are deferred.
+The v5.0 audit surface should remain CLI/API friendly and privacy-safe.
+
+Broad Multilingual Expansion
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+v5.0 may introduce locale structure, but broad multilingual tuning should wait
+until the v5 core is stable and measured.
+
+Large Presidio-Style Framework Expansion
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DataFog should not try to clone Presidio in v5.0. Custom recognizers are in
+scope, but a large enterprise recognizer framework is not.
+
+Scope Creep Test
+----------------
+
+A proposed v5.0 item should pass at least one of these tests:
+
+* Does it make first use faster or clearer?
+* Does it improve trust for a PII SDK?
+* Does it protect LLM, log, CLI, or dataset workflows?
+* Does it make the core install smaller, safer, or more predictable?
+* Does it reduce migration risk for existing users?
+
+If the answer is no, the item probably belongs in v5.1+.
+
diff --git a/docs/v5-product-brief.rst b/docs/v5-product-brief.rst
new file mode 100644
index 00000000..84fd51c7
--- /dev/null
+++ b/docs/v5-product-brief.rst
@@ -0,0 +1,114 @@
+================
+v5 Product Brief
+================
+
+Status
+------
+
+Scope lock draft for DataFog Python v5.0.0.
+
+Linear project: `DataFog Python v5.0.0: Offline PII Firewall for AI Apps <https://linear.app/threadfork/project/datafog-python-v500-offline-pii-firewall-for-ai-apps-78a1550a22f4>`_
+
+Product Thesis
+--------------
+
+DataFog v5 should be the fastest, easiest offline PII firewall for AI
+apps, logs, and datasets.
+
+The release should optimize for adoption over breadth. The first user
+experience should be a small install, a clear one-liner, and strong
+privacy defaults:
+
+.. code-block:: python
+
+   import datafog
+
+   result = datafog.redact("Email john@example.com", preset="llm")
+   safe_text = result.redacted_text
+
+In the v4.4 bridge release, ``scan``, ``redact``, and ``protect`` should default
+to the lightweight regex engine so the core install remains quiet. Higher-recall
+NLP paths can stay explicit through ``engine="smart"`` or optional extras.
+
+Primary Users
+-------------
+
+v5 is built first for developers who need to keep PII out of:
+
+* LLM prompts and model outputs.
+* Agent tool calls and MCP-style tool results.
+* Application logs, traces, analytics events, and support dumps.
+* JSONL, CSV, and text datasets used for evals or fine-tuning.
+* CI checks that should fail when accidental PII is present.
+
+Adoption Principles
+-------------------
+
+* **Time to first value under 60 seconds.** A new user should be able to
+  install DataFog and redact text without reading architecture docs.
+* **No network surprises.** A core install should not send telemetry,
+  download models, install packages, or open network connections by
+  default.
+* **One obvious API path.** The primary path is ``scan``, ``redact``,
+  ``protect``, and ``restore``.
+* **Safe metadata by default.** Redaction results should not expose
+  original PII mappings unless the user explicitly asks for a reversible
+  session.
+* **Workflow-first docs.** The first examples should be LLM protection,
+  logging filters, FastAPI middleware, CLI JSONL redaction, and custom
+  recognizers.
+* **Honest positioning.** DataFog should not try to out-Presidio
+  Presidio. It should win on lightweight offline protection for AI apps,
+  logs, and datasets.
+
+v5.0 Must-Haves
+---------------
+
+* Stable top-level APIs: ``datafog.scan``, ``datafog.redact``,
+  ``datafog.protect``, and ``datafog.restore``.
+* Public typed result objects for scans, entities, redaction results,
+  and reversible sessions.
+* Privacy-safe defaults: telemetry opt-in or no-network-by-default,
+  no runtime package installation, no implicit model downloads.
+* Policy-based redaction with presets for ``llm``, ``logs``,
+  ``strict``, and ``dataset``.
+* Reversible token sessions for workflows that need explicit
+  restore support.
+* LLM guardrails for sync, async, and streaming responses.
+* CLI support for stdin, files, directories, CSV, JSONL, machine-readable
+  output, and CI-friendly exit codes.
+* Custom recognizer registry for domain-specific PII and secrets.
+* Expanded corpus tests for app, log, and secret-like data.
+* Modern packaging, typed package marker, install-profile tests, and
+  release gates for accuracy, coverage, import time, wheel size, and
+  no-network behavior.
+
+Non-Goals for v5.0
+------------------
+
+The following work is intentionally outside the v5.0 critical path:
+
+* OCR overhaul.
+* Spark overhaul.
+* Cloud DLP integrations.
+* Enterprise dashboards or analytics products.
+* Broad multilingual model tuning.
+* A large Presidio-style recognizer framework.
+* Hosted service features.
+
+Success Metrics
+---------------
+
+* A clean core install can redact text in under 60 seconds from a fresh
+  environment.
+* ``pip install datafog`` stays lightweight and does not require spaCy,
+  PyTorch, OCR, Spark, or model downloads.
+* ``import datafog`` and core ``scan`` / ``redact`` calls make no network
+  requests by default.
+* The README quickstart uses the v5 API first and does not require legacy
+  classes.
+* At least five adoption workflows are documented and copy-pasteable:
+  LLM prompts/outputs, streaming outputs, logging, FastAPI, and JSONL CLI.
+* Accuracy and regression checks run in CI and produce readable artifacts.
+* Existing v4 users have a clear migration path with compatibility shims
+  and warnings instead of surprising breakage.
diff --git a/setup.py b/setup.py
index 97b5c909..75284180 100644
--- a/setup.py
+++ b/setup.py
@@ -95,7 +95,7 @@
     packages=find_packages(exclude=["tests", "tests.*"]),
     install_requires=core_deps,
     extras_require=extras_require,
-    python_requires=">=3.10,<3.13",
+    python_requires=">=3.10,<3.14",
     entry_points={
         "console_scripts": [
             "datafog=datafog.client:app [cli]",  # Requires cli extra
@@ -109,6 +109,7 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Text Processing",
         "Topic :: Security",
diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py
index bd20e21f..9c69e178 100644
--- a/tests/test_telemetry.py
+++ b/tests/test_telemetry.py
@@ -356,7 +356,8 @@ class TestIntegration:
     def test_detect_triggers_telemetry(self, mock_urlopen):
         from datafog import detect
 
-        detect("Contact john@example.com")
+        with pytest.warns(FutureWarning, match=r"Use datafog\.scan\(\) instead"):
+            detect("Contact john@example.com")
         time.sleep(0.3)
 
         events = []
@@ -369,7 +370,11 @@ def test_detect_triggers_telemetry(self, mock_urlopen):
     def test_process_triggers_telemetry(self, mock_urlopen):
         from datafog import process
 
-        process("Contact john@example.com", anonymize=True)
+        with pytest.warns(
+            FutureWarning,
+            match=r"datafog\.scan\(\) or datafog\.redact\(\)",
+        ):
+            process("Contact john@example.com", anonymize=True)
         time.sleep(0.3)
 
         events = []
diff --git a/tests/test_v44_bridge_api.py b/tests/test_v44_bridge_api.py
new file mode 100644
index 00000000..a26099a9
--- /dev/null
+++ b/tests/test_v44_bridge_api.py
@@ -0,0 +1,82 @@
+"""Tests for v4.4 bridge APIs that preview the v5 surface."""
+
+from __future__ import annotations
+
+import importlib
+import warnings
+
+import pytest
+
+import datafog
+
+
+def test_top_level_scan_is_available_without_optional_extras() -> None:
+    with warnings.catch_warnings(record=True) as captured:
+        warnings.simplefilter("always")
+        result = datafog.scan("Email jane@example.com")
+
+    assert result.engine_used == "regex"
+    assert any(entity.type == "EMAIL" for entity in result.entities)
+    assert not captured
+
+
+def test_top_level_redact_scans_by_default() -> None:
+    with warnings.catch_warnings(record=True) as captured:
+        warnings.simplefilter("always")
+        result = datafog.redact("Email jane@example.com")
+
+    assert result.redacted_text != "Email jane@example.com"
+    assert "[EMAIL_1]" in result.redacted_text
+    assert result.mapping
+    assert not captured
+
+
+def test_top_level_redact_accepts_precomputed_entities() -> None:
+    text = "Email jane@example.com"
+    scan_result = datafog.scan(text, engine="regex")
+
+    result = datafog.redact(text, entities=scan_result.entities, strategy="mask")
+
+    assert "jane@example.com" not in result.redacted_text
+    assert "*" in result.redacted_text
+
+
+def test_top_level_redact_supports_preview_presets() -> None:
+    result = datafog.redact("Email jane@example.com", preset="llm")
+
+    assert "[EMAIL_1]" in result.redacted_text
+
+
+def test_top_level_protect_returns_guardrail() -> None:
+    guardrail = datafog.protect(on_detect="redact")
+
+    with warnings.catch_warnings(record=True) as captured:
+        warnings.simplefilter("always")
+        filtered = guardrail.filter("Email jane@example.com")
+
+    assert filtered.redacted_text != "Email jane@example.com"
+    assert not captured
+
+
+def test_legacy_detect_warns_with_replacement() -> None:
+    with pytest.warns(FutureWarning, match=r"Use datafog\.scan\(\) instead"):
+        result = datafog.detect("Email jane@example.com")
+
+    assert result
+
+
+def test_legacy_process_warns_with_replacement() -> None:
+    with pytest.warns(FutureWarning, match=r"datafog\.scan\(\) or datafog\.redact\(\)"):
+        result = datafog.process("Email jane@example.com", anonymize=True)
+
+    assert result["anonymized"] != result["original"]
+
+
+def test_import_does_not_emit_migration_warnings() -> None:
+    with warnings.catch_warnings(record=True) as captured:
+        warnings.simplefilter("always")
+        importlib.reload(datafog)
+
+    assert not [
+        warning for warning in captured if issubclass(warning.category, FutureWarning)
+    ]

From 4f22b6a12e6b80431f32514ddaa6e6304e9594f5 Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sat, 25 Apr 2026 20:12:55 -0700
Subject: [PATCH 25/70] ci: allow prerelease base override (#131)

---
 .github/workflows/release.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f99b874c..98034d7e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -27,7 +27,7 @@ on:
         default: false
         type: boolean
       version_override:
-        description: "Override version (e.g. 4.4.0) — stable only"
+        description: "Override stable version or prerelease base (e.g. 4.4.0)"
         required: false
         type: string
 
@@ -214,6 +214,13 @@ jobs:
 
           # Strip any pre-release suffix to get base version
           BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//')
+          if [ -n "${{ inputs.version_override }}" ]; then
+            BASE="${{ inputs.version_override }}"
+            if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then
+              echo "version_override must be a stable base version like 4.4.0, not a prerelease"
+              exit 1
+            fi
+          fi
           echo "Base version: $BASE"
 
           if [ "$TYPE" = "alpha" ]; then
@@ -231,12 +238,7 @@ jobs:
             VERSION="${BASE}b${BETA_NUM}"
 
           else
-            # Stable: use override or base version
-            if [ -n "${{ inputs.version_override }}" ]; then
-              VERSION="${{ inputs.version_override }}"
-            else
-              VERSION="$BASE"
-            fi
+            VERSION="$BASE"
           fi
 
           echo "version=$VERSION" >> $GITHUB_OUTPUT

From 6845ef7f38987508ccdb715d1e4db65f4fb3ca4a Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Sun, 26 Apr 2026 03:31:05 +0000
Subject: [PATCH 26/70] chore: bump version to 4.4.0b1 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index bc4f11a2..e8957a0b 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.3.0b11"
+__version__ = "4.4.0b1"

From 8ff0618bda62f3d497484aaf1c80214336fa9dca Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sun, 26 Apr 2026 14:40:21 -0700
Subject: [PATCH 27/70] Make telemetry opt-in for v4.4 (#132)

---
 README.md                        | 14 ++++-
 datafog/telemetry.py             | 20 +++++--
 docs/v5-compatibility-matrix.rst |  4 +-
 scripts/generate_changelog.py    | 37 +++++++++++++
 tests/test_telemetry.py          | 95 +++++++++++++++++++++++---------
 5 files changed, 135 insertions(+), 35 deletions(-)

diff --git a/README.md b/README.md
index 794defcb..e3a211b3 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,10 @@ pip install datafog[nlp-advanced]
 pip install datafog[all]
 ```
 
+Python 3.13 support is certified for the core SDK and CLI. Optional extras such
+as `nlp`, `nlp-advanced`, `ocr`, `distributed`, and `all` are available but not
+yet certified on Python 3.13.
+
 ## Quick Start
 
 ```python
@@ -132,9 +136,15 @@ datafog hash-text "john@example.com"
 
 ## Telemetry
 
-DataFog includes anonymous telemetry by default.
+DataFog telemetry is disabled by default.
+
+To opt in:
+
+```bash
+export DATAFOG_TELEMETRY=1
+```
 
-To opt out:
+To force telemetry off:
 
 ```bash
 export DATAFOG_NO_TELEMETRY=1
diff --git a/datafog/telemetry.py b/datafog/telemetry.py
index fb7e3137..6b3885a3 100644
--- a/datafog/telemetry.py
+++ b/datafog/telemetry.py
@@ -1,10 +1,13 @@
 """
-Anonymous, opt-out usage telemetry for DataFog.
+Anonymous, opt-in usage telemetry for DataFog.
 
 Collects anonymous usage data to help the DataFog team understand which engines,
 functions, and features are actually used. No text content is ever sent.
 
-Opt out by setting either environment variable:
+Telemetry is disabled by default. Opt in by setting:
+    DATAFOG_TELEMETRY=1
+
+Force telemetry off by setting either environment variable:
     DATAFOG_NO_TELEMETRY=1
     DO_NOT_TRACK=1
 """
@@ -29,13 +32,18 @@
 _scope = threading.local()
 
 
+def _env_truthy(name: str) -> bool:
+    """Return True when an environment variable explicitly opts in/out."""
+    return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
+
+
 def _is_telemetry_enabled() -> bool:
-    """Check if telemetry is enabled (opt-out via env vars)."""
-    if os.environ.get("DATAFOG_NO_TELEMETRY", "").strip() == "1":
+    """Check if telemetry is enabled (opt-in, with opt-out overrides)."""
+    if _env_truthy("DATAFOG_NO_TELEMETRY"):
         return False
-    if os.environ.get("DO_NOT_TRACK", "").strip() == "1":
+    if _env_truthy("DO_NOT_TRACK"):
         return False
-    return True
+    return _env_truthy("DATAFOG_TELEMETRY")
 
 
 def _get_anonymous_id() -> str:
diff --git a/docs/v5-compatibility-matrix.rst b/docs/v5-compatibility-matrix.rst
index 41c9e339..b95483e6 100644
--- a/docs/v5-compatibility-matrix.rst
+++ b/docs/v5-compatibility-matrix.rst
@@ -127,9 +127,9 @@ Compatibility Matrix
      - Deferred for v5.1+ overhaul.
      - Keep compatibility where practical; no runtime package installs.
    * - Telemetry
-     - Default-on opt-out telemetry.
+     - Opt-in telemetry.
      - Trust-critical behavior change.
-     - Make opt-in or no-network-by-default.
+     - Keep no-network-by-default.
    * - ``*_lean`` and ``*_original`` modules
      - Parallel historical implementations.
      - Remove or make private after migration path.
diff --git a/scripts/generate_changelog.py b/scripts/generate_changelog.py
index 293ac5b8..23babcf8 100755
--- a/scripts/generate_changelog.py
+++ b/scripts/generate_changelog.py
@@ -7,6 +7,16 @@
 from datetime import datetime
 
 
+def get_current_version():
+    """Read the current package version from datafog/__about__.py."""
+    try:
+        with open("datafog/__about__.py") as f:
+            match = re.search(r'^__version__ = "([^"]+)"', f.read(), re.M)
+            return match.group(1) if match else None
+    except OSError:
+        return None
+
+
 def get_latest_tag():
     """Get the latest git tag."""
     try:
@@ -65,6 +75,7 @@ def generate_changelog(beta=False, alpha=False):
     """Generate changelog content."""
     latest_tag = get_latest_tag()
     commits = get_commits_since_tag(latest_tag)
+    current_version = get_current_version()
 
     if not commits:
         return "No changes since last release."
@@ -85,6 +96,32 @@ def generate_changelog(beta=False, alpha=False):
         changelog = "# What's New\n\n"
         changelog += f"*Released: {datetime.now().strftime('%Y-%m-%d')}*\n\n"
 
+    if not alpha and not beta and current_version == "4.4.0":
+        changelog += "## Python 3.13 Support Scope\n\n"
+        changelog += (
+            "Python 3.13 support is certified for the core SDK and CLI: "
+            "`pip install datafog` and `pip install datafog[cli]`.\n\n"
+        )
+        changelog += (
+            "Optional extras including `nlp`, `nlp-advanced`, `ocr`, "
+            "`distributed`, and `all` are available but not yet certified on "
+            "Python 3.13. They will be validated separately based on user "
+            "demand.\n\n"
+        )
+        changelog += "## v5 Migration Bridge\n\n"
+        changelog += (
+            "This release adds the v5-preview top-level APIs `datafog.scan`, "
+            "`datafog.redact`, and `datafog.protect` while keeping the legacy "
+            "`datafog.detect` and `datafog.process` APIs working with targeted "
+            "migration warnings.\n\n"
+        )
+        changelog += "## Privacy Defaults\n\n"
+        changelog += (
+            "Telemetry is now opt-in. DataFog does not send telemetry unless "
+            "`DATAFOG_TELEMETRY=1` is explicitly set. `DATAFOG_NO_TELEMETRY=1` "
+            "and `DO_NOT_TRACK=1` continue to force telemetry off.\n\n"
+        )
+
     if categories["features"]:
         changelog += "## 🚀 New Features\n"
         for commit in categories["features"]:
diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py
index 9c69e178..3886a1dc 100644
--- a/tests/test_telemetry.py
+++ b/tests/test_telemetry.py
@@ -28,13 +28,20 @@ def _reset_telemetry_state():
 def _clean_state(monkeypatch):
     """Ensure clean telemetry state for every test and disable network."""
     _reset_telemetry_state()
-    # Default: telemetry enabled but network mocked
+    # Default: telemetry disabled unless a test opts in explicitly.
+    monkeypatch.delenv("DATAFOG_TELEMETRY", raising=False)
     monkeypatch.delenv("DATAFOG_NO_TELEMETRY", raising=False)
     monkeypatch.delenv("DO_NOT_TRACK", raising=False)
     yield
     _reset_telemetry_state()
 
 
+@pytest.fixture
+def enable_telemetry(monkeypatch):
+    """Opt telemetry in for tests that assert payload behavior."""
+    monkeypatch.setenv("DATAFOG_TELEMETRY", "1")
+
+
 @pytest.fixture
 def mock_urlopen():
     """Mock urllib.request.urlopen to capture payloads without network."""
@@ -51,26 +58,62 @@ class TestOptOut:
     def test_datafog_no_telemetry_disables(self, monkeypatch):
         from datafog.telemetry import _is_telemetry_enabled
 
+        monkeypatch.setenv("DATAFOG_TELEMETRY", "1")
         monkeypatch.setenv("DATAFOG_NO_TELEMETRY", "1")
         assert _is_telemetry_enabled() is False
 
     def test_do_not_track_disables(self, monkeypatch):
         from datafog.telemetry import _is_telemetry_enabled
 
+        monkeypatch.setenv("DATAFOG_TELEMETRY", "1")
         monkeypatch.setenv("DO_NOT_TRACK", "1")
         assert _is_telemetry_enabled() is False
 
-    def test_enabled_by_default(self):
+    def test_disabled_by_default(self):
+        from datafog.telemetry import _is_telemetry_enabled
+
+        assert _is_telemetry_enabled() is False
+
+    def test_datafog_telemetry_enables(self, monkeypatch):
         from datafog.telemetry import _is_telemetry_enabled
 
+        monkeypatch.setenv("DATAFOG_TELEMETRY", "1")
         assert _is_telemetry_enabled() is True
 
-    def test_non_one_value_does_not_disable(self, monkeypatch):
+    @pytest.mark.parametrize("value", ["true", "yes", "on"])
+    def test_truthy_values_enable(self, monkeypatch, value):
         from datafog.telemetry import _is_telemetry_enabled
 
-        monkeypatch.setenv("DATAFOG_NO_TELEMETRY", "true")
+        monkeypatch.setenv("DATAFOG_TELEMETRY", value)
         assert _is_telemetry_enabled() is True
 
+    def test_falsey_value_does_not_enable(self, monkeypatch):
+        from datafog.telemetry import _is_telemetry_enabled
+
+        monkeypatch.setenv("DATAFOG_TELEMETRY", "0")
+        assert _is_telemetry_enabled() is False
+
+    def test_truthy_opt_out_overrides_opt_in(self, monkeypatch):
+        from datafog.telemetry import _is_telemetry_enabled
+
+        monkeypatch.setenv("DATAFOG_TELEMETRY", "1")
+        monkeypatch.setenv("DATAFOG_NO_TELEMETRY", "true")
+        assert _is_telemetry_enabled() is False
+
+    def test_send_event_noop_by_default(self, mock_urlopen):
+        from datafog.telemetry import _send_event
+
+        _send_event("test_event", {"key": "value"})
+        time.sleep(0.1)
+        mock_urlopen.assert_not_called()
+
+    def test_track_function_call_noop_by_default(self, mock_urlopen):
+        from datafog.telemetry import track_function_call
+
+        track_function_call("test_fn", "test_module")
+        time.sleep(0.1)
+        mock_urlopen.assert_not_called()
+
     def test_send_event_noop_when_disabled(self, monkeypatch, mock_urlopen):
         from datafog.telemetry import _send_event
 
@@ -166,7 +209,7 @@ def test_anonymous_id_persisted(self, tmp_path, monkeypatch):
         id2 = tel._get_anonymous_id()
         assert id1 == id2
 
-    def test_payload_never_contains_text_content(self, mock_urlopen):
+    def test_payload_never_contains_text_content(self, mock_urlopen, enable_telemetry):
         """Verify that tracked events don't leak text content."""
         from datafog.telemetry import track_function_call
 
@@ -197,7 +240,7 @@ def test_payload_never_contains_text_content(self, mock_urlopen):
 
 
 class TestNonBlocking:
-    def test_send_event_returns_immediately(self, mock_urlopen):
+    def test_send_event_returns_immediately(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _send_event
 
         # Make urlopen block
@@ -210,7 +253,9 @@ def test_send_event_returns_immediately(self, mock_urlopen):
         # Should return in <100ms even though urlopen blocks for 10s
         assert elapsed < 0.1
 
-    def test_track_function_call_returns_immediately(self, mock_urlopen):
+    def test_track_function_call_returns_immediately(
+        self, mock_urlopen, enable_telemetry
+    ):
         from datafog.telemetry import track_function_call
 
         mock_urlopen.side_effect = lambda *a, **k: time.sleep(10)
@@ -221,7 +266,7 @@ def test_track_function_call_returns_immediately(self, mock_urlopen):
 
         assert elapsed < 0.1
 
-    def test_network_failure_is_silent(self, mock_urlopen):
+    def test_network_failure_is_silent(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import track_function_call
 
         mock_urlopen.side_effect = Exception("Network down")
@@ -229,7 +274,7 @@ def test_network_failure_is_silent(self, mock_urlopen):
         track_function_call("fn", "mod")
         time.sleep(0.3)
 
-    def test_urlopen_timeout_is_bounded(self, mock_urlopen):
+    def test_urlopen_timeout_is_bounded(self, mock_urlopen, enable_telemetry):
         """Verify we pass a timeout to urlopen."""
         from datafog.telemetry import _send_event
 
@@ -248,7 +293,7 @@ def test_urlopen_timeout_is_bounded(self, mock_urlopen):
 
 
 class TestPayloadCorrectness:
-    def test_init_event_sent_once(self, mock_urlopen):
+    def test_init_event_sent_once(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _ensure_initialized
 
         _ensure_initialized()
@@ -259,7 +304,7 @@ def test_init_event_sent_once(self, mock_urlopen):
         # Should only create one thread/call for init
         assert mock_urlopen.call_count <= 1
 
-    def test_init_event_has_required_properties(self, mock_urlopen):
+    def test_init_event_has_required_properties(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _ensure_initialized
 
         _ensure_initialized()
@@ -281,7 +326,7 @@ def test_init_event_has_required_properties(self, mock_urlopen):
         assert "is_ci" in props
         assert "distinct_id" in props
 
-    def test_function_call_event_properties(self, mock_urlopen):
+    def test_function_call_event_properties(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import track_function_call
 
         track_function_call(
@@ -308,7 +353,7 @@ def test_function_call_event_properties(self, mock_urlopen):
                 found = True
         assert found, "datafog_function_called event not found"
 
-    def test_error_event_properties(self, mock_urlopen):
+    def test_error_event_properties(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import track_error
 
         track_error("detect", "ValueError", engine="regex")
@@ -326,7 +371,7 @@ def test_error_event_properties(self, mock_urlopen):
                 found = True
         assert found, "datafog_error event not found"
 
-    def test_posthog_endpoint_url(self, mock_urlopen):
+    def test_posthog_endpoint_url(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _send_event
 
         _send_event("test_event", {"k": "v"})
@@ -336,7 +381,7 @@ def test_posthog_endpoint_url(self, mock_urlopen):
         req = mock_urlopen.call_args[0][0]
         assert req.full_url == "https://us.i.posthog.com/capture/"
 
-    def test_content_type_is_json(self, mock_urlopen):
+    def test_content_type_is_json(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _send_event
 
         _send_event("test_event", {"k": "v"})
@@ -353,7 +398,7 @@ def test_content_type_is_json(self, mock_urlopen):
 
 
 class TestIntegration:
-    def test_detect_triggers_telemetry(self, mock_urlopen):
+    def test_detect_triggers_telemetry(self, mock_urlopen, enable_telemetry):
         from datafog import detect
 
         with pytest.warns(FutureWarning, match=r"Use datafog\.scan\(\) instead"):
@@ -367,7 +412,7 @@ def test_detect_triggers_telemetry(self, mock_urlopen):
             events.append(body["event"])
         assert "datafog_function_called" in events
 
-    def test_process_triggers_telemetry(self, mock_urlopen):
+    def test_process_triggers_telemetry(self, mock_urlopen, enable_telemetry):
         from datafog import process
 
         with pytest.warns(
@@ -384,7 +429,7 @@ def test_process_triggers_telemetry(self, mock_urlopen):
             events.append(body["event"])
         assert "datafog_function_called" in events
 
-    def test_datafog_class_triggers_telemetry(self, mock_urlopen):
+    def test_datafog_class_triggers_telemetry(self, mock_urlopen, enable_telemetry):
         from datafog.main import DataFog
 
         df = DataFog()
@@ -398,7 +443,7 @@ def test_datafog_class_triggers_telemetry(self, mock_urlopen):
             events.append(body["event"])
         assert "datafog_function_called" in events
 
-    def test_text_service_triggers_telemetry(self, mock_urlopen):
+    def test_text_service_triggers_telemetry(self, mock_urlopen, enable_telemetry):
         try:
             from datafog.services.text_service import TextService
         except ImportError:
@@ -415,7 +460,7 @@ def test_text_service_triggers_telemetry(self, mock_urlopen):
             events.append(body["event"])
         assert "datafog_function_called" in events
 
-    def test_core_detect_pii_triggers_telemetry(self, mock_urlopen):
+    def test_core_detect_pii_triggers_telemetry(self, mock_urlopen, enable_telemetry):
         try:
             from datafog.core import detect_pii
 
@@ -440,7 +485,7 @@ def test_core_detect_pii_triggers_telemetry(self, mock_urlopen):
 
 
 class TestEdgeCases:
-    def test_empty_text(self, mock_urlopen):
+    def test_empty_text(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _get_text_length_bucket, track_function_call
 
         track_function_call(
@@ -456,7 +501,7 @@ def test_large_text_bucket(self, mock_urlopen):
 
         assert _get_text_length_bucket(10_000_000) == "100k+"
 
-    def test_concurrent_init(self, mock_urlopen):
+    def test_concurrent_init(self, mock_urlopen, enable_telemetry):
         """Multiple threads calling _ensure_initialized should only init once."""
         from datafog.telemetry import _ensure_initialized
 
@@ -492,7 +537,7 @@ def fake_home():
         anon_id = tel._get_anonymous_id()
         assert len(anon_id) == 64
 
-    def test_dedup_nested_calls(self, mock_urlopen):
+    def test_dedup_nested_calls(self, mock_urlopen, enable_telemetry):
         """Nested track_function_call should only record the outer call."""
         from datafog.telemetry import track_function_call
 
@@ -531,7 +576,7 @@ def test_services_init_does_not_require_aiohttp(self):
         ts = TextService(engine="regex")
         assert ts.engine == "regex"
 
-    def test_track_error_sent_on_exception(self, mock_urlopen):
+    def test_track_error_sent_on_exception(self, mock_urlopen, enable_telemetry):
         """track_error should fire a datafog_error event."""
         from datafog.telemetry import track_error
 
@@ -550,7 +595,7 @@ def test_track_error_sent_on_exception(self, mock_urlopen):
         assert error_events[0]["error_type"] == "ValueError"
         assert error_events[0]["engine"] == "regex"
 
-    def test_pipeline_error_triggers_track_error(self, mock_urlopen):
+    def test_pipeline_error_triggers_track_error(self, mock_urlopen, enable_telemetry):
         """DataFog.run_text_pipeline_sync should fire datafog_error on failure."""
         from datafog.main import DataFog
 

From f577a4ea60639c94c72c7f2dc5d65e613b8b521b Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sun, 26 Apr 2026 15:20:07 -0700
Subject: [PATCH 28/70] chore: sync dev to v4.4.0 release (#134)

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index e8957a0b..ecdb1cef 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b1"
+__version__ = "4.4.0"

From 2b9d0461fe38972b90ca3a97c2f59e34575c7399 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 15:33:28 -0700
Subject: [PATCH 29/70] chore(deps): bump aiohttp from 3.11.18 to 3.13.4

Security cleanup tracked in DFPY-60.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2078e115..cb7a72c8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,7 +7,7 @@ Pillow==12.1.1
 sentencepiece==0.2.1
 protobuf==6.33.5
 pytesseract==0.3.13
-aiohttp==3.11.18
+aiohttp==3.13.4
 numpy==1.26.4
 fastapi==0.115.12
 asyncio==3.4.3

From f74e7508931e2eca3a105ff343273c23b0dd60cb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 15:39:21 -0700
Subject: [PATCH 30/70] chore(deps): bump pillow from 12.1.1 to 12.2.0

Security cleanup tracked in DFPY-61.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index cb7a72c8..ae4c573e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ pandas==2.2.3
 requests==2.32.3
 spacy==3.7.5
 pydantic==2.11.4
-Pillow==12.1.1
+Pillow==12.2.0
 sentencepiece==0.2.1
 protobuf==6.33.5
 pytesseract==0.3.13

From 2730317dcd45156e3da4b93a2737fbc26d4f4103 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 15:56:35 -0700
Subject: [PATCH 31/70] chore(deps): bump cryptography from 46.0.5 to 46.0.7

Security cleanup tracked in DFPY-62.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index ae4c573e..c797cab5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,7 @@ setuptools>=68.0.0
 pydantic-settings==2.3.4
 typer==0.12.3
 sphinx==7.2.6
-cryptography==46.0.5
+cryptography==46.0.7
 
 # Testing dependencies
 pytest==7.4.0

From 4633802e9d197d02fb7c67ce0eb8ab77e00097f2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 16:02:04 -0700
Subject: [PATCH 32/70] chore(deps): bump requests from 2.32.3 to 2.33.0

Security cleanup tracked in DFPY-63.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c797cab5..012e3a7a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 # Core dependencies with pinned versions
 pandas==2.2.3
-requests==2.32.3
+requests==2.33.0
 spacy==3.7.5
 pydantic==2.11.4
 Pillow==12.2.0

From 21d64fcb1e9c192e0d0166c15bb8407f4073df60 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 17:33:19 -0700
Subject: [PATCH 33/70] chore(deps-dev): bump black from 24.3.0 to 26.3.1

Security cleanup tracked in DFPY-64.
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 796afbc4..7c650063 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -4,7 +4,7 @@
 # Development and testing dependencies
 just==0.8.162
 isort==5.13.2
-black==24.3.0
+black==26.3.1
 blacken-docs==1.13.0
 certifi==2025.4.26
 flake8==7.0.0

From 8150eb38273950fa471f01a6e0c6df3e4ac35f84 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 26 Apr 2026 17:38:59 -0700
Subject: [PATCH 34/70] chore(deps): bump pytest from 7.4.0 to 9.0.3

Security cleanup tracked in DFPY-65.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 012e3a7a..f4d96bc1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,6 +18,6 @@ sphinx==7.2.6
 cryptography==46.0.7
 
 # Testing dependencies
-pytest==7.4.0
+pytest==9.0.3
 pytest-asyncio==0.21.0
 pytest-cov==4.1.0

From 76cc5efbc32870151bbaaa7b098e4d715ae3abb7 Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sun, 26 Apr 2026 18:05:25 -0700
Subject: [PATCH 35/70] chore: clarify dependency requirement scopes (#135)

---
 .github/workflows/benchmark.yml |   2 +-
 .github/workflows/ci.yml        |   9 +--
 .github/workflows/release.yml   |   7 +-
 CONTRIBUTING.md                 |   7 ++
 README.md                       |   1 +
 justfile                        |   6 +-
 requirements-benchmark.txt      |   3 +
 requirements-dev.txt            |  12 ++-
 requirements-docs.txt           |   2 +
 requirements-test.txt           |   4 +
 requirements.txt                |  26 ++-----
 setup.py                        | 128 ++++++++++++++++++--------------
 tox.ini                         |  20 +++--
 13 files changed, 125 insertions(+), 102 deletions(-)
 create mode 100644 requirements-benchmark.txt
 create mode 100644 requirements-docs.txt
 create mode 100644 requirements-test.txt

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 4bac97d1..e0bc7d5c 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -27,7 +27,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[nlp]"
-          pip install -r requirements-dev.txt
+          pip install -r requirements-benchmark.txt
           # Verify pytest-benchmark is installed and working
           python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)"
           python -m pytest --version
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4ffb6fda..8af5182c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,26 +47,25 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
 
-      - name: Install base tooling
+      - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
 
       - name: Install dependencies (core)
         if: matrix.install-profile == 'core'
         run: |
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Install dependencies (nlp)
         if: matrix.install-profile == 'nlp'
         run: |
-          pip install -e ".[dev,cli,nlp]"
+          pip install -e ".[test,cli,nlp]" -r requirements-test.txt
           python -m spacy download en_core_web_sm
 
       - name: Install dependencies (nlp-advanced)
         if: matrix.install-profile == 'nlp-advanced'
         run: |
-          pip install -e ".[dev,cli,nlp,nlp-advanced]"
+          pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt
           python -m spacy download en_core_web_sm
 
       - name: Run tests (core)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 98034d7e..53a39dfc 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -126,8 +126,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[all,dev]"
-          pip install -r requirements-dev.txt
+          pip install -e ".[all,test]"
+          pip install -r requirements-test.txt
           pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
 
       - name: Run tests with segfault protection
@@ -158,8 +158,7 @@ jobs:
       - name: Install core + CLI dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Run Python 3.13 core + CLI tests
         run: |
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 285dd51e..483a2b3e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -36,6 +36,13 @@ python -m pip install --upgrade pip
 pip install -e ".[dev,cli]"
 ```
 
+For pinned local tooling, install the development requirements after the editable
+package install:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
 For optional NLP or OCR work, install the relevant extras:
 
 ```bash
diff --git a/README.md b/README.md
index e3a211b3..62f7e10d 100644
--- a/README.md
+++ b/README.md
@@ -162,5 +162,6 @@ cd datafog-python
 python -m venv .venv
 source .venv/bin/activate  # Windows: .venv\Scripts\activate
 pip install -e ".[all,dev]"
+pip install -r requirements-dev.txt
 pytest tests/
 ```
diff --git a/justfile b/justfile
index 2e66b4d8..34415cb4 100644
--- a/justfile
+++ b/justfile
@@ -21,8 +21,8 @@ default:
 # Set up the development environment
 setup:
     python -m venv {{venv_dir}}
-    {{pip}} install -e ".[dev]"
-    {{pip}} install isort black blacken-docs flake8 tox coverage pytest pytest-cov
+    {{pip}} install -e ".[dev,cli]"
+    {{pip}} install -r requirements-dev.txt
 
 # Format the code
 format:
@@ -79,4 +79,4 @@ upload: clean
 publish: tag upload
 
 # Run all checks (format, lint, test, coverage)
-check: format lint test coverage
\ No newline at end of file
+check: format lint test coverage
diff --git a/requirements-benchmark.txt b/requirements-benchmark.txt
new file mode 100644
index 00000000..b0f3fd98
--- /dev/null
+++ b/requirements-benchmark.txt
@@ -0,0 +1,3 @@
+# Benchmark-only dependencies.
+-r requirements-test.txt
+pytest-benchmark==4.0.0
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 7c650063..1657794c 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,7 +1,11 @@
-# Include the base requirements
--r requirements.txt
+# Local development tooling.
+#
+# Install the package itself with the extras you need, for example:
+#   pip install -e ".[dev,cli]"
+#   pip install -e ".[all,dev]"
+-r requirements-test.txt
+-r requirements-docs.txt
 
-# Development and testing dependencies
 just==0.8.162
 isort==5.13.2
 black==26.3.1
@@ -14,4 +18,4 @@ pytest-benchmark==4.0.0
 mypy==1.9.0
 autoflake==2.2.1
 pre-commit==3.7.0
-ruff==0.3.4
\ No newline at end of file
+ruff==0.3.4
diff --git a/requirements-docs.txt b/requirements-docs.txt
new file mode 100644
index 00000000..d3080cf5
--- /dev/null
+++ b/requirements-docs.txt
@@ -0,0 +1,2 @@
+# Documentation-only dependencies.
+sphinx==7.2.6
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 00000000..3f3b434f
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,4 @@
+# Test-only dependencies.
+pytest==9.0.3
+pytest-asyncio==1.3.0
+pytest-cov==7.1.0
diff --git a/requirements.txt b/requirements.txt
index f4d96bc1..caa5de09 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,23 +1,7 @@
-# Core dependencies with pinned versions
-pandas==2.2.3
-requests==2.33.0
-spacy==3.7.5
+# Minimal runtime dependencies for local installs.
+#
+# Published package metadata in setup.py is the source of truth. Optional
+# feature dependencies live behind package extras such as nlp, ocr, web, and all.
 pydantic==2.11.4
-Pillow==12.2.0
-sentencepiece==0.2.1
-protobuf==6.33.5
-pytesseract==0.3.13
-aiohttp==3.13.4
-numpy==1.26.4
-fastapi==0.115.12
-asyncio==3.4.3
-setuptools>=68.0.0
 pydantic-settings==2.3.4
-typer==0.12.3
-sphinx==7.2.6
-cryptography==46.0.7
-
-# Testing dependencies
-pytest==9.0.3
-pytest-asyncio==0.21.0
-pytest-cov==4.1.0
+typing-extensions>=4.0
diff --git a/setup.py b/setup.py
index 75284180..43eaff3b 100644
--- a/setup.py
+++ b/setup.py
@@ -24,64 +24,80 @@
 ]
 
 # Optional heavy dependencies
+nlp_deps = [
+    "spacy>=3.7.0,<4.0",
+]
+
+nlp_advanced_deps = [
+    "gliner>=0.2.5",
+    "torch>=2.1.0,<2.7",
+    "transformers>=4.20.0",
+    "huggingface-hub>=0.16.0",
+]
+
+ocr_deps = [
+    "pytesseract>=0.3.0",
+    "Pillow>=12.2.0",
+    "sentencepiece>=0.2.0",
+    "protobuf>=4.0.0",
+]
+
+distributed_deps = [
+    "pandas>=2.0.0",
+    "numpy>=1.24.0",
+]
+
+web_deps = [
+    "fastapi>=0.100.0",
+    "aiohttp>=3.13.4",
+    "requests>=2.33.0",
+]
+
+cli_deps = [
+    "typer>=0.12.0",
+    "pydantic-settings>=2.0.0",
+]
+
+crypto_deps = [
+    "cryptography>=46.0.7",
+]
+
+test_deps = [
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.1.0",
+]
+
+docs_deps = [
+    "sphinx>=7.2.6",
+]
+
+benchmark_deps = [
+    "pytest-benchmark>=4.0.0",
+]
+
 extras_require = {
-    "nlp": [
-        "spacy>=3.7.0,<4.0",
-    ],
-    "nlp-advanced": [
-        "gliner>=0.2.5",
-        "torch>=2.1.0,<2.7",
-        "transformers>=4.20.0",
-        "huggingface-hub>=0.16.0",
-    ],
-    "ocr": [
-        "pytesseract>=0.3.0",
-        "Pillow>=10.0.0",
-        "sentencepiece>=0.2.0",
-        "protobuf>=4.0.0",
-    ],
-    "distributed": [
-        "pandas>=2.0.0",
-        "numpy>=1.24.0",
-    ],
-    "web": [
-        "fastapi>=0.100.0",
-        "aiohttp>=3.8.0",
-        "requests>=2.30.0",
-    ],
-    "cli": [
-        "typer>=0.12.0",
-        "pydantic-settings>=2.0.0",
-    ],
-    "crypto": [
-        "cryptography>=40.0.0",
-    ],
-    "dev": [
-        "pytest>=7.0.0",
-        "pytest-asyncio>=0.21.0",
-        "pytest-cov>=4.0.0",
-        "sphinx>=7.0.0",
-    ],
+    "nlp": nlp_deps,
+    "nlp-advanced": nlp_advanced_deps,
+    "ocr": ocr_deps,
+    "distributed": distributed_deps,
+    "web": web_deps,
+    "cli": cli_deps,
+    "crypto": crypto_deps,
+    "test": test_deps,
+    "docs": docs_deps,
+    "benchmark": benchmark_deps,
+    "dev": test_deps + docs_deps,
     # Convenience bundles
-    "all": [
-        "spacy>=3.7.0,<4.0",
-        "gliner>=0.2.5",
-        "torch>=2.1.0,<2.7",
-        "transformers>=4.20.0",
-        "huggingface-hub>=0.16.0",
-        "pytesseract>=0.3.0",
-        "Pillow>=10.0.0",
-        "sentencepiece>=0.2.0",
-        "protobuf>=4.0.0",
-        "pandas>=2.0.0",
-        "numpy>=1.24.0",
-        "fastapi>=0.100.0",
-        "aiohttp>=3.8.0",
-        "requests>=2.30.0",
-        "typer>=0.12.0",
-        "pydantic-settings>=2.0.0",
-        "cryptography>=40.0.0",
-    ],
+    "all": (
+        nlp_deps
+        + nlp_advanced_deps
+        + ocr_deps
+        + distributed_deps
+        + web_deps
+        + cli_deps
+        + crypto_deps
+    ),
 }
 
 setup(
diff --git a/tox.ini b/tox.ini
index 5e81c1f4..83d3f793 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,9 +4,11 @@ isolated_build = True
 
 [testenv:integration]
 deps =
-    -r requirements-dev.txt
+    -r requirements-test.txt
     pyspark>=3.0.0
-extras = all
+extras =
+    all
+    test
 allowlist_externals =
     tesseract
     pip
@@ -16,8 +18,10 @@ commands =
 
 [testenv]
 deps =
-    -r requirements-dev.txt
-extras = all
+    -r requirements-test.txt
+extras =
+    all
+    test
 allowlist_externals =
     tesseract
     pip
@@ -29,9 +33,9 @@ commands =
 [testenv:lint]
 skip_install = true
 deps =
-    black
-    isort
-    flake8
+    black==26.3.1
+    isort==5.13.2
+    flake8==7.0.0
 commands =
     black --check .
     isort --check-only .
@@ -39,7 +43,7 @@ commands =
 
 [testenv:typecheck]
 deps =
-    mypy
+    mypy==1.9.0
 commands =
     mypy datafog tests
 

From 8bdd56eb24f7f7cc080e24605c78b5b767a6c7a3 Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sun, 26 Apr 2026 19:00:21 -0700
Subject: [PATCH 36/70] fix: remove runtime dependency installation (#136)

* fix: remove runtime dependency installation

* fix: preserve explicit spaCy model behavior
---
 .github/workflows/ci.yml                      |  5 +-
 .github/workflows/release.yml                 |  3 +-
 datafog/client.py                             |  2 +-
 datafog/models/spacy_nlp.py                   | 24 ++++--
 .../image_processing/donut_processor.py       | 65 +++++++--------
 .../spark_processing/pyspark_udfs.py          | 24 ++++--
 .../text_processing/gliner_annotator.py       |  8 +-
 .../text_processing/spacy_pii_annotator.py    | 39 ++++-----
 datafog/services/spark_service.py             | 29 ++-----
 setup.py                                      |  1 +
 tests/test_donut_lazy_import.py               | 21 +----
 tests/test_gliner_annotator.py                | 40 +++++----
 tests/test_runtime_dependency_safety.py       | 83 +++++++++++++++++++
 13 files changed, 214 insertions(+), 130 deletions(-)
 create mode 100644 tests/test_runtime_dependency_safety.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8af5182c..2daa81ba 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -60,13 +60,14 @@ jobs:
         if: matrix.install-profile == 'nlp'
         run: |
           pip install -e ".[test,cli,nlp]" -r requirements-test.txt
-          python -m spacy download en_core_web_sm
+          python -m spacy download en_core_web_lg
 
       - name: Install dependencies (nlp-advanced)
         if: matrix.install-profile == 'nlp-advanced'
         run: |
           pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt
-          python -m spacy download en_core_web_sm
+          python -m spacy download en_core_web_lg
+          datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests (core)
         if: matrix.install-profile == 'core'
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 53a39dfc..91a459aa 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -128,7 +128,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e ".[all,test]"
           pip install -r requirements-test.txt
-          pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
+          python -m spacy download en_core_web_lg
+          datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests with segfault protection
         run: |
diff --git a/datafog/client.py b/datafog/client.py
index a76a30dd..c3d493c8 100644
--- a/datafog/client.py
+++ b/datafog/client.py
@@ -181,7 +181,7 @@ def download_model(
     Download a model for specified engine.
 
     Examples:
-        spaCy: datafog download-model en_core_web_sm --engine spacy
+        spaCy: datafog download-model en_core_web_lg --engine spacy
         GLiNER: datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
     """
     if engine == "spacy":
diff --git a/datafog/models/spacy_nlp.py b/datafog/models/spacy_nlp.py
index 7b473a17..5257ba3d 100644
--- a/datafog/models/spacy_nlp.py
+++ b/datafog/models/spacy_nlp.py
@@ -13,6 +13,8 @@
 
 from .annotator import AnnotationResult, AnnotatorRequest
 
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
+
 
 class SpacyAnnotator:
     """
@@ -22,14 +24,18 @@ class SpacyAnnotator:
     Supports various NLP tasks including entity recognition and model management.
     """
 
-    def __init__(self, model_name: str = "en_core_web_lg"):
+    def __init__(self, model_name: str = DEFAULT_SPACY_MODEL):
         self.model_name = model_name
         self.nlp = None
 
     def load_model(self):
-        if not spacy.util.is_package(self.model_name):
-            spacy.cli.download(self.model_name)
-        self.nlp = spacy.load(self.model_name)
+        try:
+            self.nlp = spacy.load(self.model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {self.model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {self.model_name} --engine spacy"
+            ) from exc
 
     def annotate_text(self, text: str, language: str = "en") -> List[AnnotationResult]:
         if not self.nlp:
@@ -72,6 +78,12 @@ def list_models() -> List[str]:
         return spacy.util.get_installed_models()
 
     @staticmethod
-    def list_entities() -> List[str]:
-        nlp = spacy.load("en_core_web_lg")
+    def list_entities(model_name: str = DEFAULT_SPACY_MODEL) -> List[str]:
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
+            ) from exc
         return [ent for ent in nlp.pipe_labels["ner"]]
diff --git a/datafog/processing/image_processing/donut_processor.py b/datafog/processing/image_processing/donut_processor.py
index 7e100585..50022b6a 100644
--- a/datafog/processing/image_processing/donut_processor.py
+++ b/datafog/processing/image_processing/donut_processor.py
@@ -6,14 +6,10 @@
 from images of documents.
 """
 
-import importlib
-import importlib.util
 import json
 import logging
 import os
 import re
-import subprocess
-import sys
 from typing import TYPE_CHECKING, Any
 
 from .image_downloader import ImageDownloader
@@ -43,13 +39,12 @@ def __init__(self, model_path="naver-clova-ix/donut-base-finetuned-cord-v2"):
         self.model_path = model_path
         self.downloader = ImageDownloader()
 
-    def ensure_installed(self, package_name):
-        try:
-            importlib.import_module(package_name)
-        except ImportError:
-            subprocess.check_call(
-                [sys.executable, "-m", "pip", "install", package_name]
-            )
+    @staticmethod
+    def _missing_dependency_message(package_name: str) -> str:
+        return (
+            f"Donut OCR requires {package_name}. "
+            "Install with: pip install datafog[nlp-advanced,ocr]"
+        )
 
     def preprocess_image(self, image: "Image.Image") -> Any:
         import numpy as np
@@ -86,40 +81,40 @@ async def extract_text_from_image(self, image: "Image.Image") -> str:
                 "PYTEST_DONUT=yes is set, running actual OCR in test environment"
             )
 
-        # Only import torch and transformers when actually needed and not in test environment
         try:
-            # Check if torch is available before trying to import it
-            try:
-                # Try to find the module without importing it
-                spec = importlib.util.find_spec("torch")
-                if spec is None:
-                    # If we're in a test that somehow bypassed the IN_TEST_ENV check,
-                    # still return a mock result instead of failing
-                    logging.warning("torch module not found, returning mock result")
-                    return json.dumps({"text": "Mock OCR text (torch not available)"})
-
-                # Ensure dependencies are installed
-                self.ensure_installed("torch")
-                self.ensure_installed("transformers")
-            except ImportError:
-                # If importlib.util is not available, fall back to direct try/except
-                pass
-
-            # Import dependencies only when needed
             try:
                 import torch
+            except ImportError as exc:
+                raise ImportError(self._missing_dependency_message("torch")) from exc
+
+            try:
                 from transformers import DonutProcessor as TransformersDonutProcessor
                 from transformers import VisionEncoderDecoderModel
             except ImportError as e:
-                logging.warning(f"Import error: {e}, returning mock result")
-                return json.dumps({"text": f"Mock OCR text (import error: {e})"})
+                raise ImportError(
+                    self._missing_dependency_message("transformers")
+                ) from e
 
             # Preprocess the image
             image_np = self.preprocess_image(image)
 
             # Initialize model components
-            processor = TransformersDonutProcessor.from_pretrained(self.model_path)
-            model = VisionEncoderDecoderModel.from_pretrained(self.model_path)
+            try:
+                processor = TransformersDonutProcessor.from_pretrained(
+                    self.model_path,
+                    local_files_only=True,
+                )
+                model = VisionEncoderDecoderModel.from_pretrained(
+                    self.model_path,
+                    local_files_only=True,
+                )
+            except OSError as exc:
+                raise RuntimeError(
+                    f"Donut model {self.model_path!r} is not available locally. "
+                    "Download it explicitly before using Donut OCR, or pass a local "
+                    "model path."
+                ) from exc
+
             device = "cuda" if torch.cuda.is_available() else "cpu"
             model.to(device)
             model.eval()
@@ -153,6 +148,8 @@ async def extract_text_from_image(self, image: "Image.Image") -> str:
             result = processor.token2json(sequence)
             return json.dumps(result)
 
+        except (ImportError, RuntimeError):
+            raise
         except Exception as e:
             logging.error(f"Error in extract_text_from_image: {e}")
             # Return a placeholder in case of error
diff --git a/datafog/processing/spark_processing/pyspark_udfs.py b/datafog/processing/spark_processing/pyspark_udfs.py
index 413e6ef3..2d7e2bc5 100644
--- a/datafog/processing/spark_processing/pyspark_udfs.py
+++ b/datafog/processing/spark_processing/pyspark_udfs.py
@@ -2,17 +2,16 @@
 PySpark UDFs for PII annotation and related utilities.
 
 This module provides functions for PII (Personally Identifiable Information) annotation
-using SpaCy models in a PySpark environment. It includes utilities for installing
-dependencies, creating and broadcasting PII annotator UDFs, and performing PII annotation
-on text data.
+using SpaCy models in a PySpark environment. It includes utilities for validating
+dependencies, creating and broadcasting PII annotator UDFs, and performing PII
+annotation on text data.
 """
 
 import importlib
-import subprocess
-import sys
 
 PII_ANNOTATION_LABELS = ["DATE_TIME", "LOC", "NRP", "ORG", "PER"]
 MAXIMAL_STRING_SIZE = 1000000
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]:
@@ -45,7 +44,7 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]:
 
 
 def broadcast_pii_annotator_udf(
-    spark_session=None, spacy_model: str = "en_core_web_lg"
+    spark_session=None, spacy_model: str = DEFAULT_SPACY_MODEL
 ):
     """Broadcast PII annotator across Spark cluster and create UDF"""
     ensure_installed("pyspark")
@@ -69,5 +68,14 @@ def broadcast_pii_annotator_udf(
 def ensure_installed(package_name):
     try:
         importlib.import_module(package_name)
-    except ImportError:
-        subprocess.check_call([sys.executable, "-m", "pip", "install", package_name])
+    except ImportError as exc:
+        if package_name == "pyspark":
+            extra = "distributed"
+        elif package_name == "spacy":
+            extra = "nlp"
+        else:
+            extra = "all"
+        raise ImportError(
+            f"{package_name} is required for Spark PII UDF support. "
+            f"Install with: pip install datafog[{extra}]"
+        ) from exc
diff --git a/datafog/processing/text_processing/gliner_annotator.py b/datafog/processing/text_processing/gliner_annotator.py
index cbaeca8c..5a195f89 100644
--- a/datafog/processing/text_processing/gliner_annotator.py
+++ b/datafog/processing/text_processing/gliner_annotator.py
@@ -79,14 +79,18 @@ def create(
 
         try:
             # Load the GLiNER model
-            model = GLiNER.from_pretrained(model_name)
+            model = GLiNER.from_pretrained(model_name, local_files_only=True)
             logging.info(f"Successfully loaded GLiNER model: {model_name}")
 
             return cls(model=model, entity_types=entity_types, model_name=model_name)
 
         except Exception as e:
             logging.error(f"Failed to load GLiNER model {model_name}: {str(e)}")
-            raise
+            raise RuntimeError(
+                f"GLiNER model {model_name!r} is not available locally. "
+                "Download it explicitly with: "
+                f"datafog download-model {model_name} --engine gliner"
+            ) from e
 
     def annotate(self, text: str) -> Dict[str, List[str]]:
         """
diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py
index e871db8a..d2d1d765 100644
--- a/datafog/processing/text_processing/spacy_pii_annotator.py
+++ b/datafog/processing/text_processing/spacy_pii_annotator.py
@@ -24,39 +24,34 @@
     "WORK_OF_ART",
 ]
 MAXIMAL_STRING_SIZE = 1000000
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 class SpacyPIIAnnotator(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     nlp: Any
+    model_name: str = DEFAULT_SPACY_MODEL
 
     @classmethod
-    def create(cls) -> "SpacyPIIAnnotator":
-        import spacy
-
+    def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator":
         try:
-            nlp = spacy.load("en_core_web_lg")
-        except OSError:
-            import subprocess
-            import sys
+            import spacy
+        except ImportError as exc:
+            raise ImportError(
+                "SpaCy engine requires the nlp extra. "
+                "Install with: pip install datafog[nlp]"
+            ) from exc
 
-            interpreter_location = sys.executable
-            subprocess.run(
-                [
-                    interpreter_location,
-                    "-m",
-                    "pip",
-                    "install",
-                    "--no-deps",
-                    "--no-cache-dir",
-                    "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.7.1/en_core_web_lg-3.7.1-py3-none-any.whl",
-                ],
-                check=True,
-            )
-            nlp = spacy.load("en_core_web_lg")
+        try:
+            nlp = spacy.load(model_name)
+        except OSError as exc:
+            raise ImportError(
+                f"spaCy model {model_name!r} is not installed. "
+                f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
+            ) from exc
 
-        return cls(nlp=nlp)
+        return cls(nlp=nlp, model_name=model_name)
 
     def annotate(self, text: str) -> Dict[str, List[str]]:
         try:
diff --git a/datafog/services/spark_service.py b/datafog/services/spark_service.py
index 5b7db28b..bf7d2e48 100644
--- a/datafog/services/spark_service.py
+++ b/datafog/services/spark_service.py
@@ -1,14 +1,12 @@
 """
 Spark service for data processing and analysis.
 
-Provides a wrapper around PySpark functionality, including session creation,
-JSON reading, and package management.
+Provides a wrapper around PySpark functionality, including session creation and
+JSON reading.
 """
 
 import importlib
 import os
-import subprocess
-import sys
 from typing import List
 
 
@@ -16,14 +14,13 @@ class SparkService:
     """
     Manages Spark operations and dependencies.
 
-    Initializes a Spark session, handles imports, and provides methods for
-    data reading and package installation.
+    Initializes a Spark session, handles imports, and provides methods for data
+    reading.
     """
 
     def __init__(self, master=None):
         self.master = master
 
-        # Ensure pyspark is installed first
         self.ensure_installed("pyspark")
 
         # Now import necessary modules after ensuring pyspark is installed
@@ -84,16 +81,8 @@ def read_json(self, path: str) -> List[dict]:
     def ensure_installed(self, package_name):
         try:
             importlib.import_module(package_name)
-        except ImportError:
-            print(f"Installing {package_name}...")
-            try:
-                subprocess.check_call(
-                    [sys.executable, "-m", "pip", "install", package_name]
-                )
-                print(f"{package_name} installed successfully.")
-            except subprocess.CalledProcessError as e:
-                print(f"Failed to install {package_name}: {e}")
-                raise ImportError(
-                    f"Could not install {package_name}. "
-                    f"Please install it manually with 'pip install {package_name}'."
-                )
+        except ImportError as exc:
+            raise ImportError(
+                f"{package_name} is required for Spark support. "
+                "Install with: pip install datafog[distributed]"
+            ) from exc
diff --git a/setup.py b/setup.py
index 43eaff3b..7c3b7992 100644
--- a/setup.py
+++ b/setup.py
@@ -45,6 +45,7 @@
 distributed_deps = [
     "pandas>=2.0.0",
     "numpy>=1.24.0",
+    "pyspark>=3.5.0",
 ]
 
 web_deps = [
diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py
index a0b62ab9..80c9ec09 100644
--- a/tests/test_donut_lazy_import.py
+++ b/tests/test_donut_lazy_import.py
@@ -1,5 +1,4 @@
 import sys
-from unittest.mock import patch
 
 from datafog.services.image_service import ImageService
 
@@ -46,21 +45,5 @@ def test_lazy_import_mechanism():
     # Verify that the extract_text_from_image method exists
     assert hasattr(processor, "extract_text_from_image")
 
-    # Mock importlib.import_module to prevent actual imports
-    with patch("importlib.import_module") as mock_import:
-        # Set up the mock to return a dummy module
-        mock_import.return_value = type("DummyModule", (), {})
-
-        # Mock the ensure_installed method to prevent actual installation
-        with patch.object(processor, "ensure_installed"):
-            # Try to call extract_text_from_image which should trigger imports
-            try:
-                # We don't actually need to run it asynchronously for this test
-                # Just call the method directly to see if it tries to import
-                processor.ensure_installed("torch")
-            except Exception:
-                # Ignore any exceptions
-                pass
-
-            # Verify ensure_installed was called
-            assert processor.ensure_installed.called
+    # Runtime package installation helpers should not exist on the processor.
+    assert not hasattr(processor, "ensure_installed")
diff --git a/tests/test_gliner_annotator.py b/tests/test_gliner_annotator.py
index bde66d02..fbf0bbdf 100644
--- a/tests/test_gliner_annotator.py
+++ b/tests/test_gliner_annotator.py
@@ -53,7 +53,8 @@ def test_gliner_annotator_creation_with_dependencies(self, mock_gliner_module):
         assert "person" in annotator.entity_types
         assert "email" in annotator.entity_types
         mock_gliner_class.from_pretrained.assert_called_with(
-            "urchade/gliner_multi_pii-v1"
+            "urchade/gliner_multi_pii-v1",
+            local_files_only=True,
         )
 
     def test_gliner_annotator_custom_model(self, mock_gliner_module):
@@ -69,7 +70,10 @@ def test_gliner_annotator_custom_model(self, mock_gliner_module):
 
         assert annotator.model_name == "urchade/gliner_base"
         assert annotator.entity_types == custom_entities
-        mock_gliner_class.from_pretrained.assert_called_with("urchade/gliner_base")
+        mock_gliner_class.from_pretrained.assert_called_with(
+            "urchade/gliner_base",
+            local_files_only=True,
+        )
 
     def test_gliner_annotate_text(self, mock_gliner_module):
         """Test GLiNER text annotation."""
@@ -353,23 +357,29 @@ def test_text_service_valid_engines(self):
 
             elif engine in ["spacy", "auto"]:
                 # Mock spaCy dependencies
-                with patch(
-                    "datafog.processing.text_processing.spacy_pii_annotator.SpacyPIIAnnotator"
-                ):
-                    from datafog.services.text_service import TextService
+                from datafog.services.text_service import TextService
 
-                    service = TextService(engine=engine)
-                    assert service.engine == engine
+                with patch.object(TextService, "_ensure_spacy_available"):
+                    with patch.object(
+                        TextService,
+                        "_create_spacy_annotator",
+                        return_value=Mock(),
+                    ):
+                        service = TextService(engine=engine)
+                        assert service.engine == engine
 
             elif engine in ["gliner", "smart"]:
                 # Mock GLiNER dependencies
-                with patch(
-                    "datafog.processing.text_processing.gliner_annotator.GLiNERAnnotator"
-                ):
-                    from datafog.services.text_service import TextService
+                from datafog.services.text_service import TextService
 
-                    service = TextService(engine=engine)
-                    assert service.engine == engine
+                with patch.object(TextService, "_ensure_gliner_available"):
+                    with patch.object(
+                        TextService,
+                        "_create_gliner_annotator",
+                        return_value=Mock(),
+                    ):
+                        service = TextService(engine=engine)
+                        assert service.engine == engine
 
     def test_text_service_invalid_engine(self):
         """Test that invalid engines raise AssertionError."""
@@ -445,7 +455,7 @@ def test_download_model_cli_output_fix(self):
         # Capture stdout
         captured_output = io.StringIO()
 
-        with patch("datafog.models.spacy_nlp.SpacyAnnotator.download_model"):
+        with patch("datafog.client.SpacyAnnotator.download_model"):
             with patch("sys.stdout", captured_output):
                 with patch("typer.echo") as mock_echo:
                     try:
diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py
new file mode 100644
index 00000000..adc787ff
--- /dev/null
+++ b/tests/test_runtime_dependency_safety.py
@@ -0,0 +1,83 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+
+def test_runtime_code_does_not_install_packages() -> None:
+    blocked_snippets = [
+        "subprocess.check_call",
+        "subprocess.run",
+        '"-m", "pip"',
+        '"pip", "install"',
+        "'pip', 'install'",
+    ]
+    offenders = []
+
+    for path in Path("datafog").rglob("*.py"):
+        source = path.read_text()
+        for snippet in blocked_snippets:
+            if snippet in source:
+                offenders.append(f"{path}: {snippet}")
+
+    assert offenders == []
+
+
+def test_spacy_pii_missing_model_requires_explicit_download(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    class FakeSpacy:
+        @staticmethod
+        def load(_model_name):
+            raise OSError("model not installed")
+
+    monkeypatch.setitem(sys.modules, "spacy", FakeSpacy())
+
+    from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
+
+    with pytest.raises(ImportError, match="Download it explicitly"):
+        SpacyPIIAnnotator.create()
+
+
+def test_spark_missing_dependency_requires_explicit_install(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from datafog.services import spark_service
+
+    def missing_module(_package_name):
+        raise ImportError("missing")
+
+    monkeypatch.setattr(spark_service.importlib, "import_module", missing_module)
+
+    service = spark_service.SparkService.__new__(spark_service.SparkService)
+    with pytest.raises(ImportError, match=r"datafog\[distributed\]"):
+        service.ensure_installed("pyspark")
+
+
+def test_spark_udf_missing_dependency_requires_explicit_install(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from datafog.processing.spark_processing import pyspark_udfs
+
+    def missing_module(_package_name):
+        raise ImportError("missing")
+
+    monkeypatch.setattr(pyspark_udfs.importlib, "import_module", missing_module)
+
+    with pytest.raises(ImportError, match=r"datafog\[nlp\]"):
+        pyspark_udfs.ensure_installed("spacy")
+
+
+@pytest.mark.asyncio
+async def test_donut_missing_dependency_requires_explicit_install(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from datafog.processing.image_processing import donut_processor
+
+    monkeypatch.setattr(donut_processor, "IN_TEST_ENV", False)
+    monkeypatch.setitem(sys.modules, "torch", None)
+
+    processor = donut_processor.DonutProcessor()
+
+    with pytest.raises(ImportError, match=r"datafog\[nlp-advanced,ocr\]"):
+        await processor.extract_text_from_image(object())

From 7a8572de5bc87f5dd0973ba0174ebf9e4923e9bc Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Sun, 26 Apr 2026 19:52:49 -0700
Subject: [PATCH 37/70] test: add no-network and install profile gates (#137)

* test: add no-network and install profile gates

* fix: avoid importing optional modules in telemetry probes

* fix: make telemetry init fully non-blocking

* test: harden telemetry nonblocking assertions

* test: cover optional dependency fallback paths
---
 .github/workflows/ci.yml                |  41 ++++++
 datafog/engine.py                       |  26 ++--
 datafog/models/spacy_nlp.py             |   3 +-
 datafog/telemetry.py                    | 161 ++++++++++++------------
 docs/v5-model-requirements.md           | 159 +++++++++++++++++++++++
 setup.py                                |   2 +
 tests/test_install_profiles.py          |  67 ++++++++++
 tests/test_no_network_core.py           |  96 ++++++++++++++
 tests/test_runtime_dependency_safety.py |  72 +++++++++++
 tests/test_telemetry.py                 | 141 +++++++++++++++++++--
 10 files changed, 655 insertions(+), 113 deletions(-)
 create mode 100644 docs/v5-model-requirements.md
 create mode 100644 tests/test_install_profiles.py
 create mode 100644 tests/test_no_network_core.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2daa81ba..a3020390 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -154,6 +154,47 @@ jobs:
           flags: ${{ matrix.install-profile }}-py${{ matrix.python-version }}
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  profile-smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        install-profile:
+          - core
+          - cli
+          - nlp
+          - nlp-advanced
+          - ocr
+          - distributed
+          - web
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install dependencies (core)
+        if: matrix.install-profile == 'core'
+        run: |
+          pip install -e ".[test]"
+
+      - name: Install dependencies (profile)
+        if: matrix.install-profile != 'core'
+        run: |
+          pip install -e ".[test,${{ matrix.install-profile }}]"
+
+      - name: Run install profile smoke test
+        env:
+          DATAFOG_INSTALL_PROFILE: ${{ matrix.install-profile }}
+        run: |
+          pytest tests/test_install_profiles.py -q
+
   wheel-size:
     runs-on: ubuntu-latest
     steps:
diff --git a/datafog/engine.py b/datafog/engine.py
index 6687c24e..1a94e634 100644
--- a/datafog/engine.py
+++ b/datafog/engine.py
@@ -171,17 +171,13 @@ def _gliner_entities(text: str) -> list[Entity]:
 def _get_spacy_annotator():
     try:
         from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
-    except ImportError:
-        return _UnavailableAnnotator(
-            "SpaCy engine requires the nlp extra. Install with: pip install datafog[nlp]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
 
     try:
         return SpacyPIIAnnotator.create()
-    except ImportError:
-        return _UnavailableAnnotator(
-            "SpaCy engine requires the nlp extra. Install with: pip install datafog[nlp]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
     except Exception as exc:
         return _UnavailableAnnotator(
             f"SpaCy engine initialization failed: {type(exc).__name__}: {exc}"
@@ -192,19 +188,13 @@ def _get_spacy_annotator():
 def _get_gliner_annotator():
     try:
         from .processing.text_processing.gliner_annotator import GLiNERAnnotator
-    except ImportError:
-        return _UnavailableAnnotator(
-            "GLiNER engine requires the nlp-advanced extra. "
-            "Install with: pip install datafog[nlp-advanced]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
 
     try:
         annotator = GLiNERAnnotator.create()
-    except ImportError:
-        return _UnavailableAnnotator(
-            "GLiNER engine requires the nlp-advanced extra. "
-            "Install with: pip install datafog[nlp-advanced]"
-        )
+    except ImportError as exc:
+        return _UnavailableAnnotator(str(exc))
     except Exception as exc:
         return _UnavailableAnnotator(
             f"GLiNER engine initialization failed: {type(exc).__name__}: {exc}"
diff --git a/datafog/models/spacy_nlp.py b/datafog/models/spacy_nlp.py
index 5257ba3d..15ca8868 100644
--- a/datafog/models/spacy_nlp.py
+++ b/datafog/models/spacy_nlp.py
@@ -9,7 +9,6 @@
 from uuid import uuid4
 
 import spacy
-from rich.progress import track
 
 from .annotator import AnnotationResult, AnnotatorRequest
 
@@ -53,7 +52,7 @@ def annotate_text(self, text: str, language: str = "en") -> List[AnnotationResul
         )
         doc = self.nlp(annotator_request.text)
         results = []
-        for ent in track(doc.ents, description="Processing entities"):
+        for ent in doc.ents:
             result = AnnotationResult(
                 start=ent.start_char,
                 end=ent.end_char,
diff --git a/datafog/telemetry.py b/datafog/telemetry.py
index 6b3885a3..5b7201df 100644
--- a/datafog/telemetry.py
+++ b/datafog/telemetry.py
@@ -13,9 +13,11 @@
 """
 
 import hashlib
+import importlib.util
 import json
 import os
 import platform
+import sys
 import threading
 import time
 import urllib.request
@@ -114,44 +116,28 @@ def _get_duration_bucket(duration_ms: float) -> str:
 
 def _detect_installed_extras() -> list:
     """Probe which optional extras are installed."""
-    extras = []
 
-    try:
-        import spacy  # noqa: F401
-
-        extras.append("nlp")
-    except ImportError:
-        pass
-
-    try:
-        import gliner  # noqa: F401
-
-        extras.append("nlp-advanced")
-    except ImportError:
-        pass
-
-    try:
-        import pytesseract  # noqa: F401
-
-        extras.append("ocr")
-    except ImportError:
-        pass
-
-    try:
-        import typer  # noqa: F401
-
-        extras.append("cli")
-    except ImportError:
-        pass
-
-    try:
-        import pyspark  # noqa: F401
-
-        extras.append("distributed")
-    except ImportError:
-        pass
-
-    return extras
+    def _module_available(module_name: str) -> bool:
+        module = sys.modules.get(module_name)
+        if module is not None and getattr(module, "__spec__", None) is None:
+            return True
+        try:
+            return importlib.util.find_spec(module_name) is not None
+        except (ImportError, ValueError):
+            return False
+
+    module_to_extra = {
+        "spacy": "nlp",
+        "gliner": "nlp-advanced",
+        "pytesseract": "ocr",
+        "typer": "cli",
+        "pyspark": "distributed",
+    }
+    return [
+        extra
+        for module_name, extra in module_to_extra.items()
+        if _module_available(module_name)
+    ]
 
 
 def _detect_ci() -> bool:
@@ -170,39 +156,69 @@ def _detect_ci() -> bool:
     return any(os.environ.get(v) for v in ci_vars)
 
 
-def _send_event(event_name: str, properties: dict) -> None:
-    """POST event to PostHog /capture/ endpoint in a daemon thread.
+def _post_event(event_name: str, properties: dict) -> None:
+    """POST event to PostHog /capture/ endpoint.
 
-    Fire-and-forget: failures are silently ignored.
+    Fire-and-forget callers run this in daemon threads. Failures are silently
+    ignored so telemetry can never affect SDK behavior.
     """
+    try:
+        payload = json.dumps(
+            {
+                "api_key": _POSTHOG_API_KEY,
+                "event": event_name,
+                "properties": {
+                    "distinct_id": _get_anonymous_id(),
+                    **properties,
+                },
+                "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
+            }
+        ).encode("utf-8")
+
+        req = urllib.request.Request(
+            f"{_POSTHOG_HOST}/capture/",
+            data=payload,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        urllib.request.urlopen(req, timeout=5)
+    except Exception:
+        pass
+
+
+def _send_event(event_name: str, properties: dict) -> None:
+    """POST event to PostHog /capture/ endpoint in a daemon thread."""
     if not _is_telemetry_enabled():
         return
 
-    def _post():
+    t = threading.Thread(target=_post_event, args=(event_name, properties), daemon=True)
+    t.start()
+
+
+def _send_init_event() -> None:
+    """Build and send the process init event without blocking API calls."""
+
+    def _post_init():
         try:
-            payload = json.dumps(
-                {
-                    "api_key": _POSTHOG_API_KEY,
-                    "event": event_name,
-                    "properties": {
-                        "distinct_id": _get_anonymous_id(),
-                        **properties,
-                    },
-                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()),
-                }
-            ).encode("utf-8")
-
-            req = urllib.request.Request(
-                f"{_POSTHOG_HOST}/capture/",
-                data=payload,
-                headers={"Content-Type": "application/json"},
-                method="POST",
-            )
-            urllib.request.urlopen(req, timeout=5)
+            from .__about__ import __version__
         except Exception:
-            pass
-
-    t = threading.Thread(target=_post, daemon=True)
+            __version__ = "unknown"
+
+        uname = platform.uname()
+        _post_event(
+            "datafog_init",
+            {
+                "package_version": __version__,
+                "python_version": platform.python_version(),
+                "os": uname.system,
+                "os_version": uname.release,
+                "arch": uname.machine,
+                "installed_extras": _detect_installed_extras(),
+                "is_ci": _detect_ci(),
+            },
+        )
+
+    t = threading.Thread(target=_post_init, daemon=True)
     t.start()
 
 
@@ -220,24 +236,7 @@ def _ensure_initialized() -> None:
     if not _is_telemetry_enabled():
         return
 
-    try:
-        from .__about__ import __version__
-    except Exception:
-        __version__ = "unknown"
-
-    uname = platform.uname()
-    _send_event(
-        "datafog_init",
-        {
-            "package_version": __version__,
-            "python_version": platform.python_version(),
-            "os": uname.system,
-            "os_version": uname.release,
-            "arch": uname.machine,
-            "installed_extras": _detect_installed_extras(),
-            "is_ci": _detect_ci(),
-        },
-    )
+    _send_init_event()
 
 
 def track_function_call(function_name: str, module: str, **kwargs) -> None:
diff --git a/docs/v5-model-requirements.md b/docs/v5-model-requirements.md
new file mode 100644
index 00000000..ab8d496f
--- /dev/null
+++ b/docs/v5-model-requirements.md
@@ -0,0 +1,159 @@
+# v5 Model Selection Requirements
+
+This sheet defines requirements for revisiting DataFog's optional model stack before
+locking the v5 core API around specific NLP/OCR backends. It is intentionally a
+requirements document, not a model recommendation list.
+
+## Decision Goals
+
+- Pick models that improve adoption by making the first successful result easy,
+  trustworthy, and local by default.
+- Keep the core SDK fast and lightweight; model-backed engines remain optional.
+- Make model behavior explicit enough that users can defend it in privacy,
+  security, and compliance reviews.
+- Preserve a clean path for future backend swaps without breaking the top-level
+  v5 API.
+
+## Must-Haves
+
+### Runtime And Packaging
+
+- No model downloads during import, install, or ordinary SDK calls.
+- All model downloads must be explicit CLI/API actions or user-provided local
+  paths.
+- The core install must not require ML, OCR, Torch, TensorFlow, Java, Spark, or
+  system OCR binaries.
+- Optional extras must map cleanly to real imports:
+  - `nlp` for lightweight NLP engines.
+  - `nlp-advanced` for heavier ML NER engines.
+  - `ocr` for local image/OCR processing.
+  - `distributed` for Spark-style processing.
+- Missing dependency and missing model errors must explain the exact install or
+  download command.
+- Python 3.10, 3.11, and 3.12 must be supported for advertised optional model
+  profiles. Python 3.13 support should be advertised only after explicit profile
+  validation.
+- Models must work in offline mode after explicit download/cache preparation.
+
+### Privacy And Trust
+
+- No network access during inference.
+- No telemetry, remote callbacks, model hub lookups, or license checks during
+  inference.
+- No raw PII should be written to logs, cache names, telemetry, exceptions, or
+  debug traces by default.
+- Model metadata exposed by DataFog should identify model name/version/source
+  without storing detected raw PII.
+- Reversible workflows must be opt-in and clearly separated from ordinary
+  redaction.
+
+### Detection Contract
+
+- Model outputs must include enough structure for the public result contract:
+  entity type, text/span, start/end offsets, confidence when available, and
+  engine/source.
+- Spans must be deterministic for the same model, text, and settings.
+- Entity labels must be mappable into DataFog's canonical entity taxonomy without
+  surprising users.
+- Model-backed engines must compose with regex detection without duplicating or
+  overwriting high-confidence structured entities.
+- Failure modes must be predictable: unsupported language, missing model, missing
+  optional dependency, and low-confidence results should all be distinguishable.
+
+### Quality Gates
+
+- Candidate models must be benchmarked on DataFog's target corpora before
+  adoption.
+- Benchmarks must include precision/recall by entity type, not only aggregate F1.
+- Structured PII such as email, phone, IP address, SSN, credit cards, dates, and
+  ZIP/postal codes should remain regex/validator-first unless a model clearly
+  improves quality.
+- NER-style entities such as person, organization, location, address, and
+  domain-specific identifiers need regression tests with realistic app/log data.
+- OCR models must be evaluated separately for text extraction quality and PII
+  extraction quality after OCR.
+
+### Operational Fit
+
+- CPU inference must be acceptable for the default advertised workflow.
+- GPU-only models are not acceptable as default engines.
+- Model size, cold-start time, memory use, and cache footprint must be measured.
+- The model must have a usable open license for commercial SDK users.
+- The model or provider must have credible maintenance signals and versioned
+  artifacts.
+
+## Nice-To-Haves
+
+- Strong multilingual support with per-language quality reporting.
+- Quantized or small variants that keep local inference practical.
+- ONNX or other portable runtime support for future non-Torch deployments.
+- Streaming/chunked inference support or predictable behavior across chunk
+  boundaries.
+- Custom entity hints or user-provided label sets.
+- Confidence calibration good enough to expose threshold controls.
+- Batch inference APIs for logs, CSV, and JSONL workflows.
+- Clear model cards with training data notes, limitations, and intended use.
+- Support for local cache directories that can be controlled by environment
+  variable or explicit config.
+- Graceful operation on Apple Silicon and common Linux CI runners.
+
+## Disqualifiers
+
+- Requires network access for inference.
+- Downloads weights implicitly from ordinary SDK calls.
+- License is unclear, non-commercial, or incompatible with SDK distribution.
+- Requires a hosted API for core value.
+- Requires GPU for reasonable first-use behavior.
+- Cannot return stable spans or forces only label-level output.
+- Emits raw text or entities through logging, telemetry, or callbacks.
+- Adds heavyweight dependencies to the core install.
+- Breaks Python version support we already advertise.
+
+## Evaluation Matrix
+
+Each candidate backend should be scored before adoption:
+
+| Area | Required Evidence |
+| --- | --- |
+| Install footprint | Extra name, package deps, wheel size impact, system deps |
+| Runtime footprint | Cold start, warm latency, memory, CPU/GPU requirements |
+| Offline behavior | Explicit download path, local cache path, no-network test |
+| Quality | Precision/recall by entity type on DataFog corpora |
+| Span quality | Offset correctness and deduplication behavior |
+| Privacy | No raw PII logs/cache/telemetry, safe error messages |
+| Licensing | Model license, dependency licenses, commercial use notes |
+| Maintenance | Release cadence, Python compatibility, issue activity |
+| API fit | Entity taxonomy mapping, confidence support, batch/chunk support |
+| Docs fit | Model card, limitations, user-facing setup instructions |
+
+## Candidate Backend Categories To Evaluate
+
+- Regex plus validators for structured PII and secrets.
+- Lightweight NLP NER for person, organization, location, and address entities.
+- Advanced local NER models for broader entity coverage and multilingual support.
+- OCR text extraction engines for local images/PDF-derived images.
+- Document understanding models only if they beat OCR plus text PII extraction
+  enough to justify their footprint.
+- User-provided backend hooks for teams that already have a preferred model.
+
+## Recommended Selection Policy
+
+- Default v5 behavior should remain regex/validator-first.
+- Model-backed engines should be opt-in by engine, policy, or extra.
+- DataFog should prefer smaller, reliable local models over maximum leaderboard
+  scores if they improve install success and first-use latency.
+- Model choices should be version-pinned in docs and CI once advertised.
+- A model can be experimental in docs/examples before it becomes part of the
+  supported contract.
+
+## Open Questions
+
+- Do we want one recommended advanced NER model, or a pluggable registry with a
+  default?
+- Should OCR stay Tesseract-first, or should v5 introduce a newer local OCR
+  default after benchmarking?
+- How much multilingual quality is required for v5.0.0 versus a later release?
+- Should Python 3.13 optional-profile support be a v4.5 compatibility release,
+  a v5 launch requirement, or both?
+- What maximum model download size is acceptable for the default recommended
+  advanced profile?
diff --git a/setup.py b/setup.py
index 7c3b7992..f84c241a 100644
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,7 @@
 ]
 
 ocr_deps = [
+    "numpy>=1.24.0",
     "pytesseract>=0.3.0",
     "Pillow>=12.2.0",
     "sentencepiece>=0.2.0",
@@ -51,6 +52,7 @@
 web_deps = [
     "fastapi>=0.100.0",
     "aiohttp>=3.13.4",
+    "certifi>=2025.4.26",
     "requests>=2.33.0",
 ]
 
diff --git a/tests/test_install_profiles.py b/tests/test_install_profiles.py
new file mode 100644
index 00000000..e17261be
--- /dev/null
+++ b/tests/test_install_profiles.py
@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+    not os.environ.get("DATAFOG_INSTALL_PROFILE"),
+    reason="install profile smoke tests run only in profile-specific CI jobs",
+)
+
+
+def test_install_profile_import_surface() -> None:
+    profile = os.environ["DATAFOG_INSTALL_PROFILE"]
+
+    if profile == "core":
+        import datafog
+
+        assert datafog.scan("Email jane@example.com").entities
+        assert datafog.redact("Email jane@example.com").redacted_text
+    elif profile == "cli":
+        from datafog.client import app
+
+        assert app is not None
+    elif profile == "nlp":
+        import spacy  # noqa: F401
+
+        from datafog.models.spacy_nlp import SpacyAnnotator
+        from datafog.processing.text_processing.spacy_pii_annotator import (
+            SpacyPIIAnnotator,
+        )
+
+        assert SpacyAnnotator is not None
+        assert SpacyPIIAnnotator is not None
+    elif profile == "nlp-advanced":
+        import gliner  # noqa: F401
+        import torch  # noqa: F401
+        import transformers  # noqa: F401
+
+        from datafog.processing.text_processing.gliner_annotator import GLiNERAnnotator
+
+        assert GLiNERAnnotator is not None
+    elif profile == "ocr":
+        import numpy  # noqa: F401
+        import pytesseract  # noqa: F401
+        from PIL import Image  # noqa: F401
+
+        from datafog.processing.image_processing.donut_processor import DonutProcessor
+        from datafog.processing.image_processing.pytesseract_processor import (
+            PytesseractProcessor,
+        )
+        from datafog.services.image_service import ImageService
+
+        assert DonutProcessor is not None
+        assert ImageService is not None
+        assert PytesseractProcessor is not None
+    elif profile == "distributed":
+        from datafog.processing.spark_processing import pyspark_udfs
+        from datafog.services.spark_service import SparkService
+
+        pyspark_udfs.ensure_installed("pyspark")
+        assert SparkService is not None
+    elif profile == "web":
+        import aiohttp  # noqa: F401
+        import certifi  # noqa: F401
+        import fastapi  # noqa: F401
+        import requests  # noqa: F401
+    else:
+        raise AssertionError(f"unknown DATAFOG_INSTALL_PROFILE: {profile}")
diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py
new file mode 100644
index 00000000..905984f4
--- /dev/null
+++ b/tests/test_no_network_core.py
@@ -0,0 +1,96 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]:
+    env = dict(os.environ)
+    env["PYTHONPATH"] = str(Path.cwd())
+    env.pop("DATAFOG_TELEMETRY", None)
+    env["DATAFOG_NO_TELEMETRY"] = "1"
+    env["DO_NOT_TRACK"] = "1"
+    return subprocess.run(
+        [sys.executable, "-c", script],
+        check=True,
+        env=env,
+        text=True,
+        capture_output=True,
+    )
+
+
+def test_import_scan_and_redact_do_not_open_network() -> None:
+    _run_isolated_python(
+        """
+import socket
+import urllib.request
+
+def blocked(*_args, **_kwargs):
+    raise AssertionError("network access is blocked in this test")
+
+socket.create_connection = blocked
+urllib.request.urlopen = blocked
+
+import datafog
+
+scan_result = datafog.scan("Email jane@example.com or call 415-555-1212")
+assert {entity.type for entity in scan_result.entities} >= {"EMAIL", "PHONE"}
+
+redact_result = datafog.redact("Email jane@example.com or call 415-555-1212")
+assert "jane@example.com" not in redact_result.redacted_text
+assert "415-555-1212" not in redact_result.redacted_text
+"""
+    )
+
+
+def test_core_defaults_do_not_initialize_optional_engines(monkeypatch) -> None:
+    import datafog
+    import datafog.engine as engine
+
+    def fail_optional_engine_probe():
+        raise AssertionError("core defaults should not initialize optional engines")
+
+    monkeypatch.setattr(engine, "_get_spacy_annotator", fail_optional_engine_probe)
+    monkeypatch.setattr(engine, "_get_gliner_annotator", fail_optional_engine_probe)
+
+    scan_result = datafog.scan("Email jane@example.com")
+    assert [entity.type for entity in scan_result.entities] == ["EMAIL"]
+
+    redact_result = datafog.redact("Email jane@example.com")
+    assert redact_result.redacted_text == "Email [EMAIL_1]"
+
+    guardrail = datafog.protect()
+    guarded = guardrail.filter("Email jane@example.com")
+    assert guarded.redacted_text == "Email [EMAIL_1]"
+
+
+def test_import_probes_do_not_load_optional_models() -> None:
+    _run_isolated_python(
+        """
+import sys
+import types
+
+def blocked_model_load(*_args, **_kwargs):
+    raise AssertionError("import should not load optional models")
+
+spacy = types.ModuleType("spacy")
+spacy.load = blocked_model_load
+spacy.cli = types.SimpleNamespace(download=blocked_model_load)
+spacy.util = types.SimpleNamespace(get_installed_models=lambda: [])
+sys.modules["spacy"] = spacy
+
+gliner = types.ModuleType("gliner")
+
+class GLiNER:
+    @staticmethod
+    def from_pretrained(*_args, **_kwargs):
+        blocked_model_load()
+
+gliner.GLiNER = GLiNER
+sys.modules["gliner"] = gliner
+
+import datafog
+
+assert datafog.scan("Email jane@example.com").entities
+"""
+    )
diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py
index adc787ff..9410ddc6 100644
--- a/tests/test_runtime_dependency_safety.py
+++ b/tests/test_runtime_dependency_safety.py
@@ -1,4 +1,6 @@
+import importlib
 import sys
+import types
 from pathlib import Path
 
 import pytest
@@ -39,6 +41,76 @@ def load(_model_name):
         SpacyPIIAnnotator.create()
 
 
+def test_spacy_engine_missing_model_surfaces_download_guidance(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    class FakeSpacy:
+        @staticmethod
+        def load(_model_name):
+            raise OSError("model not installed")
+
+    monkeypatch.setitem(sys.modules, "spacy", FakeSpacy())
+
+    from datafog import engine
+    from datafog.exceptions import EngineNotAvailable
+
+    engine._get_spacy_annotator.cache_clear()
+    try:
+        with pytest.raises(EngineNotAvailable, match="Download it explicitly"):
+            engine.scan("Jane Doe", engine="spacy")
+    finally:
+        engine._get_spacy_annotator.cache_clear()
+
+
+def test_spacy_engine_missing_module_surfaces_import_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from datafog import engine
+    from datafog.exceptions import EngineNotAvailable
+
+    module_name = "datafog.processing.text_processing.spacy_pii_annotator"
+    monkeypatch.setitem(sys.modules, module_name, None)
+
+    engine._get_spacy_annotator.cache_clear()
+    try:
+        with pytest.raises(EngineNotAvailable, match="spacy_pii_annotator"):
+            engine.scan("Jane Doe", engine="spacy")
+    finally:
+        engine._get_spacy_annotator.cache_clear()
+
+
+def test_gliner_engine_missing_module_surfaces_import_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from datafog import engine
+    from datafog.exceptions import EngineNotAvailable
+
+    module_name = "datafog.processing.text_processing.gliner_annotator"
+    monkeypatch.setitem(sys.modules, module_name, None)
+
+    engine._get_gliner_annotator.cache_clear()
+    try:
+        with pytest.raises(EngineNotAvailable, match="gliner_annotator"):
+            engine.scan("Jane Doe", engine="gliner")
+    finally:
+        engine._get_gliner_annotator.cache_clear()
+
+
+def test_spacy_helper_does_not_require_rich(monkeypatch: pytest.MonkeyPatch) -> None:
+    module_name = "datafog.models.spacy_nlp"
+    monkeypatch.delitem(sys.modules, module_name, raising=False)
+
+    fake_spacy = types.ModuleType("spacy")
+    fake_spacy.load = lambda _model_name: None
+    fake_spacy.cli = types.SimpleNamespace(download=lambda _model_name: None)
+    fake_spacy.util = types.SimpleNamespace(get_installed_models=lambda: [])
+    monkeypatch.setitem(sys.modules, "spacy", fake_spacy)
+
+    module = importlib.import_module(module_name)
+
+    assert module.SpacyAnnotator is not None
+
+
 def test_spark_missing_dependency_requires_explicit_install(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py
index 3886a1dc..ce5651a0 100644
--- a/tests/test_telemetry.py
+++ b/tests/test_telemetry.py
@@ -1,6 +1,8 @@
 """Tests for datafog.telemetry module."""
 
+import builtins
 import json
+import sys
 import threading
 import time
 from pathlib import Path
@@ -243,28 +245,88 @@ class TestNonBlocking:
     def test_send_event_returns_immediately(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import _send_event
 
-        # Make urlopen block
-        mock_urlopen.side_effect = lambda *a, **k: time.sleep(10)
+        release_network = threading.Event()
 
-        start = time.monotonic()
-        _send_event("test", {"k": "v"})
-        elapsed = time.monotonic() - start
+        def block_until_released(*args, **kwargs):
+            release_network.wait(5)
 
-        # Should return in <100ms even though urlopen blocks for 10s
-        assert elapsed < 0.1
+        mock_urlopen.side_effect = block_until_released
+
+        call_done = threading.Event()
+        caller = threading.Thread(
+            target=lambda: (_send_event("test", {"k": "v"}), call_done.set())
+        )
+
+        try:
+            caller.start()
+            assert call_done.wait(1)
+            assert not release_network.is_set()
+        finally:
+            release_network.set()
+            caller.join(1)
 
     def test_track_function_call_returns_immediately(
         self, mock_urlopen, enable_telemetry
     ):
         from datafog.telemetry import track_function_call
 
-        mock_urlopen.side_effect = lambda *a, **k: time.sleep(10)
+        release_network = threading.Event()
 
-        start = time.monotonic()
-        track_function_call("fn", "mod")
-        elapsed = time.monotonic() - start
+        def block_until_released(*args, **kwargs):
+            release_network.wait(5)
+
+        mock_urlopen.side_effect = block_until_released
+
+        call_done = threading.Event()
+        caller = threading.Thread(
+            target=lambda: (track_function_call("fn", "mod"), call_done.set())
+        )
+
+        try:
+            caller.start()
+            assert call_done.wait(1)
+            assert not release_network.is_set()
+        finally:
+            release_network.set()
+            caller.join(1)
+
+    def test_track_function_call_does_not_wait_for_init_metadata(
+        self, monkeypatch, enable_telemetry
+    ):
+        import datafog.telemetry as tel
 
-        assert elapsed < 0.1
+        call_done = threading.Event()
+        detect_started = threading.Event()
+        release_detect = threading.Event()
+        init_posted = threading.Event()
+
+        def slow_detect_installed_extras():
+            detect_started.set()
+            release_detect.wait(5)
+            return []
+
+        def fake_post_event(event_name, properties):
+            if event_name == "datafog_init":
+                init_posted.set()
+
+        monkeypatch.setattr(
+            tel, "_detect_installed_extras", slow_detect_installed_extras
+        )
+        monkeypatch.setattr(tel, "_post_event", fake_post_event)
+
+        caller = threading.Thread(
+            target=lambda: (tel.track_function_call("fn", "mod"), call_done.set())
+        )
+
+        try:
+            caller.start()
+            assert detect_started.wait(1)
+            assert call_done.wait(1)
+        finally:
+            release_detect.set()
+            caller.join(1)
+
+        assert init_posted.wait(1)
 
     def test_network_failure_is_silent(self, mock_urlopen, enable_telemetry):
         from datafog.telemetry import track_function_call
@@ -569,6 +631,61 @@ def test_detect_installed_extras_returns_list(self):
         result = _detect_installed_extras()
         assert isinstance(result, list)
 
+    def test_detect_installed_extras_does_not_import_optional_modules(
+        self, monkeypatch
+    ):
+        from datafog.telemetry import _detect_installed_extras
+
+        real_import = builtins.__import__
+        optional_modules = {"spacy", "gliner", "pytesseract", "typer", "pyspark"}
+
+        def guarded_import(name, *args, **kwargs):
+            if name.split(".", 1)[0] in optional_modules:
+                raise AssertionError(f"imported optional module {name}")
+            return real_import(name, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "__import__", guarded_import)
+
+        result = _detect_installed_extras()
+        assert isinstance(result, list)
+
+    def test_detect_installed_extras_handles_probe_errors(self, monkeypatch):
+        import datafog.telemetry as tel
+
+        optional_modules = {"spacy", "gliner", "pytesseract", "typer", "pyspark"}
+        for module_name in optional_modules:
+            monkeypatch.delitem(sys.modules, module_name, raising=False)
+
+        def broken_find_spec(_module_name):
+            raise ValueError("invalid module state")
+
+        monkeypatch.setattr(tel.importlib.util, "find_spec", broken_find_spec)
+
+        assert tel._detect_installed_extras() == []
+
+    def test_send_init_event_uses_unknown_version_when_about_import_fails(
+        self, monkeypatch
+    ):
+        import datafog.telemetry as tel
+
+        init_posted = threading.Event()
+        captured = {}
+
+        monkeypatch.setitem(sys.modules, "datafog.__about__", None)
+
+        def fake_post_event(event_name, properties):
+            captured["event_name"] = event_name
+            captured["properties"] = properties
+            init_posted.set()
+
+        monkeypatch.setattr(tel, "_post_event", fake_post_event)
+
+        tel._send_init_event()
+
+        assert init_posted.wait(1)
+        assert captured["event_name"] == "datafog_init"
+        assert captured["properties"]["package_version"] == "unknown"
+
     def test_services_init_does_not_require_aiohttp(self):
         """TextService should be importable without aiohttp/PIL (services/__init__.py fix)."""
         from datafog.services.text_service import TextService

From 9c59e42076be1066db0f0de16408b958a4731992 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 27 Apr 2026 03:18:35 +0000
Subject: [PATCH 38/70] chore: bump version to 4.4.0a1 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index ecdb1cef..6ab08e5a 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0"
+__version__ = "4.4.0a1"

From ea042768420449c4b1ef2f16cfc35535c9f179f6 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 30 Apr 2026 03:16:11 +0000
Subject: [PATCH 39/70] chore: bump version to 4.4.0b2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 6ab08e5a..350a141c 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0a1"
+__version__ = "4.4.0b2"

From cd2eb79169d918e9dc561094843f62c49f1a9cbc Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 4 May 2026 03:18:56 +0000
Subject: [PATCH 40/70] chore: bump version to 4.4.0a2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 350a141c..d01ea1b9 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b2"
+__version__ = "4.4.0a2"

From 8c9514891bf83d783776ff2d01458b3d7f376abb Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 7 May 2026 03:16:24 +0000
Subject: [PATCH 41/70] chore: bump version to 4.4.0b3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index d01ea1b9..ed916ea7 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0a2"
+__version__ = "4.4.0b3"

From 3768a66b94d0cf524ef4a2b7bde4d296f4e842c9 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 11 May 2026 03:21:01 +0000
Subject: [PATCH 42/70] chore: bump version to 4.4.0a3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index ed916ea7..99ab8a37 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b3"
+__version__ = "4.4.0a3"

From c81619847f6424fce1d7b3b8649095df6bcafd88 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 14 May 2026 03:16:23 +0000
Subject: [PATCH 43/70] chore: bump version to 4.4.0b4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 99ab8a37..96070546 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0a3"
+__version__ = "4.4.0b4"

From 47212ddb6544429c5ec5d4d3f62721e9ec21814f Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 18 May 2026 03:21:46 +0000
Subject: [PATCH 44/70] chore: bump version to 4.4.0a4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 96070546..0601d44c 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b4"
+__version__ = "4.4.0a4"

From f4dfb0656c109227cfa05529a71966665e1be0b6 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 21 May 2026 03:18:08 +0000
Subject: [PATCH 45/70] chore: bump version to 4.4.0b5 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 0601d44c..f8bdfa5b 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0a4"
+__version__ = "4.4.0b5"

From c1093d636d941920e291b25c836346519083ef26 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 25 May 2026 03:22:02 +0000
Subject: [PATCH 46/70] chore: bump version to 4.4.0a5 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index f8bdfa5b..829fa6a5 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b5"
+__version__ = "4.4.0a5"

From a79cd8f67d0aedb421dae9f07128cafeeaf5637f Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 19:00:34 +0200
Subject: [PATCH 47/70] chore: land v4.5 release prep baseline

Refs DFPY-71
---
 .bumpversion.cfg                |  6 +--
 .gitignore                      |  5 +-
 Claude.md => AGENTS.md          | 93 ++++++++++++++++++++++++--------
 datafog/__init__.py             | 95 +++++++++++++++------------------
 docs/agents/domain.md           | 31 +++++++++++
 docs/agents/issue-tracker.md    | 24 +++++++++
 docs/agents/triage-labels.md    | 13 +++++
 docs/conf.py                    |  7 ++-
 setup.py                        |  2 +
 tests/test_donut_lazy_import.py | 67 ++++++++++++-----------
 tests/test_install_profiles.py  |  3 ++
 11 files changed, 231 insertions(+), 115 deletions(-)
 rename Claude.md => AGENTS.md (82%)
 create mode 100644 docs/agents/domain.md
 create mode 100644 docs/agents/issue-tracker.md
 create mode 100644 docs/agents/triage-labels.md

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 463cf894..0f1713f9 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.3.0
+current_version = 4.4.0a5
 commit = True
 tag = True
 tag_name = v{new_version}
@@ -20,7 +20,3 @@ values =
 [bumpversion:file:datafog/__about__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-
-[bumpversion:file:setup.py]
-search = version="{current_version}"
-replace = version="{new_version}"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 2f62eff9..bb9f8105 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ error_log.txt
 # Environment
 .env
 .venv
+.venv*/
 venv/
 env/
 examples/venv/
@@ -58,14 +59,14 @@ docs/*
 !docs/conf.py
 !docs/Makefile
 !docs/make.bat
+!docs/agents/
+!docs/agents/**
 !docs/audit/
 !docs/audit/**
 
 # Keep all directories but ignore their contents
 */**/__pycache__/
 
-# Keep all files but ignore their contents
-Claude.md
 notes/benchmarking_notes.md
 Roadmap.md
 notes/*
diff --git a/Claude.md b/AGENTS.md
similarity index 82%
rename from Claude.md
rename to AGENTS.md
index dcbe7934..a46c4402 100644
--- a/Claude.md
+++ b/AGENTS.md
@@ -1,18 +1,26 @@
-# DataFog - Claude Development Guide
+# DataFog - Agent Development Guide
 
 ## Project Overview
+
 **DataFog** is an open-source Python library for PII detection and anonymization with a focus on speed and lightweight architecture.
 
 ## Core Value Proposition
+
 - **Ultra-Fast Performance**: 190x faster than spaCy for structured PII, 32x faster with GLiNER
 - **Lightweight Core**: <2MB package with optional ML extras
 - **Modern Engine Options**: Regex, GLiNER, spaCy, and smart cascading
 - **Production Ready**: Comprehensive testing, CI/CD, and performance validation
 
 ## Current Project Status
-**Version: 4.3.0**
+
+**Stable version: 4.4.0**
+
+**Development version: 4.4.0a5**
+
+**Next minor target: 4.5.0**
 
 ### ✅ Recently Completed (Latest)
+
 - **GLiNER Integration**: Modern NER engine with PII-specialized models
 - **Smart Cascading**: Intelligent regex → GLiNER → spaCy progression
 - **Enhanced CLI**: Model management with `--engine` flags
@@ -43,6 +51,7 @@ python -c "from datafog.services.text_service import TextService; print('✅ All
 ## Architecture Overview
 
 ### Engine Ecosystem (Updated with GLiNER)
+
 ```python
 from datafog.services.text_service import TextService
 
@@ -59,21 +68,23 @@ auto_service = TextService(engine="auto")        # Legacy: regex→spaCy
 ```
 
 ### Performance Comparison (Validated)
-| Engine  | Speed vs spaCy | Accuracy | Use Case | Install |
-|---------|----------------|----------|----------|---------|
-| `regex` | **190x faster** | High (structured) | Emails, phones, SSNs | Core only |
-| `gliner` | **32x faster** | Very High | Modern NER, custom entities | `[nlp-advanced]` |
-| `spacy` | 1x (baseline) | Good | Traditional NLP | `[nlp]` |
-| `smart` | **60x faster** | Highest | Best balance | `[nlp-advanced]` |
+
+| Engine   | Speed vs spaCy  | Accuracy          | Use Case                    | Install          |
+| -------- | --------------- | ----------------- | --------------------------- | ---------------- |
+| `regex`  | **190x faster** | High (structured) | Emails, phones, SSNs        | Core only        |
+| `gliner` | **32x faster**  | Very High         | Modern NER, custom entities | `[nlp-advanced]` |
+| `spacy`  | 1x (baseline)   | Good              | Traditional NLP             | `[nlp]`          |
+| `smart`  | **60x faster**  | Highest           | Best balance                | `[nlp-advanced]` |
 
 ### Dependency Strategy
+
 ```python
 # Lightweight core (<2MB)
 pip install datafog
 
 # Optional ML engines
 pip install datafog[nlp]           # spaCy (traditional NLP)
-pip install datafog[nlp-advanced]  # GLiNER (modern NER) 
+pip install datafog[nlp-advanced]  # GLiNER (modern NER)
 pip install datafog[ocr]           # Image processing
 pip install datafog[all]           # Everything
 ```
@@ -81,15 +92,18 @@ pip install datafog[all]           # Everything
 ## GLiNER Integration (NEW)
 
 ### Overview
+
 GLiNER (Generalist Model for Named Entity Recognition) provides modern, accurate NER capabilities optimized for PII detection.
 
 ### Key Features
+
 - **PII-Specialized Models**: `urchade/gliner_multi_pii-v1` trained specifically for PII
 - **Custom Entity Types**: Configurable entity detection beyond default PII types
 - **Smart Cascading**: Automatically tries regex first, GLiNER second, spaCy last
 - **CLI Management**: Download and manage GLiNER models via CLI
 
 ### Usage Examples
+
 ```python
 # GLiNER engine
 from datafog.services.text_service import TextService
@@ -108,6 +122,7 @@ subprocess.run(["datafog", "list-models", "--engine", "gliner"])
 ```
 
 ### Available GLiNER Models
+
 - `urchade/gliner_multi_pii-v1` - PII-specialized (recommended)
 - `urchade/gliner_base` - General purpose starter
 - `urchade/gliner_large-v2` - Higher accuracy
@@ -116,17 +131,19 @@ subprocess.run(["datafog", "list-models", "--engine", "gliner"])
 ## Development Workflow
 
 ### Git Branch Strategy
+
 - **main**: Production releases only
 - **dev**: Main development branch (use this)
-- **feature/***: New features from dev
-- **fix/***: Bug fixes from dev
+- **feature/\***: New features from dev
+- **fix/\***: Bug fixes from dev
 
 ### Making Changes
+
 ```bash
 # Start from dev
 git checkout dev && git pull origin dev
 
-# Create feature branch  
+# Create feature branch
 git checkout -b feature/your-change
 
 # Make changes, test, commit
@@ -137,6 +154,7 @@ git push -u origin feature/your-change
 ```
 
 ### Testing
+
 ```bash
 # Run specific test suites
 pytest tests/test_text_service.py -v           # Core functionality
@@ -149,13 +167,14 @@ PYTEST_DONUT=yes pytest tests/test_ocr_integration.py  # OCR with real models
 
 # Performance requirements
 # - Regex: 150x+ faster than spaCy
-# - GLiNER: 25x+ faster than spaCy  
+# - GLiNER: 25x+ faster than spaCy
 # - Package size: Core <2MB, full <8MB
 ```
 
 ## Key Implementation Patterns
 
 ### Simple API (Recommended)
+
 ```python
 # Always available, lightweight
 from datafog import detect, process
@@ -164,6 +183,7 @@ result = process("john@example.com", method="redact")
 ```
 
 ### Advanced Engine Selection
+
 ```python
 # For specialized use cases
 from datafog.services.text_service import TextService
@@ -173,7 +193,7 @@ service = TextService(engine="regex")
 
 # Modern NER with custom entities
 service = TextService(
-    engine="gliner", 
+    engine="gliner",
     gliner_model="urchade/gliner_base"
 )
 
@@ -182,6 +202,7 @@ service = TextService(engine="smart")
 ```
 
 ### Graceful Degradation
+
 ```python
 # Handles missing dependencies elegantly
 try:
@@ -194,18 +215,21 @@ except ImportError:
 ## Common Tasks
 
 ### Adding New Entity Types
+
 1. Update regex patterns in `regex_annotator.py`
 2. Add GLiNER entity types in `gliner_annotator.py`
 3. Update tests and benchmarks
 4. Validate performance doesn't regress >10%
 
 ### Performance Optimization
+
 1. Profile with existing benchmarks
 2. Maintain speed thresholds (regex 150x+, GLiNER 25x+)
 3. Update baselines when making improvements
 4. Test across all engines
 
 ### CLI Enhancements
+
 1. Update `client.py` with new commands
 2. Support `--engine` flag for multi-engine commands
 3. Add comprehensive help text and examples
@@ -215,31 +239,36 @@ except ImportError:
 
 ### Workflow Architecture (3 workflows)
 
-| Workflow | Purpose | Trigger |
-|----------|---------|---------|
-| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev |
-| `release.yml` | Alpha/Beta/Stable publishing | Schedule + manual dispatch |
-| `benchmark.yml` | Performance benchmarks | Push/PR/weekly |
+| Workflow        | Purpose                             | Trigger                    |
+| --------------- | ----------------------------------- | -------------------------- |
+| `ci.yml`        | Lint + Test + Coverage + Wheel size | Push/PR to main/dev        |
+| `release.yml`   | Alpha/Beta/Stable publishing        | Schedule + manual dispatch |
+| `benchmark.yml` | Performance benchmarks              | Push/PR/weekly             |
 
 ### Release Cadence
+
 - **Alpha** (Mon-Wed 2AM UTC): Automatic from `dev`, date+commit versioning
 - **Beta** (Thursday 2AM UTC): Automatic from `dev`, incremental beta numbers
 - **Stable** (manual dispatch): From `main`, base version or override
 
 ### Release Pipeline
+
 `determine-release` → `test` → `publish` → `cleanup`
+
 - Tests are a hard gate — no tests = no publish
 - Stable releases check out `main`; alpha/beta check out `dev`
 - Old alphas pruned to 7, betas to 5
 - `[skip ci]` in version bump commits to prevent loops
 
 ### Pre-commit Hooks
+
 - **isort**, **black**, **flake8**, **ruff**: Code formatting and linting
 - **prettier**: Markdown, JSON, YAML formatting
 - **gitleaks**: Secret scanning
 - **pre-commit-hooks**: Large file checks, merge conflict detection, YAML validation
 
 ## Environment Variables
+
 ```bash
 # Testing configuration
 export PYTEST_DONUT=yes              # Enable real OCR testing
@@ -250,33 +279,51 @@ export PYTHONPATH=$(pwd)             # Local development imports
 ```
 
 ## Performance Requirements
+
 - **Core Package**: <2MB (from ~8MB in v4.0.x)
 - **Regex Engine**: 150x+ faster than spaCy (currently 190x)
-- **GLiNER Engine**: 25x+ faster than spaCy (currently 32x)  
+- **GLiNER Engine**: 25x+ faster than spaCy (currently 32x)
 - **Memory Usage**: Graceful handling of large texts (1MB+ chunks)
 - **Model Loading**: Cache GLiNER models to avoid repeated downloads
 
-## Best Practices for Claude Agents
+## Agent skills
+
+### Issue tracker
+
+Issues and PRDs are tracked in Linear under the DFPY team. See `docs/agents/issue-tracker.md`.
+
+### Triage labels
+
+Use the default five-label triage vocabulary. See `docs/agents/triage-labels.md`.
+
+### Domain docs
+
+Single-context repo: use root `CONTEXT.md` and root `docs/adr/` when present. See `docs/agents/domain.md`.
+
+## Best Practices for Agents
 
 Before beginning any task please checkout a branch from `dev` and create a pull request to `dev`.
 
 ### Code Quality
+
 - Follow existing patterns before implementing new approaches
 - Add comprehensive tests for all new functionality
 - Update documentation immediately with code changes
 - Run benchmarks for any text processing modifications
 
 ### GLiNER Development
+
 - Use PII-specialized models when available (`urchade/gliner_multi_pii-v1`)
 - Test graceful degradation when GLiNER dependencies missing
 - Validate smart cascading thresholds with real data
 - Consider model download time and caching strategies
 
 ### Release Preparation
+
 - Alpha/beta releases are automated via `release.yml` schedule
 - Stable releases: merge `dev` → `main`, then trigger `release.yml` with `stable` type
 - Use `dry_run: true` to validate before actual publish
 - Performance validation on realistic data sets
-- In Release Notes or Comments, do not reference that it was authored by Claude (all code is anonymously authored)
+- In Release Notes or Comments, do not reference that it was authored by an AI agent (all code is anonymously authored)
 
-This guide provides the essential information for DataFog development while maintaining focus on current priorities and recent GLiNER integration work.
\ No newline at end of file
+This guide provides the essential information for DataFog development while maintaining focus on current priorities and recent GLiNER integration work.
diff --git a/datafog/__init__.py b/datafog/__init__.py
index e3974ad7..b7d8e4e7 100644
--- a/datafog/__init__.py
+++ b/datafog/__init__.py
@@ -61,24 +61,6 @@ def _lazy_import_regex_annotator():
         globals()["RegexAnnotator"] = RegexAnnotator
 
 
-# Optional imports with graceful fallback
-try:
-    from .client import app
-except ImportError:
-    app = None
-
-try:
-    from .main import DataFog, TextPIIAnnotator
-except ImportError:
-    DataFog = None
-    TextPIIAnnotator = None
-
-try:
-    from .services.text_service import TextService
-except ImportError:
-    TextService = None
-
-
 def __getattr__(name: str):
     """Handle lazy imports for better lightweight performance."""
     # Lazy import core models when first accessed
@@ -98,46 +80,53 @@ def __getattr__(name: str):
         _lazy_import_regex_annotator()
         return globals()[name]
 
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    elif name in _LAZY_EXPORTS:
+        module_path, attr_name, extra_name = _LAZY_EXPORTS[name]
+        try:
+            module = __import__(module_path, fromlist=[attr_name])
+            value = getattr(module, attr_name)
+        except ImportError:
+            if extra_name is None:
+                value = None
+            else:
 
+                def _missing_dependency(*args, **kwargs):
+                    raise ImportError(
+                        f"{name} requires additional dependencies. "
+                        f"Install with: pip install datafog[{extra_name}]"
+                    )
 
-# Optional heavy features - only import if dependencies available
-def _optional_import(name, module_path, extra_name):
-    """Helper to import optional modules with helpful error messages."""
-    try:
-        module = __import__(module_path, fromlist=[name])
-        return getattr(module, name)
-    except ImportError:
-
-        def _missing_dependency(*args, **kwargs):
-            raise ImportError(
-                f"{name} requires additional dependencies. "
-                f"Install with: pip install datafog[{extra_name}]"
-            )
+                value = _missing_dependency
+
+        globals()[name] = value
+        return value
 
-        return _missing_dependency
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
-# OCR/Image processing - requires 'ocr' extra
-DonutProcessor = _optional_import(
-    "DonutProcessor", "datafog.processing.image_processing.donut_processor", "ocr"
-)
-PytesseractProcessor = _optional_import(
-    "PytesseractProcessor",
-    "datafog.processing.image_processing.pytesseract_processor",
-    "ocr",
-)
-ImageService = _optional_import("ImageService", "datafog.services.image_service", "ocr")
-
-# NLP processing - requires 'nlp' extra
-SpacyPIIAnnotator = _optional_import(
-    "SpacyPIIAnnotator", "datafog.processing.text_processing.spacy_pii_annotator", "nlp"
-)
-
-# Distributed processing - requires 'distributed' extra
-SparkService = _optional_import(
-    "SparkService", "datafog.services.spark_service", "distributed"
-)
+_LAZY_EXPORTS = {
+    "app": ("datafog.client", "app", None),
+    "DataFog": ("datafog.main", "DataFog", None),
+    "TextPIIAnnotator": ("datafog.main", "TextPIIAnnotator", None),
+    "TextService": ("datafog.services.text_service", "TextService", None),
+    "DonutProcessor": (
+        "datafog.processing.image_processing.donut_processor",
+        "DonutProcessor",
+        "ocr",
+    ),
+    "PytesseractProcessor": (
+        "datafog.processing.image_processing.pytesseract_processor",
+        "PytesseractProcessor",
+        "ocr",
+    ),
+    "ImageService": ("datafog.services.image_service", "ImageService", "ocr"),
+    "SpacyPIIAnnotator": (
+        "datafog.processing.text_processing.spacy_pii_annotator",
+        "SpacyPIIAnnotator",
+        "nlp",
+    ),
+    "SparkService": ("datafog.services.spark_service", "SparkService", "distributed"),
+}
 
 
 _REDACT_PRESETS = {
diff --git a/docs/agents/domain.md b/docs/agents/domain.md
new file mode 100644
index 00000000..4fbc0445
--- /dev/null
+++ b/docs/agents/domain.md
@@ -0,0 +1,31 @@
+# Domain Docs
+
+How the engineering skills should consume this repo's domain documentation when exploring the codebase.
+
+Configured layout: single-context.
+
+## Before exploring, read these
+
+- **`CONTEXT.md`** at the repo root.
+- **`docs/adr/`** for ADRs that touch the area you're about to work in.
+
+If any of these files don't exist, proceed silently. Don't flag their absence; don't suggest creating them upfront. The producer skill (`/grill-with-docs`) creates them lazily when terms or decisions actually get resolved.
+
+## File structure
+
+```text
+/
+|-- CONTEXT.md
+|-- docs/adr/
+`-- datafog/
+```
+
+## Use the glossary's vocabulary
+
+When your output names a domain concept in an issue title, refactor proposal, hypothesis, or test name, use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.
+
+If the concept you need isn't in the glossary yet, that's a signal: either you're inventing language the project doesn't use, or there's a real gap to note for `/grill-with-docs`.
+
+## Flag ADR conflicts
+
+If your output contradicts an existing ADR, surface it explicitly rather than silently overriding.
diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md
new file mode 100644
index 00000000..b07dfca3
--- /dev/null
+++ b/docs/agents/issue-tracker.md
@@ -0,0 +1,24 @@
+# Issue tracker: Linear
+
+Issues and PRDs for this repo live in Linear under the DFPY team:
+
+https://linear.app/threadfork/team/DFPY/all
+
+Use the Linear connector/app when available. Do not create GitHub or GitLab issues for this repo unless the user explicitly asks for that.
+
+## Conventions
+
+- Create new issues in the DFPY team.
+- Use the triage labels mapped in `docs/agents/triage-labels.md`.
+- Keep issue titles concise and action-oriented.
+- Include enough context, acceptance criteria, and verification notes for an AFK agent or human implementer to pick up the work.
+- When referencing code, include repo-relative file paths and relevant symbols.
+- When a task comes from a PRD, link related Linear issues together where possible.
+
+## When a skill says "publish to the issue tracker"
+
+Create a Linear issue in the DFPY team.
+
+## When a skill says "fetch the relevant ticket"
+
+Use the Linear connector/app to read the referenced Linear issue, including description, labels, status, comments, and linked issues.
diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md
new file mode 100644
index 00000000..0806b2f8
--- /dev/null
+++ b/docs/agents/triage-labels.md
@@ -0,0 +1,13 @@
+# Triage Labels
+
+The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker.
+
+| Label in mattpocock/skills | Label in our tracker | Meaning                                  |
+| -------------------------- | -------------------- | ---------------------------------------- |
+| `needs-triage`             | `needs-triage`       | Maintainer needs to evaluate this issue  |
+| `needs-info`               | `needs-info`         | Waiting on reporter for more information |
+| `ready-for-agent`          | `ready-for-agent`    | Fully specified, ready for an AFK agent  |
+| `ready-for-human`          | `ready-for-human`    | Requires human implementation            |
+| `wontfix`                  | `wontfix`            | Will not be actioned                     |
+
+When a skill mentions a role, use the corresponding label string from this table.
diff --git a/docs/conf.py b/docs/conf.py
index 1cb1c895..d71e76b0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -3,13 +3,18 @@
 # For the full list of built-in configuration values, see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
+import re
+from pathlib import Path
+
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = "DataFog"
 copyright = "2024, DataFog Inc."
 author = "Sid Mohan"
-release = "v4.1.1"
+_version_file = Path(__file__).resolve().parents[1] / "datafog" / "__about__.py"
+_version_match = re.search(r'^__version__ = "([^"]+)"', _version_file.read_text(), re.M)
+release = f"v{_version_match.group(1)}" if _version_match else "v0.0.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
diff --git a/setup.py b/setup.py
index f84c241a..39f01651 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,7 @@
 
 # Optional heavy dependencies
 nlp_deps = [
+    "click>=8.0,<9.0",
     "spacy>=3.7.0,<4.0",
 ]
 
@@ -57,6 +58,7 @@
 ]
 
 cli_deps = [
+    "click>=8.0,<9.0",
     "typer>=0.12.0",
     "pydantic-settings>=2.0.0",
 ]
diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py
index 80c9ec09..9b2a28f1 100644
--- a/tests/test_donut_lazy_import.py
+++ b/tests/test_donut_lazy_import.py
@@ -1,23 +1,36 @@
+import os
+import subprocess
 import sys
+from pathlib import Path
 
-from datafog.services.image_service import ImageService
+
+def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]:
+    env = dict(os.environ)
+    env["PYTHONPATH"] = str(Path.cwd())
+    env["DATAFOG_NO_TELEMETRY"] = "1"
+    env["DO_NOT_TRACK"] = "1"
+    return subprocess.run(
+        [sys.executable, "-c", script],
+        check=True,
+        env=env,
+        text=True,
+        capture_output=True,
+    )
 
 
 def test_no_torch_import_when_donut_disabled():
     """Test that torch is not imported when use_donut is False"""
-    # Remove torch and transformers from sys.modules if they're already imported
-    if "torch" in sys.modules:
-        del sys.modules["torch"]
-    if "transformers" in sys.modules:
-        del sys.modules["transformers"]
+    _run_isolated_python(
+        """
+import sys
+from datafog.services.image_service import ImageService
 
-    # Create ImageService with use_donut=False
-    # The variable is used indirectly by creating the service which affects sys.modules
-    _ = ImageService(use_donut=False, use_tesseract=True)
+_ = ImageService(use_donut=False, use_tesseract=True)
 
-    # Verify that torch and transformers were not imported
-    assert "torch" not in sys.modules
-    assert "transformers" not in sys.modules
+assert "torch" not in sys.modules
+assert "transformers" not in sys.modules
+"""
+    )
 
 
 def test_lazy_import_mechanism():
@@ -26,24 +39,16 @@ def test_lazy_import_mechanism():
     # to use lazy imports. We don't need to actually test the imports themselves,
     # just that the structure is correct.
 
-    # First, ensure torch and transformers are not in sys.modules
-    if "torch" in sys.modules:
-        del sys.modules["torch"]
-    if "transformers" in sys.modules:
-        del sys.modules["transformers"]
-
-    # Import the DonutProcessor directly
-    from datafog.processing.image_processing.donut_processor import DonutProcessor
-
-    # Create a processor instance
-    processor = DonutProcessor()
-
-    # Verify that torch and transformers were not imported just by creating the processor
-    assert "torch" not in sys.modules
-    assert "transformers" not in sys.modules
+    _run_isolated_python(
+        """
+import sys
+from datafog.processing.image_processing.donut_processor import DonutProcessor
 
-    # Verify that the extract_text_from_image method exists
-    assert hasattr(processor, "extract_text_from_image")
+processor = DonutProcessor()
 
-    # Runtime package installation helpers should not exist on the processor.
-    assert not hasattr(processor, "ensure_installed")
+assert "torch" not in sys.modules
+assert "transformers" not in sys.modules
+assert hasattr(processor, "extract_text_from_image")
+assert not hasattr(processor, "ensure_installed")
+"""
+    )
diff --git a/tests/test_install_profiles.py b/tests/test_install_profiles.py
index e17261be..7222186e 100644
--- a/tests/test_install_profiles.py
+++ b/tests/test_install_profiles.py
@@ -17,10 +17,13 @@ def test_install_profile_import_surface() -> None:
         assert datafog.scan("Email jane@example.com").entities
         assert datafog.redact("Email jane@example.com").redacted_text
     elif profile == "cli":
+        import click  # noqa: F401
+
         from datafog.client import app
 
         assert app is not None
     elif profile == "nlp":
+        import click  # noqa: F401
         import spacy  # noqa: F401
 
         from datafog.models.spacy_nlp import SpacyAnnotator

From 216b7380647604c7a74acb3bdbdb5828c233099e Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 19:27:54 +0200
Subject: [PATCH 48/70] fix: keep agent helpers on regex core path

Refs DFPY-74
---
 datafog/agent.py              | 16 +++++------
 tests/test_no_network_core.py | 52 +++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/datafog/agent.py b/datafog/agent.py
index 58a84ed7..6603ba11 100644
--- a/datafog/agent.py
+++ b/datafog/agent.py
@@ -51,7 +51,7 @@ class Guardrail:
     """Reusable text guardrail for wrapping LLM prompts and outputs."""
 
     entity_types: Optional[list[str]] = None
-    engine: str = "smart"
+    engine: str = "regex"
     strategy: str = "token"
     on_detect: str = "redact"
 
@@ -111,33 +111,33 @@ def watch(self) -> Iterator[GuardrailWatch]:
         yield watcher
 
 
-def sanitize(text: str, **kwargs: Any) -> str:
+def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str:
     """
     One-liner PII removal.
 
     Returns the redacted text only.
     """
-    result = scan_and_redact(text=text, **kwargs)
+    result = scan_and_redact(text=text, engine=engine, **kwargs)
     return result.redacted_text
 
 
-def scan_prompt(prompt: str, **kwargs: Any) -> ScanResult:
+def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult:
     """
     Scan an LLM prompt for PII without modifying the input text.
     """
-    return scan(prompt, **kwargs)
+    return scan(prompt, engine=engine, **kwargs)
 
 
-def filter_output(output: str, **kwargs: Any) -> RedactResult:
+def filter_output(output: str, engine: str = "regex", **kwargs: Any) -> RedactResult:
     """
     Scan and redact PII from model output before returning to users.
     """
-    return scan_and_redact(output, **kwargs)
+    return scan_and_redact(output, engine=engine, **kwargs)
 
 
 def create_guardrail(
     entity_types: Optional[list[str]] = None,
-    engine: str = "smart",
+    engine: str = "regex",
     strategy: str = "token",
     on_detect: str = "redact",
 ) -> Guardrail:
diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py
index 905984f4..f06e8360 100644
--- a/tests/test_no_network_core.py
+++ b/tests/test_no_network_core.py
@@ -63,6 +63,19 @@ def fail_optional_engine_probe():
     guarded = guardrail.filter("Email jane@example.com")
     assert guarded.redacted_text == "Email [EMAIL_1]"
 
+    sanitized = datafog.sanitize("Email jane@example.com")
+    assert sanitized == "Email [EMAIL_1]"
+
+    prompt_result = datafog.scan_prompt("Email jane@example.com")
+    assert [entity.type for entity in prompt_result.entities] == ["EMAIL"]
+
+    output_result = datafog.filter_output("Email jane@example.com")
+    assert output_result.redacted_text == "Email [EMAIL_1]"
+
+    agent_guardrail = datafog.create_guardrail()
+    agent_guarded = agent_guardrail.filter("Email jane@example.com")
+    assert agent_guarded.redacted_text == "Email [EMAIL_1]"
+
 
 def test_import_probes_do_not_load_optional_models() -> None:
     _run_isolated_python(
@@ -94,3 +107,42 @@ def from_pretrained(*_args, **_kwargs):
 assert datafog.scan("Email jane@example.com").entities
 """
     )
+
+
+def test_core_path_does_not_import_optional_dependency_modules() -> None:
+    _run_isolated_python(
+        """
+import importlib.abc
+import sys
+
+blocked = {
+    "aiohttp",
+    "certifi",
+    "gliner",
+    "PIL",
+    "pyspark",
+    "pytesseract",
+    "spacy",
+    "torch",
+    "transformers",
+}
+
+class BlockOptionalImports(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname.split(".", 1)[0] in blocked:
+            raise AssertionError(f"optional dependency imported: {fullname}")
+        return None
+
+sys.meta_path.insert(0, BlockOptionalImports())
+
+import datafog
+
+assert datafog.scan("Email jane@example.com").entities
+assert datafog.redact("Email jane@example.com").redacted_text == "Email [EMAIL_1]"
+assert datafog.protect().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]"
+assert datafog.sanitize("Email jane@example.com") == "Email [EMAIL_1]"
+assert datafog.scan_prompt("Email jane@example.com").entities
+assert datafog.filter_output("Email jane@example.com").redacted_text == "Email [EMAIL_1]"
+assert datafog.create_guardrail().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]"
+"""
+    )

From 90c426d320dab289d3ce77c25a6cb3ad923f6202 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 19:38:56 +0200
Subject: [PATCH 49/70] docs: clarify optional OCR and Spark surfaces

Refs DFPY-75
---
 .gitignore                              |   1 +
 README.md                               |  29 +++++++
 docs/cli.rst                            |  12 +++
 docs/index.rst                          |  38 ++++-----
 docs/optional-surfaces.rst              | 109 ++++++++++++++++++++++++
 docs/python-sdk.rst                     |  42 ++++++++-
 docs/roadmap.rst                        |  34 ++++++--
 tests/test_runtime_dependency_safety.py |  53 ++++++++++++
 8 files changed, 287 insertions(+), 31 deletions(-)
 create mode 100644 docs/optional-surfaces.rst

diff --git a/.gitignore b/.gitignore
index bb9f8105..1316a1f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,6 +59,7 @@ docs/*
 !docs/conf.py
 !docs/Makefile
 !docs/make.bat
+!docs/optional-surfaces.rst
 !docs/agents/
 !docs/agents/**
 !docs/audit/
diff --git a/README.md b/README.md
index 62f7e10d..2ff68cd2 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,12 @@ pip install datafog[nlp]
 # Add GLiNER + spaCy support
 pip install datafog[nlp-advanced]
 
+# Add local OCR support
+pip install datafog[ocr]
+
+# Add Spark/distributed support
+pip install datafog[distributed]
+
 # Everything
 pip install datafog[all]
 ```
@@ -95,6 +101,29 @@ Use the engine that matches your accuracy and dependency constraints:
   - Cascades regex with optional NER engines.
   - If optional deps are missing, it degrades gracefully and warns.
 
+## Optional OCR And Spark Surfaces
+
+DataFog 4.5 keeps the main package story centered on lightweight text PII
+screening. OCR and Spark remain supported optional surfaces for users who
+already rely on them, but they are not required for the core import, default
+scan/redact helpers, or guardrail helpers.
+
+- OCR:
+  - Install `datafog[ocr]` for local image OCR helpers.
+  - URL-based image downloading also needs `datafog[web,ocr]`.
+  - Tesseract usage requires the system `tesseract` binary.
+  - Donut OCR requires `datafog[nlp-advanced,ocr]` and a model already available
+    locally.
+- Spark:
+  - Install `datafog[distributed]` for `SparkService`.
+  - Spark PII UDF helpers also require `datafog[nlp]` and an installed spaCy
+    model.
+  - A Java runtime is required by PySpark.
+
+OCR and Spark are not deprecated. Their broader API and packaging overhaul is
+deferred; the 4.5 goal is to keep them explicit, documented, and isolated from
+the lightweight core path.
+
 ## Backward-Compatible APIs
 
 The existing public API remains available.
diff --git a/docs/cli.rst b/docs/cli.rst
index a4c67272..ec452f53 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -7,6 +7,18 @@ Overview
 The main entrypoint for the CLI is through the DataFog client file, defined in :mod:`datafog.client`.
 We use Typer to build the CLI, with each command defined as a separate function.
 
+Core text commands such as ``scan-text``, ``redact-text``, ``replace-text``,
+and ``hash-text`` are the primary 4.5 CLI path. OCR commands remain available
+for existing users, but they are optional:
+
+* Local image OCR requires ``datafog[ocr]`` and any needed system OCR binaries
+  such as Tesseract.
+* URL-based image OCR also requires ``datafog[web,ocr]``.
+* Donut OCR requires ``datafog[nlp-advanced,ocr]`` and a local model.
+
+Spark/distributed workflows are Python SDK surfaces rather than first-path CLI
+commands. Install ``datafog[distributed]`` when using ``SparkService``.
+
 Definitions
 -----------
 .. automodule:: datafog.client
diff --git a/docs/index.rst b/docs/index.rst
index a22af1c5..7e05a867 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2,7 +2,9 @@
 DataFog Documentation
 =====================
 
-DataFog is an open-source tool for PII detection and anonymization of unstructured data. This documentation covers the CLI and Python SDK.
+DataFog is an open-source tool for lightweight text PII detection and
+anonymization. The core install focuses on fast regex-based scanning and
+redaction, with optional extras for NLP, OCR, and Spark-style workflows.
 
 .. toctree::
    :maxdepth: 2
@@ -10,6 +12,7 @@ DataFog is an open-source tool for PII detection and anonymization of unstructur
    important-concepts
    cli
    python-sdk
+   optional-surfaces
    definitions
    roadmap
    v44-bridge-release
@@ -24,13 +27,14 @@ Getting Started
 Installation
 ------------
 
-Install DataFog via pip:
+Install the lightweight text screening core via pip:
 
 .. code-block:: bash
 
     pip install datafog
 
-This installs the latest stable version with CLI support.
+Optional extras such as ``nlp``, ``nlp-advanced``, ``ocr``, ``distributed``,
+and ``web`` are installed only when you need those surfaces.
 
 ---------------------
 CLI Usage
@@ -48,19 +52,21 @@ Scan text for PII:
 
     datafog scan-text "Your text here"
 
-Extract text from image:
+Image/OCR commands are optional. Local OCR requires ``datafog[ocr]``; URL-based
+image downloading requires ``datafog[web,ocr]``.
 
 .. code-block:: bash
 
     datafog scan-image "path/to/image.png" --operations extract
 
-Scan for PII in image:
+Scan for PII in image text:
 
 .. code-block:: bash
 
     datafog scan-image "path/to/image.png" --operations scan
 
-For more information on the CLI, see :doc:`cli`.
+For more information on optional OCR and Spark surfaces, see
+:doc:`optional-surfaces`.
 
 ---------------------
 Python SDK Usage
@@ -71,22 +77,14 @@ Scan text for PII:
 .. code-block:: python
 
    
-   import requests
-   from datafog import DataFog
-
-   # For text annotation
-   client = DataFog(operations="scan")
+   import datafog
 
-   # Fetch sample medical record
-   doc_url = "https://gist.githubusercontent.com/sidmohan0/b43b72693226422bac5f083c941ecfdb/raw/b819affb51796204d59987893f89dee18428ed5d/note1.txt"
-   response = requests.get(doc_url)
-   text_lines = [line for line in response.text.splitlines() if line.strip()]
+   text = "Contact jane@example.com or call 415-555-1212"
+   result = datafog.scan(text, engine="regex")
+   print(result.entities)
+   print(datafog.redact(text, engine="regex").redacted_text)
 
-   # Run annotation
-   annotations = client.run_text_pipeline_sync(str_list=text_lines)
-   print(annotations)
-    
-Scan image for PII:
+Run OCR and then scan extracted text only when the OCR extra is installed:
 
 .. code-block:: python
 
diff --git a/docs/optional-surfaces.rst b/docs/optional-surfaces.rst
new file mode 100644
index 00000000..aa9b725e
--- /dev/null
+++ b/docs/optional-surfaces.rst
@@ -0,0 +1,109 @@
+=========================
+Optional OCR And Spark
+=========================
+
+DataFog 4.5 keeps the core package focused on lightweight text PII screening.
+The default path is:
+
+.. code-block:: bash
+
+   pip install datafog
+
+.. code-block:: python
+
+   import datafog
+
+   result = datafog.redact("Email jane@example.com", engine="regex")
+   print(result.redacted_text)
+
+OCR and Spark are supported optional surfaces. They are useful for image and
+distributed workflows, but they should not be treated as required for the core
+install, package import, text scanning, text redaction, or guardrail helpers.
+
+OCR
+---
+
+Use OCR when you need to extract text from images before running PII detection.
+
+Install local OCR support:
+
+.. code-block:: bash
+
+   pip install "datafog[ocr]"
+
+Use URL-based image downloads:
+
+.. code-block:: bash
+
+   pip install "datafog[web,ocr]"
+
+Use Donut OCR:
+
+.. code-block:: bash
+
+   pip install "datafog[nlp-advanced,ocr]"
+
+Notes:
+
+* Tesseract usage requires the system ``tesseract`` binary in addition to the
+  Python extra.
+* Donut OCR requires a model that is already available locally. DataFog should
+  not download models implicitly during normal runtime usage.
+* OCR is not deprecated. A broader OCR API and packaging overhaul is deferred
+  beyond the 4.5 focus release.
+
+Example local OCR flow:
+
+.. code-block:: python
+
+   import asyncio
+   from datafog.services.image_service import ImageService
+
+   async def main():
+       service = ImageService(use_tesseract=True, use_donut=False)
+       extracted = await service.ocr_extract(["./invoice.png"])
+       print(extracted)
+
+   asyncio.run(main())
+
+Spark
+------
+
+Use Spark when you need distributed processing around DataFog PII detection.
+
+Install Spark support:
+
+.. code-block:: bash
+
+   pip install "datafog[distributed]"
+
+Use Spark PII UDF helpers:
+
+.. code-block:: bash
+
+   pip install "datafog[distributed,nlp]"
+
+Notes:
+
+* ``SparkService`` requires PySpark and a Java runtime.
+* Spark PII UDF helpers also require spaCy and an installed spaCy model.
+* Spark is not deprecated. A broader Spark overhaul is deferred beyond the 4.5
+  focus release.
+
+Example local Spark flow:
+
+.. code-block:: python
+
+   from datafog.services.spark_service import SparkService
+
+   service = SparkService(master="local[1]")
+   rows = service.read_json("./records.json")
+   print(rows)
+
+Core-path verification
+----------------------
+
+The repository includes tests that block optional dependency imports while
+importing ``datafog`` and running the default text helpers. These checks verify
+that OCR, Spark, NLP, model-loading, and web dependencies are not required for
+the core path.
diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst
index dbf1982d..a1adec5d 100644
--- a/docs/python-sdk.rst
+++ b/docs/python-sdk.rst
@@ -4,8 +4,46 @@ DataFog Python SDK
 
 Overview
 --------
-The main entrypoint for the SDK is through the DataFog class, defined in :mod:`datafog.main`.
-Here you can initialize the different services, including TextService, ImageService, and SparkService.
+The primary 4.5 SDK path is lightweight text PII screening through the
+top-level ``datafog`` helpers. These helpers use the regex engine by default
+and do not require OCR, Spark, model downloads, or distributed dependencies.
+
+.. code-block:: python
+
+   import datafog
+
+   text = "Contact jane@example.com or call 415-555-1212"
+
+   scan_result = datafog.scan(text, engine="regex")
+   print(scan_result.entities)
+
+   redact_result = datafog.redact(text, engine="regex")
+   print(redact_result.redacted_text)
+
+   print(datafog.sanitize(text))
+
+The backward-compatible ``DataFog`` and ``TextService`` classes remain
+available for existing users. ``TextService(engine="regex")`` is the
+dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark
+surfaces require their explicit extras.
+
+Optional services
+-----------------
+
+OCR and Spark are supported optional surfaces, not the primary 4.5 path:
+
+* Use ``datafog[ocr]`` for local OCR helpers such as ``ImageService`` and
+  ``PytesseractProcessor``.
+* Use ``datafog[web,ocr]`` when OCR inputs must be downloaded from URLs.
+* Use ``datafog[nlp-advanced,ocr]`` for Donut OCR, with the model already
+  available locally.
+* Use ``datafog[distributed]`` for ``SparkService``.
+* Use ``datafog[distributed,nlp]`` plus an installed spaCy model for Spark PII
+  UDF helpers.
+
+OCR and Spark are not deprecated. Their broader overhaul is deferred so the
+4.5 release can keep the core package tight while preserving existing optional
+usage. See :doc:`optional-surfaces` for install notes and limitations.
 
 Definitions
 -----------
diff --git a/docs/roadmap.rst b/docs/roadmap.rst
index acf8b6a0..9db4d433 100644
--- a/docs/roadmap.rst
+++ b/docs/roadmap.rst
@@ -134,13 +134,29 @@ All features will remain backward compatible with the lightweight architecture.
 
 4.5.0
 ------
-Version ``4.5.0`` will introduce:
-
-* **Enterprise features** in dedicated extras
-* **Advanced analytics** for PII detection patterns
-* **Multi-language support** for international PII types
-* **Cloud integration** helpers for AWS, GCP, Azure
-* **Performance monitoring** and metrics collection
+Version ``4.5.0`` is a focus release for lightweight text PII screening. It
+should make the core package easier to install, reason about, test, and use
+before larger v5 middleware work.
+
+4.5.0 should focus on:
+
+* Core text scanning, redaction, and guardrail helpers that stay dependency
+  light by default.
+* Clear install-profile documentation for core, NLP, OCR, Spark, CLI, and web
+  surfaces.
+* OCR and Spark as supported optional surfaces, not the main 4.5 adoption path.
+* Documentation cleanup so users and contributors can find the current package
+  story without reading historical planning material first.
+* German PII regex support if the external PR passes review and does not
+  compromise core precision.
+
+Deferred beyond 4.5.0:
+
+* Full middleware adapters for Sentry, OpenTelemetry, logging frameworks, or
+  cloud DLP services.
+* OCR architecture overhaul.
+* Spark architecture overhaul.
+* Enterprise dashboards and analytics.
 
-The lightweight core will remain unchanged, ensuring existing
-integrations continue to work without modification.
+The lightweight core remains the first path; optional surfaces should stay
+explicit and isolated from default import, scan, redact, and guardrail usage.
diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py
index 9410ddc6..d34ceb8a 100644
--- a/tests/test_runtime_dependency_safety.py
+++ b/tests/test_runtime_dependency_safety.py
@@ -1,4 +1,6 @@
 import importlib
+import os
+import subprocess
 import sys
 import types
 from pathlib import Path
@@ -6,6 +8,20 @@
 import pytest
 
 
+def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]:
+    env = dict(os.environ)
+    env["PYTHONPATH"] = str(Path.cwd())
+    env["DATAFOG_NO_TELEMETRY"] = "1"
+    env["DO_NOT_TRACK"] = "1"
+    return subprocess.run(
+        [sys.executable, "-c", script],
+        check=True,
+        env=env,
+        text=True,
+        capture_output=True,
+    )
+
+
 def test_runtime_code_does_not_install_packages() -> None:
     blocked_snippets = [
         "subprocess.check_call",
@@ -25,6 +41,43 @@ def test_runtime_code_does_not_install_packages() -> None:
     assert offenders == []
 
 
+def test_ocr_and_spark_public_services_do_not_require_optional_imports() -> None:
+    _run_isolated_python(
+        """
+import importlib.abc
+import sys
+
+blocked = {
+    "aiohttp",
+    "certifi",
+    "PIL",
+    "pyspark",
+    "pytesseract",
+    "torch",
+    "transformers",
+}
+
+class BlockOptionalImports(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname.split(".", 1)[0] in blocked:
+            raise AssertionError(f"optional dependency imported: {fullname}")
+        return None
+
+sys.meta_path.insert(0, BlockOptionalImports())
+
+import datafog
+from datafog.services import ImageService, SparkService, TextService
+
+assert datafog.scan("Email jane@example.com").entities
+assert ImageService is not None
+assert SparkService is not None
+assert TextService is not None
+assert datafog.ImageService is ImageService
+assert datafog.SparkService is SparkService
+"""
+    )
+
+
 def test_spacy_pii_missing_model_requires_explicit_download(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

From 6f57af2dd14bafd9c7403acf8d4ee6041449f4bb Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 19:51:59 +0200
Subject: [PATCH 50/70] docs: make v4.5 core story obvious

Refs DFPY-72
---
 README.md                 |   8 +++
 docs/getting-started.rst  | 112 +++++++++++++++++++++++++++++++++++
 docs/index.rst            | 120 ++++++++++----------------------------
 docs/planning-history.rst |  30 ++++++++++
 4 files changed, 181 insertions(+), 89 deletions(-)
 create mode 100644 docs/getting-started.rst
 create mode 100644 docs/planning-history.rst

diff --git a/README.md b/README.md
index 2ff68cd2..3f2473e9 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,14 @@ It provides:
 - A simple agent-oriented API for LLM applications
 - Backward-compatible `DataFog` and `TextService` classes
 
+## 4.5 Focus
+
+DataFog 4.5 is focused on lightweight text PII screening: a small core install,
+fast regex-based scan/redact helpers, explicit optional extras, and a clearer
+path toward future middleware use cases. Dedicated Sentry, OpenTelemetry,
+logging-framework, and cloud DLP adapters are future-facing work and are not
+part of the 4.5 release.
+
 ## Installation
 
 ```bash
diff --git a/docs/getting-started.rst b/docs/getting-started.rst
new file mode 100644
index 00000000..69637611
--- /dev/null
+++ b/docs/getting-started.rst
@@ -0,0 +1,112 @@
+================================
+Getting Started With DataFog 4.5
+================================
+
+DataFog 4.5 focuses on lightweight text PII screening. A core install should
+let you scan and redact common structured PII without installing OCR, Spark,
+large NLP models, or middleware integrations.
+
+Install Profiles
+================
+
+Core text screening:
+
+.. code-block:: bash
+
+   pip install datafog
+
+Optional extras are explicit:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Profile
+     - Install command
+     - Use when
+   * - Core
+     - ``pip install datafog``
+     - You need regex-based text scanning, redaction, and guardrail helpers.
+   * - NLP
+     - ``pip install "datafog[nlp]"``
+     - You need spaCy-backed named entity recognition.
+   * - Advanced NLP
+     - ``pip install "datafog[nlp-advanced]"``
+     - You need GLiNER-backed named entity recognition.
+   * - OCR
+     - ``pip install "datafog[ocr]"``
+     - You need local image text extraction before PII scanning.
+   * - OCR from URLs
+     - ``pip install "datafog[web,ocr]"``
+     - You need DataFog to download image inputs before OCR.
+   * - Spark
+     - ``pip install "datafog[distributed]"``
+     - You need the optional ``SparkService`` surface.
+   * - Everything
+     - ``pip install "datafog[all]"``
+     - You are developing or deliberately want every optional surface.
+
+Python Usage
+============
+
+Use the top-level helpers for the 4.5 core path:
+
+.. code-block:: python
+
+   import datafog
+
+   text = "Contact jane@example.com or call 415-555-1212"
+
+   scan_result = datafog.scan(text, engine="regex")
+   print(scan_result.entities)
+
+   redact_result = datafog.redact(text, engine="regex")
+   print(redact_result.redacted_text)
+
+   print(datafog.sanitize("Card: 4111-1111-1111-1111"))
+
+Agent-oriented helpers use the same lightweight text path:
+
+.. code-block:: python
+
+   import datafog
+
+   prompt = "My SSN is 123-45-6789"
+   scan_result = datafog.scan_prompt(prompt, engine="regex")
+
+   if scan_result.entities:
+       print("PII detected before sending the prompt")
+
+   output = "Email me at jane.doe@example.com"
+   safe_output = datafog.filter_output(output, engine="regex")
+   print(safe_output.redacted_text)
+
+CLI Usage
+=========
+
+The CLI core path is text-first:
+
+.. code-block:: bash
+
+   datafog scan-text "Contact jane@example.com"
+   datafog redact-text "Contact jane@example.com"
+   datafog replace-text "Contact jane@example.com"
+   datafog hash-text "Contact jane@example.com"
+
+Image commands are optional. Install ``datafog[ocr]`` for local OCR and
+``datafog[web,ocr]`` when the CLI needs to download image inputs.
+
+What 4.5 Is Not
+===============
+
+DataFog 4.5 prepares the package for future middleware use cases, but it does
+not ship dedicated Sentry, OpenTelemetry, logging-framework, or cloud DLP
+adapters. Those integrations are future-facing work built on the same core
+text screening path.
+
+Next Pages
+==========
+
+* :doc:`python-sdk` documents the Python API surface.
+* :doc:`cli` documents command-line usage.
+* :doc:`optional-surfaces` documents OCR and Spark install notes.
+* :doc:`roadmap` explains how 4.5 leads toward later middleware work.
diff --git a/docs/index.rst b/docs/index.rst
index 7e05a867..2c4dab59 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2,105 +2,47 @@
 DataFog Documentation
 =====================
 
-DataFog is an open-source tool for lightweight text PII detection and
-anonymization. The core install focuses on fast regex-based scanning and
-redaction, with optional extras for NLP, OCR, and Spark-style workflows.
+DataFog 4.5 is a lightweight text PII screening package for Python. The
+primary path is a small core install, fast regex-based scanning and redaction,
+agent-friendly guardrail helpers, and explicit optional extras when you need
+NLP, OCR, Spark, or web inputs.
+
+Start with :doc:`getting-started` if you want the shortest route from install
+to scanning text. The roadmap and historical planning pages remain available,
+but the live user docs are the first path for 4.5.
+
+Use DataFog 4.5
+===============
 
 .. toctree::
    :maxdepth: 2
+   :caption: Use DataFog 4.5
 
-   important-concepts
-   cli
+   getting-started
    python-sdk
+   cli
    optional-surfaces
-   definitions
-   roadmap
-   v44-bridge-release
-   v5-product-brief
-   v5-compatibility-matrix
-   v5-cut-line
-
-=====================
-Getting Started
-=====================
-
-Installation
-------------
-
-Install the lightweight text screening core via pip:
-
-.. code-block:: bash
-
-    pip install datafog
-
-Optional extras such as ``nlp``, ``nlp-advanced``, ``ocr``, ``distributed``,
-and ``web`` are installed only when you need those surfaces.
-
----------------------
-CLI Usage
----------------------
-
-For a list of available operations, run:
-
-.. code-block:: bash
-
-    datafog --help
-
-Scan text for PII:
-
-.. code-block:: bash
-
-    datafog scan-text "Your text here"
-
-Image/OCR commands are optional. Local OCR requires ``datafog[ocr]``; URL-based
-image downloading requires ``datafog[web,ocr]``.
-
-.. code-block:: bash
-
-    datafog scan-image "path/to/image.png" --operations extract
-
-Scan for PII in image text:
-
-.. code-block:: bash
-
-    datafog scan-image "path/to/image.png" --operations scan
-
-For more information on optional OCR and Spark surfaces, see
-:doc:`optional-surfaces`.
-
----------------------
-Python SDK Usage
----------------------
-
-Scan text for PII:
-
-.. code-block:: python
-
-   
-   import datafog
-
-   text = "Contact jane@example.com or call 415-555-1212"
-   result = datafog.scan(text, engine="regex")
-   print(result.entities)
-   print(datafog.redact(text, engine="regex").redacted_text)
+   important-concepts
 
-Run OCR and then scan extracted text only when the OCR extra is installed:
+Reference
+=========
 
-.. code-block:: python
+.. toctree::
+   :maxdepth: 2
+   :caption: Reference
 
-   
-   import asyncio
-   from datafog import DataFog
+   definitions
 
-   # For OCR and PII annotation
-   ocr_client = DataFog(operations="extract,scan")
+Planning And History
+====================
 
-   async def run_ocr_pipeline_demo():
-       image_url = "https://s3.amazonaws.com/thumbnails.venngage.com/template/dc377004-1c2d-49f2-8ddf-d63f11c8d9c2.png"
-       results = await ocr_client.run_ocr_pipeline(image_urls=[image_url])
-       print("OCR Pipeline Results:", results)
+The pages below document release planning, migration history, and future
+direction. They are useful context, but they are secondary to the live 4.5
+usage path above.
 
-   # Run the async function
-   asyncio.run(run_ocr_pipeline_demo())
+.. toctree::
+   :maxdepth: 1
+   :caption: Planning and history
 
-For detailed information on the Python SDK, see :doc:`python-sdk`.
+   roadmap
+   planning-history
diff --git a/docs/planning-history.rst b/docs/planning-history.rst
new file mode 100644
index 00000000..20025573
--- /dev/null
+++ b/docs/planning-history.rst
@@ -0,0 +1,30 @@
+====================
+Planning And History
+====================
+
+These pages and artifacts are preserved for context, but they are not the
+first path for using DataFog 4.5. Start with :doc:`getting-started` for live
+user docs.
+
+Release Planning
+================
+
+.. toctree::
+   :maxdepth: 1
+
+   v44-bridge-release
+   v5-product-brief
+   v5-compatibility-matrix
+   v5-cut-line
+
+Audit Artifacts
+===============
+
+Historical audit notes remain available in the repository for maintainers who
+need the detailed background:
+
+* :download:`Reconnaissance notes <audit/00-reconnaissance.md>`
+* :download:`Coverage baseline <audit/01-coverage-baseline.md>`
+* :download:`Detection accuracy review <audit/02-detection-accuracy.md>`
+* :download:`Architecture review <audit/03-architecture-review.md>`
+* :download:`Final coverage notes <audit/06-final-coverage.md>`

From 3befe2f24250dd80d1feb844770aa2838767366f Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 20:00:34 +0200
Subject: [PATCH 51/70] docs: add contributor 4.5 runbook

Refs DFPY-73
---
 CONTRIBUTING.md       |  30 ++++++-
 docs/contributing.rst | 201 ++++++++++++++++++++++++++++++++++++++++++
 docs/index.rst        |   9 ++
 3 files changed, 237 insertions(+), 3 deletions(-)
 create mode 100644 docs/contributing.rst

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 483a2b3e..6e7e416e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -27,6 +27,11 @@ pushes except for explicit emergency maintenance.
 
 ## Local Development
 
+The current contributor runbook for DataFog 4.5 lives in
+[`docs/contributing.rst`](docs/contributing.rst). It includes supported Python
+versions, install profiles, focused and broad test commands, docs-build
+verification, and the 4.5 release-flow boundary.
+
 ```bash
 git clone https://github.com/datafog/datafog-python
 cd datafog-python
@@ -43,17 +48,33 @@ package install:
 pip install -r requirements-dev.txt
 ```
 
-For optional NLP or OCR work, install the relevant extras:
+For optional NLP, OCR, or distributed work, install the relevant extras:
 
 ```bash
 pip install -e ".[dev,cli,nlp]"
 pip install -e ".[dev,cli,nlp,nlp-advanced]"
+pip install -e ".[dev,cli,ocr]"
+pip install -e ".[dev,cli,distributed]"
 pip install -e ".[all,dev]"
 ```
 
 ## Tests
 
-Run the core test suite before opening a pull request:
+Run focused tests for the area you changed before opening a pull request. For
+core import and dependency-boundary work, use:
+
+```bash
+DATAFOG_NO_TELEMETRY=1 DO_NOT_TRACK=1 \
+  pytest tests/test_runtime_dependency_safety.py tests/test_no_network_core.py -q
+```
+
+For broader local confidence, run the non-slow suite:
+
+```bash
+pytest -m "not slow" -q
+```
+
+To mimic the core CI profile, run:
 
 ```bash
 pytest tests/ -m "not slow" \
@@ -68,9 +89,12 @@ Run the focused test file for the area you changed whenever possible. For
 documentation-only changes, build the docs:
 
 ```bash
-sphinx-build -b html docs docs/_build/html
+python -m sphinx -b html docs docs/_build/html
 ```
 
+See [`docs/contributing.rst`](docs/contributing.rst) for optional-profile smoke
+commands and release-prep checks.
+
 ## Pull Request Checklist
 
 Before requesting review:
diff --git a/docs/contributing.rst b/docs/contributing.rst
new file mode 100644
index 00000000..2016e6cb
--- /dev/null
+++ b/docs/contributing.rst
@@ -0,0 +1,201 @@
+=======================================
+Contributor Setup And 4.5 Release Flow
+=======================================
+
+This page is the contributor runbook for DataFog 4.5 work. It is meant for
+humans and agents preparing local changes, choosing verification commands, and
+understanding where the 4.5 release boundary sits.
+
+Version Frame
+=============
+
+Current release planning uses this frame:
+
+* Stable package release: ``4.4.0``.
+* Current development package version: ``4.4.0a5``.
+* Next minor target: ``4.5.0``.
+
+Do not bump routine feature, documentation, or cleanup branches directly to
+``4.5.0``. Keep the version stable during local release-prep work, then handle
+the final version and release-note alignment in the release-readiness slice.
+
+Python Environments
+===================
+
+DataFog currently declares support for Python ``>=3.10,<3.14``. The CI matrix
+tests core installs on Python 3.10, 3.11, 3.12, and 3.13. Optional NLP and
+NLP-advanced profiles are tested on Python 3.10, 3.11, and 3.12; Python 3.13
+optional-profile validation is tracked separately for 4.5.
+
+Create one virtual environment per Python version when you need to compare
+profiles locally:
+
+.. code-block:: bash
+
+   python3.12 -m venv .venv312
+   source .venv312/bin/activate
+   python -m pip install --upgrade pip
+
+For another version, keep the environment name explicit:
+
+.. code-block:: bash
+
+   python3.10 -m venv .venv310
+   python3.11 -m venv .venv311
+   python3.13 -m venv .venv313
+
+Install Profiles
+================
+
+Install the package in editable mode with the smallest profile that matches the
+work you are doing:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Profile
+     - Command
+     - Notes
+   * - Core
+     - ``pip install -e .``
+     - Lightweight regex engine and package import path.
+   * - Core test + CLI
+     - ``pip install -e ".[test,cli]" -r requirements-test.txt``
+     - Matches the core CI test profile.
+   * - Docs
+     - ``pip install -e ".[docs]" -r requirements-docs.txt``
+     - Enough to build Sphinx docs locally.
+   * - Local dev
+     - ``pip install -e ".[dev,cli]" && pip install -r requirements-dev.txt``
+     - Test, docs, lint, formatting, and pre-commit tooling.
+   * - NLP
+     - ``pip install -e ".[test,cli,nlp]" -r requirements-test.txt``
+     - Also install the spaCy model needed for NLP tests.
+   * - NLP advanced
+     - ``pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt``
+     - Also install spaCy and GLiNER models explicitly.
+   * - OCR
+     - ``pip install -e ".[test,ocr]" -r requirements-test.txt``
+     - Tesseract workflows also need the system ``tesseract`` binary.
+   * - Distributed
+     - ``pip install -e ".[test,distributed]" -r requirements-test.txt``
+     - Spark workflows also need a Java runtime.
+   * - All extras
+     - ``pip install -e ".[all,dev]"``
+     - Use only when you deliberately want every optional surface locally.
+
+Optional model setup is explicit:
+
+.. code-block:: bash
+
+   python -m spacy download en_core_web_lg
+   datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
+
+Focused Verification
+====================
+
+Use focused checks for the area you touched before running broader suites.
+Set the no-telemetry environment variables when testing core privacy and import
+behavior:
+
+.. code-block:: bash
+
+   export DATAFOG_NO_TELEMETRY=1
+   export DO_NOT_TRACK=1
+
+Core dependency and no-network checks:
+
+.. code-block:: bash
+
+   python -m pytest tests/test_runtime_dependency_safety.py tests/test_no_network_core.py -q
+
+Run a changed test file directly when behavior changes:
+
+.. code-block:: bash
+
+   python -m pytest tests/test_engine_api.py -q
+   python -m pytest tests/test_agent_api.py -q
+   python -m pytest tests/test_cli_smoke.py -q
+
+Docs build:
+
+.. code-block:: bash
+
+   python -m sphinx -b html docs docs/_build/html
+
+Pre-commit on touched files:
+
+.. code-block:: bash
+
+   pre-commit run --files README.md docs/index.rst --show-diff-on-failure
+   git diff --check
+
+Broad Verification
+==================
+
+Run the broad non-slow suite when a change affects shared behavior,
+public docs, imports, packaging, or release confidence:
+
+.. code-block:: bash
+
+   python -m pytest -m "not slow" -q
+
+To mimic the core CI profile more closely:
+
+.. code-block:: bash
+
+   python -m pytest tests/ \
+     -m "not slow" \
+     --ignore=tests/test_gliner_annotator.py \
+     --ignore=tests/test_image_service.py \
+     --ignore=tests/test_ocr_integration.py \
+     --ignore=tests/test_spark_integration.py \
+     --ignore=tests/test_text_service_integration.py
+
+Use optional-profile smoke checks when changing extras, dependency boundaries,
+or install behavior:
+
+.. code-block:: bash
+
+   DATAFOG_INSTALL_PROFILE=core python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=cli python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=nlp python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=nlp-advanced python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=ocr python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=distributed python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=web python -m pytest tests/test_install_profiles.py -q
+
+4.5 Release Flow
+================
+
+The 4.5 work lands as focused pull requests into ``dev``. Keep feature and docs
+branches narrow, and avoid mixing local cleanup, external PR review, and final
+release mechanics in one branch.
+
+The release flow for 4.5 is:
+
+1. Land the local release-prep baseline and follow-up cleanup/docs slices.
+2. Review the external German regex PR after the local release-prep baseline is
+   in place.
+3. Integrate German regex support only if review says it fits the 4.5
+   lightweight text screening thesis.
+4. Validate optional Python 3.13 profiles before claiming support beyond core
+   SDK and CLI.
+5. Prepare release readiness: changelog/release notes, package checks, docs
+   build, CI state, and version alignment.
+6. Bump or override the final stable release to ``4.5.0`` only during the
+   release-readiness and stable-release path.
+
+The current release workflow strips prerelease suffixes from the package
+version unless a manual stable ``version_override`` is provided. For the final
+4.5 stable release, use a dedicated release-readiness change or the stable
+workflow override so the published version is ``4.5.0`` rather than another
+``4.4.0`` prerelease line.
+
+External PR Boundary
+====================
+
+The external German PII regex PR belongs after local baseline cleanup. Review
+it as a 4.5 candidate, not as a v5 planning shortcut. If accepted, adapt it in
+the German regex integration slice with tests, documentation of locale
+coverage, and no new dependency burden on the core path.
diff --git a/docs/index.rst b/docs/index.rst
index 2c4dab59..04731388 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -33,6 +33,15 @@ Reference
 
    definitions
 
+Contributing
+============
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contributing
+
+   contributing
+
 Planning And History
 ====================
 

From 2ba8e34ed5bdef8b24991c5a31a920950d2d4226 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 20:10:51 +0200
Subject: [PATCH 52/70] docs: quarantine historical shadow modules

Refs DFPY-76
---
 datafog/__init___lean.py                  |  8 +-
 datafog/__init___original.py              |  9 +++
 datafog/main_lean.py                      |  8 +-
 datafog/main_original.py                  |  9 ++-
 datafog/services/text_service_lean.py     |  9 ++-
 datafog/services/text_service_original.py |  9 ++-
 docs/contributing.rst                     |  8 ++
 docs/index.rst                            |  1 +
 docs/live-module-map.rst                  | 99 +++++++++++++++++++++++
 docs/planning-history.rst                 |  4 +
 docs/v5-compatibility-matrix.rst          |  2 +
 setup_lean.py                             |  3 +
 setup_original.py                         |  3 +
 tests/test_main.py                        |  3 +-
 tests/test_text_service.py                |  3 +-
 15 files changed, 171 insertions(+), 7 deletions(-)
 create mode 100644 docs/live-module-map.rst

diff --git a/datafog/__init___lean.py b/datafog/__init___lean.py
index 40a3f530..5c25f9bc 100644
--- a/datafog/__init___lean.py
+++ b/datafog/__init___lean.py
@@ -1,4 +1,10 @@
-"""
+"""Historical shadow package export module.
+
+This file is not the live DataFog 4.5 package export surface. Use
+``datafog/__init__.py`` for current package exports, lazy optional imports,
+top-level helpers, and compatibility shims. This snapshot remains importable
+only as historical reference until legacy cleanup can remove it safely.
+
 DataFog: Lightning-fast PII detection and anonymization library.
 
 Core package provides regex-based PII detection with 190x performance advantage.
diff --git a/datafog/__init___original.py b/datafog/__init___original.py
index 7838dd31..380511b1 100644
--- a/datafog/__init___original.py
+++ b/datafog/__init___original.py
@@ -1,3 +1,12 @@
+"""Historical shadow package export module.
+
+This file is not the live DataFog 4.5 package export surface. Use
+``datafog/__init__.py`` for current package exports, lazy optional imports,
+top-level helpers, and compatibility shims. This eager-export snapshot remains
+importable only as historical reference until legacy cleanup can remove it
+safely.
+"""
+
 from .__about__ import __version__
 from .client import app
 from .config import OperationType, get_config
diff --git a/datafog/main_lean.py b/datafog/main_lean.py
index af61559e..f67501e6 100644
--- a/datafog/main_lean.py
+++ b/datafog/main_lean.py
@@ -1,4 +1,10 @@
-"""
+"""Historical shadow main module.
+
+This file is not the live DataFog 4.5 ``DataFog`` implementation. Use
+``datafog/main.py`` for the current backward-compatible class and text/OCR
+compatibility methods. This lean snapshot remains importable only as historical
+reference until legacy cleanup can remove it safely.
+
 Lean main module for DataFog core functionality.
 
 This module contains the lightweight core classes for DataFog:
diff --git a/datafog/main_original.py b/datafog/main_original.py
index 58224e59..9594c824 100644
--- a/datafog/main_original.py
+++ b/datafog/main_original.py
@@ -1,4 +1,11 @@
-"""
+"""Historical shadow main module.
+
+This file is not the live DataFog 4.5 ``DataFog`` implementation. Use
+``datafog/main.py`` for the current backward-compatible class and use the
+optional OCR/Spark service modules directly when those surfaces are needed.
+This original full-featured snapshot remains importable for legacy tests until
+cleanup can remove it safely.
+
 Main module for DataFog.
 
 This module contains the core classes for DataFog:
diff --git a/datafog/services/text_service_lean.py b/datafog/services/text_service_lean.py
index ce9203ec..1262db3f 100644
--- a/datafog/services/text_service_lean.py
+++ b/datafog/services/text_service_lean.py
@@ -1,4 +1,11 @@
-"""Lean text processing service for PII annotation.
+"""Historical shadow text service module.
+
+This file is not the live DataFog 4.5 text service. Use
+``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER,
+auto, and smart engine boundary. This lean snapshot remains importable only as
+historical reference until legacy cleanup can remove it safely.
+
+Lean text processing service for PII annotation.
 
 Provides synchronous and asynchronous methods for annotating text with personally
 identifiable information (PII) using regex patterns. Supports chunking long texts
diff --git a/datafog/services/text_service_original.py b/datafog/services/text_service_original.py
index 6d5dde1b..8ad576a9 100644
--- a/datafog/services/text_service_original.py
+++ b/datafog/services/text_service_original.py
@@ -1,4 +1,11 @@
-"""Text processing service for PII annotation.
+"""Historical shadow text service module.
+
+This file is not the live DataFog 4.5 text service. Use
+``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER,
+auto, and smart engine boundary. This original snapshot remains importable for
+legacy tests until cleanup can remove it safely.
+
+Text processing service for PII annotation.
 
 Provides synchronous and asynchronous methods for annotating text with personally identifiable information (PII) using SpaCy or regex patterns. Supports chunking long texts and batch processing.
 """
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2016e6cb..6037d926 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -165,6 +165,14 @@ or install behavior:
    DATAFOG_INSTALL_PROFILE=distributed python -m pytest tests/test_install_profiles.py -q
    DATAFOG_INSTALL_PROFILE=web python -m pytest tests/test_install_profiles.py -q
 
+Live Modules
+============
+
+Use :doc:`live-module-map` before changing core package structure. It lists the
+live 4.5 modules for each concept and the historical ``*_lean`` and
+``*_original`` files that are kept only as non-live compatibility/audit
+artifacts.
+
 4.5 Release Flow
 ================
 
diff --git a/docs/index.rst b/docs/index.rst
index 04731388..1e64fb5b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -41,6 +41,7 @@ Contributing
    :caption: Contributing
 
    contributing
+   live-module-map
 
 Planning And History
 ====================
diff --git a/docs/live-module-map.rst b/docs/live-module-map.rst
new file mode 100644
index 00000000..8ed98bd7
--- /dev/null
+++ b/docs/live-module-map.rst
@@ -0,0 +1,99 @@
+===============
+Live Module Map
+===============
+
+This map identifies the live modules for DataFog 4.5 and the historical
+shadow files that should not be used for new work. The goal is to make the
+current code path obvious without removing importable legacy files that still
+have test coverage or historical value.
+
+Live 4.5 Modules
+================
+
+.. list-table::
+   :header-rows: 1
+
+   * - Concept
+     - Live module
+     - Notes
+   * - Package exports
+     - ``datafog/__init__.py``
+     - Top-level ``scan``, ``redact``, guardrail helpers, compatibility shims,
+       and lazy optional exports.
+   * - Core engine
+     - ``datafog/engine.py``
+     - Dataclass-based scan/redact path used by the 4.5 core helpers.
+   * - Agent helpers
+     - ``datafog/agent.py``
+     - Prompt/output screening and guardrail helpers on the lightweight text
+       path.
+   * - Backward-compatible ``DataFog`` class
+     - ``datafog/main.py``
+     - Current public ``DataFog`` class and text/OCR compatibility methods.
+   * - Text service
+     - ``datafog/services/text_service.py``
+     - Current service boundary for regex, spaCy, GLiNER, auto, and smart
+       engines.
+   * - CLI
+     - ``datafog/client.py``
+     - Current command-line entrypoint.
+   * - OCR surface
+     - ``datafog/services/image_service.py`` and
+       ``datafog/processing/image_processing/``
+     - Optional image/OCR surface behind explicit extras.
+   * - Spark surface
+     - ``datafog/services/spark_service.py`` and
+       ``datafog/processing/spark_processing/``
+     - Optional distributed surface behind explicit extras.
+   * - Packaging
+     - ``setup.py`` and ``requirements-*.txt``
+     - Current packaging and contributor dependency inputs.
+
+Historical Shadow Files
+=======================
+
+The following files are historical snapshots or alternate implementation
+lineage. They are kept importable for now, but new work should not add behavior
+to them.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Historical file
+     - Live replacement
+     - 4.5 status
+   * - ``datafog/__init___lean.py``
+     - ``datafog/__init__.py``
+     - Historical package export snapshot.
+   * - ``datafog/__init___original.py``
+     - ``datafog/__init__.py``
+     - Historical eager-export package snapshot.
+   * - ``datafog/main_lean.py``
+     - ``datafog/main.py``
+     - Historical lightweight ``DataFog`` implementation.
+   * - ``datafog/main_original.py``
+     - ``datafog/main.py`` plus optional OCR/Spark services
+     - Historical full-featured ``DataFog`` implementation still referenced by
+       legacy tests.
+   * - ``datafog/services/text_service_lean.py``
+     - ``datafog/services/text_service.py``
+     - Historical regex-first service variant.
+   * - ``datafog/services/text_service_original.py``
+     - ``datafog/services/text_service.py``
+     - Historical spaCy/regex service still referenced by legacy tests.
+   * - ``setup_lean.py``
+     - ``setup.py``
+     - Historical packaging snapshot.
+   * - ``setup_original.py``
+     - ``setup.py``
+     - Historical packaging snapshot.
+
+Cleanup Boundary
+================
+
+This 4.5 slice marks the shadow files as non-live and documents their live
+replacements. It does not remove importable modules because ``main_original``
+and ``text_service_original`` still have explicit legacy tests. A future
+breaking cleanup can remove the shadow files after any remaining tested
+behavior is either migrated to live modules or intentionally dropped with a
+compatibility note.
diff --git a/docs/planning-history.rst b/docs/planning-history.rst
index 20025573..b0cab0f9 100644
--- a/docs/planning-history.rst
+++ b/docs/planning-history.rst
@@ -17,6 +17,10 @@ Release Planning
    v5-compatibility-matrix
    v5-cut-line
 
+Additional planning artifacts:
+
+* :download:`v5 model selection requirements <v5-model-requirements.md>`
+
 Audit Artifacts
 ===============
 
diff --git a/docs/v5-compatibility-matrix.rst b/docs/v5-compatibility-matrix.rst
index b95483e6..a0e2691c 100644
--- a/docs/v5-compatibility-matrix.rst
+++ b/docs/v5-compatibility-matrix.rst
@@ -134,6 +134,8 @@ Compatibility Matrix
      - Parallel historical implementations.
      - Remove or make private after migration path.
      - Consolidate around the v5 core and delete duplicate runtime surfaces.
+       In 4.5 these files are marked as non-live shadow modules; removal is
+       deferred until legacy tests are migrated or intentionally dropped.
 
 Warning Policy
 --------------
diff --git a/setup_lean.py b/setup_lean.py
index 9cd06dd6..ece73a1f 100644
--- a/setup_lean.py
+++ b/setup_lean.py
@@ -1,5 +1,8 @@
 from setuptools import find_packages, setup
 
+# Historical shadow packaging snapshot. The live DataFog 4.5 packaging input is
+# setup.py. Do not use this file for builds, releases, or dependency changes.
+
 # Read README for the long description
 with open("README.md", "r") as f:
     long_description = f.read()
diff --git a/setup_original.py b/setup_original.py
index 1e6ca3af..d70db319 100644
--- a/setup_original.py
+++ b/setup_original.py
@@ -1,5 +1,8 @@
 from setuptools import find_packages, setup
 
+# Historical shadow packaging snapshot. The live DataFog 4.5 packaging input is
+# setup.py. Do not use this file for builds, releases, or dependency changes.
+
 # Read README for the long description
 with open("README.md", "r") as f:
     long_description = f.read()
diff --git a/tests/test_main.py b/tests/test_main.py
index c35ed505..e9b1e385 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -29,7 +29,8 @@
     ImageService = None
     TextService = None
 
-# Try to import the full-featured DataFog for integration tests
+# Keep historical main_original importable while legacy tests still cover it.
+# New behavior belongs in datafog.main.DataFog.
 try:
     from datafog.main_original import DataFog as FullDataFog
 
diff --git a/tests/test_text_service.py b/tests/test_text_service.py
index 9f02f3c8..74156082 100644
--- a/tests/test_text_service.py
+++ b/tests/test_text_service.py
@@ -2,7 +2,8 @@
 
 import pytest
 
-# Test the full-featured TextService from text_service_original
+# Legacy coverage for the historical shadow TextService. New behavior belongs
+# in datafog.services.text_service.
 from datafog.services.text_service_original import TextService
 
 

From cb71a59b84aca76a05cf2deb52361fa825a479d7 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 20:34:06 +0200
Subject: [PATCH 53/70] feat: add German regex locale support

---
 README.md                                     |  25 ++-
 datafog/__init__.py                           |   7 +-
 datafog/agent.py                              |  17 +-
 datafog/client.py                             |  53 ++++-
 datafog/core.py                               |  41 ++--
 datafog/engine.py                             |  74 ++++++-
 datafog/main.py                               |  11 +-
 .../regex_annotator/regex_annotator.py        | 207 +++++++++++++++++-
 datafog/services/text_service.py              |   6 +-
 docs/cli.rst                                  |  11 +
 docs/getting-started.rst                      |  29 +++
 docs/python-sdk.rst                           |  33 +++
 tests/corpus/structured_pii.json              |  24 ++
 tests/test_de_pii_regex.py                    | 146 ++++++++++++
 tests/test_detection_accuracy.py              |   2 +
 tests/test_regex_annotator.py                 |   6 +-
 16 files changed, 637 insertions(+), 55 deletions(-)
 create mode 100644 tests/test_de_pii_regex.py

diff --git a/README.md b/README.md
index 3f2473e9..adc7ff58 100644
--- a/README.md
+++ b/README.md
@@ -76,6 +76,25 @@ print(datafog.sanitize("Card: 4111-1111-1111-1111", engine="regex"))
 # Card: [CREDIT_CARD_1]
 ```
 
+## German Structured PII
+
+German VAT IDs and German IBANs are detected by the default regex path because
+their country-code structure is specific enough for default-on screening.
+Broader German identifiers are available with explicit locale selection or
+explicit entity-type filtering.
+
+```python
+import datafog
+
+text = "Steuer-ID 12345678901 liegt vor."
+
+print(datafog.scan(text, engine="regex").entities)
+# []
+
+print(datafog.scan(text, engine="regex", locales=["de"]).entities)
+# [Entity(type='DE_TAX_ID', text='12345678901', ...)]
+```
+
 ### Guardrails
 
 ```python
@@ -98,7 +117,8 @@ Use the engine that matches your accuracy and dependency constraints:
 
 - `regex`:
   - Fastest and always available.
-  - Best for structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE`.
+  - Best for structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DE_VAT_ID`, `DE_IBAN`, `DATE`, `ZIP_CODE`.
+  - Use `locales=["de"]` for broader German structured IDs such as `DE_TAX_ID`, `DE_POSTAL_CODE`, and passport or residence permit numbers.
 - `spacy`:
   - Requires `pip install datafog[nlp]`.
   - Useful for unstructured entities like person and organization names.
@@ -169,6 +189,9 @@ datafog replace-text "john@example.com"
 
 # Hash detected entities
 datafog hash-text "john@example.com"
+
+# Enable broader German regex identifiers
+datafog redact-text "Steuer-ID 12345678901" --locale de
 ```
 
 ## Telemetry
diff --git a/datafog/__init__.py b/datafog/__init__.py
index b7d8e4e7..4f7567a8 100644
--- a/datafog/__init__.py
+++ b/datafog/__init__.py
@@ -152,6 +152,7 @@ def scan(
     text: str,
     engine: str = "regex",
     entity_types: list[str] | None = None,
+    locales: list[str] | None = None,
 ) -> ScanResult:
     """
     v5-preview scan entrypoint.
@@ -159,7 +160,7 @@ def scan(
     Defaults to the lightweight regex engine so the core install works without
     optional dependency fallback warnings.
     """
-    return _scan(text=text, engine=engine, entity_types=entity_types)
+    return _scan(text=text, engine=engine, entity_types=entity_types, locales=locales)
 
 
 def redact(
@@ -169,6 +170,7 @@ def redact(
     entity_types: list[str] | None = None,
     strategy: str = "token",
     preset: str | None = None,
+    locales: list[str] | None = None,
 ) -> RedactResult:
     """
     v5-preview redaction entrypoint.
@@ -191,6 +193,7 @@ def redact(
         engine=engine,
         entity_types=entity_types,
         strategy=strategy,
+        locales=locales,
     )
 
 
@@ -199,6 +202,7 @@ def protect(
     engine: str = "regex",
     strategy: str = "token",
     on_detect: str = "redact",
+    locales: list[str] | None = None,
 ):
     """
     v5-preview guardrail factory.
@@ -208,6 +212,7 @@ def protect(
         engine=engine,
         strategy=strategy,
         on_detect=on_detect,
+        locales=locales,
     )
 
 
diff --git a/datafog/agent.py b/datafog/agent.py
index 6603ba11..a031a147 100644
--- a/datafog/agent.py
+++ b/datafog/agent.py
@@ -27,11 +27,7 @@ class GuardrailWatch:
 
     def scan(self, text: str) -> ScanResult:
         """Scan text and increment detection counters."""
-        result = scan(
-            text=text,
-            engine=self.guardrail.engine,
-            entity_types=self.guardrail.entity_types,
-        )
+        result = self.guardrail.scan(text)
         if result.entities:
             self.detections += len(result.entities)
         return result
@@ -51,6 +47,7 @@ class Guardrail:
     """Reusable text guardrail for wrapping LLM prompts and outputs."""
 
     entity_types: Optional[list[str]] = None
+    locales: Optional[list[str]] = None
     engine: str = "regex"
     strategy: str = "token"
     on_detect: str = "redact"
@@ -61,7 +58,12 @@ def __post_init__(self) -> None:
 
     def scan(self, text: str) -> ScanResult:
         """Scan a text value for entities."""
-        return scan(text=text, engine=self.engine, entity_types=self.entity_types)
+        return scan(
+            text=text,
+            engine=self.engine,
+            entity_types=self.entity_types,
+            locales=self.locales,
+        )
 
     def filter(self, text: str) -> RedactResult:
         """Scan then enforce configured behavior."""
@@ -70,6 +72,7 @@ def filter(self, text: str) -> RedactResult:
             engine=self.engine,
             entity_types=self.entity_types,
             strategy=self.strategy,
+            locales=self.locales,
         )
         if not result.entities:
             return result
@@ -137,6 +140,7 @@ def filter_output(output: str, engine: str = "regex", **kwargs: Any) -> RedactRe
 
 def create_guardrail(
     entity_types: Optional[list[str]] = None,
+    locales: Optional[list[str]] = None,
     engine: str = "regex",
     strategy: str = "token",
     on_detect: str = "redact",
@@ -146,6 +150,7 @@ def create_guardrail(
     """
     return Guardrail(
         entity_types=entity_types,
+        locales=locales,
         engine=engine,
         strategy=strategy,
         on_detect=on_detect,
diff --git a/datafog/client.py b/datafog/client.py
index c3d493c8..eab203b8 100644
--- a/datafog/client.py
+++ b/datafog/client.py
@@ -6,7 +6,7 @@
 
 import asyncio
 import logging
-from typing import List
+from typing import List, Optional
 
 import typer
 
@@ -104,6 +104,11 @@ def scan_text(
         None, help="List of texts to extract text from"
     ),
     operations: str = typer.Option("scan", help="Operation to perform"),
+    locale: Optional[List[str]] = typer.Option(
+        None,
+        "--locale",
+        help="Regex locale to enable, for example de.",
+    ),
 ):
     """
     Scan texts for PII.
@@ -123,7 +128,7 @@ def scan_text(
     logging.basicConfig(level=logging.INFO)
     # Convert comma-separated string operations to a list of OperationType objects
     operation_list = [OperationType(op.strip()) for op in operations.split(",")]
-    text_client = DataFog(operations=operation_list)
+    text_client = DataFog(operations=operation_list, locales=locale)
     try:
         results = text_client.run_text_pipeline_sync(str_list=str_list)
         typer.echo(f"Text Pipeline Results: {results}")
@@ -316,7 +321,14 @@ def list_entities():
 
 
 @app.command()
-def redact_text(text: str = typer.Argument(None, help="Text to redact")):
+def redact_text(
+    text: str = typer.Argument(None, help="Text to redact"),
+    locale: Optional[List[str]] = typer.Option(
+        None,
+        "--locale",
+        help="Regex locale to enable, for example de.",
+    ),
+):
     """
     Redact PII in text.
 
@@ -329,7 +341,12 @@ def redact_text(text: str = typer.Argument(None, help="Text to redact")):
         typer.echo("No text provided to redact.")
         raise typer.Exit(code=1)
 
-    result = scan_and_redact(text=text, engine="smart", strategy="token")
+    result = scan_and_redact(
+        text=text,
+        engine="smart",
+        strategy="token",
+        locales=locale,
+    )
     typer.echo(result.redacted_text)
 
     try:
@@ -346,7 +363,14 @@ def redact_text(text: str = typer.Argument(None, help="Text to redact")):
 
 
 @app.command()
-def replace_text(text: str = typer.Argument(None, help="Text to replace PII")):
+def replace_text(
+    text: str = typer.Argument(None, help="Text to replace PII"),
+    locale: Optional[List[str]] = typer.Option(
+        None,
+        "--locale",
+        help="Regex locale to enable, for example de.",
+    ),
+):
     """
     Replace PII in text with anonymized values.
 
@@ -359,7 +383,12 @@ def replace_text(text: str = typer.Argument(None, help="Text to replace PII")):
         typer.echo("No text provided to replace PII.")
         raise typer.Exit(code=1)
 
-    result = scan_and_redact(text=text, engine="smart", strategy="pseudonymize")
+    result = scan_and_redact(
+        text=text,
+        engine="smart",
+        strategy="pseudonymize",
+        locales=locale,
+    )
     typer.echo(result.redacted_text)
 
     try:
@@ -379,6 +408,11 @@ def replace_text(text: str = typer.Argument(None, help="Text to replace PII")):
 def hash_text(
     text: str = typer.Argument(None, help="Text to hash PII"),
     hash_type: HashType = typer.Option(HashType.SHA256, help="Hash algorithm to use"),
+    locale: Optional[List[str]] = typer.Option(
+        None,
+        "--locale",
+        help="Regex locale to enable, for example de.",
+    ),
 ):
     """
     Choose from SHA256, MD5, or SHA3-256 algorithms to hash detected PII in text.
@@ -395,7 +429,12 @@ def hash_text(
 
     # HashType is retained for backward-compatible CLI signature.
     _ = hash_type
-    result = scan_and_redact(text=text, engine="smart", strategy="hash")
+    result = scan_and_redact(
+        text=text,
+        engine="smart",
+        strategy="hash",
+        locales=locale,
+    )
     typer.echo(result.redacted_text)
 
     try:
diff --git a/datafog/core.py b/datafog/core.py
index f4e17850..aa207f6d 100644
--- a/datafog/core.py
+++ b/datafog/core.py
@@ -9,6 +9,7 @@
 
 from datafog.engine import scan, scan_and_redact
 from datafog.models.anonymizer import AnonymizerType
+from datafog.processing.text_processing.regex_annotator import RegexAnnotator
 
 # Engine types as constants
 REGEX_ENGINE = "regex"
@@ -16,7 +17,10 @@
 AUTO_ENGINE = "auto"
 
 
-def detect_pii(text: str) -> Dict[str, List[str]]:
+def detect_pii(
+    text: str,
+    locales: List[str] | None = None,
+) -> Dict[str, List[str]]:
     """
     Simple PII detection using lightweight regex engine.
 
@@ -37,7 +41,7 @@ def detect_pii(text: str) -> Dict[str, List[str]]:
 
     try:
         # Use engine boundary for canonical scan behavior.
-        scan_result = scan(text=text, engine=REGEX_ENGINE)
+        scan_result = scan(text=text, engine=REGEX_ENGINE, locales=locales)
         pii_dict: Dict[str, List[str]] = {}
         for entity in scan_result.entities:
             if not entity.text.strip():
@@ -81,7 +85,11 @@ def detect_pii(text: str) -> Dict[str, List[str]]:
         ) from e
 
 
-def anonymize_text(text: str, method: Union[str, AnonymizerType] = "redact") -> str:
+def anonymize_text(
+    text: str,
+    method: Union[str, AnonymizerType] = "redact",
+    locales: List[str] | None = None,
+) -> str:
     """
     Simple text anonymization using lightweight regex engine.
 
@@ -120,6 +128,7 @@ def anonymize_text(text: str, method: Union[str, AnonymizerType] = "redact") ->
             text=text,
             engine=REGEX_ENGINE,
             strategy=strategy_map[method],
+            locales=locales,
         )
 
         try:
@@ -155,7 +164,9 @@ def anonymize_text(text: str, method: Union[str, AnonymizerType] = "redact") ->
 
 
 def scan_text(
-    text: str, return_entities: bool = False
+    text: str,
+    return_entities: bool = False,
+    locales: List[str] | None = None,
 ) -> Union[bool, Dict[str, List[str]]]:
     """
     Quick scan to check if text contains any PII.
@@ -180,7 +191,7 @@ def scan_text(
 
     _start = _time.monotonic()
 
-    entities = detect_pii(text)
+    entities = detect_pii(text, locales=locales)
 
     result = entities if return_entities else len(entities) > 0
 
@@ -200,7 +211,7 @@ def scan_text(
     return result
 
 
-def get_supported_entities() -> List[str]:
+def get_supported_entities(locales: List[str] | None = None) -> List[str]:
     """
     Get list of PII entity types supported by the regex engine.
 
@@ -210,17 +221,15 @@ def get_supported_entities() -> List[str]:
     Example:
         >>> entities = get_supported_entities()
         >>> print(entities)
-        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DOB', 'ZIP']
+        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DE_VAT_ID', 'DE_IBAN', 'DATE', 'ZIP_CODE']
     """
-    result = [
-        "EMAIL",
-        "PHONE",
-        "SSN",
-        "CREDIT_CARD",
-        "IP_ADDRESS",
-        "DATE",
-        "ZIP_CODE",
-    ]
+    annotator = RegexAnnotator(locales=locales)
+    legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"}
+    result = []
+    for label in annotator.active_labels:
+        canonical = legacy_map.get(label, label)
+        if canonical not in result:
+            result.append(canonical)
 
     try:
         from datafog.telemetry import track_function_call
diff --git a/datafog/engine.py b/datafog/engine.py
index 1a94e634..250cd6fe 100644
--- a/datafog/engine.py
+++ b/datafog/engine.py
@@ -31,6 +31,13 @@
     "SSN",
     "CREDIT_CARD",
     "IP_ADDRESS",
+    "DE_VAT_ID",
+    "DE_IBAN",
+    "DE_TAX_ID",
+    "DE_SOCIAL_SECURITY_NUMBER",
+    "DE_POSTAL_CODE",
+    "DE_PASSPORT_NUMBER",
+    "DE_RESIDENCE_PERMIT_NUMBER",
     "DATE",
     "ZIP_CODE",
     "PERSON",
@@ -41,6 +48,20 @@
 
 NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"}
 
+ENTITY_TYPE_PRIORITY = {
+    "DE_IBAN": 100,
+    "DE_VAT_ID": 100,
+    "DE_TAX_ID": 100,
+    "DE_SOCIAL_SECURITY_NUMBER": 100,
+    "DE_POSTAL_CODE": 100,
+    "DE_PASSPORT_NUMBER": 100,
+    "DE_RESIDENCE_PERMIT_NUMBER": 100,
+    "CREDIT_CARD": 90,
+    "IP_ADDRESS": 80,
+    "SSN": 70,
+    "PHONE": 60,
+}
+
 
 @dataclass(frozen=True)
 class _UnavailableAnnotator:
@@ -131,8 +152,40 @@ def _entities_from_dict(
     return entities
 
 
-def _regex_entities(text: str) -> list[Entity]:
-    annotator = RegexAnnotator()
+def _entity_length(entity: Entity) -> int:
+    return max(entity.end - entity.start, 0)
+
+
+def _entities_overlap(left: Entity, right: Entity) -> bool:
+    if left.start < 0 or right.start < 0:
+        return False
+    return left.start < right.end and right.start < left.end
+
+
+def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]:
+    selected: list[Entity] = []
+    for entity in sorted(
+        entities,
+        key=lambda item: (
+            -_entity_length(item),
+            -ENTITY_TYPE_PRIORITY.get(item.type, 0),
+            item.start,
+            item.end,
+            item.type,
+        ),
+    ):
+        if any(_entities_overlap(entity, kept) for kept in selected):
+            continue
+        selected.append(entity)
+    return sorted(selected, key=lambda item: (item.start, item.end, item.type))
+
+
+def _regex_entities(
+    text: str,
+    entity_types: Optional[list[str]] = None,
+    locales: Optional[list[str]] = None,
+) -> list[Entity]:
+    annotator = RegexAnnotator(locales=locales, enabled_labels=entity_types)
     _, structured = annotator.annotate_with_spans(text)
     entities: list[Entity] = []
     for span in structured.spans:
@@ -148,7 +201,7 @@ def _regex_entities(text: str) -> list[Entity]:
                 engine="regex",
             )
         )
-    return entities
+    return _suppress_overlapping_entities(entities)
 
 
 def _spacy_entities(text: str) -> list[Entity]:
@@ -235,6 +288,7 @@ def scan(
     text: str,
     engine: str = "smart",
     entity_types: Optional[list[str]] = None,
+    locales: Optional[list[str]] = None,
 ) -> ScanResult:
     """Scan text for PII entities."""
     if not isinstance(text, str):
@@ -243,7 +297,11 @@ def scan(
     if engine not in {"regex", "spacy", "gliner", "smart"}:
         raise ValueError("engine must be one of: regex, spacy, gliner, smart")
 
-    regex_entities = _regex_entities(text)
+    regex_entities = _regex_entities(
+        text,
+        entity_types=entity_types,
+        locales=locales,
+    )
 
     if engine == "regex":
         filtered = _filter_entity_types(regex_entities, entity_types)
@@ -378,7 +436,13 @@ def scan_and_redact(
     engine: str = "smart",
     entity_types: Optional[list[str]] = None,
     strategy: str = "token",
+    locales: Optional[list[str]] = None,
 ) -> RedactResult:
     """Convenience wrapper: scan then redact."""
-    scan_result = scan(text=text, engine=engine, entity_types=entity_types)
+    scan_result = scan(
+        text=text,
+        engine=engine,
+        entity_types=entity_types,
+        locales=locales,
+    )
     return redact(text=text, entities=scan_result.entities, strategy=strategy)
diff --git a/datafog/main.py b/datafog/main.py
index 31ac22e5..a8b1bcba 100644
--- a/datafog/main.py
+++ b/datafog/main.py
@@ -39,8 +39,10 @@ def __init__(
         operations: List[OperationType] = [OperationType.SCAN],
         hash_type: HashType = HashType.SHA256,
         anonymizer_type: AnonymizerType = AnonymizerType.REPLACE,
+        locales: List[str] | None = None,
     ):
-        self.regex_annotator = RegexAnnotator()
+        self.locales = locales
+        self.regex_annotator = RegexAnnotator(locales=locales)
         normalized_ops: List[OperationType] = []
         for op in operations:
             if isinstance(op, OperationType):
@@ -181,7 +183,7 @@ def detect(self, text: str) -> dict:
 
         _start = _time.monotonic()
 
-        scan_result = scan(text=text, engine="regex")
+        scan_result = scan(text=text, engine="regex", locales=self.locales)
         result = {label: [] for label in RegexAnnotator.LABELS}
         legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"}
         for entity in scan_result.entities:
@@ -246,6 +248,7 @@ def process(
                 text=text,
                 engine="regex",
                 strategy=strategy,
+                locales=self.locales,
             )
             result["anonymized"] = redact_result.redacted_text
 
@@ -288,8 +291,8 @@ class TextPIIAnnotator:
         regex_annotator: RegexAnnotator instance for text annotation.
     """
 
-    def __init__(self):
-        self.regex_annotator = RegexAnnotator()
+    def __init__(self, locales: List[str] | None = None):
+        self.regex_annotator = RegexAnnotator(locales=locales)
 
     def run(self, text, output_path=None):
         """
diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
index a843a8d8..d478b43d 100644
--- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py
+++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
@@ -1,5 +1,6 @@
 import re
-from typing import Dict, List, Pattern, Tuple
+from collections.abc import Iterable
+from typing import Dict, List, Match, Pattern, Tuple
 
 from pydantic import BaseModel
 
@@ -27,12 +28,36 @@ class RegexAnnotator:
     performance, targeting ≤ 20 µs / kB on a MacBook M-series.
     """
 
-    # Labels for PII entities
-    LABELS = ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"]
+    BASE_LABELS = ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"]
+    GERMAN_LABELS = [
+        "DE_VAT_ID",
+        "DE_IBAN",
+        "DE_TAX_ID",
+        "DE_SOCIAL_SECURITY_NUMBER",
+        "DE_POSTAL_CODE",
+        "DE_PASSPORT_NUMBER",
+        "DE_RESIDENCE_PERMIT_NUMBER",
+    ]
+    DEFAULT_LOCALIZED_LABELS = ["DE_VAT_ID", "DE_IBAN"]
+    LABELS = BASE_LABELS + GERMAN_LABELS
+    DEFAULT_LABELS = BASE_LABELS + DEFAULT_LOCALIZED_LABELS
+    SUPPORTED_LOCALES = {"de", "de-de", "de_de"}
+    LOCALE_LABELS = {
+        "de": GERMAN_LABELS,
+        "de-de": GERMAN_LABELS,
+        "de_de": GERMAN_LABELS,
+    }
+
+    def __init__(
+        self,
+        locales: str | Iterable[str] | None = None,
+        enabled_labels: Iterable[str] | None = None,
+    ):
+        self.locales = self._normalize_locales(locales)
+        self.active_labels = self._resolve_active_labels(enabled_labels)
 
-    def __init__(self):
         # Compile all patterns once at initialization
-        self.patterns: Dict[str, Pattern] = {
+        all_patterns: Dict[str, Pattern] = {
             # Email pattern - RFC 5322 subset
             # Intentionally permissive to favor false positives over false negatives
             # Allows for multiple dots, special characters in local part, and subdomains
@@ -175,12 +200,171 @@ def __init__(self):
                 """,
                 re.IGNORECASE | re.MULTILINE | re.VERBOSE,
             ),
+            # German VAT ID (USt-IdNr) - DE followed by 9 digits.
+            "DE_VAT_ID": re.compile(
+                r"""
+                (?<![A-Za-z0-9])
+                DE
+                [\s-]?
+                \d{9}
+                (?![A-Za-z0-9])
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German IBAN - DE followed by 20 digits, often grouped.
+            "DE_IBAN": re.compile(
+                r"""
+                (?<![A-Za-z0-9])
+                DE
+                \d{2}
+                (?:\s?\d{4}){4}
+                \s?\d{2}
+                (?![A-Za-z0-9])
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German Tax ID (Steuer-ID) - context required to avoid generic IDs.
+            "DE_TAX_ID": re.compile(
+                r"""
+                (?:
+                    Steuer[\s-]?ID |
+                    Steueridentifikationsnummer |
+                    Identifikationsnummer |
+                    IdNr\.? |
+                    Tax[\s-]?ID
+                )
+                \s*[:#-]?\s*
+                (?P<value>
+                    (?<![A-Za-z0-9])
+                    (?:\d{11}|\d{2}\s?\d{3}\s?\d{3}\s?\d{3})
+                    (?![A-Za-z0-9])
+                )
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German pension insurance number (Rentenversicherungsnummer).
+            "DE_SOCIAL_SECURITY_NUMBER": re.compile(
+                r"""
+                (?:
+                    Rentenversicherungsnummer |
+                    Sozialversicherungsnummer |
+                    RVNR |
+                    SVNR
+                )
+                \s*[:#-]?\s*
+                (?P<value>
+                    (?<![A-Za-z0-9])
+                    \d{2}
+                    \s?
+                    \d{6}
+                    \s?
+                    [A-Z]
+                    \s?
+                    \d{3}
+                    (?![A-Za-z0-9])
+                )
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German postal code with an explicit German/PLZ prefix.
+            "DE_POSTAL_CODE": re.compile(
+                r"""
+                (?<![A-Za-z0-9])
+                (?P<value>
+                    (?:
+                        PLZ[\s:-]? |
+                        DE[\s-] |
+                        D[\s-]
+                    )
+                    \d{5}
+                )
+                (?![A-Za-z0-9])
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German passport number - context required; bare A12345678 is too broad.
+            "DE_PASSPORT_NUMBER": re.compile(
+                r"""
+                (?:
+                    Passnummer |
+                    Reisepass(?:nummer)? |
+                    Passport(?:\s+No\.?|\s+Number)?
+                )
+                \s*[:#-]?\s*
+                (?P<value>
+                    (?<![A-Za-z0-9])
+                    [A-Z]
+                    \d{8}
+                    (?![A-Za-z0-9])
+                )
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
+            # German residence permit number - context required; AT IDs are common.
+            "DE_RESIDENCE_PERMIT_NUMBER": re.compile(
+                r"""
+                (?:
+                    Aufenthaltstitel |
+                    Aufenthaltserlaubnis |
+                    Residence\s+Permit |
+                    eAT
+                )
+                \s*[:#-]?\s*
+                (?P<value>
+                    (?<![A-Za-z0-9])
+                    AT
+                    \d{7}
+                    (?![A-Za-z0-9])
+                )
+                """,
+                re.IGNORECASE | re.MULTILINE | re.VERBOSE,
+            ),
         }
+        self.patterns = {
+            label: pattern
+            for label, pattern in all_patterns.items()
+            if label in self.active_labels
+        }
+
+    @classmethod
+    def _normalize_locales(cls, locales: str | Iterable[str] | None) -> tuple[str, ...]:
+        if locales is None:
+            return ()
+        if isinstance(locales, str):
+            locales = [locales]
+        normalized = []
+        for locale in locales:
+            value = locale.strip().lower()
+            if not value:
+                continue
+            if value not in cls.SUPPORTED_LOCALES:
+                allowed = ", ".join(sorted(cls.SUPPORTED_LOCALES))
+                raise ValueError(f"locale must be one of: {allowed}")
+            normalized.append(value)
+        return tuple(dict.fromkeys(normalized))
+
+    def _resolve_active_labels(self, enabled_labels: Iterable[str] | None) -> list[str]:
+        active = set(self.DEFAULT_LABELS)
+        for locale in self.locales:
+            active.update(self.LOCALE_LABELS[locale])
+        if enabled_labels is not None:
+            active.update(label.strip().upper() for label in enabled_labels)
+        return [label for label in self.LABELS if label in active]
+
+    @staticmethod
+    def _match_text(match: Match[str]) -> str:
+        return match.groupdict().get("value") or match.group()
+
+    @staticmethod
+    def _match_span(match: Match[str]) -> tuple[int, int]:
+        if "value" in match.groupdict() and match.group("value") is not None:
+            return match.start("value"), match.end("value")
+        return match.start(), match.end()
 
     @classmethod
-    def create(cls) -> "RegexAnnotator":
+    def create(cls, **kwargs) -> "RegexAnnotator":
         """Factory method to create a new RegexAnnotator instance."""
-        return cls()
+        return cls(**kwargs)
 
     def annotate(self, text: str) -> Dict[str, List[str]]:
         """Annotate text with PII entities using regex patterns.
@@ -200,7 +384,7 @@ def annotate(self, text: str) -> Dict[str, List[str]]:
         # Process with each pattern
         for label, pattern in self.patterns.items():
             for match in pattern.finditer(text):
-                result[label].append(match.group())
+                result[label].append(self._match_text(match))
 
         return result
 
@@ -225,11 +409,12 @@ def annotate_with_spans(
 
         for label, pattern in self.patterns.items():
             for match in pattern.finditer(text):
+                start, end = self._match_span(match)
                 span = Span(
                     label=label,
-                    start=match.start(),
-                    end=match.end(),
-                    text=match.group(),
+                    start=start,
+                    end=end,
+                    text=self._match_text(match),
                 )
                 spans_by_label[label].append(span)
                 all_spans.append(span)
diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py
index 0956256f..1057eb6a 100644
--- a/datafog/services/text_service.py
+++ b/datafog/services/text_service.py
@@ -43,6 +43,7 @@ def __init__(
         text_chunk_length: int = 1000,
         engine: str = "regex",
         gliner_model: str = "urchade/gliner_multi_pii-v1",
+        locales: List[str] | None = None,
     ):
         """
         Initialize the TextService with specified chunk length and annotation engine.
@@ -56,6 +57,8 @@ def __init__(
                 - "auto": Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities found
                 - "smart": Try RegexAnnotator → GLiNER → SpaCy cascade (requires nlp-advanced extra)
             gliner_model: GLiNER model name to use when engine is "gliner" or "smart"
+            locales: Optional locale tags for regex detection. Use ["de"] to enable
+                broader German structured identifiers.
 
         Raises:
             AssertionError: If an invalid engine type is provided
@@ -65,6 +68,7 @@ def __init__(
         self.engine = engine
         self.text_chunk_length = text_chunk_length
         self.gliner_model = gliner_model
+        self.locales = locales
 
         # Lazy initialization - annotators created only when needed
         self._regex_annotator = None
@@ -102,7 +106,7 @@ def regex_annotator(self):
                 RegexAnnotator,
             )
 
-            self._regex_annotator = RegexAnnotator()
+            self._regex_annotator = RegexAnnotator(locales=self.locales)
         return self._regex_annotator
 
     @property
diff --git a/docs/cli.rst b/docs/cli.rst
index ec452f53..9742887d 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -19,6 +19,17 @@ for existing users, but they are optional:
 Spark/distributed workflows are Python SDK surfaces rather than first-path CLI
 commands. Install ``datafog[distributed]`` when using ``SparkService``.
 
+German locale support
+---------------------
+
+German VAT IDs and German IBANs are detected by the default regex path. Broader
+German identifiers are opt-in through ``--locale de`` on the core text commands:
+
+.. code-block:: bash
+
+   datafog scan-text "Steuer-ID 12345678901" --locale de
+   datafog redact-text "Passnummer C12345678" --locale de
+
 Definitions
 -----------
 .. automodule:: datafog.client
diff --git a/docs/getting-started.rst b/docs/getting-started.rst
index 69637611..bfc42290 100644
--- a/docs/getting-started.rst
+++ b/docs/getting-started.rst
@@ -80,6 +80,34 @@ Agent-oriented helpers use the same lightweight text path:
    safe_output = datafog.filter_output(output, engine="regex")
    print(safe_output.redacted_text)
 
+German Structured PII
+=====================
+
+The core regex engine includes German VAT IDs and German IBANs by default
+because they carry strong country-code structure:
+
+.. code-block:: python
+
+   import datafog
+
+   result = datafog.scan("USt-IdNr DE 123456789", engine="regex")
+   print([(entity.type, entity.text) for entity in result.entities])
+
+Broader German identifiers such as ``DE_TAX_ID``,
+``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
+``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER`` require explicit
+German locale selection or explicit ``entity_types`` filtering. This keeps
+ordinary ticket, SKU, order, and invoice IDs from becoming default-on false
+positives.
+
+.. code-block:: python
+
+   text = "Steuer-ID 12345678901 liegt vor."
+
+   print(datafog.scan(text, engine="regex").entities)
+   print(datafog.scan(text, engine="regex", locales=["de"]).entities)
+   print(datafog.scan(text, engine="regex", entity_types=["DE_TAX_ID"]).entities)
+
 CLI Usage
 =========
 
@@ -91,6 +119,7 @@ The CLI core path is text-first:
    datafog redact-text "Contact jane@example.com"
    datafog replace-text "Contact jane@example.com"
    datafog hash-text "Contact jane@example.com"
+   datafog redact-text "Steuer-ID 12345678901" --locale de
 
 Image commands are optional. Install ``datafog[ocr]`` for local OCR and
 ``datafog[web,ocr]`` when the CLI needs to download image inputs.
diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst
index a1adec5d..b1093de0 100644
--- a/docs/python-sdk.rst
+++ b/docs/python-sdk.rst
@@ -27,6 +27,39 @@ available for existing users. ``TextService(engine="regex")`` is the
 dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark
 surfaces require their explicit extras.
 
+German locale coverage
+----------------------
+
+DataFog 4.5 includes regex-only German structured PII support without adding
+dependencies. German VAT IDs and German IBANs are active in the default regex
+path. Broader German-only identifiers are opt-in because their raw shapes are
+common in ordinary product, ticket, invoice, and order data.
+
+Use ``locales=["de"]`` to enable the broader German set:
+
+.. code-block:: python
+
+   import datafog
+
+   text = "Steuer-ID 12345678901 liegt vor."
+   result = datafog.scan(text, engine="regex", locales=["de"])
+   print([(entity.type, entity.text) for entity in result.entities])
+
+You can also request one German entity type directly:
+
+.. code-block:: python
+
+   result = datafog.scan(
+       "Steuer-ID 12345678901 liegt vor.",
+       engine="regex",
+       entity_types=["DE_TAX_ID"],
+   )
+
+The opt-in German set currently covers ``DE_TAX_ID``,
+``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
+``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER``. The default set
+also covers ``DE_VAT_ID`` and ``DE_IBAN``.
+
 Optional services
 -----------------
 
diff --git a/tests/corpus/structured_pii.json b/tests/corpus/structured_pii.json
index 672e7483..9c13dda1 100644
--- a/tests/corpus/structured_pii.json
+++ b/tests/corpus/structured_pii.json
@@ -733,5 +733,29 @@
         "end": 5
       }
     ]
+  },
+  {
+    "id": "de-vat-id-default",
+    "input": "USt-IdNr DE 123456789 ist gesetzt.",
+    "expected_entities": [
+      {
+        "type": "DE_VAT_ID",
+        "text": "DE 123456789",
+        "start": 9,
+        "end": 21
+      }
+    ]
+  },
+  {
+    "id": "de-iban-default",
+    "input": "IBAN DE44 5001 0517 5407 3249 31 ist gueltig.",
+    "expected_entities": [
+      {
+        "type": "DE_IBAN",
+        "text": "DE44 5001 0517 5407 3249 31",
+        "start": 5,
+        "end": 32
+      }
+    ]
   }
 ]
diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py
new file mode 100644
index 00000000..f24d605c
--- /dev/null
+++ b/tests/test_de_pii_regex.py
@@ -0,0 +1,146 @@
+import pytest
+
+import datafog
+from datafog.core import get_supported_entities
+from datafog.engine import scan, scan_and_redact
+from datafog.processing.text_processing.regex_annotator import RegexAnnotator
+from datafog.services.text_service import TextService
+
+
+@pytest.mark.parametrize(
+    "label,text,expected",
+    [
+        ("DE_VAT_ID", "USt-IdNr DE 123456789 ist gesetzt.", "DE 123456789"),
+        ("DE_VAT_ID", "USt-IdNr DE-123456789 liegt vor.", "DE-123456789"),
+        (
+            "DE_IBAN",
+            "IBAN DE44500105175407324931 ist gueltig.",
+            "DE44500105175407324931",
+        ),
+        (
+            "DE_IBAN",
+            "IBAN DE44 5001 0517 5407 3249 31 ist gueltig.",
+            "DE44 5001 0517 5407 3249 31",
+        ),
+    ],
+)
+def test_high_specificity_german_regex_default_cases(
+    label: str, text: str, expected: str
+) -> None:
+    annotator = RegexAnnotator()
+    result = annotator.annotate(text)
+    assert expected in result[label]
+
+
+@pytest.mark.parametrize(
+    "label,text,expected",
+    [
+        ("DE_TAX_ID", "Steuer-ID 12345678901 liegt vor.", "12345678901"),
+        ("DE_TAX_ID", "Steuer-ID 12 345 678 901 ist gesetzt.", "12 345 678 901"),
+        (
+            "DE_SOCIAL_SECURITY_NUMBER",
+            "Rentenversicherungsnummer 65150804A123 liegt vor.",
+            "65150804A123",
+        ),
+        (
+            "DE_SOCIAL_SECURITY_NUMBER",
+            "Rentenversicherungsnummer 65 150804 A123 liegt vor.",
+            "65 150804 A123",
+        ),
+        ("DE_POSTAL_CODE", "PLZ10115 Berlin.", "PLZ10115"),
+        ("DE_POSTAL_CODE", "DE-10115 Berlin.", "DE-10115"),
+        ("DE_PASSPORT_NUMBER", "Passnummer C12345678 wurde geprueft.", "C12345678"),
+        (
+            "DE_RESIDENCE_PERMIT_NUMBER",
+            "Aufenthaltstitel AT1234567 gueltig.",
+            "AT1234567",
+        ),
+    ],
+)
+def test_broad_german_regex_cases_require_german_locale(
+    label: str, text: str, expected: str
+) -> None:
+    default_result = RegexAnnotator().annotate(text)
+    assert expected not in default_result[label]
+
+    german_result = RegexAnnotator(locales=["de"]).annotate(text)
+    assert expected in german_result[label]
+
+
+@pytest.mark.parametrize(
+    "label,text",
+    [
+        ("DE_VAT_ID", "USt-IdNr DE12345678 liegt vor."),
+        ("DE_VAT_ID", "USt-IdNr DE1234567890 liegt vor."),
+        ("DE_VAT_ID", "USt-IdNr DE123456789A should not prefix-match."),
+        ("DE_IBAN", "IBAN DE4450010517540732493 ist gueltig."),
+        ("DE_IBAN", "IBAN DE44 5001 0517 5407 3249 3X ist gueltig."),
+        ("DE_TAX_ID", "Invoice 12345678901 was paid."),
+        ("DE_SOCIAL_SECURITY_NUMBER", "Build 65150804A123 failed."),
+        ("DE_POSTAL_CODE", "SKU D12345 is not a postcode."),
+        ("DE_POSTAL_CODE", "Release DE12345 shipped."),
+        ("DE_PASSPORT_NUMBER", "Ticket A12345678 was shipped."),
+        ("DE_RESIDENCE_PERMIT_NUMBER", "Order AT1234567 is internal."),
+    ],
+)
+def test_german_regex_false_positive_guards(label: str, text: str) -> None:
+    result = RegexAnnotator(locales=["de"]).annotate(text)
+    assert not result[label]
+
+
+def test_scan_locale_and_explicit_entity_type_activation() -> None:
+    text = "Steuer-ID 12345678901 liegt vor."
+
+    default_result = scan(text, engine="regex")
+    assert "DE_TAX_ID" not in {entity.type for entity in default_result.entities}
+
+    locale_result = scan(text, engine="regex", locales=["de"])
+    assert [
+        entity.text for entity in locale_result.entities if entity.type == "DE_TAX_ID"
+    ] == ["12345678901"]
+
+    explicit_result = scan(text, engine="regex", entity_types=["DE_TAX_ID"])
+    assert [(entity.type, entity.text) for entity in explicit_result.entities] == [
+        ("DE_TAX_ID", "12345678901")
+    ]
+
+
+def test_redaction_and_service_locale_support() -> None:
+    text = "Passnummer C12345678 wurde geprueft."
+
+    default_redaction = scan_and_redact(text, engine="regex")
+    assert default_redaction.redacted_text == text
+
+    locale_redaction = scan_and_redact(text, engine="regex", locales=["de"])
+    assert "[DE_PASSPORT_NUMBER_1]" in locale_redaction.redacted_text
+
+    service_result = TextService(locales=["de"]).annotate_text_sync(text)
+    assert service_result["DE_PASSPORT_NUMBER"] == ["C12345678"]
+
+
+def test_german_vat_redaction_suppresses_inner_generic_ssn_match() -> None:
+    text = "USt-IdNr DE123456789 ist gesetzt."
+
+    scan_result = scan(text, engine="regex")
+    assert [(entity.type, entity.text) for entity in scan_result.entities] == [
+        ("DE_VAT_ID", "DE123456789")
+    ]
+
+    redaction = scan_and_redact(text, engine="regex")
+    assert redaction.redacted_text == "USt-IdNr [DE_VAT_ID_1] ist gesetzt."
+
+
+def test_top_level_helpers_and_supported_entities_respect_locale() -> None:
+    default_entities = get_supported_entities()
+    assert "DE_VAT_ID" in default_entities
+    assert "DE_IBAN" in default_entities
+    assert "DE_TAX_ID" not in default_entities
+
+    german_entities = get_supported_entities(locales=["de"])
+    assert "DE_TAX_ID" in german_entities
+    assert "DE_RESIDENCE_PERMIT_NUMBER" in german_entities
+
+    result = datafog.scan("Aufenthaltstitel AT1234567 gueltig.", locales=["de"])
+    assert [(entity.type, entity.text) for entity in result.entities] == [
+        ("DE_RESIDENCE_PERMIT_NUMBER", "AT1234567")
+    ]
diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py
index 852a7937..6777d9dd 100644
--- a/tests/test_detection_accuracy.py
+++ b/tests/test_detection_accuracy.py
@@ -22,6 +22,8 @@
     "SSN",
     "CREDIT_CARD",
     "IP_ADDRESS",
+    "DE_VAT_ID",
+    "DE_IBAN",
     "DATE",
     "ZIP_CODE",
 }
diff --git a/tests/test_regex_annotator.py b/tests/test_regex_annotator.py
index 5916bfae..fc63dc2f 100644
--- a/tests/test_regex_annotator.py
+++ b/tests/test_regex_annotator.py
@@ -40,9 +40,9 @@ def test_regex_annotator_initialization():
     """Test that the RegexAnnotator can be initialized."""
     annotator = RegexAnnotator()
     assert annotator is not None
-    assert (
-        len(annotator.LABELS) == 7
-    )  # EMAIL, PHONE, SSN, CREDIT_CARD, IP_ADDRESS, DOB, ZIP
+    assert set(RegexAnnotator.BASE_LABELS).issubset(annotator.LABELS)
+    assert {"DE_VAT_ID", "DE_IBAN"}.issubset(annotator.active_labels)
+    assert "DE_TAX_ID" not in annotator.active_labels
 
 
 def test_regex_annotator_create_method():

From 44a1ae145248c2dd5a9a078d69163464d846f76e Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 20:45:32 +0200
Subject: [PATCH 54/70] ci: validate python 313 optional profiles

---
 .github/workflows/ci.yml       | 25 ++++++++++++++++---------
 README.md                      |  9 ++++++---
 docs/contributing.rst          |  8 +++++---
 docs/optional-surfaces.rst     | 34 ++++++++++++++++++++++++++++++++++
 tests/test_install_profiles.py |  2 ++
 5 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3020390..dfcae0f4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,14 +31,6 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
         install-profile: ["core", "nlp", "nlp-advanced"]
-        exclude:
-          # v4.4.0 claims Python 3.13 support for core + CLI first.
-          # Optional heavyweight profiles remain validated separately before
-          # we advertise Python 3.13 support for them.
-          - python-version: "3.13"
-            install-profile: "nlp"
-          - python-version: "3.13"
-            install-profile: "nlp-advanced"
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -159,6 +151,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        python-version: ["3.11"]
         install-profile:
           - core
           - cli
@@ -167,18 +160,31 @@ jobs:
           - ocr
           - distributed
           - web
+        include:
+          - python-version: "3.13"
+            install-profile: nlp
+          - python-version: "3.13"
+            install-profile: nlp-advanced
+          - python-version: "3.13"
+            install-profile: ocr
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
 
+      - name: Install Tesseract OCR
+        if: matrix.install-profile == 'ocr'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr libtesseract-dev
+
       - name: Install dependencies (core)
         if: matrix.install-profile == 'core'
         run: |
@@ -192,6 +198,7 @@ jobs:
       - name: Run install profile smoke test
         env:
           DATAFOG_INSTALL_PROFILE: ${{ matrix.install-profile }}
+          DATAFOG_REQUIRE_TESSERACT: ${{ matrix.install-profile == 'ocr' && '1' || '' }}
         run: |
           pytest tests/test_install_profiles.py -q
 
diff --git a/README.md b/README.md
index adc7ff58..6f080e30 100644
--- a/README.md
+++ b/README.md
@@ -39,9 +39,10 @@ pip install datafog[distributed]
 pip install datafog[all]
 ```
 
-Python 3.13 support is certified for the core SDK and CLI. Optional extras such
-as `nlp`, `nlp-advanced`, `ocr`, `distributed`, and `all` are available but not
-yet certified on Python 3.13.
+Python 3.13 support is certified for the core SDK, CLI, `nlp`,
+`nlp-advanced`, and `ocr` install profiles. Donut OCR still requires a model
+that is available locally before runtime use. `distributed` and `all` are not
+newly certified on Python 3.13 in the 4.5 line.
 
 ## Quick Start
 
@@ -140,6 +141,8 @@ scan/redact helpers, or guardrail helpers.
   - Install `datafog[ocr]` for local image OCR helpers.
   - URL-based image downloading also needs `datafog[web,ocr]`.
   - Tesseract usage requires the system `tesseract` binary.
+  - Python 3.13 is validated for the OCR install profile, Pillow,
+    pytesseract, and system Tesseract smoke checks.
   - Donut OCR requires `datafog[nlp-advanced,ocr]` and a model already available
     locally.
 - Spark:
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 6037d926..1565437e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -23,9 +23,10 @@ Python Environments
 ===================
 
 DataFog currently declares support for Python ``>=3.10,<3.14``. The CI matrix
-tests core installs on Python 3.10, 3.11, 3.12, and 3.13. Optional NLP and
-NLP-advanced profiles are tested on Python 3.10, 3.11, and 3.12; Python 3.13
-optional-profile validation is tracked separately for 4.5.
+tests core, NLP, and NLP-advanced installs on Python 3.10, 3.11, 3.12, and
+3.13. OCR profile smoke checks also run on Python 3.13 with system Tesseract
+installed. Distributed and all-profile Python 3.13 validation remain outside
+the 4.5 support claim.
 
 Create one virtual environment per Python version when you need to compare
 profiles locally:
@@ -162,6 +163,7 @@ or install behavior:
    DATAFOG_INSTALL_PROFILE=nlp python -m pytest tests/test_install_profiles.py -q
    DATAFOG_INSTALL_PROFILE=nlp-advanced python -m pytest tests/test_install_profiles.py -q
    DATAFOG_INSTALL_PROFILE=ocr python -m pytest tests/test_install_profiles.py -q
+   DATAFOG_INSTALL_PROFILE=ocr DATAFOG_REQUIRE_TESSERACT=1 python -m pytest tests/test_install_profiles.py -q
    DATAFOG_INSTALL_PROFILE=distributed python -m pytest tests/test_install_profiles.py -q
    DATAFOG_INSTALL_PROFILE=web python -m pytest tests/test_install_profiles.py -q
 
diff --git a/docs/optional-surfaces.rst b/docs/optional-surfaces.rst
index aa9b725e..57ea5994 100644
--- a/docs/optional-surfaces.rst
+++ b/docs/optional-surfaces.rst
@@ -47,6 +47,8 @@ Notes:
 
 * Tesseract usage requires the system ``tesseract`` binary in addition to the
   Python extra.
+* Python 3.13 is validated for the OCR install profile, Pillow, pytesseract,
+  and system Tesseract smoke checks.
 * Donut OCR requires a model that is already available locally. DataFog should
   not download models implicitly during normal runtime usage.
 * OCR is not deprecated. A broader OCR API and packaging overhaul is deferred
@@ -107,3 +109,35 @@ The repository includes tests that block optional dependency imports while
 importing ``datafog`` and running the default text helpers. These checks verify
 that OCR, Spark, NLP, model-loading, and web dependencies are not required for
 the core path.
+
+Python 3.13 optional-profile status
+-----------------------------------
+
+DataFog 4.5 validates Python 3.13 beyond the core/CLI path for the optional
+profiles that currently have compatible wheels in the tested dependency set.
+
+.. list-table::
+   :header-rows: 1
+
+   * - Profile
+     - Python 3.13 status
+     - Notes
+   * - ``nlp``
+     - Supported
+     - spaCy imports and the profile smoke test pass on Python 3.13.
+   * - ``nlp-advanced``
+     - Supported
+     - GLiNER, torch, transformers, and onnxruntime import successfully on
+       Python 3.13.
+   * - ``ocr``
+     - Supported
+     - Pillow, pytesseract, and the system Tesseract bridge validate on Python
+       3.13 when the ``tesseract`` binary is installed.
+   * - ``nlp-advanced,ocr``
+     - Supported with local model requirement
+     - Donut dependencies import on Python 3.13; runtime OCR still requires the
+       configured Donut model to be present locally.
+   * - ``distributed`` and ``all``
+     - Not newly certified in 4.5
+     - Keep using Python 3.10-3.12 for distributed/all-profile validation until
+       Spark and the full optional surface are audited separately.
diff --git a/tests/test_install_profiles.py b/tests/test_install_profiles.py
index 7222186e..2680543b 100644
--- a/tests/test_install_profiles.py
+++ b/tests/test_install_profiles.py
@@ -55,6 +55,8 @@ def test_install_profile_import_surface() -> None:
         assert DonutProcessor is not None
         assert ImageService is not None
         assert PytesseractProcessor is not None
+        if os.environ.get("DATAFOG_REQUIRE_TESSERACT"):
+            assert pytesseract.get_tesseract_version()
     elif profile == "distributed":
         from datafog.processing.spark_processing import pyspark_udfs
         from datafog.services.spark_service import SparkService

From e950770b4381d6a0f238c731a8457d9dfc6a24e0 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 20:58:14 +0200
Subject: [PATCH 55/70] docs: prepare 4.5 release readiness

---
 CHANGELOG.MD                   |  65 +++++++++++++++
 docs/contributing.rst          |   4 +-
 docs/index.rst                 |   1 +
 docs/planning-history.rst      |   1 +
 docs/roadmap.rst               |   3 +-
 docs/v45-release-readiness.rst | 139 +++++++++++++++++++++++++++++++++
 scripts/generate_changelog.py  |  37 +++++++++
 7 files changed, 247 insertions(+), 3 deletions(-)
 create mode 100644 docs/v45-release-readiness.rst

diff --git a/CHANGELOG.MD b/CHANGELOG.MD
index 976e9cc5..45d042da 100644
--- a/CHANGELOG.MD
+++ b/CHANGELOG.MD
@@ -1,5 +1,70 @@
 # ChangeLog
 
+## [Unreleased]
+
+### `datafog-python` [4.5.0]
+
+#### Release Thesis
+
+- Frames 4.5.0 as a focused, lightweight text PII screening release rather
+  than a v5 package overhaul.
+- Keeps the first path centered on core install, regex scanning/redaction,
+  CLI text commands, and agent-oriented guardrail helpers.
+- Defers dedicated Sentry, OpenTelemetry, logging-framework, and cloud DLP
+  middleware adapters to v5 planning.
+
+#### Core Text PII Screening
+
+- Clarifies the live top-level APIs: `scan`, `redact`, `protect`,
+  `scan_prompt`, `filter_output`, `sanitize`, and guardrail helpers.
+- Documents the current module map so users and contributors can distinguish
+  live 4.5 modules from historical compatibility and audit artifacts.
+- Preserves backward-compatible `DataFog` and `TextService` entry points.
+
+#### German Structured PII
+
+- Adds regex-only German structured PII support without adding core
+  dependencies.
+- Detects German VAT IDs and German IBANs by default because their country-code
+  structure is precise enough for default screening.
+- Enables broader German identifiers only through `locales=["de"]` or explicit
+  entity selection, including German tax IDs, pension insurance numbers,
+  postal codes, passport numbers, and residence permit numbers.
+
+#### Optional Profiles And Python 3.13
+
+- Certifies Python 3.13 support for the core SDK, CLI, `nlp`, `nlp-advanced`,
+  and `ocr` install profiles.
+- Adds CI coverage for Python 3.13 `nlp` and `nlp-advanced` test profiles plus
+  3.13 smoke checks for `nlp`, `nlp-advanced`, and `ocr`.
+- Documents Donut OCR as requiring a local model before runtime use.
+- Leaves `distributed` and `all` outside the new Python 3.13 certification
+  claim for 4.5.0.
+
+#### Optional OCR And Spark Surfaces
+
+- Documents OCR and Spark as supported optional surfaces, not deprecated
+  features and not the main 4.5 adoption path.
+- Keeps local OCR behind `datafog[ocr]`, URL image inputs behind
+  `datafog[web,ocr]`, Donut behind `datafog[nlp-advanced,ocr]`, and Spark
+  behind `datafog[distributed]`.
+
+#### Telemetry And Privacy
+
+- Documents telemetry behavior without changing defaults.
+- Telemetry remains disabled unless `DATAFOG_TELEMETRY=1` is set.
+- `DATAFOG_NO_TELEMETRY=1` and `DO_NOT_TRACK=1` continue to force telemetry
+  off for tests, CI, and privacy-sensitive environments.
+
+#### Release Readiness
+
+- Adds a 4.5 release-readiness checklist covering docs build, formatting,
+  core no-network checks, install-profile smoke checks, German regex tests,
+  broad non-slow tests, package build checks, and final CI status.
+- Clarifies the version alignment path: the development package remains
+  `4.4.0a5` until stable release promotion, and the final stable release should
+  publish as `4.5.0`.
+
 ## [2026-02-13]
 
 ### `datafog-python` [4.3.0]
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 1565437e..b1b58777 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -191,8 +191,8 @@ The release flow for 4.5 is:
    lightweight text screening thesis.
 4. Validate optional Python 3.13 profiles before claiming support beyond core
    SDK and CLI.
-5. Prepare release readiness: changelog/release notes, package checks, docs
-   build, CI state, and version alignment.
+5. Prepare release readiness with :doc:`v45-release-readiness`: changelog and
+   release notes, package checks, docs build, CI state, and version alignment.
 6. Bump or override the final stable release to ``4.5.0`` only during the
    release-readiness and stable-release path.
 
diff --git a/docs/index.rst b/docs/index.rst
index 1e64fb5b..d5cfdc66 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -41,6 +41,7 @@ Contributing
    :caption: Contributing
 
    contributing
+   v45-release-readiness
    live-module-map
 
 Planning And History
diff --git a/docs/planning-history.rst b/docs/planning-history.rst
index b0cab0f9..8237f38f 100644
--- a/docs/planning-history.rst
+++ b/docs/planning-history.rst
@@ -13,6 +13,7 @@ Release Planning
    :maxdepth: 1
 
    v44-bridge-release
+   v45-release-readiness
    v5-product-brief
    v5-compatibility-matrix
    v5-cut-line
diff --git a/docs/roadmap.rst b/docs/roadmap.rst
index 9db4d433..1ea8bbc4 100644
--- a/docs/roadmap.rst
+++ b/docs/roadmap.rst
@@ -24,9 +24,10 @@ v4.4.0 should focus on:
 * Targeted deprecation warnings with no warnings on import.
 * Migration docs and release notes that announce the v5 path.
 
-Scope artifact:
+Scope artifacts:
 
 * :doc:`v44-bridge-release`
+* :doc:`v45-release-readiness`
 
 v5.0.0 - Offline PII Firewall for AI Apps
 -----------------------------------------
diff --git a/docs/v45-release-readiness.rst b/docs/v45-release-readiness.rst
new file mode 100644
index 00000000..6d655c3b
--- /dev/null
+++ b/docs/v45-release-readiness.rst
@@ -0,0 +1,139 @@
+======================
+v4.5 Release Readiness
+======================
+
+This page is the release-readiness artifact for DataFog 4.5.0. It summarizes
+the intended release story, the final version alignment path, and the checks
+that should be true before promoting the release.
+
+Release Position
+================
+
+DataFog 4.5.0 is a lightweight text PII screening focus release. It should make
+the current package easier to install, read, test, and contribute to while
+building toward a sharper v5 middleware direction.
+
+The 4.5 release includes:
+
+* Core text scanning, redaction, and guardrail helpers that stay dependency
+  light by default.
+* Regex-only German structured PII support with broad German identifiers gated
+  behind explicit locale or entity selection.
+* Clear optional-profile documentation for NLP, OCR, Spark, CLI, web, and
+  install-profile testing.
+* Python 3.13 validation for the core SDK, CLI, ``nlp``, ``nlp-advanced``, and
+  ``ocr`` profiles.
+* Telemetry documentation that states the existing opt-in behavior and opt-out
+  controls without changing runtime defaults.
+
+The 4.5 release does not include:
+
+* A v5 package break.
+* Dedicated Sentry, OpenTelemetry, logging-framework, or cloud DLP middleware
+  adapters.
+* An OCR or Spark architecture overhaul.
+* Full certification of ``distributed`` or ``all`` install profiles on
+  Python 3.13.
+
+Release Notes Draft
+===================
+
+Use this framing for the GitHub release notes and package announcement:
+
+   DataFog 4.5.0 is a focused release for lightweight text PII screening. The
+   core install remains dependency-light while the text APIs, CLI, guardrail
+   helpers, German structured PII coverage, optional-profile docs, and Python
+   3.13 compatibility story become clearer and easier to verify.
+
+Call out these user-facing points:
+
+* German VAT IDs and German IBANs are detected by default in the regex engine.
+* Broader German identifiers such as tax IDs, postal codes, passport numbers,
+  residence permit numbers, and pension insurance numbers require
+  ``locales=["de"]`` or explicit entity selection.
+* OCR and Spark remain supported optional surfaces. They are not deprecated,
+  but their broader overhaul is deferred beyond 4.5.
+* Telemetry remains disabled unless ``DATAFOG_TELEMETRY=1`` is set.
+  ``DATAFOG_NO_TELEMETRY=1`` and ``DO_NOT_TRACK=1`` continue to force it off.
+* Python 3.13 is certified for core SDK, CLI, ``nlp``, ``nlp-advanced``, and
+  ``ocr``. Donut OCR still requires a model already available locally.
+
+Version Alignment
+=================
+
+The source of truth for the package version is ``datafog/__about__.py``.
+``setup.py`` reads that value during packaging, and ``docs/conf.py`` reads the
+same value for the Sphinx ``release`` field.
+
+Before stable release promotion:
+
+* Stable package release: ``4.4.0``.
+* Current development package version: ``4.4.0a5``.
+* Next stable target: ``4.5.0``.
+
+Do not bump routine feature or documentation branches directly to ``4.5.0``.
+For the stable release, promote the merged 4.5 stack and either:
+
+* trigger the release workflow with ``release_type=stable`` and
+  ``version_override=4.5.0``, or
+* make a dedicated stable-release bump that updates ``datafog/__about__.py``
+  and reruns the docs build so Sphinx reports ``v4.5.0``.
+
+After the bump path is chosen, verify:
+
+* ``python -c "import datafog; print(datafog.__version__)"`` prints
+  ``4.5.0`` from the release build environment.
+* Built package metadata reports ``Version: 4.5.0``.
+* Built docs report ``v4.5.0`` through ``docs/conf.py``.
+* ``CHANGELOG.MD`` and the GitHub release notes both describe the 4.5 focus
+  release rather than v5 planning work.
+
+Readiness Checklist
+===================
+
+Run these gates before promoting 4.5.0:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Gate
+     - Command or evidence
+   * - Formatting and static checks
+     - ``pre-commit run --all-files --show-diff-on-failure`` and
+       ``git diff --check``
+   * - Docs build
+     - ``python -m sphinx -b html docs docs/_build/html``
+   * - Core no-network and dependency boundary
+     - ``DATAFOG_NO_TELEMETRY=1 DO_NOT_TRACK=1 python -m pytest tests/test_runtime_dependency_safety.py tests/test_no_network_core.py -q``
+   * - German regex behavior
+     - ``python -m pytest tests/test_de_pii_regex.py tests/test_regex_annotator.py -q``
+   * - Broad non-slow suite
+     - ``DATAFOG_NO_TELEMETRY=1 DO_NOT_TRACK=1 python -m pytest -m "not slow" -q``
+   * - Install-profile smoke checks
+     - ``DATAFOG_INSTALL_PROFILE=<profile> python -m pytest tests/test_install_profiles.py -q`` for ``core``, ``cli``, ``nlp``, ``nlp-advanced``, ``ocr``, ``distributed``, and ``web``
+   * - OCR system smoke
+     - ``DATAFOG_INSTALL_PROFILE=ocr DATAFOG_REQUIRE_TESSERACT=1 python -m pytest tests/test_install_profiles.py -q``
+   * - Package build
+     - ``python -m build`` and ``python scripts/check_wheel_size.py``
+   * - GitHub CI
+     - The final release-readiness PR and the merged release branch have green
+       CI, including Python 3.13 profile coverage.
+   * - Stable release dry run
+     - Trigger ``release_type=stable``, ``version_override=4.5.0``,
+       ``dry_run=true`` before publishing.
+
+Review Notes
+============
+
+German regex support is included in 4.5 with documented default and opt-in
+behavior. The external PR was treated as review input rather than merged
+unchanged, because broad German identifiers need locale or contextual gating to
+avoid noisy default detection.
+
+OCR and Spark remain documented as optional surfaces. They should not be
+described as deprecated, and they should not be positioned as the primary 4.5
+adoption path.
+
+The v5 planning pages remain useful context, but the 4.5 release should not
+claim middleware adapters, a package break, or a complete optional-surface
+redesign.
diff --git a/scripts/generate_changelog.py b/scripts/generate_changelog.py
index 23babcf8..114d037a 100755
--- a/scripts/generate_changelog.py
+++ b/scripts/generate_changelog.py
@@ -122,6 +122,43 @@ def generate_changelog(beta=False, alpha=False):
             "and `DO_NOT_TRACK=1` continue to force telemetry off.\n\n"
         )
 
+    if not alpha and not beta and current_version == "4.5.0":
+        changelog += "## 4.5 Release Focus\n\n"
+        changelog += (
+            "DataFog 4.5.0 is a focused release for lightweight text PII "
+            "screening. The core install remains dependency-light while the "
+            "text APIs, CLI, guardrail helpers, German structured PII coverage, "
+            "optional-profile docs, and Python 3.13 compatibility story become "
+            "clearer and easier to verify.\n\n"
+        )
+        changelog += "## German Structured PII\n\n"
+        changelog += (
+            "German VAT IDs and German IBANs are detected by default in the "
+            "regex engine. Broader German identifiers such as tax IDs, postal "
+            "codes, passport numbers, residence permit numbers, and pension "
+            'insurance numbers require `locales=["de"]` or explicit entity '
+            "selection.\n\n"
+        )
+        changelog += "## Python 3.13 Optional Profiles\n\n"
+        changelog += (
+            "Python 3.13 is certified for the core SDK, CLI, `nlp`, "
+            "`nlp-advanced`, and `ocr` install profiles. Donut OCR still "
+            "requires a model already available locally. `distributed` and "
+            "`all` are not newly certified on Python 3.13 in 4.5.0.\n\n"
+        )
+        changelog += "## Optional OCR And Spark Surfaces\n\n"
+        changelog += (
+            "OCR and Spark remain supported optional surfaces. They are not "
+            "deprecated, but their broader overhaul is deferred beyond 4.5.0 "
+            "so the core package can stay tight and text-first.\n\n"
+        )
+        changelog += "## Telemetry Defaults\n\n"
+        changelog += (
+            "Telemetry remains disabled unless `DATAFOG_TELEMETRY=1` is set. "
+            "`DATAFOG_NO_TELEMETRY=1` and `DO_NOT_TRACK=1` continue to force "
+            "telemetry off.\n\n"
+        )
+
     if categories["features"]:
         changelog += "## 🚀 New Features\n"
         for commit in categories["features"]:

From 978ab7f6dfa01af7a52ed14b1d29eb0ce4e8cb57 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 28 May 2026 03:19:08 +0000
Subject: [PATCH 56/70] chore: bump version to 4.4.0b6 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 829fa6a5..8a13ad68 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0a5"
+__version__ = "4.4.0b6"

From 2b6dc2e707c9164352f35036f7fede5255786072 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Tue, 26 May 2026 19:05:12 +0200
Subject: [PATCH 57/70] ci: upgrade actions for Node 24 runtime

Refs DFPY-67
---
 .github/workflows/benchmark.yml |  8 ++++----
 .github/workflows/ci.yml        | 18 +++++++++---------
 .github/workflows/release.yml   | 28 ++++++++++++++--------------
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index e0bc7d5c..bcc51e1b 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -13,12 +13,12 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0 # Fetch all history for proper comparison
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"
@@ -34,7 +34,7 @@ jobs:
           python -m pytest --collect-only tests/benchmark_text_service.py::test_regex_performance
 
       - name: Restore benchmark data
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: .benchmarks
           # Updated cache key to reset baseline due to performance optimization changes
@@ -101,7 +101,7 @@ jobs:
           # fi
 
       - name: Upload benchmark results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: benchmark-results
           path: |
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dfcae0f4..01fc9c15 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,8 +14,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"
@@ -32,9 +32,9 @@ jobs:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
         install-profile: ["core", "nlp", "nlp-advanced"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -140,7 +140,7 @@ jobs:
           PY
 
       - name: Upload coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v6
         with:
           files: ./coverage.xml
           flags: ${{ matrix.install-profile }}-py${{ matrix.python-version }}
@@ -168,9 +168,9 @@ jobs:
           - python-version: "3.13"
             install-profile: ocr
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -205,8 +205,8 @@ jobs:
   wheel-size:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 91a459aa..a2d5a95c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -61,11 +61,11 @@ jobs:
             BRANCH="dev"
           fi
 
-          echo "release_type=$TYPE" >> $GITHUB_OUTPUT
-          echo "target_branch=$BRANCH" >> $GITHUB_OUTPUT
+          echo "release_type=$TYPE" >> "$GITHUB_OUTPUT"
+          echo "target_branch=$BRANCH" >> "$GITHUB_OUTPUT"
           echo "Release type: $TYPE from $BRANCH"
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ steps.resolve.outputs.target_branch }}
@@ -86,15 +86,15 @@ jobs:
           if [ -z "$LAST_TAG" ]; then
             COMMIT_COUNT=$(git rev-list --count --since="7 days ago" HEAD)
           else
-            COMMIT_COUNT=$(git rev-list --count ${LAST_TAG}..HEAD)
+            COMMIT_COUNT=$(git rev-list --count "${LAST_TAG}..HEAD")
           fi
 
           echo "Commits since ${LAST_TAG:-'(none)'}: $COMMIT_COUNT"
 
           if [ "$COMMIT_COUNT" -gt 0 ] || [ "${{ inputs.force_build }}" = "true" ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
           else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
             echo "No changes detected, skipping release"
           fi
 
@@ -107,13 +107,13 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -145,13 +145,13 @@ jobs:
     if: needs.determine-release.outputs.has_changes == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
           cache: "pip"
@@ -177,14 +177,14 @@ jobs:
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
           token: ${{ secrets.GH_PAT }}
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.11"
 
@@ -241,7 +241,7 @@ jobs:
             VERSION="$BASE"
           fi
 
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
           echo "Publishing version: $VERSION"
 
           sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py
@@ -322,7 +322,7 @@ jobs:
     if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Prune old alpha releases (keep 7)
         if: needs.determine-release.outputs.release_type == 'alpha'

From 86a215c5e0f4062f1e91ad1e15afe057c5848530 Mon Sep 17 00:00:00 2001
From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com>
Date: Thu, 28 May 2026 00:58:25 +0200
Subject: [PATCH 58/70] Adopt locale-gated German regex defaults

Co-authored-by: Pranjal Parmar <pranjalrparmar@gmail.com>
---
 README.md                                     | 13 +++--
 datafog/core.py                               |  2 +-
 .../regex_annotator/regex_annotator.py        | 12 +++--
 datafog/services/text_service.py              |  2 +-
 docs/cli.rst                                  |  4 +-
 docs/getting-started.rst                      | 17 ++++---
 docs/python-sdk.rst                           | 15 +++---
 tests/corpus/structured_pii.json              |  6 ++-
 tests/test_de_pii_regex.py                    | 47 +++++++++++++------
 tests/test_detection_accuracy.py              | 16 ++++---
 tests/test_regex_annotator.py                 |  8 +++-
 11 files changed, 86 insertions(+), 56 deletions(-)

diff --git a/README.md b/README.md
index 6f080e30..856fafd4 100644
--- a/README.md
+++ b/README.md
@@ -79,10 +79,9 @@ print(datafog.sanitize("Card: 4111-1111-1111-1111", engine="regex"))
 
 ## German Structured PII
 
-German VAT IDs and German IBANs are detected by the default regex path because
-their country-code structure is specific enough for default-on screening.
-Broader German identifiers are available with explicit locale selection or
-explicit entity-type filtering.
+German structured PII is country-specific and opt-in. Use explicit locale
+selection or entity-type filtering when you want German VAT IDs, German IBANs,
+tax IDs, postal codes, passports, or residence permits.
 
 ```python
 import datafog
@@ -118,8 +117,8 @@ Use the engine that matches your accuracy and dependency constraints:
 
 - `regex`:
   - Fastest and always available.
-  - Best for structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DE_VAT_ID`, `DE_IBAN`, `DATE`, `ZIP_CODE`.
-  - Use `locales=["de"]` for broader German structured IDs such as `DE_TAX_ID`, `DE_POSTAL_CODE`, and passport or residence permit numbers.
+  - Best for default structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE`.
+  - Use `locales=["de"]` for German structured IDs such as `DE_VAT_ID`, `DE_IBAN`, `DE_TAX_ID`, `DE_POSTAL_CODE`, and passport or residence permit numbers.
 - `spacy`:
   - Requires `pip install datafog[nlp]`.
   - Useful for unstructured entities like person and organization names.
@@ -193,7 +192,7 @@ datafog replace-text "john@example.com"
 # Hash detected entities
 datafog hash-text "john@example.com"
 
-# Enable broader German regex identifiers
+# Enable German regex identifiers
 datafog redact-text "Steuer-ID 12345678901" --locale de
 ```
 
diff --git a/datafog/core.py b/datafog/core.py
index aa207f6d..e56c2aa5 100644
--- a/datafog/core.py
+++ b/datafog/core.py
@@ -221,7 +221,7 @@ def get_supported_entities(locales: List[str] | None = None) -> List[str]:
     Example:
         >>> entities = get_supported_entities()
         >>> print(entities)
-        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DE_VAT_ID', 'DE_IBAN', 'DATE', 'ZIP_CODE']
+        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE']
     """
     annotator = RegexAnnotator(locales=locales)
     legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"}
diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
index d478b43d..0b904830 100644
--- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py
+++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
@@ -38,9 +38,8 @@ class RegexAnnotator:
         "DE_PASSPORT_NUMBER",
         "DE_RESIDENCE_PERMIT_NUMBER",
     ]
-    DEFAULT_LOCALIZED_LABELS = ["DE_VAT_ID", "DE_IBAN"]
     LABELS = BASE_LABELS + GERMAN_LABELS
-    DEFAULT_LABELS = BASE_LABELS + DEFAULT_LOCALIZED_LABELS
+    DEFAULT_LABELS = BASE_LABELS
     SUPPORTED_LOCALES = {"de", "de-de", "de_de"}
     LOCALE_LABELS = {
         "de": GERMAN_LABELS,
@@ -103,13 +102,18 @@ def __init__(
             # Supports dashed and no-dash formats.
             "SSN": re.compile(
                 r"""
-                (?<!\d)
                 (?:
+                    (?<!\d)
                     (?!000|666)\d{3}-(?!00)\d{2}-(?!0000)\d{4}
+                    (?!\d)
                     |
+                    (?<![A-Za-z0-9])
+                    (?<!DE)
+                    (?<!DE\s)
+                    (?<!DE-)
                     (?!000|666)\d{3}(?!00)\d{2}(?!0000)\d{4}
+                    (?![A-Za-z0-9])
                 )
-                (?!\d)
                 """,
                 re.IGNORECASE | re.MULTILINE | re.VERBOSE,
             ),
diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py
index 1057eb6a..6b80b26d 100644
--- a/datafog/services/text_service.py
+++ b/datafog/services/text_service.py
@@ -58,7 +58,7 @@ def __init__(
                 - "smart": Try RegexAnnotator → GLiNER → SpaCy cascade (requires nlp-advanced extra)
             gliner_model: GLiNER model name to use when engine is "gliner" or "smart"
             locales: Optional locale tags for regex detection. Use ["de"] to enable
-                broader German structured identifiers.
+                German structured identifiers.
 
         Raises:
             AssertionError: If an invalid engine type is provided
diff --git a/docs/cli.rst b/docs/cli.rst
index 9742887d..8dfcc9f8 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -22,8 +22,8 @@ commands. Install ``datafog[distributed]`` when using ``SparkService``.
 German locale support
 ---------------------
 
-German VAT IDs and German IBANs are detected by the default regex path. Broader
-German identifiers are opt-in through ``--locale de`` on the core text commands:
+German structured PII is opt-in through ``--locale de`` on the core text
+commands:
 
 .. code-block:: bash
 
diff --git a/docs/getting-started.rst b/docs/getting-started.rst
index bfc42290..eeeeb46d 100644
--- a/docs/getting-started.rst
+++ b/docs/getting-started.rst
@@ -83,22 +83,21 @@ Agent-oriented helpers use the same lightweight text path:
 German Structured PII
 =====================
 
-The core regex engine includes German VAT IDs and German IBANs by default
-because they carry strong country-code structure:
+German structured PII is country-specific and opt-in, including German VAT IDs
+and German IBANs:
 
 .. code-block:: python
 
    import datafog
 
-   result = datafog.scan("USt-IdNr DE 123456789", engine="regex")
+   result = datafog.scan("USt-IdNr DE 123456789", engine="regex", locales=["de"])
    print([(entity.type, entity.text) for entity in result.entities])
 
-Broader German identifiers such as ``DE_TAX_ID``,
-``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
-``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER`` require explicit
-German locale selection or explicit ``entity_types`` filtering. This keeps
-ordinary ticket, SKU, order, and invoice IDs from becoming default-on false
-positives.
+German identifiers such as ``DE_VAT_ID``, ``DE_IBAN``, ``DE_TAX_ID``,
+``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``, ``DE_PASSPORT_NUMBER``, and
+``DE_RESIDENCE_PERMIT_NUMBER`` require explicit German locale selection or
+explicit ``entity_types`` filtering. This keeps ordinary ticket, SKU, order,
+and invoice IDs from becoming default-on false positives.
 
 .. code-block:: python
 
diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst
index b1093de0..ce70f577 100644
--- a/docs/python-sdk.rst
+++ b/docs/python-sdk.rst
@@ -31,11 +31,11 @@ German locale coverage
 ----------------------
 
 DataFog 4.5 includes regex-only German structured PII support without adding
-dependencies. German VAT IDs and German IBANs are active in the default regex
-path. Broader German-only identifiers are opt-in because their raw shapes are
-common in ordinary product, ticket, invoice, and order data.
+dependencies. German-only identifiers are opt-in because their raw shapes are
+country-specific or common in ordinary product, ticket, invoice, and order
+data.
 
-Use ``locales=["de"]`` to enable the broader German set:
+Use ``locales=["de"]`` to enable the German set:
 
 .. code-block:: python
 
@@ -55,10 +55,9 @@ You can also request one German entity type directly:
        entity_types=["DE_TAX_ID"],
    )
 
-The opt-in German set currently covers ``DE_TAX_ID``,
-``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
-``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER``. The default set
-also covers ``DE_VAT_ID`` and ``DE_IBAN``.
+The opt-in German set currently covers ``DE_VAT_ID``, ``DE_IBAN``,
+``DE_TAX_ID``, ``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
+``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER``.
 
 Optional services
 -----------------
diff --git a/tests/corpus/structured_pii.json b/tests/corpus/structured_pii.json
index 9c13dda1..f1b4a29f 100644
--- a/tests/corpus/structured_pii.json
+++ b/tests/corpus/structured_pii.json
@@ -735,8 +735,9 @@
     ]
   },
   {
-    "id": "de-vat-id-default",
+    "id": "de-vat-id-locale",
     "input": "USt-IdNr DE 123456789 ist gesetzt.",
+    "locales": ["de"],
     "expected_entities": [
       {
         "type": "DE_VAT_ID",
@@ -747,8 +748,9 @@
     ]
   },
   {
-    "id": "de-iban-default",
+    "id": "de-iban-locale",
     "input": "IBAN DE44 5001 0517 5407 3249 31 ist gueltig.",
+    "locales": ["de"],
     "expected_entities": [
       {
         "type": "DE_IBAN",
diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py
index f24d605c..e5b6077a 100644
--- a/tests/test_de_pii_regex.py
+++ b/tests/test_de_pii_regex.py
@@ -24,12 +24,17 @@
         ),
     ],
 )
-def test_high_specificity_german_regex_default_cases(
+def test_german_regex_cases_require_german_locale_or_explicit_entity_type(
     label: str, text: str, expected: str
 ) -> None:
-    annotator = RegexAnnotator()
-    result = annotator.annotate(text)
-    assert expected in result[label]
+    default_result = RegexAnnotator().annotate(text)
+    assert expected not in default_result[label]
+
+    german_result = RegexAnnotator(locales=["de"]).annotate(text)
+    assert expected in german_result[label]
+
+    explicit_result = RegexAnnotator(enabled_labels=[label]).annotate(text)
+    assert expected in explicit_result[label]
 
 
 @pytest.mark.parametrize(
@@ -118,25 +123,39 @@ def test_redaction_and_service_locale_support() -> None:
     assert service_result["DE_PASSPORT_NUMBER"] == ["C12345678"]
 
 
-def test_german_vat_redaction_suppresses_inner_generic_ssn_match() -> None:
-    text = "USt-IdNr DE123456789 ist gesetzt."
-
+@pytest.mark.parametrize(
+    "text,vat_text",
+    [
+        ("USt-IdNr DE123456789 ist gesetzt.", "DE123456789"),
+        ("USt-IdNr DE 123456789 ist gesetzt.", "DE 123456789"),
+        ("USt-IdNr DE-123456789 ist gesetzt.", "DE-123456789"),
+    ],
+)
+def test_german_vat_redaction_suppresses_inner_generic_ssn_match(
+    text: str, vat_text: str
+) -> None:
     scan_result = scan(text, engine="regex")
-    assert [(entity.type, entity.text) for entity in scan_result.entities] == [
-        ("DE_VAT_ID", "DE123456789")
+    assert scan_result.entities == []
+
+    locale_scan_result = scan(text, engine="regex", locales=["de"])
+    assert [(entity.type, entity.text) for entity in locale_scan_result.entities] == [
+        ("DE_VAT_ID", vat_text)
     ]
 
-    redaction = scan_and_redact(text, engine="regex")
-    assert redaction.redacted_text == "USt-IdNr [DE_VAT_ID_1] ist gesetzt."
+    default_redaction = scan_and_redact(text, engine="regex")
+    assert default_redaction.redacted_text == text
+
+    redaction = scan_and_redact(text, engine="regex", locales=["de"])
+    assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]")
 
 
 def test_top_level_helpers_and_supported_entities_respect_locale() -> None:
     default_entities = get_supported_entities()
-    assert "DE_VAT_ID" in default_entities
-    assert "DE_IBAN" in default_entities
-    assert "DE_TAX_ID" not in default_entities
+    assert all(not entity.startswith("DE_") for entity in default_entities)
 
     german_entities = get_supported_entities(locales=["de"])
+    assert "DE_VAT_ID" in german_entities
+    assert "DE_IBAN" in german_entities
     assert "DE_TAX_ID" in german_entities
     assert "DE_RESIDENCE_PERMIT_NUMBER" in german_entities
 
diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py
index 6777d9dd..61dec7ba 100644
--- a/tests/test_detection_accuracy.py
+++ b/tests/test_detection_accuracy.py
@@ -280,9 +280,11 @@ def _canon_type(entity_type: str) -> str:
     return TYPE_ALIASES.get(raw, raw)
 
 
-def _extract_entities(text: str, engine: str) -> list[dict[str, Any]]:
+def _extract_entities(
+    text: str, engine: str, locales: list[str] | None = None
+) -> list[dict[str, Any]]:
     try:
-        result = scan(text=text, engine=engine)
+        result = scan(text=text, engine=engine, locales=locales)
     except (ImportError, EngineNotAvailable) as exc:
         pytest.skip(f"{engine} engine unavailable in this environment: {exc}")
 
@@ -347,7 +349,7 @@ def _assert_expected_found(
     case: dict[str, Any], engine: str, corpus_kind: str
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     text = case["input"]
-    actual = _extract_entities(text, engine)
+    actual = _extract_entities(text, engine, locales=case.get("locales"))
     expected = _required_expected(case["expected_entities"], engine, corpus_kind)
 
     for exp in expected:
@@ -403,7 +405,9 @@ def _compute_metrics(
     for engine in engines:
         for corpus_kind, cases in corpora:
             for case in cases:
-                actual = _extract_entities(case["input"], engine)
+                actual = _extract_entities(
+                    case["input"], engine, locales=case.get("locales")
+                )
                 expected = _required_expected(
                     case["expected_entities"], engine, corpus_kind
                 )
@@ -490,7 +494,7 @@ def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> Non
 @pytest.mark.parametrize("engine", FAST_ENGINES)
 def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None:
     _xfail_if_known_limitation(case, engine, "negative")
-    actual = _extract_entities(case["input"], engine)
+    actual = _extract_entities(case["input"], engine, locales=case.get("locales"))
     assert not actual, f"{case['id']} ({engine}) false positives: {actual}"
 
 
@@ -501,7 +505,7 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None:
 @pytest.mark.parametrize("engine", SLOW_ENGINES)
 def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None:
     _xfail_if_known_limitation(case, engine, "negative")
-    actual = _extract_entities(case["input"], engine)
+    actual = _extract_entities(case["input"], engine, locales=case.get("locales"))
     assert not actual, f"{case['id']} ({engine}) false positives: {actual}"
 
 
diff --git a/tests/test_regex_annotator.py b/tests/test_regex_annotator.py
index fc63dc2f..8f834bc0 100644
--- a/tests/test_regex_annotator.py
+++ b/tests/test_regex_annotator.py
@@ -41,8 +41,12 @@ def test_regex_annotator_initialization():
     annotator = RegexAnnotator()
     assert annotator is not None
     assert set(RegexAnnotator.BASE_LABELS).issubset(annotator.LABELS)
-    assert {"DE_VAT_ID", "DE_IBAN"}.issubset(annotator.active_labels)
-    assert "DE_TAX_ID" not in annotator.active_labels
+    assert all(
+        label not in annotator.active_labels for label in RegexAnnotator.GERMAN_LABELS
+    )
+
+    german_annotator = RegexAnnotator(locales=["de"])
+    assert set(RegexAnnotator.GERMAN_LABELS).issubset(german_annotator.active_labels)
 
 
 def test_regex_annotator_create_method():

From ccfa3b846c74156875f1723fc43930025ea98878 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Sun, 31 May 2026 21:19:25 +0000
Subject: [PATCH 59/70] chore: bump version to 4.5.0b1 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 8a13ad68..84faed07 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.4.0b6"
+__version__ = "4.5.0b1"

From b919af9fb6b7fad6c62e0b2f3b7dba1cbb63ce56 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 1 Jun 2026 03:30:09 +0000
Subject: [PATCH 60/70] chore: bump version to 4.5.0a1 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 84faed07..04187482 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0b1"
+__version__ = "4.5.0a1"

From c513b651de3c4d68b8b8d07ecafbd2e26df276a0 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 11 Jun 2026 03:16:44 +0000
Subject: [PATCH 61/70] chore: bump version to 4.5.0b2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 04187482..6e49aee7 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0a1"
+__version__ = "4.5.0b2"

From 0b931da0c5a33912d56a10cd590cb1c37ff58636 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 15 Jun 2026 03:23:46 +0000
Subject: [PATCH 62/70] chore: bump version to 4.5.0a2 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 6e49aee7..92525dd1 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0b2"
+__version__ = "4.5.0a2"

From a6e4792afe84c663def71a8e0ff7a0cbae004206 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 18 Jun 2026 03:18:24 +0000
Subject: [PATCH 63/70] chore: bump version to 4.5.0b3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 92525dd1..a161ce32 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0a2"
+__version__ = "4.5.0b3"

From 2017a5d9b2a44a87a236ce67f63a4392f92b2196 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 22 Jun 2026 03:23:13 +0000
Subject: [PATCH 64/70] chore: bump version to 4.5.0a3 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index a161ce32..25351ab6 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0b3"
+__version__ = "4.5.0a3"

From a4392183c5f81601e3efad6377793ab70c560c0c Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 25 Jun 2026 03:16:27 +0000
Subject: [PATCH 65/70] chore: bump version to 4.5.0b4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 25351ab6..885b52e7 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0a3"
+__version__ = "4.5.0b4"

From c8fc70733681768c17546471d746ef4d61a4854d Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 29 Jun 2026 03:22:03 +0000
Subject: [PATCH 66/70] chore: bump version to 4.5.0a4 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 885b52e7..21650231 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0b4"
+__version__ = "4.5.0a4"

From 76d164577a968481239c3eea73403ed8d8e89d13 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Thu, 2 Jul 2026 03:15:26 +0000
Subject: [PATCH 67/70] chore: bump version to 4.5.0b5 [skip ci]

---
 datafog/__about__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafog/__about__.py b/datafog/__about__.py
index 21650231..2f36ec70 100644
--- a/datafog/__about__.py
+++ b/datafog/__about__.py
@@ -1 +1 @@
-__version__ = "4.5.0a4"
+__version__ = "4.5.0b5"

From 2fea5ad86fce325b06432642fa90f14a7d413dcb Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Thu, 2 Jul 2026 09:55:23 -0700
Subject: [PATCH 68/70] fix: restore v4.4.0 detection parity for SSN and
 detect() output keys

Two silent behavior regressions found in the 4.5.0 pre-release review:

- The SSN pattern had gained (?<!DE) lookbehinds to stop German VAT IDs
  double-matching as SSNs, but the pattern is always active, so default
  (no-locale) users silently lost detection of nine-digit runs preceded
  by 'DE', 'DE ', or 'DE-'. Restore the exact v4.4.0 pattern and let the
  engine's span-overlap suppression resolve the DE_VAT_ID overlap only
  when German locale support is active.

- DataFog.detect() pre-populated its result dict from the full LABELS
  list, adding seven always-empty DE_* keys for every caller. Scope the
  keys to the labels active under the configured locales via a new
  RegexAnnotator.active_labels_for() classmethod.

Both changes are covered by new regression tests pinning the default
(v4.4.0-parity) behavior.
---
 datafog/main.py                               |  5 ++-
 .../regex_annotator/regex_annotator.py        | 31 +++++++++++--------
 tests/test_de_pii_regex.py                    | 10 ++++--
 tests/test_regex_annotator.py                 | 15 +++++++++
 4 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/datafog/main.py b/datafog/main.py
index a8b1bcba..62abaaff 100644
--- a/datafog/main.py
+++ b/datafog/main.py
@@ -184,7 +184,10 @@ def detect(self, text: str) -> dict:
         _start = _time.monotonic()
 
         scan_result = scan(text=text, engine="regex", locales=self.locales)
-        result = {label: [] for label in RegexAnnotator.LABELS}
+        # Only pre-populate keys for labels active under the configured
+        # locales so the default output shape matches v4.4.0 (no DE_* keys
+        # unless German locale support is enabled).
+        result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)}
         legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"}
         for entity in scan_result.entities:
             label = legacy_map.get(entity.type, entity.type)
diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
index 0b904830..44dfa2e0 100644
--- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py
+++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py
@@ -53,7 +53,7 @@ def __init__(
         enabled_labels: Iterable[str] | None = None,
     ):
         self.locales = self._normalize_locales(locales)
-        self.active_labels = self._resolve_active_labels(enabled_labels)
+        self.active_labels = self.active_labels_for(self.locales, enabled_labels)
 
         # Compile all patterns once at initialization
         all_patterns: Dict[str, Pattern] = {
@@ -100,20 +100,19 @@ def __init__(
             ),
             # SSN pattern - U.S. Social Security Number
             # Supports dashed and no-dash formats.
+            # Note: overlaps with locale-gated labels (e.g. the nine-digit run
+            # inside a DE_VAT_ID) are resolved by the engine's span-overlap
+            # suppression, not here, so default (EN) detection keeps v4.4.0
+            # behavior even when German labels are active.
             "SSN": re.compile(
                 r"""
+                (?<!\d)
                 (?:
-                    (?<!\d)
                     (?!000|666)\d{3}-(?!00)\d{2}-(?!0000)\d{4}
-                    (?!\d)
                     |
-                    (?<![A-Za-z0-9])
-                    (?<!DE)
-                    (?<!DE\s)
-                    (?<!DE-)
                     (?!000|666)\d{3}(?!00)\d{2}(?!0000)\d{4}
-                    (?![A-Za-z0-9])
                 )
+                (?!\d)
                 """,
                 re.IGNORECASE | re.MULTILINE | re.VERBOSE,
             ),
@@ -347,13 +346,19 @@ def _normalize_locales(cls, locales: str | Iterable[str] | None) -> tuple[str, .
             normalized.append(value)
         return tuple(dict.fromkeys(normalized))
 
-    def _resolve_active_labels(self, enabled_labels: Iterable[str] | None) -> list[str]:
-        active = set(self.DEFAULT_LABELS)
-        for locale in self.locales:
-            active.update(self.LOCALE_LABELS[locale])
+    @classmethod
+    def active_labels_for(
+        cls,
+        locales: str | Iterable[str] | None = None,
+        enabled_labels: Iterable[str] | None = None,
+    ) -> list[str]:
+        """Resolve the labels active for the given locales and explicit labels."""
+        active = set(cls.DEFAULT_LABELS)
+        for locale in cls._normalize_locales(locales):
+            active.update(cls.LOCALE_LABELS[locale])
         if enabled_labels is not None:
             active.update(label.strip().upper() for label in enabled_labels)
-        return [label for label in self.LABELS if label in active]
+        return [label for label in cls.LABELS if label in active]
 
     @staticmethod
     def _match_text(match: Match[str]) -> str:
diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py
index e5b6077a..96901cf2 100644
--- a/tests/test_de_pii_regex.py
+++ b/tests/test_de_pii_regex.py
@@ -134,16 +134,22 @@ def test_redaction_and_service_locale_support() -> None:
 def test_german_vat_redaction_suppresses_inner_generic_ssn_match(
     text: str, vat_text: str
 ) -> None:
+    # Default (no locale): v4.4.0 parity — the bare nine-digit run still
+    # matches the generic SSN pattern even when prefixed by a country code.
     scan_result = scan(text, engine="regex")
-    assert scan_result.entities == []
+    assert [(entity.type, entity.text) for entity in scan_result.entities] == [
+        ("SSN", "123456789")
+    ]
 
+    # German locale: the longer DE_VAT_ID span wins via the engine's
+    # span-overlap suppression, so the inner SSN match is dropped.
     locale_scan_result = scan(text, engine="regex", locales=["de"])
     assert [(entity.type, entity.text) for entity in locale_scan_result.entities] == [
         ("DE_VAT_ID", vat_text)
     ]
 
     default_redaction = scan_and_redact(text, engine="regex")
-    assert default_redaction.redacted_text == text
+    assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]")
 
     redaction = scan_and_redact(text, engine="regex", locales=["de"])
     assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]")
diff --git a/tests/test_regex_annotator.py b/tests/test_regex_annotator.py
index 8f834bc0..ec0363c1 100644
--- a/tests/test_regex_annotator.py
+++ b/tests/test_regex_annotator.py
@@ -371,3 +371,18 @@ def test_annotation_result_format():
 
     assert len(ssn_spans) >= 1
     assert ssn_spans[0].text == "123-45-6789"
+
+
+def test_ssn_detection_keeps_v44_behavior_for_country_prefixed_digits():
+    """Regression guard: bare nine-digit runs after a country prefix must
+    still match SSN when no locale is configured (v4.4.0 parity). The
+    DE_VAT_ID overlap is resolved by engine-level span suppression only
+    when German locale support is active, never by weakening the base
+    SSN pattern."""
+    annotator = RegexAnnotator()
+    for text in (
+        "Reference DE 123456789 was issued.",
+        "Reference DE-123456789 was issued.",
+        "Reference DE123456789 was issued.",
+    ):
+        assert annotator.annotate(text)["SSN"] == ["123456789"], text

From 2cab1a3951e3122c4d05b9291a81f555252f37b8 Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Thu, 2 Jul 2026 09:55:33 -0700
Subject: [PATCH 69/70] chore: finalize 4.5.0 changelog and document guardrail
 engine defaults

Release-readiness cleanups from the 4.5.0 pre-release review:

- Date the 4.5.0 changelog entry and add a 'Behavior Changes Since
  4.4.0' section covering the guardrail helpers' regex default (with
  engine="smart" migration guidance), the locale-scoped detect() output
  shape, and the intentional Python <3.14 cap.
- Fix the stale claim that German VAT IDs/IBANs are detected by default
  (all German labels are locale-gated) in the changelog and the release
  readiness doc, and fix the stale 4.4.0a5 version-alignment note.
- Document the regex default on Guardrail and the sanitize/scan_prompt/
  filter_output helpers, and pin the defaults with a regression test so
  they cannot drift silently again.
- Sync .bumpversion.cfg current_version (4.4.0a5 -> 4.5.0b5) with
  __about__.py.
---
 .bumpversion.cfg               |  2 +-
 CHANGELOG.MD                   | 40 ++++++++++++++++++++++++++--------
 datafog/agent.py               | 17 ++++++++++++++-
 docs/v45-release-readiness.rst | 11 ++++++----
 tests/test_agent_api.py        | 14 +++++++++++-
 5 files changed, 68 insertions(+), 16 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 0f1713f9..bce21f1b 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.4.0a5
+current_version = 4.5.0b5
 commit = True
 tag = True
 tag_name = v{new_version}
diff --git a/CHANGELOG.MD b/CHANGELOG.MD
index 45d042da..2004ad63 100644
--- a/CHANGELOG.MD
+++ b/CHANGELOG.MD
@@ -1,9 +1,27 @@
 # ChangeLog
 
-## [Unreleased]
+## [2026-07-02]
 
 ### `datafog-python` [4.5.0]
 
+#### Behavior Changes Since 4.4.0
+
+- Agent guardrail helpers (`sanitize`, `scan_prompt`, `filter_output`,
+  `create_guardrail`, and the `Guardrail` class) now default to
+  `engine="regex"` instead of `engine="smart"`, matching the top-level
+  `scan`/`redact`/`protect` defaults. This keeps the core install from
+  probing optional NLP dependencies, but means NER-backed entities
+  (PERSON, ORGANIZATION, LOCATION) are no longer detected by these
+  helpers unless requested. **Migration:** pass `engine="smart"` to
+  restore the 4.4.0 behavior (requires `datafog[nlp]` or
+  `datafog[nlp-advanced]`).
+- `DataFog.detect()` / `DataFog.scan_text()` result dictionaries now
+  contain keys only for the labels active under the configured locales:
+  the seven base labels by default, plus the `DE_*` labels when
+  constructed with `locales=["de"]`.
+- Supported Python versions are `>=3.10,<3.14`. Python 3.14 is not yet
+  certified and is intentionally excluded from this release.
+
 #### Release Thesis
 
 - Frames 4.5.0 as a focused, lightweight text PII screening release rather
@@ -25,11 +43,15 @@
 
 - Adds regex-only German structured PII support without adding core
   dependencies.
-- Detects German VAT IDs and German IBANs by default because their country-code
-  structure is precise enough for default screening.
-- Enables broader German identifiers only through `locales=["de"]` or explicit
-  entity selection, including German tax IDs, pension insurance numbers,
-  postal codes, passport numbers, and residence permit numbers.
+- All German identifiers are locale-gated: they activate only through
+  `locales=["de"]` or explicit entity selection. This covers German VAT
+  IDs, IBANs, tax IDs, pension insurance numbers, postal codes, passport
+  numbers, and residence permit numbers. Default (no-locale) detection
+  behavior is unchanged from 4.4.0.
+- When German locale support is active, overlapping matches (e.g. the
+  nine-digit run inside a VAT ID also matching the generic SSN pattern)
+  are resolved by engine-level span-overlap suppression in favor of the
+  more specific German label.
 
 #### Optional Profiles And Python 3.13
 
@@ -61,9 +83,9 @@
 - Adds a 4.5 release-readiness checklist covering docs build, formatting,
   core no-network checks, install-profile smoke checks, German regex tests,
   broad non-slow tests, package build checks, and final CI status.
-- Clarifies the version alignment path: the development package remains
-  `4.4.0a5` until stable release promotion, and the final stable release should
-  publish as `4.5.0`.
+- Clarifies the version alignment path: development prereleases publish as
+  `4.5.0aN`/`4.5.0bN` from `dev`, and the stable release publishes as
+  `4.5.0` via the Release workflow's `stable` dispatch from `main`.
 
 ## [2026-02-13]
 
diff --git a/datafog/agent.py b/datafog/agent.py
index a031a147..e3cd7bdb 100644
--- a/datafog/agent.py
+++ b/datafog/agent.py
@@ -44,7 +44,12 @@ def filter(self, text: str) -> RedactResult:
 
 @dataclass
 class Guardrail:
-    """Reusable text guardrail for wrapping LLM prompts and outputs."""
+    """Reusable text guardrail for wrapping LLM prompts and outputs.
+
+    Defaults to the lightweight regex engine (changed from "smart" in 4.5.0)
+    so the core install never probes optional NLP dependencies; pass
+    ``engine="smart"`` to restore NER-backed detection.
+    """
 
     entity_types: Optional[list[str]] = None
     locales: Optional[list[str]] = None
@@ -119,6 +124,10 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str:
     One-liner PII removal.
 
     Returns the redacted text only.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection, which requires
+    the optional NLP extras.
     """
     result = scan_and_redact(text=text, engine=engine, **kwargs)
     return result.redacted_text
@@ -127,6 +136,9 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str:
 def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult:
     """
     Scan an LLM prompt for PII without modifying the input text.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection.
     """
     return scan(prompt, engine=engine, **kwargs)
 
@@ -134,6 +146,9 @@ def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult
 def filter_output(output: str, engine: str = "regex", **kwargs: Any) -> RedactResult:
     """
     Scan and redact PII from model output before returning to users.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection.
     """
     return scan_and_redact(output, engine=engine, **kwargs)
 
diff --git a/docs/v45-release-readiness.rst b/docs/v45-release-readiness.rst
index 6d655c3b..62de2cc3 100644
--- a/docs/v45-release-readiness.rst
+++ b/docs/v45-release-readiness.rst
@@ -47,10 +47,13 @@ Use this framing for the GitHub release notes and package announcement:
 
 Call out these user-facing points:
 
-* German VAT IDs and German IBANs are detected by default in the regex engine.
-* Broader German identifiers such as tax IDs, postal codes, passport numbers,
-  residence permit numbers, and pension insurance numbers require
-  ``locales=["de"]`` or explicit entity selection.
+* German structured identifiers — VAT IDs, IBANs, tax IDs, postal codes,
+  passport numbers, residence permit numbers, and pension insurance numbers —
+  are locale-gated and require ``locales=["de"]`` or explicit entity
+  selection. Default (no-locale) detection behavior is unchanged from 4.4.0.
+* Guardrail helpers (``sanitize``, ``scan_prompt``, ``filter_output``,
+  ``create_guardrail``) now default to the regex engine; pass
+  ``engine="smart"`` to restore 4.4.0's NER-backed helper behavior.
 * OCR and Spark remain supported optional surfaces. They are not deprecated,
   but their broader overhaul is deferred beyond 4.5.
 * Telemetry remains disabled unless ``DATAFOG_TELEMETRY=1`` is set.
diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py
index ff72e9fa..fd34a63c 100644
--- a/tests/test_agent_api.py
+++ b/tests/test_agent_api.py
@@ -2,10 +2,22 @@
 
 from __future__ import annotations
 
+import inspect
+
 import pytest
 
 import datafog
-from datafog.agent import GuardrailBlockedError
+from datafog.agent import Guardrail, GuardrailBlockedError
+
+
+def test_agent_helpers_default_to_regex_engine() -> None:
+    """4.5 lean-core contract: guardrail helpers stay on the regex path
+    unless an engine is explicitly requested (changed from "smart" in 4.5.0;
+    see CHANGELOG behavior changes)."""
+    assert Guardrail().engine == "regex"
+    assert datafog.create_guardrail().engine == "regex"
+    for helper in (datafog.sanitize, datafog.scan_prompt, datafog.filter_output):
+        assert inspect.signature(helper).parameters["engine"].default == "regex"
 
 
 def test_sanitize_redacts_structured_pii() -> None:

From d01a01ede06f79fb892e72c2aeda7c169718968f Mon Sep 17 00:00:00 2001
From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com>
Date: Thu, 2 Jul 2026 10:02:42 -0700
Subject: [PATCH 70/70] ci: exclude historical audit artifacts from prettier

The lint job started failing because prettier now reformats
docs/audit/01-coverage-baseline.md, a UTF-16LE-encoded captured test
log committed as a historical audit artifact (#121). The file hasn't
changed since the last green run on dev, so this is hook-environment
drift on the runner, not a content change.

Exclude docs/audit/ from the prettier hook: these are quarantined
historical artifacts and letting prettier rewrite a UTF-16 test log
would corrupt it. Verified pre-commit run --all-files passes locally.
---
 .pre-commit-config.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 65a35656..4063257e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -36,6 +36,7 @@ repos:
         exclude: |
           (?x)^(
             .venv|
+            docs/audit/.*|
             .*\.github/workflows/.*\.ya?ml$
           )$