diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f1713f9..bce21f1b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.4.0a5 +current_version = 4.5.0b5 commit = True tag = True tag_name = v{new_version} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65a35656..4063257e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,7 @@ repos: exclude: | (?x)^( .venv| + docs/audit/.*| .*\.github/workflows/.*\.ya?ml$ )$ diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 45d042da..2004ad63 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,9 +1,27 @@ # ChangeLog -## [Unreleased] +## [2026-07-02] ### `datafog-python` [4.5.0] +#### Behavior Changes Since 4.4.0 + +- Agent guardrail helpers (`sanitize`, `scan_prompt`, `filter_output`, + `create_guardrail`, and the `Guardrail` class) now default to + `engine="regex"` instead of `engine="smart"`, matching the top-level + `scan`/`redact`/`protect` defaults. This keeps the core install from + probing optional NLP dependencies, but means NER-backed entities + (PERSON, ORGANIZATION, LOCATION) are no longer detected by these + helpers unless requested. **Migration:** pass `engine="smart"` to + restore the 4.4.0 behavior (requires `datafog[nlp]` or + `datafog[nlp-advanced]`). +- `DataFog.detect()` / `DataFog.scan_text()` result dictionaries now + contain keys only for the labels active under the configured locales: + the seven base labels by default, plus the `DE_*` labels when + constructed with `locales=["de"]`. +- Supported Python versions are `>=3.10,<3.14`. Python 3.14 is not yet + certified and is intentionally excluded from this release. + #### Release Thesis - Frames 4.5.0 as a focused, lightweight text PII screening release rather @@ -25,11 +43,15 @@ - Adds regex-only German structured PII support without adding core dependencies. -- Detects German VAT IDs and German IBANs by default because their country-code - structure is precise enough for default screening. -- Enables broader German identifiers only through `locales=["de"]` or explicit - entity selection, including German tax IDs, pension insurance numbers, - postal codes, passport numbers, and residence permit numbers. +- All German identifiers are locale-gated: they activate only through + `locales=["de"]` or explicit entity selection. This covers German VAT + IDs, IBANs, tax IDs, pension insurance numbers, postal codes, passport + numbers, and residence permit numbers. Default (no-locale) detection + behavior is unchanged from 4.4.0. +- When German locale support is active, overlapping matches (e.g. the + nine-digit run inside a VAT ID also matching the generic SSN pattern) + are resolved by engine-level span-overlap suppression in favor of the + more specific German label. #### Optional Profiles And Python 3.13 @@ -61,9 +83,9 @@ - Adds a 4.5 release-readiness checklist covering docs build, formatting, core no-network checks, install-profile smoke checks, German regex tests, broad non-slow tests, package build checks, and final CI status. -- Clarifies the version alignment path: the development package remains - `4.4.0a5` until stable release promotion, and the final stable release should - publish as `4.5.0`. +- Clarifies the version alignment path: development prereleases publish as + `4.5.0aN`/`4.5.0bN` from `dev`, and the stable release publishes as + `4.5.0` via the Release workflow's `stable` dispatch from `main`. ## [2026-02-13] diff --git a/datafog/agent.py b/datafog/agent.py index a031a147..e3cd7bdb 100644 --- a/datafog/agent.py +++ b/datafog/agent.py @@ -44,7 +44,12 @@ def filter(self, text: str) -> RedactResult: @dataclass class Guardrail: - """Reusable text guardrail for wrapping LLM prompts and outputs.""" + """Reusable text guardrail for wrapping LLM prompts and outputs. + + Defaults to the lightweight regex engine (changed from "smart" in 4.5.0) + so the core install never probes optional NLP dependencies; pass + ``engine="smart"`` to restore NER-backed detection. + """ entity_types: Optional[list[str]] = None locales: Optional[list[str]] = None @@ -119,6 +124,10 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str: One-liner PII removal. Returns the redacted text only. + + Uses the lightweight regex engine by default (changed from "smart" in + 4.5.0); pass ``engine="smart"`` for NER-backed detection, which requires + the optional NLP extras. """ result = scan_and_redact(text=text, engine=engine, **kwargs) return result.redacted_text @@ -127,6 +136,9 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str: def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult: """ Scan an LLM prompt for PII without modifying the input text. + + Uses the lightweight regex engine by default (changed from "smart" in + 4.5.0); pass ``engine="smart"`` for NER-backed detection. """ return scan(prompt, engine=engine, **kwargs) @@ -134,6 +146,9 @@ def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult def filter_output(output: str, engine: str = "regex", **kwargs: Any) -> RedactResult: """ Scan and redact PII from model output before returning to users. + + Uses the lightweight regex engine by default (changed from "smart" in + 4.5.0); pass ``engine="smart"`` for NER-backed detection. """ return scan_and_redact(output, engine=engine, **kwargs) diff --git a/datafog/main.py b/datafog/main.py index a8b1bcba..62abaaff 100644 --- a/datafog/main.py +++ b/datafog/main.py @@ -184,7 +184,10 @@ def detect(self, text: str) -> dict: _start = _time.monotonic() scan_result = scan(text=text, engine="regex", locales=self.locales) - result = {label: [] for label in RegexAnnotator.LABELS} + # Only pre-populate keys for labels active under the configured + # locales so the default output shape matches v4.4.0 (no DE_* keys + # unless German locale support is enabled). + result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)} legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"} for entity in scan_result.entities: label = legacy_map.get(entity.type, entity.type) diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 0b904830..44dfa2e0 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -53,7 +53,7 @@ def __init__( enabled_labels: Iterable[str] | None = None, ): self.locales = self._normalize_locales(locales) - self.active_labels = self._resolve_active_labels(enabled_labels) + self.active_labels = self.active_labels_for(self.locales, enabled_labels) # Compile all patterns once at initialization all_patterns: Dict[str, Pattern] = { @@ -100,20 +100,19 @@ def __init__( ), # SSN pattern - U.S. Social Security Number # Supports dashed and no-dash formats. + # Note: overlaps with locale-gated labels (e.g. the nine-digit run + # inside a DE_VAT_ID) are resolved by the engine's span-overlap + # suppression, not here, so default (EN) detection keeps v4.4.0 + # behavior even when German labels are active. "SSN": re.compile( r""" + (? tuple[str, . normalized.append(value) return tuple(dict.fromkeys(normalized)) - def _resolve_active_labels(self, enabled_labels: Iterable[str] | None) -> list[str]: - active = set(self.DEFAULT_LABELS) - for locale in self.locales: - active.update(self.LOCALE_LABELS[locale]) + @classmethod + def active_labels_for( + cls, + locales: str | Iterable[str] | None = None, + enabled_labels: Iterable[str] | None = None, + ) -> list[str]: + """Resolve the labels active for the given locales and explicit labels.""" + active = set(cls.DEFAULT_LABELS) + for locale in cls._normalize_locales(locales): + active.update(cls.LOCALE_LABELS[locale]) if enabled_labels is not None: active.update(label.strip().upper() for label in enabled_labels) - return [label for label in self.LABELS if label in active] + return [label for label in cls.LABELS if label in active] @staticmethod def _match_text(match: Match[str]) -> str: diff --git a/docs/v45-release-readiness.rst b/docs/v45-release-readiness.rst index 6d655c3b..62de2cc3 100644 --- a/docs/v45-release-readiness.rst +++ b/docs/v45-release-readiness.rst @@ -47,10 +47,13 @@ Use this framing for the GitHub release notes and package announcement: Call out these user-facing points: -* German VAT IDs and German IBANs are detected by default in the regex engine. -* Broader German identifiers such as tax IDs, postal codes, passport numbers, - residence permit numbers, and pension insurance numbers require - ``locales=["de"]`` or explicit entity selection. +* German structured identifiers — VAT IDs, IBANs, tax IDs, postal codes, + passport numbers, residence permit numbers, and pension insurance numbers — + are locale-gated and require ``locales=["de"]`` or explicit entity + selection. Default (no-locale) detection behavior is unchanged from 4.4.0. +* Guardrail helpers (``sanitize``, ``scan_prompt``, ``filter_output``, + ``create_guardrail``) now default to the regex engine; pass + ``engine="smart"`` to restore 4.4.0's NER-backed helper behavior. * OCR and Spark remain supported optional surfaces. They are not deprecated, but their broader overhaul is deferred beyond 4.5. * Telemetry remains disabled unless ``DATAFOG_TELEMETRY=1`` is set. diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index ff72e9fa..fd34a63c 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -2,10 +2,22 @@ from __future__ import annotations +import inspect + import pytest import datafog -from datafog.agent import GuardrailBlockedError +from datafog.agent import Guardrail, GuardrailBlockedError + + +def test_agent_helpers_default_to_regex_engine() -> None: + """4.5 lean-core contract: guardrail helpers stay on the regex path + unless an engine is explicitly requested (changed from "smart" in 4.5.0; + see CHANGELOG behavior changes).""" + assert Guardrail().engine == "regex" + assert datafog.create_guardrail().engine == "regex" + for helper in (datafog.sanitize, datafog.scan_prompt, datafog.filter_output): + assert inspect.signature(helper).parameters["engine"].default == "regex" def test_sanitize_redacts_structured_pii() -> None: diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index e5b6077a..96901cf2 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -134,16 +134,22 @@ def test_redaction_and_service_locale_support() -> None: def test_german_vat_redaction_suppresses_inner_generic_ssn_match( text: str, vat_text: str ) -> None: + # Default (no locale): v4.4.0 parity — the bare nine-digit run still + # matches the generic SSN pattern even when prefixed by a country code. scan_result = scan(text, engine="regex") - assert scan_result.entities == [] + assert [(entity.type, entity.text) for entity in scan_result.entities] == [ + ("SSN", "123456789") + ] + # German locale: the longer DE_VAT_ID span wins via the engine's + # span-overlap suppression, so the inner SSN match is dropped. locale_scan_result = scan(text, engine="regex", locales=["de"]) assert [(entity.type, entity.text) for entity in locale_scan_result.entities] == [ ("DE_VAT_ID", vat_text) ] default_redaction = scan_and_redact(text, engine="regex") - assert default_redaction.redacted_text == text + assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]") redaction = scan_and_redact(text, engine="regex", locales=["de"]) assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]") diff --git a/tests/test_regex_annotator.py b/tests/test_regex_annotator.py index 8f834bc0..ec0363c1 100644 --- a/tests/test_regex_annotator.py +++ b/tests/test_regex_annotator.py @@ -371,3 +371,18 @@ def test_annotation_result_format(): assert len(ssn_spans) >= 1 assert ssn_spans[0].text == "123-45-6789" + + +def test_ssn_detection_keeps_v44_behavior_for_country_prefixed_digits(): + """Regression guard: bare nine-digit runs after a country prefix must + still match SSN when no locale is configured (v4.4.0 parity). The + DE_VAT_ID overlap is resolved by engine-level span suppression only + when German locale support is active, never by weakening the base + SSN pattern.""" + annotator = RegexAnnotator() + for text in ( + "Reference DE 123456789 was issued.", + "Reference DE-123456789 was issued.", + "Reference DE123456789 was issued.", + ): + assert annotator.annotate(text)["SSN"] == ["123456789"], text