Prepare v4.5.0 stable release

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.3.0
+current_version = 4.5.0b5
 commit = True
 tag = True
 tag_name = v{new_version}
@@ -20,7 +20,3 @@ values =
 [bumpversion:file:datafog/__about__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-
-[bumpversion:file:setup.py]
-search = version="{current_version}"
-replace = version="{new_version}"

.github/workflows/benchmark.yml

@@ -13,12 +13,12 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0 # Fetch all history for proper comparison
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"
@@ -27,14 +27,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[nlp]"
-          pip install -r requirements-dev.txt
+          pip install -r requirements-benchmark.txt
           # Verify pytest-benchmark is installed and working
           python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)"
           python -m pytest --version
           python -m pytest --collect-only tests/benchmark_text_service.py::test_regex_performance
 
       - name: Restore benchmark data
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: .benchmarks
           # Updated cache key to reset baseline due to performance optimization changes
@@ -101,7 +101,7 @@ jobs:
           # fi
 
       - name: Upload benchmark results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: benchmark-results
           path: |

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"
@@ -31,43 +31,35 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
         install-profile: ["core", "nlp", "nlp-advanced"]
-        exclude:
-          # v4.4.0 claims Python 3.13 support for core + CLI first.
-          # Optional heavyweight profiles remain validated separately before
-          # we advertise Python 3.13 support for them.
-          - python-version: "3.13"
-            install-profile: "nlp"
-          - python-version: "3.13"
-            install-profile: "nlp-advanced"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
 
-      - name: Install base tooling
+      - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
 
       - name: Install dependencies (core)
         if: matrix.install-profile == 'core'
         run: |
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Install dependencies (nlp)
         if: matrix.install-profile == 'nlp'
         run: |
-          pip install -e ".[dev,cli,nlp]"
-          python -m spacy download en_core_web_sm
+          pip install -e ".[test,cli,nlp]" -r requirements-test.txt
+          python -m spacy download en_core_web_lg
 
       - name: Install dependencies (nlp-advanced)
         if: matrix.install-profile == 'nlp-advanced'
         run: |
-          pip install -e ".[dev,cli,nlp,nlp-advanced]"
-          python -m spacy download en_core_web_sm
+          pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt
+          python -m spacy download en_core_web_lg
+          datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests (core)
         if: matrix.install-profile == 'core'
@@ -148,17 +140,73 @@ jobs:
           PY
 
       - name: Upload coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v6
         with:
           files: ./coverage.xml
           flags: ${{ matrix.install-profile }}-py${{ matrix.python-version }}
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  profile-smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+        install-profile:
+          - core
+          - cli
+          - nlp
+          - nlp-advanced
+          - ocr
+          - distributed
+          - web
+        include:
+          - python-version: "3.13"
+            install-profile: nlp
+          - python-version: "3.13"
+            install-profile: nlp-advanced
+          - python-version: "3.13"
+            install-profile: ocr
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install Tesseract OCR
+        if: matrix.install-profile == 'ocr'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr libtesseract-dev
+
+      - name: Install dependencies (core)
+        if: matrix.install-profile == 'core'
+        run: |
+          pip install -e ".[test]"
+
+      - name: Install dependencies (profile)
+        if: matrix.install-profile != 'core'
+        run: |
+          pip install -e ".[test,${{ matrix.install-profile }}]"
+
+      - name: Run install profile smoke test
+        env:
+          DATAFOG_INSTALL_PROFILE: ${{ matrix.install-profile }}
+          DATAFOG_REQUIRE_TESSERACT: ${{ matrix.install-profile == 'ocr' && '1' || '' }}
+        run: |
+          pytest tests/test_install_profiles.py -q
+
   wheel-size:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"

.github/workflows/release.yml

@@ -61,11 +61,11 @@ jobs:
             BRANCH="dev"
           fi
 
-          echo "release_type=$TYPE" >> $GITHUB_OUTPUT
-          echo "target_branch=$BRANCH" >> $GITHUB_OUTPUT
+          echo "release_type=$TYPE" >> "$GITHUB_OUTPUT"
+          echo "target_branch=$BRANCH" >> "$GITHUB_OUTPUT"
           echo "Release type: $TYPE from $BRANCH"
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ steps.resolve.outputs.target_branch }}
@@ -86,15 +86,15 @@ jobs:
           if [ -z "$LAST_TAG" ]; then
             COMMIT_COUNT=$(git rev-list --count --since="7 days ago" HEAD)
           else
-            COMMIT_COUNT=$(git rev-list --count ${LAST_TAG}..HEAD)
+            COMMIT_COUNT=$(git rev-list --count "${LAST_TAG}..HEAD")
           fi
 
           echo "Commits since ${LAST_TAG:-'(none)'}: $COMMIT_COUNT"
 
           if [ "$COMMIT_COUNT" -gt 0 ] || [ "${{ inputs.force_build }}" = "true" ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
           else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
             echo "No changes detected, skipping release"
           fi
 
@@ -107,13 +107,13 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -126,9 +126,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[all,dev]"
-          pip install -r requirements-dev.txt
-          pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
+          pip install -e ".[all,test]"
+          pip install -r requirements-test.txt
+          python -m spacy download en_core_web_lg
+          datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests with segfault protection
         run: |
@@ -144,22 +145,21 @@ jobs:
     if: needs.determine-release.outputs.has_changes == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
 
       - name: Set up Python 3.13
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.13"
           cache: "pip"
 
       - name: Install core + CLI dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov coverage
-          pip install -e ".[dev,cli]"
+          pip install -e ".[test,cli]" -r requirements-test.txt
 
       - name: Run Python 3.13 core + CLI tests
         run: |
@@ -177,14 +177,14 @@ jobs:
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{ needs.determine-release.outputs.target_branch }}
           token: ${{ secrets.GH_PAT }}
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.11"
 
@@ -241,7 +241,7 @@ jobs:
             VERSION="$BASE"
           fi
 
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
           echo "Publishing version: $VERSION"
 
           sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py
@@ -322,7 +322,7 @@ jobs:
     if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Prune old alpha releases (keep 7)
         if: needs.determine-release.outputs.release_type == 'alpha'

.gitignore

@@ -24,6 +24,7 @@ error_log.txt
 # Environment
 .env
 .venv
+.venv*/
 venv/
 env/
 examples/venv/
@@ -58,14 +59,15 @@ docs/*
 !docs/conf.py
 !docs/Makefile
 !docs/make.bat
+!docs/optional-surfaces.rst
+!docs/agents/
+!docs/agents/**
 !docs/audit/
 !docs/audit/**
 
 # Keep all directories but ignore their contents
 */**/__pycache__/
 
-# Keep all files but ignore their contents
-Claude.md
 notes/benchmarking_notes.md
 Roadmap.md
 notes/*

.pre-commit-config.yaml

@@ -36,6 +36,7 @@ repos:
         exclude: |
           (?x)^(
             .venv|
+            docs/audit/.*|
             .*\.github/workflows/.*\.ya?ml$
           )$