From a0fd0f504c966d6f99e5eda5e0108a3320386c97 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 23 Oct 2025 13:58:16 +0100 Subject: [PATCH 01/15] Add ansible playbook fixes --- roles/celery/meta/main.yml | 2 +- roles/fc-backend/defaults/main.yml | 2 +- roles/fc-backend/meta/main.yml | 2 +- roles/fc-backend/tasks/main.yml | 24 ++++++++++++++++++++++++ roles/python/tasks/main.yml | 6 +++--- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/roles/celery/meta/main.yml b/roles/celery/meta/main.yml index e352f63..bebd29f 100644 --- a/roles/celery/meta/main.yml +++ b/roles/celery/meta/main.yml @@ -10,7 +10,7 @@ dependencies: - role: prep_venv user: '{{ celery_user }}' virtualenv: '{{ celery_virtualenv }}' - virtualenv_python: python3.7 + virtualenv_python: python3.8 # Then we need the FC backend available for the celery user, # since our tasks import modules from it - role: fc-backend diff --git a/roles/fc-backend/defaults/main.yml b/roles/fc-backend/defaults/main.yml index 38eed72..b7e33f4 100644 --- a/roles/fc-backend/defaults/main.yml +++ b/roles/fc-backend/defaults/main.yml @@ -7,7 +7,7 @@ fc_user: celery legacy_fc_virtualenv: "{{ user_meta[fc_user].home }}/python2_venv" # Chaste backend configuration -chaste_repo: https://chaste.cs.ox.ac.uk/git/chaste.git +chaste_repo: https://github.com/Chaste/Chaste.git fc_repo: https://chaste.cs.ox.ac.uk/svn/chaste/projects/FunctionalCuration chaste_root: "{{ user_meta[fc_user].home }}/eclipse/workspace/Chaste" fc_project_name: FunctionalCuration diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 2078f23..6c141fb 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -14,5 +14,5 @@ dependencies: - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' - virtualenv_python: python3.7 + virtualenv_python: python3.8 virtualenv_site_packages: no diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 2167cd0..4e387d4 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -68,6 +68,12 @@ version: 2.7.3 virtualenv: '{{ legacy_fc_virtualenv }}' + - name: FC | Install legacy cma that works on Python 2.7 + pip: + name: cma + version: ">=2,<3" + virtualenv: '{{ legacy_fc_virtualenv }}' + - name: FC | Install Python packages pip: name: ['numpy', 'scipy', 'cython', 'tables', 'matplotlib<2', 'lxml'] @@ -125,6 +131,24 @@ update: yes force: yes + - name: weblab-fc | Install setuptools + pip: + name: setuptools + version: ">=0,<3" + virtualenv: '{{ weblab_fc_virtualenv }}' + + - name: weblab-fc | Install Cython + pip: + name: cython + version: ">=0,<3" + virtualenv: '{{ weblab_fc_virtualenv }}' + + - name: weblab-fc | Install numpy + pip: + name: numpy + version: ">=1,<2" + virtualenv: '{{ weblab_fc_virtualenv }}' + - name: weblab-fc | Install packages needed by setup.py pip: requirements: '{{ weblab_fc_root }}/requirements/setup.txt' diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index 2d1271f..e316b7b 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -18,13 +18,13 @@ name: pyOpenSSL state: latest -- name: Install Python 3.7 +- name: Install Python 3.8 become: yes block: - name: Enable deadsnakes ppa apt_repository: repo: ppa:deadsnakes/ppa - - name: Install Python 3.7 + - name: Install Python 3.8 apt: - name: ['python3.7', 'python3.7-dev', 'python3.7-venv'] + name: ['python3.8', 'python3.8-dev', 'python3.8-venv'] state: present From 0893b8cda4383e6f3e78ea8a2ca7599e67694f71 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 23 Oct 2025 14:36:16 +0100 Subject: [PATCH 02/15] Update fc_backend playbook --- roles/fc-backend/tasks/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 4e387d4..7c8b38f 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -131,12 +131,6 @@ update: yes force: yes - - name: weblab-fc | Install setuptools - pip: - name: setuptools - version: ">=0,<3" - virtualenv: '{{ weblab_fc_virtualenv }}' - - name: weblab-fc | Install Cython pip: name: cython From cfba3007a3779e51468de2cfc7249798eea2ec8f Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 26 Nov 2025 16:45:29 +0000 Subject: [PATCH 03/15] Downgrade pip version --- roles/prep_venv/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/prep_venv/tasks/main.yml b/roles/prep_venv/tasks/main.yml index d23bb48..95834e5 100644 --- a/roles/prep_venv/tasks/main.yml +++ b/roles/prep_venv/tasks/main.yml @@ -11,7 +11,8 @@ - name: Upgrade pip in virtualenv pip: name: pip - state: latest + state: present + version: ">=20,<24.1" virtualenv: '{{ virtualenv }}' virtualenv_command: '{{ virtualenv_command | default(omit) }}' virtualenv_python: '{{ virtualenv_python | default(omit) }}' From c8dc223ae06120e635569b4005bcb53b69e04264 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 27 Nov 2025 14:29:25 +0000 Subject: [PATCH 04/15] Add Dockerfile --- Dockerfile | 104 ++++++++++++++++++++++++++++++++++++++ patches/fc/pyproject.toml | 3 ++ 2 files changed, 107 insertions(+) create mode 100755 Dockerfile create mode 100644 patches/fc/pyproject.toml diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..5c62a93 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,104 @@ +# Build the image: +# docker build -t weblab:test . + +# Run the container in interactive mode: +# docker run --init -it weblab:test /bin/bash + +FROM ubuntu:bionic + +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] + +USER root + +ENV DEFAULT_USER="celery" \ + DEFAULT_HOME="/home/celery" \ + CELERY_DIR="/opt/celery" \ + WEBLAB_FC_DIR="/opt/weblab-fc" \ + CHASTE_ROOT="/home/celery/eclipse/workspace/Chaste" + +# Add celery user and create necessary directories +RUN useradd -r -m -d ${DEFAULT_HOME} -s /bin/bash ${DEFAULT_USER} && \ + mkdir -p ${CELERY_DIR} && \ + mkdir -p ${WEBLAB_FC_DIR} && \ + chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${CELERY_DIR} && \ + chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${WEBLAB_FC_DIR} + +# Install dependencies +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + apt-utils \ + build-essential \ + ca-certificates \ + curl \ + git \ + gnupg \ + gnupg1 \ + gnupg2 \ + openssh-server \ + python-dev \ + python-pip \ + python-virtualenv \ + python3.8-dev \ + python3-pip \ + python3.8-venv \ + python-psycopg2 \ + rabbitmq-server \ + software-properties-common \ + sudo \ + ufw \ + wget && \ + echo 'deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/' > /etc/apt/sources.list.d/chaste.list && \ + apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 422C4D99 && \ + apt-get update && \ + apt-get install -y chaste-dependencies && \ + python -m pip install --upgrade pip && \ + python3.8 -m pip install --upgrade pip && \ + /tmp/tmp-runner/bin/installdependencies.sh && \ + apt-get -y clean && \ + rm -rf /var/cache/apt && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/tmp/* && \ + rm -rf /tmp/* + +# Clone repositories +USER ${DEFAULT_USER}:${DEFAULT_USER} +RUN git clone -b master --depth 1 https://github.com/ModellingWebLab/fc-runner.git ${CELERY_DIR}/repo && \ + git clone -b weblab --depth 1 https://github.com/Chaste/Chaste.git ${CHASTE_ROOT} && \ + git clone -b master --depth 1 https://github.com/ModellingWebLab/chaste-project-fitting-pints.git ${CHASTE_ROOT}/projects/AidanDaly && \ + git clone -b master --depth 1 https://github.com/ModellingWebLab/weblab-fc.git ${WEBLAB_FC_DIR}/repo + +# Apply patches +COPY patches/fc/pyproject.toml ${WEBLAB_FC_DIR}/repo/ + +# Install WebLab +RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ + source ${CELERY_DIR}/py3_venv/bin/activate && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "pip>=20,<24.1" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "numpy>=1,<2" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "cython>=0,<3" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${CELERY_DIR}/repo/requirements/base.txt && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${WEBLAB_FC_DIR}/repo/requirements/setup.txt && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ + deactivate + +# Run ansible workflows +USER root +RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ + python3.8 -m venv ~/deploy_env && \ + source ~/deploy_env/bin/activate && \ + python3.8 -m pip install --upgrade pip && \ + python3.8 -m pip install ansible && \ + cd ~/deployment && \ + ansible-playbook -i inventories/dev site.yml \ + -e 'django_git_branch=master' \ + -e 'django_superuser_email="my.email@domain"' \ + -e 'django_superuser_full_name="My Full Name"' \ + -e 'django_superuser_institution="My Institution"' && \ + deactivate && \ + rm -rf ~/deploy_env && \ + rm -rf ~/deployment + +USER ${DEFAULT_USER}:${DEFAULT_USER} +WORKDIR ${DEFAULT_HOME} diff --git a/patches/fc/pyproject.toml b/patches/fc/pyproject.toml new file mode 100644 index 0000000..106dbc8 --- /dev/null +++ b/patches/fc/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] +build-backend = "setuptools.build_meta" From 7d164754ae2688239f4c895b625627d3d96f714b Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 4 Dec 2025 11:54:20 +0000 Subject: [PATCH 05/15] Install Chaste manually in Docker --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5c62a93..3f83276 100755 --- a/Dockerfile +++ b/Dockerfile @@ -83,6 +83,14 @@ RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ deactivate +# Install Chaste +RUN python -m venv ${CELERY_DIR}/venv && \ + source ${CELERY_DIR}/venv/bin/activate && \ + cd ${CHASTE_ROOT} && \ + scons -j$(nproc) b=GccOpt co=1 cl=1 projects/FunctionalCuration && \ + scons -j$(nproc) b=GccOpt co=1 cl=1 exe=1 projects/FunctionalCuration/apps && \ + deactivate + # Run ansible workflows USER root RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ From 6c613ce60ea77cf11aedb8c85b4cbc30d845dd95 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Sun, 21 Dec 2025 14:36:33 +0000 Subject: [PATCH 06/15] Update Ansible scripts --- Vagrantfile | 19 +- broker.yml | 2 + roles/add_user/tasks/main.yml | 4 +- roles/celery/tasks/main.yml | 20 +-- roles/core/tasks/main.yml | 8 +- roles/django/meta/main.yml | 6 +- roles/django/tasks/main.yml | 31 ++-- roles/fc-backend/meta/main.yml | 4 +- roles/fc-backend/tasks/main.yml | 167 ++++++++++-------- roles/fc-web-service/tasks/main.yml | 2 +- roles/fc-web-service/vars/main.yml | 2 +- roles/geerlingguy.nginx/README.md | 6 +- roles/geerlingguy.nginx/handlers/main.yml | 4 +- roles/geerlingguy.nginx/tasks/main.yml | 21 ++- .../geerlingguy.nginx/tasks/setup-Ubuntu.yml | 2 +- roles/geerlingguy.nginx/tasks/vhosts.yml | 2 +- roles/git/tasks/main.yml | 2 +- roles/nginx/tasks/main.yml | 39 ++-- roles/nginx/vars/main.yml | 4 +- roles/ontology/tasks/main.yml | 16 +- roles/prep_venv/tasks/main.yml | 4 +- roles/python/tasks/main.yml | 57 ++++-- roles/rabbitmq/tasks/main.yml | 4 +- roles/uwsgi/tasks/main.yml | 20 +-- site.yml | 23 ++- 25 files changed, 269 insertions(+), 200 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7b7b4a7..3e66fe2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,30 +9,32 @@ Vagrant.configure("2") do |config| # Needs plugin vagrant-disksize: `vagrant plugin install vagrant-disksize` config.disksize.size = '40GB' - config.vm.network :forwarded_port, guest: 80, host: 8088 # Django front-end - config.vm.network :forwarded_port, guest: 8080, host: 8089 # Expt runner + # NOTE: This will enable public access to the opened ports. + config.vm.network "forwarded_port", guest: 80, host: 8088 # Django front-end + config.vm.network "forwarded_port", guest: 8080, host: 8089 # Experiment runner + + # NOTE: This restricts access to the opened ports via 127.0.0.1 only. + # config.vm.network "forwarded_port", guest: 80, host: 8088, host_ip: "127.0.0.1" + # config.vm.network "forwarded_port", guest: 8080, host: 8089, host_ip: "127.0.0.1" # This needs to look real enough for git to set a default identity config.vm.hostname = "weblab.local" + # Provider-specific configuration for VirtualBox config.vm.provider "virtualbox" do |vb| vb.name = "WebLab18" vb.memory = "4096" + vb.cpus = "4" end # Install the Web Lab using Ansible config.vm.provision "ansible_local" do |ansible| - # Install a specific Ansible version with pip + ansible.compatibility_mode = "2.0" ansible.install = true - ansible.install_mode = "pip" - ansible.pip_install_cmd = "sudo apt-get install -y python-pip" - # The line above is because otherwise Vagrant tries to use pip3 and breaks because OS is Python 2 - ansible.version = "2.8.0" ansible.playbook = "site.yml" ansible.inventory_path = "inventories/dev" ansible.limit = "localhost" - ansible.raw_arguments = ['--vault-id', 'dev@dev-vault-pw'] ansible.extra_vars = { django_git_branch: 'master', @@ -43,4 +45,3 @@ Vagrant.configure("2") do |config| ansible.verbose = true end end - diff --git a/broker.yml b/broker.yml index 15ba6ac..dae2ff4 100644 --- a/broker.yml +++ b/broker.yml @@ -2,5 +2,7 @@ - hosts: broker roles: + - django + - { role: uwsgi, become: yes } - core - rabbitmq diff --git a/roles/add_user/tasks/main.yml b/roles/add_user/tasks/main.yml index bd8d336..3bbcb92 100644 --- a/roles/add_user/tasks/main.yml +++ b/roles/add_user/tasks/main.yml @@ -1,13 +1,13 @@ # Create a new user account, and record meta information - name: Create user's group - become: yes + become: true group: name: '{{ user_group }}' state: present - name: Create user - become: yes + become: true user: name: '{{ user_name }}' group: '{{ user_group }}' diff --git a/roles/celery/tasks/main.yml b/roles/celery/tasks/main.yml index 857e25c..d718a55 100644 --- a/roles/celery/tasks/main.yml +++ b/roles/celery/tasks/main.yml @@ -10,31 +10,31 @@ - '{{ celery_dir }}' - '{{ celery_state_dir }}' - '{{ celery_log_dir }}' - become: yes + become: true - name: Celery | List celery state - become: yes + become: true find: paths: '{{ celery_state_dir }}' patterns: '*' register: celery_state_files - name: Celery | Checkout repository - become: yes + become: true become_user: '{{ celery_user }}' git: - clone: yes + clone: true dest: '{{ celery_checkout }}' repo: '{{ celery_git_repo }}' version: '{{ celery_git_branch }}' - update: yes - force: yes + update: true + force: true notify: - clean celery state - restart experiment runner - name: Celery | Install python packages - become: yes + become: true become_user: '{{ celery_user }}' pip: requirements: '{{ celery_checkout }}/requirements/base.txt' @@ -46,7 +46,7 @@ - restart experiment runner - name: Celery | FCWS configuration - become: yes # Because the celery_user may not be in the nginx_group + become: true # Because the celery_user may not be in the nginx_group template: src: '{{ item }}.j2' dest: '{{ celery_checkout }}/fcws/{{ item }}' @@ -59,7 +59,7 @@ - restart experiment runner - name: Install Celery service - become: yes + become: true block: - name: Celery | Systemd service definition template: @@ -83,5 +83,5 @@ # TODO: Make this an always-run notify of 'start celery' handler? service: name: '{{ celery_service_name }}' - enabled: yes + enabled: true state: started diff --git a/roles/core/tasks/main.yml b/roles/core/tasks/main.yml index 01db4f4..63d07dd 100644 --- a/roles/core/tasks/main.yml +++ b/roles/core/tasks/main.yml @@ -1,7 +1,13 @@ # Setup common to all hosts +- name: Install ufw + apt: + name: ["ufw"] + state: present + become: true + - name: Open SSH port in firewall - become: yes + become: true ufw: name: OpenSSH rule: allow diff --git a/roles/django/meta/main.yml b/roles/django/meta/main.yml index 51f9e11..2fb07c8 100644 --- a/roles/django/meta/main.yml +++ b/roles/django/meta/main.yml @@ -3,13 +3,13 @@ dependencies: - role: add_user user_name: "{{ django_user }}" user_comment: Web Lab Django user account - - { role: geerlingguy.postgresql, become: yes } + - { role: geerlingguy.postgresql, become: true } # And we need a virtualenv - role: prep_venv user: '{{ django_user }}' virtualenv: '{{ django_virtualenv }}' - virtualenv_command: pyvenv + virtualenv_python: python3.8 # Ensure node is new enough (and installed) - - { role: geerlingguy.nodejs, become: yes } + - { role: geerlingguy.nodejs, become: true } diff --git a/roles/django/tasks/main.yml b/roles/django/tasks/main.yml index 9f50237..582ccb1 100644 --- a/roles/django/tasks/main.yml +++ b/roles/django/tasks/main.yml @@ -3,7 +3,7 @@ - name: Install postfix for Django email sending if needed import_role: name: postfix - become: yes + become: true when: email_smtp_host == 'localhost' - name: Ensure the Django dirs exist with correct permissions @@ -17,10 +17,10 @@ - '{{ django_data_path }}' - '{{ django_experiments_path }}' - '{{ django_datasets_path }}' - become: yes + become: true - name: Raise limit on max open files for Django - become: yes + become: true template: src: ulimit.j2 dest: /etc/security/limits.d/weblab.conf @@ -29,27 +29,26 @@ # Install Django & dependencies - name: Install Django - become: yes + become: true become_user: "{{ django_user }}" become_method: sudo block: - name: Django | Checkout repository git: - clone: yes + clone: true dest: "{{ django_checkout }}" repo: "{{ django_git_repo }}" version: "{{ django_git_branch }}" - update: yes - force: yes + update: true + force: true notify: - restart uwsgi - - name: Django | Build psycopg2 from source + - name: Django | Install psycopg2 pip: name: psycopg2 - version: 2.7.7 + version: 2.8.6 state: present - extra_args: '--no-binary psycopg2' virtualenv: '{{ django_virtualenv }}' virtualenv_command: pyvenv become_flags: "-H" @@ -122,7 +121,7 @@ command: npm install args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - block: @@ -130,7 +129,7 @@ command: ./node_modules/gulp/bin/gulp.js args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" rescue: # Errors are often due to a node upgrade @@ -138,13 +137,13 @@ command: npm rebuild args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - name: gulp | Rebuild static assets command: ./node_modules/gulp/bin/gulp.js args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - name: Django | Collect static files @@ -153,7 +152,7 @@ app_path: "{{ django_app_path }}" settings: config.settings.deployed virtualenv: '{{ django_virtualenv }}' - become: yes + become: true become_user: "{{ django_user }}" - name: Django | Copy ontology definition @@ -174,4 +173,4 @@ weekday: "*" job: systemctl restart uwsgi state: present - become: yes + become: true diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 6c141fb..92fb265 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -10,9 +10,9 @@ dependencies: user: '{{ fc_user }}' virtualenv: '{{ legacy_fc_virtualenv }}' virtualenv_python: python2 - virtualenv_site_packages: yes + virtualenv_site_packages: true - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' virtualenv_python: python3.8 - virtualenv_site_packages: no + virtualenv_site_packages: false diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 7c8b38f..dcdbf2a 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -1,7 +1,7 @@ # Set up the (WL1) functional curation experiment runner backend - name: Install Chaste dependencies - become: yes + become: true block: - name: Chaste | Add our apt key apt_key: @@ -13,144 +13,163 @@ repo: deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/ filename: chaste state: present - update_cache: yes + update_cache: true - name: Chaste | Install the dependencies packages apt: - name: ['chaste-dependencies', 'scons', 'subversion', 'python-numpy'] + name: ["chaste-dependencies", "scons", "subversion", "python-numpy"] state: present - install_recommends: yes + install_recommends: true - name: Install Chaste & FunctionalCuration - become: yes - become_user: '{{ fc_user }}' + become: true + become_user: "{{ fc_user }}" block: - name: Checkout Chaste git: - clone: yes - dest: '{{ chaste_root }}' - repo: '{{ chaste_repo }}' - version: '{{ chaste_version }}' - update: yes - force: yes + clone: true + dest: "{{ chaste_root }}" + repo: "{{ chaste_repo }}" + version: "{{ chaste_version }}" + update: true + force: true register: code_chaste - name: Checkout FunctionalCuration subversion: - checkout: yes + checkout: true username: anonymous password: ansible@weblab.org - dest: '{{ fc_root }}' - repo: '{{ fc_repo }}' - update: yes - force: yes + dest: "{{ fc_root }}" + repo: "{{ fc_repo }}" + update: true + force: true register: code_fc notify: restart celery - name: Checkout Fitting prototype git: - clone: yes - dest: '{{ fitting_prototype_root }}' - repo: '{{ fitting_prototype_repo }}' - version: 'master' - update: yes - force: yes + clone: true + dest: "{{ fitting_prototype_root }}" + repo: "{{ fitting_prototype_repo }}" + version: "master" + update: true + force: true register: code_fitting_prototype - name: Default to ubuntu hostconfig for Chaste copy: - dest: '{{ chaste_root }}/python/hostconfig/local.py' - src: '{{ chaste_root }}/python/hostconfig/ubuntu.py' - remote_src: yes + dest: "{{ chaste_root }}/python/hostconfig/local.py" + src: "{{ chaste_root }}/python/hostconfig/ubuntu.py" + remote_src: true - name: FC | Install legacy numexpr that works on Python 2.7 pip: name: numexpr version: 2.7.3 - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install legacy cma that works on Python 2.7 pip: name: cma version: ">=2,<3" - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install Python packages pip: - name: ['numpy', 'scipy', 'cython', 'tables', 'matplotlib<2', 'lxml'] + name: + [ + "numpy>=1,<2", + "scipy", + "cython>=0,<3", + "tables", + "matplotlib<2", + "lxml", + ] state: present - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install legacy PINTS that works on Python 2.7 pip: - name: 'git+https://github.com/pints-team/pints.git@46092377397dc2a81461854a21c27df823333d7d#egg=pints' - virtualenv: '{{ legacy_fc_virtualenv }}' + name: "git+https://github.com/pints-team/pints.git@46092377397dc2a81461854a21c27df823333d7d#egg=pints" + virtualenv: "{{ legacy_fc_virtualenv }}" - name: Stat FC exe stat: - path: '{{ chaste_fc_exe }}' + path: "{{ chaste_fc_exe }}" register: fc_exe - name: Install Chaste build script template: src: build_chaste.j2 - dest: '{{ legacy_fc_virtualenv }}/bin/build_chaste' + dest: "{{ legacy_fc_virtualenv }}/bin/build_chaste" mode: 0755 register: build_script - name: Ensure Chaste is rebuilt at this point, if needed # We can't use a handler, as it's not possible to force a handler to run before the next role's tasks - command: '{{ legacy_fc_virtualenv }}/bin/build_chaste' - when: code_chaste.changed or code_fc.changed or build_script.changed or fc_exe.stat.exists == False + command: "{{ legacy_fc_virtualenv }}/bin/build_chaste" + when: + code_chaste.changed or code_fc.changed or build_script.changed or not + fc_exe.stat.exists - name: FC | Copy ontology definition import_role: name: ontology vars: - onto_destination: '{{ chaste_root }}/python/pycml/oxford-metadata.rdf' + onto_destination: "{{ chaste_root }}/python/pycml/oxford-metadata.rdf" # Moving towards WL2, we need the weblab-fc backend, although it isn't being used to run experiments yet - name: weblab-fc | Ensure parent folder exists file: - path: '{{ weblab_fc_root | dirname }}' + path: "{{ weblab_fc_root | dirname }}" state: directory - owner: '{{ fc_user }}' + owner: "{{ fc_user }}" mode: 0755 - become: yes + become: true - name: weblab-fc - become: yes - become_user: '{{ fc_user }}' + become: true + become_user: "{{ fc_user }}" block: - - name: weblab-fc | Clone repository - git: - clone: yes - dest: '{{ weblab_fc_root }}' - repo: '{{ weblab_fc_repo }}' - version: '{{ weblab_fc_branch }}' - update: yes - force: yes - - - name: weblab-fc | Install Cython - pip: - name: cython - version: ">=0,<3" - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install numpy - pip: - name: numpy - version: ">=1,<2" - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install packages needed by setup.py - pip: - requirements: '{{ weblab_fc_root }}/requirements/setup.txt' - state: present - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install packages - pip: - name: ['{{ weblab_fc_root }}'] - state: latest - virtualenv: '{{ weblab_fc_virtualenv }}' + - name: weblab-fc | Clone repository + git: + clone: true + dest: "{{ weblab_fc_root }}" + repo: "{{ weblab_fc_repo }}" + version: "{{ weblab_fc_branch }}" + update: true + force: true + + - name: weblab-fc | Add pyproject.toml to fix build issues + copy: + dest: "{{ weblab_fc_root }}/pyproject.toml" + content: | + [build-system] + requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] + build-backend = "setuptools.build_meta" + force: true + + - name: weblab-fc | Install Cython + pip: + name: cython + version: ">=0,<3" + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install numpy + pip: + name: numpy + version: ">=1,<2" + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install packages needed by setup.py + pip: + requirements: "{{ weblab_fc_root }}/requirements/setup.txt" + state: present + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install packages + pip: + name: ["{{ weblab_fc_root }}"] + state: latest + virtualenv: "{{ weblab_fc_virtualenv }}" diff --git a/roles/fc-web-service/tasks/main.yml b/roles/fc-web-service/tasks/main.yml index 8cba595..88d5d00 100644 --- a/roles/fc-web-service/tasks/main.yml +++ b/roles/fc-web-service/tasks/main.yml @@ -25,7 +25,7 @@ notify: restart fcgiwrap - name: fc-web-service | Configure nginx - become: yes + become: true import_role: name: geerlingguy.nginx diff --git a/roles/fc-web-service/vars/main.yml b/roles/fc-web-service/vars/main.yml index 02628c4..cc1357d 100644 --- a/roles/fc-web-service/vars/main.yml +++ b/roles/fc-web-service/vars/main.yml @@ -1,6 +1,6 @@ # Configure geerlingguy.nginx -nginx_remove_default_vhost: yes +nginx_remove_default_vhost: true nginx_vhosts: # Our main host configuration diff --git a/roles/geerlingguy.nginx/README.md b/roles/geerlingguy.nginx/README.md index 04223bf..8101112 100644 --- a/roles/geerlingguy.nginx/README.md +++ b/roles/geerlingguy.nginx/README.md @@ -56,7 +56,7 @@ An example of a secondary vhost which will redirect to the one shown above. *Note: The `filename` defaults to the first domain in `server_name`, if you have two vhosts with the same domain, eg. a redirect, you need to manually set the `filename` so the second one doesn't override the first one* - nginx_remove_default_vhost: false + nginx_remove_default_vhost: no Whether to remove the 'default' virtualhost configuration supplied by Nginx. Useful if you want the base `/` URL to be directed at one of your own virtual hosts configured in a separate .conf file. @@ -139,12 +139,12 @@ Configures Nginx's [`log_format`](http://nginx.org/en/docs/http/ngx_http_log_mod (For Debian/Ubuntu only) Allows you to set a different repository for the installation of Nginx. As an example, if you are running Debian's wheezy release, and want to get a newer version of Nginx, you can install the `wheezy-backports` repository and set that value here, and Ansible will use that as the `-t` option while installing Nginx. - nginx_ppa_use: false + nginx_ppa_use: no nginx_ppa_version: stable (For Ubuntu only) Allows you to use the official Nginx PPA instead of the system's package. You can set the version to `stable` or `development`. - nginx_yum_repo_enabled: true + nginx_yum_repo_enabled: yes (For RedHat/CentOS only) Set this to `false` to disable the installation of the `nginx` yum repository. This could be necessary if you want the default OS stable packages, or if you use Satellite. diff --git a/roles/geerlingguy.nginx/handlers/main.yml b/roles/geerlingguy.nginx/handlers/main.yml index 23b5afd..d09454e 100644 --- a/roles/geerlingguy.nginx/handlers/main.yml +++ b/roles/geerlingguy.nginx/handlers/main.yml @@ -1,7 +1,7 @@ --- - name: restart nginx service: name=nginx state=restarted - become: yes + become: true become_user: root - name: validate nginx configuration @@ -10,5 +10,5 @@ - name: reload nginx service: name=nginx state=reloaded - become: yes + become: true become_user: root diff --git a/roles/geerlingguy.nginx/tasks/main.yml b/roles/geerlingguy.nginx/tasks/main.yml index 4ee901a..2850eff 100644 --- a/roles/geerlingguy.nginx/tasks/main.yml +++ b/roles/geerlingguy.nginx/tasks/main.yml @@ -9,26 +9,33 @@ when: nginx_user is not defined # Setup/install tasks. -- include: setup-RedHat.yml +- name: Include RedHat setup tasks + include_tasks: setup-RedHat.yml when: ansible_os_family == 'RedHat' -- include: setup-Ubuntu.yml +- name: Include Ubuntu setup tasks + include_tasks: setup-Ubuntu.yml when: ansible_distribution == 'Ubuntu' -- include: setup-Debian.yml +- name: Include Debian setup tasks + include_tasks: setup-Debian.yml when: ansible_os_family == 'Debian' -- include: setup-FreeBSD.yml +- name: Include FreeBSD setup tasks + include_tasks: setup-FreeBSD.yml when: ansible_os_family == 'FreeBSD' -- include: setup-OpenBSD.yml +- name: Include OpenBSD setup tasks + include_tasks: setup-OpenBSD.yml when: ansible_os_family == 'OpenBSD' -- include: setup-Archlinux.yml +- name: Include Archlinux setup tasks + include_tasks: setup-Archlinux.yml when: ansible_os_family == 'Archlinux' # Vhost configuration. -- include: vhosts.yml +- name: Include vhost configuration tasks. + include_tasks: vhosts.yml # Nginx setup. - name: Copy nginx configuration in place. diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index 920dc0b..e198062 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -3,7 +3,7 @@ apt_repository: repo: 'ppa:nginx/{{ nginx_ppa_version }}' state: present - update_cache: yes + update_cache: true register: nginx_ppa_added when: nginx_ppa_use diff --git a/roles/geerlingguy.nginx/tasks/vhosts.yml b/roles/geerlingguy.nginx/tasks/vhosts.yml index 05af037..a148dec 100644 --- a/roles/geerlingguy.nginx/tasks/vhosts.yml +++ b/roles/geerlingguy.nginx/tasks/vhosts.yml @@ -16,7 +16,7 @@ template: src: "{{ item.template|default(nginx_vhost_template) }}" dest: "{{ nginx_vhost_path }}/{{ item.filename|default(item.server_name.split(' ')[0] ~ '.conf') }}" - force: yes + force: true owner: root group: root mode: 0644 diff --git a/roles/git/tasks/main.yml b/roles/git/tasks/main.yml index 4498c60..4a565f0 100644 --- a/roles/git/tasks/main.yml +++ b/roles/git/tasks/main.yml @@ -2,4 +2,4 @@ apt: name: 'git' state: present - become: yes + become: true diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 316e9af..bed14c6 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -1,28 +1,28 @@ # Our specific tasks for configuring nginx # Most is done by the imported geerlingguy.nginx role -- name: certbot | Install deps - become: yes +- name: Certbot | Install deps + become: true apt: - name: ['software-properties-common', 'python3-apt'] + name: ["software-properties-common", "python3-apt"] state: present when: inventory == 'cloud' -- name: certbot | Install ppa - become: yes +- name: Certbot | Install ppa + become: true apt_repository: repo: ppa:certbot/certbot state: present when: inventory == 'cloud' -- name: certbot | Install packages - become: yes +- name: Certbot | Install packages + become: true apt: - name: 'python-certbot-nginx' + name: "python-certbot-nginx" state: present when: inventory == 'cloud' -- name: nginx | Copy SSL certificate +- name: Nginx | Copy SSL certificate copy: src: weblab.crt dest: "{{ ssl_certificate_folder }}" @@ -33,7 +33,7 @@ - reload nginx when: inventory == 'production' -- name: nginx | Copy SSL private key +- name: Nginx | Copy SSL private key copy: src: weblab.key dest: "{{ ssl_key_folder }}" @@ -44,7 +44,7 @@ - reload nginx when: inventory == 'production' -- name: nginx | Copy DH params +- name: Nginx | Copy DH params copy: src: dhparams.pem dest: "{{ dh_param_path }}" @@ -55,22 +55,23 @@ - reload nginx when: inventory == 'production' -- name: nginx | Configure nginx - become: yes +- name: Nginx | Configure nginx + become: true import_role: name: geerlingguy.nginx -- name: nginx | Open http(s) ports +- name: Nginx | Open http(s) ports ufw: name: Nginx Full rule: allow state: enabled -- name: certbot | Configure nginx - become: yes - shell: 'certbot --non-interactive --nginx --agree-tos --email {{ certbot_email }} --no-eff-email - --keep-until-expiring - --domains {{ webserver_fqdn.split(" ") | join(",") }}' +- name: Certbot | Configure nginx + become: true + shell: + 'certbot --non-interactive --nginx --agree-tos --email {{ certbot_email }} + --no-eff-email --keep-until-expiring --domains {{ webserver_fqdn.split(" ") + | join(",") }}' register: certbot_result when: inventory == 'cloud' changed_when: '"Keeping" not in certbot_result.stdout' diff --git a/roles/nginx/vars/main.yml b/roles/nginx/vars/main.yml index a05976e..e5380ab 100644 --- a/roles/nginx/vars/main.yml +++ b/roles/nginx/vars/main.yml @@ -1,7 +1,7 @@ # Configure geerlingguy.nginx -nginx_ppa_use: yes -nginx_remove_default_vhost: yes +nginx_ppa_use: true +nginx_remove_default_vhost: true # nginx_extra_http_options: | # proxy_buffering off; # proxy_set_header X-Real-IP $remote_addr; diff --git a/roles/ontology/tasks/main.yml b/roles/ontology/tasks/main.yml index e079db7..bd4fb86 100644 --- a/roles/ontology/tasks/main.yml +++ b/roles/ontology/tasks/main.yml @@ -1,7 +1,7 @@ # Tasks to clone the Web Lab ontology definitions and install in the appropriate locations - name: Ensure folders exist with correct permissions - become: yes + become: true file: path: '{{ item }}' state: directory @@ -11,22 +11,22 @@ - '{{ onto_root }}' - name: Checkout repository - become: yes + become: true become_user: '{{ onto_user }}' become_method: sudo git: - clone: yes + clone: true dest: '{{ onto_root }}' repo: '{{ onto_repo }}' version: '{{ onto_branch }}' - update: yes - force: yes + update: true + force: true - name: Copy ontology file to destination - become: yes + become: true copy: src: '{{ onto_rdf_xml }}' - remote_src: yes + remote_src: true dest: '{{ onto_destination }}' - force: yes + force: true mode: 0644 diff --git a/roles/prep_venv/tasks/main.yml b/roles/prep_venv/tasks/main.yml index 95834e5..87a17db 100644 --- a/roles/prep_venv/tasks/main.yml +++ b/roles/prep_venv/tasks/main.yml @@ -6,7 +6,7 @@ state: directory owner: '{{ user }}' mode: 0755 - become: yes + become: true - name: Upgrade pip in virtualenv pip: @@ -17,6 +17,6 @@ virtualenv_command: '{{ virtualenv_command | default(omit) }}' virtualenv_python: '{{ virtualenv_python | default(omit) }}' virtualenv_site_packages: '{{ virtualenv_site_packages | default(omit) }}' - become: yes + become: true become_user: '{{ user }}' become_flags: "-H" diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index e316b7b..97a4a55 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -1,30 +1,49 @@ # Make sure we have our common Python tools installed -- name: Install Python +- name: Install Python 2.7 via apt apt: - name: ['python3', 'python3-pip', 'python3-venv', 'python-pip', 'python-virtualenv', 'python-psycopg2'] + name: ["python2.7-dev", "python-pip", "python-virtualenv"] state: present - become: yes + become: true -- name: Upgrade system pip - become: yes +- name: Install Python 3.8 via apt + apt: + name: ["python3.8-dev", "python3-pip", "python3.8-venv"] + state: present + become: true + +- name: Install additional Python tools via apt + apt: + name: ["python-psycopg2", "virtualenv"] + state: present + become: true + +# Set pip to versions known to work with Web Lab components +- name: Upgrade Python 2 system pip + become: true pip: name: pip - state: latest + version: ">=20,<24.1" + executable: pip2 -- name: Upgrade pyOpenSSL to avoid python-openssl breaking pip - become: yes +- name: Upgrade Python 3 system pip + become: true + pip: + name: pip + version: ">=20,<24.1" + executable: pip3 + +# Upgrade pyOpenSSL to avoid python-openssl breaking pip +- name: Upgrade Python 2 system pyOpenSSL + become: true pip: name: pyOpenSSL - state: latest + version: ">=21" + executable: pip2 -- name: Install Python 3.8 - become: yes - block: - - name: Enable deadsnakes ppa - apt_repository: - repo: ppa:deadsnakes/ppa - - name: Install Python 3.8 - apt: - name: ['python3.8', 'python3.8-dev', 'python3.8-venv'] - state: present +- name: Upgrade Python 3 system pyOpenSSL + become: true + pip: + name: pyOpenSSL + version: ">=21" + executable: pip3 diff --git a/roles/rabbitmq/tasks/main.yml b/roles/rabbitmq/tasks/main.yml index 3b55f4f..18f14fa 100644 --- a/roles/rabbitmq/tasks/main.yml +++ b/roles/rabbitmq/tasks/main.yml @@ -1,7 +1,7 @@ -# Make sure we have our common Python tools installed +# Install message broker - name: Install RabbitMQ apt: name: 'rabbitmq-server' state: present - become: yes + become: true diff --git a/roles/uwsgi/tasks/main.yml b/roles/uwsgi/tasks/main.yml index 40209bc..88ef11f 100644 --- a/roles/uwsgi/tasks/main.yml +++ b/roles/uwsgi/tasks/main.yml @@ -2,35 +2,35 @@ - name: uWSGI | Install uWSGI with Python plugin apt: - name: ['uwsgi', 'uwsgi-plugin-python3'] + name: ["uwsgi", "uwsgi-plugin-python3"] state: present - name: uWSGI | Create socket folder file: - path: '{{ uwsgi_socket_dir }}' + path: "{{ uwsgi_socket_dir }}" state: directory - owner: '{{ django_user }}' - group: '{{ nginx_group }}' + owner: "{{ django_user }}" + group: "{{ nginx_group }}" mode: 0770 - name: uWSGI | Configure logging template: src: 20-weblab.conf - dest: '/etc/rsyslog.d/20-weblab.conf' + dest: "/etc/rsyslog.d/20-weblab.conf" notify: - restart rsyslog - name: uWSGI | Configure app template: - src: django-uwsgi.ini - dest: '/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini' + src: django-uwsgi.ini + dest: "/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini" notify: - restart uwsgi - name: uWSGI | Enable app file: - dest: '/etc/uwsgi/apps-enabled/{{ uwsgi_app }}.ini' - src: '/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini' - state: link + dest: "/etc/uwsgi/apps-enabled/{{ uwsgi_app }}.ini" + src: "/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini" + state: link notify: - restart uwsgi diff --git a/site.yml b/site.yml index a9a60b0..429c290 100644 --- a/site.yml +++ b/site.yml @@ -1,6 +1,21 @@ # Main playbook -- import_playbook: broker.yml -- import_playbook: task_queue.yml -- import_playbook: workers.yml -- import_playbook: webservers.yml +- hosts: localhost + gather_facts: false + tasks: + - name: Check Ansible version + debug: + msg: "{{ ansible_version.full }}" + run_once: true + +- name: Run RabbitMQ Playbook + import_playbook: broker.yml + +- name: Run Celery Task Queue Playbook + import_playbook: task_queue.yml + +- name: Run Celery Workers Playbook + import_playbook: workers.yml + +- name: Run Django Playbook + import_playbook: webservers.yml From f710d8eae1b843091103660a2e54a624a2ee16d0 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Mon, 5 Jan 2026 22:09:36 +0000 Subject: [PATCH 07/15] Reset broker tasks --- broker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/broker.yml b/broker.yml index dae2ff4..15ba6ac 100644 --- a/broker.yml +++ b/broker.yml @@ -2,7 +2,5 @@ - hosts: broker roles: - - django - - { role: uwsgi, become: yes } - core - rabbitmq From dac485e1561c6236ef418f342fb9c0e0897ceb3a Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 3 Mar 2026 14:29:05 +0000 Subject: [PATCH 08/15] Remove Dockerfile --- Dockerfile | 112 ----------------------------------------------------- 1 file changed, 112 deletions(-) delete mode 100755 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100755 index 3f83276..0000000 --- a/Dockerfile +++ /dev/null @@ -1,112 +0,0 @@ -# Build the image: -# docker build -t weblab:test . - -# Run the container in interactive mode: -# docker run --init -it weblab:test /bin/bash - -FROM ubuntu:bionic - -SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] - -USER root - -ENV DEFAULT_USER="celery" \ - DEFAULT_HOME="/home/celery" \ - CELERY_DIR="/opt/celery" \ - WEBLAB_FC_DIR="/opt/weblab-fc" \ - CHASTE_ROOT="/home/celery/eclipse/workspace/Chaste" - -# Add celery user and create necessary directories -RUN useradd -r -m -d ${DEFAULT_HOME} -s /bin/bash ${DEFAULT_USER} && \ - mkdir -p ${CELERY_DIR} && \ - mkdir -p ${WEBLAB_FC_DIR} && \ - chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${CELERY_DIR} && \ - chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${WEBLAB_FC_DIR} - -# Install dependencies -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y \ - apt-utils \ - build-essential \ - ca-certificates \ - curl \ - git \ - gnupg \ - gnupg1 \ - gnupg2 \ - openssh-server \ - python-dev \ - python-pip \ - python-virtualenv \ - python3.8-dev \ - python3-pip \ - python3.8-venv \ - python-psycopg2 \ - rabbitmq-server \ - software-properties-common \ - sudo \ - ufw \ - wget && \ - echo 'deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/' > /etc/apt/sources.list.d/chaste.list && \ - apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 422C4D99 && \ - apt-get update && \ - apt-get install -y chaste-dependencies && \ - python -m pip install --upgrade pip && \ - python3.8 -m pip install --upgrade pip && \ - /tmp/tmp-runner/bin/installdependencies.sh && \ - apt-get -y clean && \ - rm -rf /var/cache/apt && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /var/tmp/* && \ - rm -rf /tmp/* - -# Clone repositories -USER ${DEFAULT_USER}:${DEFAULT_USER} -RUN git clone -b master --depth 1 https://github.com/ModellingWebLab/fc-runner.git ${CELERY_DIR}/repo && \ - git clone -b weblab --depth 1 https://github.com/Chaste/Chaste.git ${CHASTE_ROOT} && \ - git clone -b master --depth 1 https://github.com/ModellingWebLab/chaste-project-fitting-pints.git ${CHASTE_ROOT}/projects/AidanDaly && \ - git clone -b master --depth 1 https://github.com/ModellingWebLab/weblab-fc.git ${WEBLAB_FC_DIR}/repo - -# Apply patches -COPY patches/fc/pyproject.toml ${WEBLAB_FC_DIR}/repo/ - -# Install WebLab -RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ - source ${CELERY_DIR}/py3_venv/bin/activate && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "pip>=20,<24.1" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "numpy>=1,<2" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "cython>=0,<3" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${CELERY_DIR}/repo/requirements/base.txt && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${WEBLAB_FC_DIR}/repo/requirements/setup.txt && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ - deactivate - -# Install Chaste -RUN python -m venv ${CELERY_DIR}/venv && \ - source ${CELERY_DIR}/venv/bin/activate && \ - cd ${CHASTE_ROOT} && \ - scons -j$(nproc) b=GccOpt co=1 cl=1 projects/FunctionalCuration && \ - scons -j$(nproc) b=GccOpt co=1 cl=1 exe=1 projects/FunctionalCuration/apps && \ - deactivate - -# Run ansible workflows -USER root -RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ - python3.8 -m venv ~/deploy_env && \ - source ~/deploy_env/bin/activate && \ - python3.8 -m pip install --upgrade pip && \ - python3.8 -m pip install ansible && \ - cd ~/deployment && \ - ansible-playbook -i inventories/dev site.yml \ - -e 'django_git_branch=master' \ - -e 'django_superuser_email="my.email@domain"' \ - -e 'django_superuser_full_name="My Full Name"' \ - -e 'django_superuser_institution="My Institution"' && \ - deactivate && \ - rm -rf ~/deploy_env && \ - rm -rf ~/deployment - -USER ${DEFAULT_USER}:${DEFAULT_USER} -WORKDIR ${DEFAULT_HOME} From 00545b1f9a943c7c84d272c4a142fe11489d75b6 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 4 Mar 2026 11:35:51 +0000 Subject: [PATCH 09/15] Fix nginx setup --- README.md | 2 +- Vagrantfile | 1 + roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml | 9 ++++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 676179e..9ed1134 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Note that you will need to set up the appropriate proxying *outside* the Web Lab ### Using Vagrant -Version 2.2 of Vagrant is required for this. We recommend using the VirtualBox provider with it. +Version 2.4 of Vagrant is required for this. We recommend using the VirtualBox provider with it. You may need to edit some options in the `Vagrantfile` depending on how you have configured your local variables. For instance, remove the `raw_arguments` if you're not encrypting any secrets. diff --git a/Vagrantfile b/Vagrantfile index 3e66fe2..33dcbd2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,6 +35,7 @@ Vagrant.configure("2") do |config| ansible.playbook = "site.yml" ansible.inventory_path = "inventories/dev" ansible.limit = "localhost" + ansible.raw_arguments = ['--vault-id', 'dev@dev-vault-pw'] ansible.extra_vars = { django_git_branch: 'master', diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index e198062..6a8611d 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -1,9 +1,16 @@ --- +# This step is a workaround for the next step below which fails because +# it tries to add a key from an hkp:// address instead of an hkps:// +- name: Add PPA for Nginx manually. + command: add-apt-repository ppa:nginx/{{ nginx_ppa_version }} + when: nginx_ppa_use + - name: Add PPA for Nginx. apt_repository: - repo: 'ppa:nginx/{{ nginx_ppa_version }}' + repo: "ppa:nginx/{{ nginx_ppa_version }}" state: present update_cache: true + validate_certs: true register: nginx_ppa_added when: nginx_ppa_use From 6533e7a959d5709dca87b1ad12a7d5b697b99d79 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 10 Mar 2026 15:24:37 +0000 Subject: [PATCH 10/15] Use Python 3.6 --- patches/fc/pyproject.toml | 2 +- roles/celery/meta/main.yml | 2 +- roles/django/meta/main.yml | 2 +- roles/django/tasks/main.yml | 6 ++++++ roles/fc-backend/meta/main.yml | 2 +- roles/fc-backend/tasks/main.yml | 2 +- roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml | 2 +- roles/python/tasks/main.yml | 4 ++-- 8 files changed, 14 insertions(+), 8 deletions(-) diff --git a/patches/fc/pyproject.toml b/patches/fc/pyproject.toml index 106dbc8..5c5771b 100644 --- a/patches/fc/pyproject.toml +++ b/patches/fc/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] +requires = ["setuptools", "wheel", "numpy>=1,<2", "cython>=0,<3"] build-backend = "setuptools.build_meta" diff --git a/roles/celery/meta/main.yml b/roles/celery/meta/main.yml index bebd29f..32cb12c 100644 --- a/roles/celery/meta/main.yml +++ b/roles/celery/meta/main.yml @@ -10,7 +10,7 @@ dependencies: - role: prep_venv user: '{{ celery_user }}' virtualenv: '{{ celery_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 # Then we need the FC backend available for the celery user, # since our tasks import modules from it - role: fc-backend diff --git a/roles/django/meta/main.yml b/roles/django/meta/main.yml index 2fb07c8..e9efab2 100644 --- a/roles/django/meta/main.yml +++ b/roles/django/meta/main.yml @@ -9,7 +9,7 @@ dependencies: - role: prep_venv user: '{{ django_user }}' virtualenv: '{{ django_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 # Ensure node is new enough (and installed) - { role: geerlingguy.nodejs, become: true } diff --git a/roles/django/tasks/main.yml b/roles/django/tasks/main.yml index 582ccb1..edff083 100644 --- a/roles/django/tasks/main.yml +++ b/roles/django/tasks/main.yml @@ -44,6 +44,12 @@ notify: - restart uwsgi + - name: Django | fix git-python dependency + lineinfile: + path: '{{ django_checkout }}/requirements/base.txt' + regexp: '^gitpython==' + line: gitpython==3.1.18 + - name: Django | Install psycopg2 pip: name: psycopg2 diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 92fb265..b7a45e3 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -14,5 +14,5 @@ dependencies: - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 virtualenv_site_packages: false diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index dcdbf2a..8b85ff8 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -146,7 +146,7 @@ dest: "{{ weblab_fc_root }}/pyproject.toml" content: | [build-system] - requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] + requires = ["setuptools", "wheel", "numpy>=1,<2", "cython>=0,<3"] build-backend = "setuptools.build_meta" force: true diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index 6a8611d..e829d65 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -2,7 +2,7 @@ # This step is a workaround for the next step below which fails because # it tries to add a key from an hkp:// address instead of an hkps:// - name: Add PPA for Nginx manually. - command: add-apt-repository ppa:nginx/{{ nginx_ppa_version }} + command: add-apt-repository -y ppa:nginx/{{ nginx_ppa_version }} when: nginx_ppa_use - name: Add PPA for Nginx. diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index 97a4a55..92ba00f 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -6,9 +6,9 @@ state: present become: true -- name: Install Python 3.8 via apt +- name: Install Python 3.6 via apt apt: - name: ["python3.8-dev", "python3-pip", "python3.8-venv"] + name: ["python3.6-dev", "python3-pip", "python3.6-venv"] state: present become: true From c351f8f31bd2230dff799322dfcce2ac875add2f Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Fri, 8 May 2026 16:34:04 +0100 Subject: [PATCH 11/15] Increase Vagrant size --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 33dcbd2..5a63514 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,7 +7,7 @@ Vagrant.configure("2") do |config| config.vm.box = "ubuntu/bionic64" # Needs plugin vagrant-disksize: `vagrant plugin install vagrant-disksize` - config.disksize.size = '40GB' + config.disksize.size = '100GB' # NOTE: This will enable public access to the opened ports. config.vm.network "forwarded_port", guest: 80, host: 8088 # Django front-end From 67f34d406c185d5f88294afe04c21fe98a0448b5 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Fri, 8 May 2026 16:35:10 +0100 Subject: [PATCH 12/15] Add db restore scripts --- scripts/restore_vagrant_db.sh | 156 +++++++++++++++++++++++++++ scripts/verify_vagrant_db_restore.sh | 142 ++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100755 scripts/restore_vagrant_db.sh create mode 100755 scripts/verify_vagrant_db_restore.sh diff --git a/scripts/restore_vagrant_db.sh b/scripts/restore_vagrant_db.sh new file mode 100755 index 0000000..772156b --- /dev/null +++ b/scripts/restore_vagrant_db.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Restore the WebLab DB inside the Vagrant VM from a JSON backup fixture. +# Usage: +# scripts/restore_vagrant_db.sh [/path/in/vm/to/db.json] +# Example: +# scripts/restore_vagrant_db.sh /home/vagrant/backups/db.json + +BACKUP_PATH="${1:-/home/vagrant/backups/db.json}" +PROJECT_DIR="${PROJECT_DIR:-/opt/django/WebLab/weblab}" +SETTINGS_MODULE="${SETTINGS_MODULE:-config.settings.vagrant}" + +# Temporary artifacts are created inside the VM so host files stay untouched. +FILTERED_FIXTURE="/tmp/db_filtered_restore.json" +LOAD_LOG="/tmp/loaddata_restore.log" +MIGRATE_LOG="/tmp/migrate_restore.log" + +if ! command -v vagrant >/dev/null 2>&1; then + echo "Error: vagrant command not found in PATH." >&2 + exit 1 +fi + +if [[ ! -f Vagrantfile ]]; then + echo "Error: run this script from the deployment folder (where Vagrantfile exists)." >&2 + exit 1 +fi + +if ! vagrant status --machine-readable | grep -q ',state,running'; then + echo "Error: Vagrant VM is not running. Start it with: vagrant up" >&2 + exit 1 +fi + +echo "[1/5] Applying signal safety guards (raw fixture load)" +echo "[2/5] Clearing datasets directory" +echo "[3/5] Building filtered fixture from ${BACKUP_PATH}" +echo "[4/5] Flushing DB + loading fixture" +echo "[5/5] Running migrate" + +# Quote values for safe transport through the vagrant ssh command boundary. +backup_q=$(printf '%q' "$BACKUP_PATH") +project_q=$(printf '%q' "$PROJECT_DIR") +settings_q=$(printf '%q' "$SETTINGS_MODULE") +filtered_q=$(printf '%q' "$FILTERED_FIXTURE") +load_log_q=$(printf '%q' "$LOAD_LOG") +migrate_log_q=$(printf '%q' "$MIGRATE_LOG") + +# Run the entire restore flow as the Django app user inside the VM. +vagrant ssh -c "sudo -u weblab_django env BACKUP_PATH=${backup_q} PROJECT_DIR=${project_q} SETTINGS_MODULE=${settings_q} FILTERED_FIXTURE=${filtered_q} LOAD_LOG=${load_log_q} MIGRATE_LOG=${migrate_log_q} bash -s" <<'REMOTE_SCRIPT' +set -euo pipefail + +# Workaround for non-interactive shells because venv activation references `PS1`. +export PS1="${PS1-}" +set +u +source /opt/django/venv/bin/activate +set -u +cd "$PROJECT_DIR" + + +# During loaddata, Django sends signals with raw=True. Adding a `raw` guard +# avoids side effects (eg. repo creation) while fixture rows are being inserted. + +python - <<'PY_PATCH' +from pathlib import Path +import os + +project_dir = Path(os.environ["PROJECT_DIR"]) + +patch_targets = [ + (project_dir / "entities/signals.py", "def entity_created(sender, instance, created, **kwargs):"), + (project_dir / "accounts/signals.py", "def user_created(sender, instance, created, **kwargs):"), + (project_dir / "datasets/signals.py", "def dataset_created(sender, instance, created, **kwargs):"), +] + +guard = " if kwargs.get('raw', False):\n return\n" + +for path, signature in patch_targets: + text = path.read_text() + if "kwargs.get('raw', False)" in text: + continue + idx = text.find(signature) + if idx == -1: + raise RuntimeError(f"Could not find function signature in {path}") + insert_at = idx + len(signature) + updated = text[:insert_at] + "\n" + guard + text[insert_at:] + path.write_text(updated) +PY_PATCH + +# A clean datasets dir prevents stale files from earlier runs. +rm -rf "$PROJECT_DIR/data/datasets/"* + + +# Exclude framework-managed permission/content-type data that tends to mismatch +# across schema versions; Django will recreate the required rows. + +python - <<'PY_FILTER' +import json +import os + +backup = os.environ["BACKUP_PATH"] +filtered_path = os.environ["FILTERED_FIXTURE"] + +with open(backup) as f: + data = json.load(f) + +excluded_models = { + "auth.permission", + "contenttypes.contenttype", + "guardian.userobjectpermission", + "guardian.groupobjectpermission", +} + +filtered = [] +for obj in data: + model = obj.get("model") + if model in excluded_models: + continue + if model == "accounts.user": + # Remove direct permission links to avoid foreign key mismatches against + # regenerated auth_permission ids after flush/migrate. + fields = obj.get("fields", {}) + if "user_permissions" in fields: + fields["user_permissions"] = [] + filtered.append(obj) + +with open(filtered_path, "w") as f: + json.dump(filtered, f) + +print(f"Original records: {len(data)}") +print(f"Filtered records: {len(filtered)}") +PY_FILTER + +# Flush so imported private keys and relationships are loaded onto an empty DB. +python manage.py flush --no-input --settings="$SETTINGS_MODULE" + +# Ignore unknown fields to tolerate fixture/schema drift (for example removed fields). +python manage.py loaddata "$FILTERED_FIXTURE" --ignorenonexistent --settings="$SETTINGS_MODULE" > "$LOAD_LOG" 2>&1 + +# Treat missing "Installed ..." lines as a failed import. +if ! grep -q '^Installed ' "$LOAD_LOG"; then + echo "loaddata failed. Last lines from $LOAD_LOG:" >&2 + tail -n 60 "$LOAD_LOG" >&2 + exit 1 +fi + +grep '^Installed ' "$LOAD_LOG" + +# Ensure schema is fully current and post_migrate hooks run for regenerated metadata. +python manage.py migrate --settings="$SETTINGS_MODULE" > "$MIGRATE_LOG" 2>&1 +tail -n 8 "$MIGRATE_LOG" +REMOTE_SCRIPT + +echo +echo "Restore completed successfully." +echo "loaddata log: $LOAD_LOG" +echo "migrate log: $MIGRATE_LOG" diff --git a/scripts/verify_vagrant_db_restore.sh b/scripts/verify_vagrant_db_restore.sh new file mode 100755 index 0000000..5c5dd6a --- /dev/null +++ b/scripts/verify_vagrant_db_restore.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Sanity checks for a restored WebLab DB inside the Vagrant VM. +# Usage: +# scripts/verify_vagrant_db_restore.sh +# Optional env vars: +# PROJECT_DIR=/opt/django/WebLab/weblab +# SETTINGS_MODULE=config.settings.vagrant +# EXPECT_MIN_USERS=1 +# EXPECT_MIN_ENTITIES=1 +# EXPECT_MIN_DATASETS=0 + +PROJECT_DIR="${PROJECT_DIR:-/opt/django/WebLab/weblab}" +SETTINGS_MODULE="${SETTINGS_MODULE:-config.settings.vagrant}" + +EXPECT_MIN_USERS="${EXPECT_MIN_USERS:-1}" +EXPECT_MIN_ENTITIES="${EXPECT_MIN_ENTITIES:-1}" +EXPECT_MIN_DATASETS="${EXPECT_MIN_DATASETS:-0}" + +if ! command -v vagrant >/dev/null 2>&1; then + echo "Error: vagrant command not found in PATH." >&2 + exit 1 +fi + +if [[ ! -f Vagrantfile ]]; then + echo "Error: run this script from the deployment folder (where Vagrantfile exists)." >&2 + exit 1 +fi + +if ! vagrant status --machine-readable | grep -q ',state,running'; then + echo "Error: Vagrant VM is not running. Start it with: vagrant up" >&2 + exit 1 +fi + +# Quote values before passing them through ssh to avoid shell parsing issues. +project_q=$(printf '%q' "$PROJECT_DIR") +settings_q=$(printf '%q' "$SETTINGS_MODULE") +min_users_q=$(printf '%q' "$EXPECT_MIN_USERS") +min_entities_q=$(printf '%q' "$EXPECT_MIN_ENTITIES") +min_datasets_q=$(printf '%q' "$EXPECT_MIN_DATASETS") + +echo "Running restore verification checks inside VM..." + +# Execute checks as the Django runtime user to mirror app behavior. +vagrant ssh -c "sudo -u weblab_django env PROJECT_DIR=${project_q} SETTINGS_MODULE=${settings_q} EXPECT_MIN_USERS=${min_users_q} EXPECT_MIN_ENTITIES=${min_entities_q} EXPECT_MIN_DATASETS=${min_datasets_q} bash -s" <<'REMOTE_VERIFY' +set -euo pipefail + +# Workaround for non-interactive shells because venv activation references `PS1`. +export PS1="${PS1-}" +set +u +source /opt/django/venv/bin/activate +set -u +cd "$PROJECT_DIR" + +python - <<'PY_VERIFY' +import os +import sys +import json + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", os.environ["SETTINGS_MODULE"]) + +import django +django.setup() + +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission +from django.contrib.contenttypes.models import ContentType +from django.db import connection + +checks = [] +failures = [] + +min_users = int(os.environ["EXPECT_MIN_USERS"]) +min_entities = int(os.environ["EXPECT_MIN_ENTITIES"]) +min_datasets = int(os.environ["EXPECT_MIN_DATASETS"]) + +# Sanity checking for core user/permission tables +User = get_user_model() +user_count = User.objects.count() +perm_count = Permission.objects.count() +ctype_count = ContentType.objects.count() + +# Core metadata tables must exist after a valid restore/migrate sequence. +checks.append(("users.count", user_count, f">= {min_users}")) +if user_count < min_users: + failures.append(f"users.count={user_count} < {min_users}") + +checks.append(("auth_permission.count", perm_count, "> 0")) +if perm_count <= 0: + failures.append("auth_permission is empty") + +checks.append(("contenttypes.count", ctype_count, "> 0")) +if ctype_count <= 0: + failures.append("contenttypes is empty") + +# Check model counts. Missing models are reported as SKIPPED instead of failing. +for label, minimum in [("entities.Entity", min_entities), ("datasets.Dataset", min_datasets), ("stories.StoryGraph", 0)]: + try: + model = django.apps.apps.get_model(label) + count = model.objects.count() + checks.append((f"{label}.count", count, f">= {minimum}")) + if count < minimum: + failures.append(f"{label}.count={count} < {minimum}") + except Exception as exc: + checks.append((f"{label}.count", "SKIPPED", str(exc))) + +# Check foreign key sanity for accounts_user_user_permissions -> auth_permission. +with connection.cursor() as cursor: + cursor.execute( + """ + SELECT COUNT(*) + FROM accounts_user_user_permissions up + LEFT JOIN auth_permission p ON p.id = up.permission_id + WHERE p.id IS NULL + """ + ) + orphan_perm_links = cursor.fetchone()[0] + +checks.append(("accounts_user_user_permissions.orphans", orphan_perm_links, "== 0")) +if orphan_perm_links != 0: + failures.append(f"accounts_user_user_permissions has {orphan_perm_links} orphan permission links") + +# A tiny smoke check: fetch one active user and print basic identifying fields. +smoke_user = User.objects.filter(is_active=True).order_by("id").values("id", "email").first() +checks.append(("smoke.active_user", smoke_user if smoke_user else "NONE", "any active user preferred")) + +print("=== Restore Verification Summary ===") +for name, value, expected in checks: + print(f"- {name}: {value} (expected {expected})") + +if failures: + print("\nFAILURES:") + for item in failures: + print(f"- {item}") + sys.exit(1) + +print("\nAll checks passed.") +PY_VERIFY +REMOTE_VERIFY + +echo "Verification completed successfully." From ccd930158f32c2ad50f74aba185e0de6614562b4 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Fri, 8 May 2026 17:23:07 +0100 Subject: [PATCH 13/15] Dump skipped records to logs --- scripts/restore_vagrant_db.sh | 76 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/scripts/restore_vagrant_db.sh b/scripts/restore_vagrant_db.sh index 772156b..4adbae7 100755 --- a/scripts/restore_vagrant_db.sh +++ b/scripts/restore_vagrant_db.sh @@ -15,6 +15,9 @@ SETTINGS_MODULE="${SETTINGS_MODULE:-config.settings.vagrant}" FILTERED_FIXTURE="/tmp/db_filtered_restore.json" LOAD_LOG="/tmp/loaddata_restore.log" MIGRATE_LOG="/tmp/migrate_restore.log" +IGNORED_LOG="/tmp/restore_ignored_data.log" +IGNORED_JSON="/tmp/restore_ignored_data.json" +IGNORED_RECORDS_JSON="/tmp/restore_ignored_records.json" if ! command -v vagrant >/dev/null 2>&1; then echo "Error: vagrant command not found in PATH." >&2 @@ -44,9 +47,12 @@ settings_q=$(printf '%q' "$SETTINGS_MODULE") filtered_q=$(printf '%q' "$FILTERED_FIXTURE") load_log_q=$(printf '%q' "$LOAD_LOG") migrate_log_q=$(printf '%q' "$MIGRATE_LOG") +ignored_log_q=$(printf '%q' "$IGNORED_LOG") +ignored_json_q=$(printf '%q' "$IGNORED_JSON") +ignored_records_json_q=$(printf '%q' "$IGNORED_RECORDS_JSON") # Run the entire restore flow as the Django app user inside the VM. -vagrant ssh -c "sudo -u weblab_django env BACKUP_PATH=${backup_q} PROJECT_DIR=${project_q} SETTINGS_MODULE=${settings_q} FILTERED_FIXTURE=${filtered_q} LOAD_LOG=${load_log_q} MIGRATE_LOG=${migrate_log_q} bash -s" <<'REMOTE_SCRIPT' +vagrant ssh -c "sudo -u weblab_django env BACKUP_PATH=${backup_q} PROJECT_DIR=${project_q} SETTINGS_MODULE=${settings_q} FILTERED_FIXTURE=${filtered_q} LOAD_LOG=${load_log_q} MIGRATE_LOG=${migrate_log_q} IGNORED_LOG=${ignored_log_q} IGNORED_JSON=${ignored_json_q} IGNORED_RECORDS_JSON=${ignored_records_json_q} bash -s" <<'REMOTE_SCRIPT' set -euo pipefail # Workaround for non-interactive shells because venv activation references `PS1`. @@ -96,9 +102,13 @@ rm -rf "$PROJECT_DIR/data/datasets/"* python - <<'PY_FILTER' import json import os +from datetime import datetime backup = os.environ["BACKUP_PATH"] filtered_path = os.environ["FILTERED_FIXTURE"] +ignored_log_path = os.environ["IGNORED_LOG"] +ignored_json_path = os.environ["IGNORED_JSON"] +ignored_records_json_path = os.environ["IGNORED_RECORDS_JSON"] with open(backup) as f: data = json.load(f) @@ -110,24 +120,85 @@ excluded_models = { "guardian.groupobjectpermission", } +ignored_counts = {model: 0 for model in excluded_models} +user_permission_links_removed = 0 +user_records_touched = 0 +ignored_records = [] +stripped_user_permission_links = [] + filtered = [] for obj in data: model = obj.get("model") if model in excluded_models: + ignored_counts[model] += 1 + ignored_records.append(obj) continue if model == "accounts.user": # Remove direct permission links to avoid foreign key mismatches against # regenerated auth_permission ids after flush/migrate. fields = obj.get("fields", {}) if "user_permissions" in fields: + removed_links = fields.get("user_permissions", []) + user_permission_links_removed += len(removed_links) + if removed_links: + stripped_user_permission_links.append( + { + "model": model, + "pk": obj.get("pk"), + "removed_permission_ids": removed_links, + } + ) fields["user_permissions"] = [] + user_records_touched += 1 filtered.append(obj) with open(filtered_path, "w") as f: json.dump(filtered, f) +ignored_total = sum(ignored_counts.values()) +report = { + "generated_at_utc": datetime.utcnow().isoformat() + "Z", + "backup_path": backup, + "original_records": len(data), + "filtered_records": len(filtered), + "ignored_records_total": ignored_total, + "ignored_records_by_model": dict(sorted(ignored_counts.items())), + "accounts_user_permission_links_removed": user_permission_links_removed, + "accounts_user_records_touched": user_records_touched, +} + +with open(ignored_json_path, "w") as f: + json.dump(report, f, indent=2, sort_keys=True) + +ignored_records_dump = { + "generated_at_utc": report["generated_at_utc"], + "backup_path": backup, + "excluded_models": sorted(excluded_models), + "ignored_fixture_records": ignored_records, + "stripped_accounts_user_permission_links": stripped_user_permission_links, +} + +with open(ignored_records_json_path, "w") as f: + json.dump(ignored_records_dump, f, indent=2) + +with open(ignored_log_path, "w") as f: + f.write("Restore ignored-data summary\n") + f.write(f"Generated (UTC): {report['generated_at_utc']}\n") + f.write(f"Backup fixture: {backup}\n") + f.write(f"Original records: {len(data)}\n") + f.write(f"Filtered records: {len(filtered)}\n") + f.write(f"Ignored records total: {ignored_total}\n") + f.write("Ignored records by model:\n") + for model, count in sorted(ignored_counts.items()): + f.write(f" - {model}: {count}\n") + f.write(f"accounts.user records touched: {user_records_touched}\n") + f.write(f"accounts.user permission links removed: {user_permission_links_removed}\n") + print(f"Original records: {len(data)}") print(f"Filtered records: {len(filtered)}") +print(f"Ignored data log: {ignored_log_path}") +print(f"Ignored data JSON: {ignored_json_path}") +print(f"Ignored records JSON: {ignored_records_json_path}") PY_FILTER # Flush so imported private keys and relationships are loaded onto an empty DB. @@ -154,3 +225,6 @@ echo echo "Restore completed successfully." echo "loaddata log: $LOAD_LOG" echo "migrate log: $MIGRATE_LOG" +echo "ignored log: $IGNORED_LOG" +echo "ignored json: $IGNORED_JSON" +echo "ignored records json: $IGNORED_RECORDS_JSON" From 22d6d8483b8a31a2b70128999a6e82af979c5541 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 12 May 2026 09:49:53 +0100 Subject: [PATCH 14/15] Split python dat load script and disable emailsg --- .gitignore | 2 + scripts/restore_vagrant_db.sh | 206 ++---------- scripts/restore_vagrant_db_remote.py | 468 +++++++++++++++++++++++++++ 3 files changed, 500 insertions(+), 176 deletions(-) create mode 100644 scripts/restore_vagrant_db_remote.py diff --git a/.gitignore b/.gitignore index dff9b4d..f3103cc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ dev-vault-pw # Local-dev-only passwords group_vars/dev/vault.yml + +__pycache__ diff --git a/scripts/restore_vagrant_db.sh b/scripts/restore_vagrant_db.sh index 4adbae7..9c9f09e 100755 --- a/scripts/restore_vagrant_db.sh +++ b/scripts/restore_vagrant_db.sh @@ -11,13 +11,17 @@ BACKUP_PATH="${1:-/home/vagrant/backups/db.json}" PROJECT_DIR="${PROJECT_DIR:-/opt/django/WebLab/weblab}" SETTINGS_MODULE="${SETTINGS_MODULE:-config.settings.vagrant}" +REMOTE_PYTHON_SCRIPT="scripts/restore_vagrant_db_remote.py" + # Temporary artifacts are created inside the VM so host files stay untouched. FILTERED_FIXTURE="/tmp/db_filtered_restore.json" LOAD_LOG="/tmp/loaddata_restore.log" MIGRATE_LOG="/tmp/migrate_restore.log" IGNORED_LOG="/tmp/restore_ignored_data.log" IGNORED_JSON="/tmp/restore_ignored_data.json" -IGNORED_RECORDS_JSON="/tmp/restore_ignored_records.json" +IGNORED_RECORDS_DIR="/tmp/restore_ignored_records" +IGNORED_NONEXISTENT_FIELDS_JSON="/tmp/restore_ignored_nonexistent_fields.json" +CONVERTED_FIELDS_JSON="/tmp/restore_converted_fields.json" if ! command -v vagrant >/dev/null 2>&1; then echo "Error: vagrant command not found in PATH." >&2 @@ -34,11 +38,11 @@ if ! vagrant status --machine-readable | grep -q ',state,running'; then exit 1 fi -echo "[1/5] Applying signal safety guards (raw fixture load)" -echo "[2/5] Clearing datasets directory" -echo "[3/5] Building filtered fixture from ${BACKUP_PATH}" -echo "[4/5] Flushing DB + loading fixture" -echo "[5/5] Running migrate" + +if [[ ! -f "$REMOTE_PYTHON_SCRIPT" ]]; then + echo "Error: remote script not found: $REMOTE_PYTHON_SCRIPT" >&2 + exit 1 +fi # Quote values for safe transport through the vagrant ssh command boundary. backup_q=$(printf '%q' "$BACKUP_PATH") @@ -49,177 +53,25 @@ load_log_q=$(printf '%q' "$LOAD_LOG") migrate_log_q=$(printf '%q' "$MIGRATE_LOG") ignored_log_q=$(printf '%q' "$IGNORED_LOG") ignored_json_q=$(printf '%q' "$IGNORED_JSON") -ignored_records_json_q=$(printf '%q' "$IGNORED_RECORDS_JSON") - -# Run the entire restore flow as the Django app user inside the VM. -vagrant ssh -c "sudo -u weblab_django env BACKUP_PATH=${backup_q} PROJECT_DIR=${project_q} SETTINGS_MODULE=${settings_q} FILTERED_FIXTURE=${filtered_q} LOAD_LOG=${load_log_q} MIGRATE_LOG=${migrate_log_q} IGNORED_LOG=${ignored_log_q} IGNORED_JSON=${ignored_json_q} IGNORED_RECORDS_JSON=${ignored_records_json_q} bash -s" <<'REMOTE_SCRIPT' -set -euo pipefail - -# Workaround for non-interactive shells because venv activation references `PS1`. -export PS1="${PS1-}" -set +u -source /opt/django/venv/bin/activate -set -u -cd "$PROJECT_DIR" - - -# During loaddata, Django sends signals with raw=True. Adding a `raw` guard -# avoids side effects (eg. repo creation) while fixture rows are being inserted. - -python - <<'PY_PATCH' -from pathlib import Path -import os - -project_dir = Path(os.environ["PROJECT_DIR"]) - -patch_targets = [ - (project_dir / "entities/signals.py", "def entity_created(sender, instance, created, **kwargs):"), - (project_dir / "accounts/signals.py", "def user_created(sender, instance, created, **kwargs):"), - (project_dir / "datasets/signals.py", "def dataset_created(sender, instance, created, **kwargs):"), -] - -guard = " if kwargs.get('raw', False):\n return\n" - -for path, signature in patch_targets: - text = path.read_text() - if "kwargs.get('raw', False)" in text: - continue - idx = text.find(signature) - if idx == -1: - raise RuntimeError(f"Could not find function signature in {path}") - insert_at = idx + len(signature) - updated = text[:insert_at] + "\n" + guard + text[insert_at:] - path.write_text(updated) -PY_PATCH - -# A clean datasets dir prevents stale files from earlier runs. -rm -rf "$PROJECT_DIR/data/datasets/"* - - -# Exclude framework-managed permission/content-type data that tends to mismatch -# across schema versions; Django will recreate the required rows. +ignored_records_dir_q=$(printf '%q' "$IGNORED_RECORDS_DIR") +ignored_nonexistent_fields_json_q=$(printf '%q' "$IGNORED_NONEXISTENT_FIELDS_JSON") +converted_fields_json_q=$(printf '%q' "$CONVERTED_FIELDS_JSON") -python - <<'PY_FILTER' -import json -import os -from datetime import datetime -backup = os.environ["BACKUP_PATH"] -filtered_path = os.environ["FILTERED_FIXTURE"] -ignored_log_path = os.environ["IGNORED_LOG"] -ignored_json_path = os.environ["IGNORED_JSON"] -ignored_records_json_path = os.environ["IGNORED_RECORDS_JSON"] - -with open(backup) as f: - data = json.load(f) - -excluded_models = { - "auth.permission", - "contenttypes.contenttype", - "guardian.userobjectpermission", - "guardian.groupobjectpermission", -} - -ignored_counts = {model: 0 for model in excluded_models} -user_permission_links_removed = 0 -user_records_touched = 0 -ignored_records = [] -stripped_user_permission_links = [] - -filtered = [] -for obj in data: - model = obj.get("model") - if model in excluded_models: - ignored_counts[model] += 1 - ignored_records.append(obj) - continue - if model == "accounts.user": - # Remove direct permission links to avoid foreign key mismatches against - # regenerated auth_permission ids after flush/migrate. - fields = obj.get("fields", {}) - if "user_permissions" in fields: - removed_links = fields.get("user_permissions", []) - user_permission_links_removed += len(removed_links) - if removed_links: - stripped_user_permission_links.append( - { - "model": model, - "pk": obj.get("pk"), - "removed_permission_ids": removed_links, - } - ) - fields["user_permissions"] = [] - user_records_touched += 1 - filtered.append(obj) - -with open(filtered_path, "w") as f: - json.dump(filtered, f) - -ignored_total = sum(ignored_counts.values()) -report = { - "generated_at_utc": datetime.utcnow().isoformat() + "Z", - "backup_path": backup, - "original_records": len(data), - "filtered_records": len(filtered), - "ignored_records_total": ignored_total, - "ignored_records_by_model": dict(sorted(ignored_counts.items())), - "accounts_user_permission_links_removed": user_permission_links_removed, - "accounts_user_records_touched": user_records_touched, -} - -with open(ignored_json_path, "w") as f: - json.dump(report, f, indent=2, sort_keys=True) - -ignored_records_dump = { - "generated_at_utc": report["generated_at_utc"], - "backup_path": backup, - "excluded_models": sorted(excluded_models), - "ignored_fixture_records": ignored_records, - "stripped_accounts_user_permission_links": stripped_user_permission_links, -} - -with open(ignored_records_json_path, "w") as f: - json.dump(ignored_records_dump, f, indent=2) - -with open(ignored_log_path, "w") as f: - f.write("Restore ignored-data summary\n") - f.write(f"Generated (UTC): {report['generated_at_utc']}\n") - f.write(f"Backup fixture: {backup}\n") - f.write(f"Original records: {len(data)}\n") - f.write(f"Filtered records: {len(filtered)}\n") - f.write(f"Ignored records total: {ignored_total}\n") - f.write("Ignored records by model:\n") - for model, count in sorted(ignored_counts.items()): - f.write(f" - {model}: {count}\n") - f.write(f"accounts.user records touched: {user_records_touched}\n") - f.write(f"accounts.user permission links removed: {user_permission_links_removed}\n") - -print(f"Original records: {len(data)}") -print(f"Filtered records: {len(filtered)}") -print(f"Ignored data log: {ignored_log_path}") -print(f"Ignored data JSON: {ignored_json_path}") -print(f"Ignored records JSON: {ignored_records_json_path}") -PY_FILTER - -# Flush so imported private keys and relationships are loaded onto an empty DB. -python manage.py flush --no-input --settings="$SETTINGS_MODULE" - -# Ignore unknown fields to tolerate fixture/schema drift (for example removed fields). -python manage.py loaddata "$FILTERED_FIXTURE" --ignorenonexistent --settings="$SETTINGS_MODULE" > "$LOAD_LOG" 2>&1 - -# Treat missing "Installed ..." lines as a failed import. -if ! grep -q '^Installed ' "$LOAD_LOG"; then - echo "loaddata failed. Last lines from $LOAD_LOG:" >&2 - tail -n 60 "$LOAD_LOG" >&2 - exit 1 -fi - -grep '^Installed ' "$LOAD_LOG" - -# Ensure schema is fully current and post_migrate hooks run for regenerated metadata. -python manage.py migrate --settings="$SETTINGS_MODULE" > "$MIGRATE_LOG" 2>&1 -tail -n 8 "$MIGRATE_LOG" -REMOTE_SCRIPT +# Run the entire restore flow as the Django app user inside the VM. +vagrant ssh -c "sudo -u weblab_django env \ + BACKUP_PATH=${backup_q} \ + PROJECT_DIR=${project_q} \ + SETTINGS_MODULE=${settings_q} \ + FILTERED_FIXTURE=${filtered_q} \ + LOAD_LOG=${load_log_q} \ + MIGRATE_LOG=${migrate_log_q} \ + IGNORED_LOG=${ignored_log_q} \ + IGNORED_JSON=${ignored_json_q} \ + IGNORED_RECORDS_DIR=${ignored_records_dir_q} \ + IGNORED_NONEXISTENT_FIELDS_JSON=${ignored_nonexistent_fields_json_q} \ + CONVERTED_FIELDS_JSON=${converted_fields_json_q} \ + /opt/django/venv/bin/python -" < "$REMOTE_PYTHON_SCRIPT" echo echo "Restore completed successfully." @@ -227,4 +79,6 @@ echo "loaddata log: $LOAD_LOG" echo "migrate log: $MIGRATE_LOG" echo "ignored log: $IGNORED_LOG" echo "ignored json: $IGNORED_JSON" -echo "ignored records json: $IGNORED_RECORDS_JSON" +echo "ignored records per-model dir: $IGNORED_RECORDS_DIR" +echo "ignored --ignorenonexistent fields json: $IGNORED_NONEXISTENT_FIELDS_JSON" +echo "converted fields json: $CONVERTED_FIELDS_JSON" diff --git a/scripts/restore_vagrant_db_remote.py b/scripts/restore_vagrant_db_remote.py new file mode 100644 index 0000000..ea882c5 --- /dev/null +++ b/scripts/restore_vagrant_db_remote.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import subprocess +import sys +from contextlib import redirect_stderr, redirect_stdout +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +import django +from django.apps import apps +from django.core.management import CommandError, call_command +from django.test.utils import override_settings + + +def patch_signal_handlers(project_dir: Path) -> None: + """ + During loaddata, Django sends signals with raw=True. Adding a `raw` guard + avoids side effects (eg. repo creation) while fixture rows are being inserted. + """ + patch_targets = [ + (project_dir / "entities/signals.py", "def entity_created(sender, instance, created, **kwargs):"), + (project_dir / "accounts/signals.py", "def user_created(sender, instance, created, **kwargs):"), + (project_dir / "datasets/signals.py", "def dataset_created(sender, instance, created, **kwargs):"), + ] + guard = " if kwargs.get('raw', False):\n return\n" + + for path, signature in patch_targets: + text = path.read_text() + if "kwargs.get('raw', False)" in text: + continue + idx = text.find(signature) + if idx == -1: + raise RuntimeError(f"Could not find function signature in {path}") + insert_at = idx + len(signature) + updated = text[:insert_at] + "\n" + guard + text[insert_at:] + path.write_text(updated) + + +def clear_datasets_dir(project_dir: Path) -> None: + """ + A clean datasets dir prevents stale files from earlier runs. + """ + datasets_dir = project_dir / "data/datasets" + if not datasets_dir.exists(): + return + for child in datasets_dir.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + + +def run_manage_py(args: List[str], project_dir: Path, settings_module: str, log_path: Optional[Path] = None) -> int: + cmd = [sys.executable, "manage.py", *args, f"--settings={settings_module}"] + if log_path is None: + subprocess.run(cmd, cwd=project_dir, check=True) + return 0 + + with log_path.open("w") as handle: + result = subprocess.run(cmd, cwd=project_dir, stdout=handle, stderr=subprocess.STDOUT, check=False) + return result.returncode + + +def run_loaddata_no_email(fixture_path: Path, log_path: Path) -> int: + # Use Django's dummy backend to guarantee fixture imports never emit emails. + with log_path.open("w") as handle: + with redirect_stdout(handle), redirect_stderr(handle): + try: + with override_settings(EMAIL_BACKEND="django.core.mail.backends.dummy.EmailBackend"): + call_command("loaddata", str(fixture_path), ignorenonexistent=True) + except CommandError: + return 1 + return 0 + + +def run_migrate_no_email(log_path: Path) -> int: + # Keep migrations side-effect free from outbound email. + with log_path.open("w") as handle: + with redirect_stdout(handle), redirect_stderr(handle): + try: + with override_settings(EMAIL_BACKEND="django.core.mail.backends.dummy.EmailBackend"): + call_command("migrate") + except CommandError: + return 1 + return 0 + + +def print_tail(path: Path, lines: int) -> None: + if not path.exists(): + return + content = path.read_text().splitlines() + for line in content[-lines:]: + print(line) + + +def filter_fixture( + backup: str, + excluded_models: set, + model_path_rename_from: str, + model_path_rename_to: str, +) -> tuple: + """ + Load backup fixture and filter it: exclude framework models, remove stale + permission links from accounts.user, rename model paths, and detect + nonexistent schema fields. + + Returns tuple of: + (original_record_count, filtered_data, ignored_counts, ignored_records_by_model, + user_permission_links_removed, user_records_touched, model_path_replacements, + converted_field_records, ignored_nonexistent_fields_records, + ignored_nonexistent_fields_by_model) + """ + with open(backup) as handle: + data = json.load(handle) + + original_record_count = len(data) + ignored_counts = {model: 0 for model in excluded_models} + user_permission_links_removed = 0 + user_records_touched = 0 + ignored_records = [] + ignored_records_by_model = {model: [] for model in excluded_models} + stripped_user_permission_links = [] + model_path_replacements = 0 + converted_field_records = [] + + def replace_model_path(value, path=None, changes=None): + # Walk any nested fixture structure (dict/list/scalar), replace exact + # occurrences of the old model path in both values and dict keys, + # and record each replacement with its location for reporting. + nonlocal model_path_replacements + if path is None: + path = [] + if changes is None: + changes = [] + if isinstance(value, str): + if value == model_path_rename_from: + model_path_replacements += 1 + changes.append( + { + "kind": "value", + "path": "/".join(str(p) for p in path), + "from": model_path_rename_from, + "to": model_path_rename_to, + } + ) + return model_path_rename_to + return value + if isinstance(value, list): + return [replace_model_path(item, path + [idx], changes) for idx, item in enumerate(value)] + if isinstance(value, dict): + updated = {} + for key, item in value.items(): + new_key = model_path_rename_to if key == model_path_rename_from else key + if new_key != key: + model_path_replacements += 1 + changes.append( + { + "kind": "key", + "path": "/".join(str(p) for p in path + [str(key)]), + "from": model_path_rename_from, + "to": model_path_rename_to, + } + ) + converted_item = replace_model_path(item, path + [str(new_key)], changes) + if key == model_path_rename_from and not isinstance(converted_item, list): + # The old field stores a single value; the new field expects a list. + converted_item = [] if converted_item is None else [converted_item] + changes.append( + { + "kind": "value_to_list", + "path": "/".join(str(p) for p in path + [str(new_key)]), + "from_type": type(item).__name__, + "to_type": "list", + } + ) + updated[new_key] = converted_item + return updated + return value + + filtered = [] + for index, obj in enumerate(data): + original_model = obj.get("model") + original_pk = obj.get("pk") + record_changes = [] + obj = replace_model_path(obj, path=[], changes=record_changes) + if record_changes: + converted_field_records.append( + { + "record_index": index, + "model_before": original_model, + "pk": original_pk, + "change_count": len(record_changes), + "changes": record_changes, + } + ) + + model = obj.get("model") + if model in excluded_models: + ignored_counts[model] += 1 + ignored_records.append(obj) + ignored_records_by_model[model].append(obj) + continue + + if model == "accounts.user": + # Remove direct permission links to avoid foreign key mismatches against + # regenerated auth_permission ids after flush/migrate. + fields = obj.get("fields", {}) + if "user_permissions" in fields: + removed_links = fields.get("user_permissions", []) + user_permission_links_removed += len(removed_links) + if removed_links: + stripped_user_permission_links.append( + { + "model": model, + "pk": obj.get("pk"), + "removed_permission_ids": removed_links, + } + ) + fields["user_permissions"] = [] + user_records_touched += 1 + + filtered.append(obj) + + ignored_nonexistent_fields_records = [] + ignored_nonexistent_fields_by_model = defaultdict(lambda: defaultdict(int)) + model_fields_cache = {} + + for obj in filtered: + model_label = obj.get("model", "") + if "." not in model_label: + continue + app_label, model_name = model_label.split(".", 1) + cache_key = f"{app_label}.{model_name}" + + if cache_key not in model_fields_cache: + try: + model_cls = apps.get_model(app_label, model_name) + except LookupError: + model_fields_cache[cache_key] = None + else: + allowed = set() + for field in model_cls._meta.get_fields(): + if field.auto_created and not field.concrete: + continue + allowed.add(field.name) + attname = getattr(field, "attname", None) + if attname: + allowed.add(attname) + model_fields_cache[cache_key] = allowed + + allowed_fields = model_fields_cache[cache_key] + if allowed_fields is None: + continue + + fixture_fields = obj.get("fields", {}) + ignored_field_names = sorted(name for name in fixture_fields if name not in allowed_fields) + if not ignored_field_names: + continue + + ignored_values = {name: fixture_fields.get(name) for name in ignored_field_names} + ignored_nonexistent_fields_records.append( + { + "model": model_label, + "pk": obj.get("pk"), + "ignored_fields": ignored_values, + } + ) + for name in ignored_field_names: + ignored_nonexistent_fields_by_model[model_label][name] += 1 + + return ( + original_record_count, + filtered, + ignored_counts, + ignored_records_by_model, + user_permission_links_removed, + user_records_touched, + model_path_replacements, + converted_field_records, + ignored_nonexistent_fields_records, + ignored_nonexistent_fields_by_model, + ) + + +def main() -> int: + backup = os.environ["BACKUP_PATH"] + project_dir = Path(os.environ["PROJECT_DIR"]) + settings_module = os.environ["SETTINGS_MODULE"] + filtered_path = Path(os.environ["FILTERED_FIXTURE"]) + load_log_path = Path(os.environ["LOAD_LOG"]) + migrate_log_path = Path(os.environ["MIGRATE_LOG"]) + ignored_log_path = Path(os.environ["IGNORED_LOG"]) + ignored_json_path = Path(os.environ["IGNORED_JSON"]) + ignored_records_dir = Path(os.environ["IGNORED_RECORDS_DIR"]) + ignored_nonexistent_fields_json_path = Path(os.environ["IGNORED_NONEXISTENT_FIELDS_JSON"]) + converted_fields_json_path = Path(os.environ["CONVERTED_FIELDS_JSON"]) + + os.chdir(project_dir) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module) + django.setup() + + print("[1/6] Applying signal safety guards (raw fixture load)") + patch_signal_handlers(project_dir) + + print("[2/6] Clearing datasets directory") + clear_datasets_dir(project_dir) + + print(f"[3/6] Building filtered fixture from {backup}") + + # Exclude framework-managed permission/content-type data that tends to mismatch + # across schema versions; Django will recreate the required rows. + excluded_models = { + "auth.permission", + "contenttypes.contenttype", + "guardian.userobjectpermission", + "guardian.groupobjectpermission", + } + + model_path_rename_from = "stories.storygraph.modelgroup" + model_path_rename_to = "stories.storygraph.modelgroups" + + ignored_records_dir.mkdir(parents=True, exist_ok=True) + + ( + original_record_count, + filtered, + ignored_counts, + ignored_records_by_model, + user_permission_links_removed, + user_records_touched, + model_path_replacements, + converted_field_records, + ignored_nonexistent_fields_records, + ignored_nonexistent_fields_by_model, + ) = filter_fixture(backup, excluded_models, model_path_rename_from, model_path_rename_to) + + with filtered_path.open("w") as handle: + json.dump(filtered, handle) + + ignored_total = sum(ignored_counts.values()) + generated_at = datetime.utcnow().isoformat() + "Z" + + report = { + "generated_at_utc": generated_at, + "backup_path": backup, + "original_records": original_record_count, + "filtered_records": len(filtered), + "ignored_records_total": ignored_total, + "ignored_records_by_model": dict(sorted(ignored_counts.items())), + "renamed_model_path_from": model_path_rename_from, + "renamed_model_path_to": model_path_rename_to, + "renamed_model_path_replacements": model_path_replacements, + "accounts_user_permission_links_removed": user_permission_links_removed, + "accounts_user_records_touched": user_records_touched, + } + + ignored_records_files = {} + for model in sorted(excluded_models): + model_filename = model.replace(".", "_") + ".json" + model_path = ignored_records_dir / model_filename + ignored_records_files[model] = str(model_path) + model_dump = { + "generated_at_utc": generated_at, + "backup_path": backup, + "model": model, + "ignored_records_total": len(ignored_records_by_model[model]), + "ignored_fixture_records": ignored_records_by_model[model], + } + with model_path.open("w") as handle: + json.dump(model_dump, handle, indent=2) + + report["ignored_records_per_model_files"] = ignored_records_files + report["ignored_nonexistent_fields_total_records"] = len(ignored_nonexistent_fields_records) + report["converted_fields_json"] = str(converted_fields_json_path) + report["converted_fields_total_records"] = len(converted_field_records) + + converted_fields_report = { + "generated_at_utc": generated_at, + "backup_path": backup, + "conversion_from": model_path_rename_from, + "conversion_to": model_path_rename_to, + "replacement_count": model_path_replacements, + "records_with_conversions": len(converted_field_records), + "converted_records": converted_field_records, + } + with converted_fields_json_path.open("w") as handle: + json.dump(converted_fields_report, handle, indent=2) + + ignored_nonexistent_fields_report = { + "generated_at_utc": generated_at, + "backup_path": backup, + "analyzed_fixture_records": len(filtered), + "records_with_ignored_nonexistent_fields": len(ignored_nonexistent_fields_records), + "ignored_nonexistent_fields_by_model": { + model: dict(sorted(field_counts.items())) + for model, field_counts in sorted(ignored_nonexistent_fields_by_model.items()) + }, + "ignored_nonexistent_fields_records": ignored_nonexistent_fields_records, + } + with ignored_nonexistent_fields_json_path.open("w") as handle: + json.dump(ignored_nonexistent_fields_report, handle, indent=2) + + with ignored_json_path.open("w") as handle: + json.dump(report, handle, indent=2, sort_keys=True) + + with ignored_log_path.open("w") as handle: + handle.write("Restore ignored-data summary\n") + handle.write(f"Generated (UTC): {generated_at}\n") + handle.write(f"Backup fixture: {backup}\n") + handle.write(f"Original records: {original_record_count}\n") + handle.write(f"Filtered records: {len(filtered)}\n") + handle.write(f"Ignored records total: {ignored_total}\n") + handle.write( + f"Renamed model path replacements ({model_path_rename_from} -> {model_path_rename_to}): {model_path_replacements}\n" + ) + handle.write(f"Ignored records per-model JSON files: {ignored_records_dir}\n") + handle.write("Ignored records by model:\n") + for model, count in sorted(ignored_counts.items()): + handle.write(f" - {model}: {count}\n") + handle.write(f"accounts.user records touched: {user_records_touched}\n") + handle.write(f"accounts.user permission links removed: {user_permission_links_removed}\n") + + print(f"Original records: {original_record_count}") + print(f"Filtered records: {len(filtered)}") + print(f"Ignored data log: {ignored_log_path}") + print(f"Ignored data JSON: {ignored_json_path}") + print(f"Ignored records per-model JSON dir: {ignored_records_dir}") + print(f"Ignored --ignorenonexistent fields JSON: {ignored_nonexistent_fields_json_path}") + print(f"Converted fields JSON: {converted_fields_json_path}") + + + print("[4/6] Flushing DB") + # Flush so imported private keys and relationships are loaded onto an empty DB. + run_manage_py(["flush", "--no-input"], project_dir, settings_module) + + print("[5/6] Loading fixture") + # Ignore unknown fields to tolerate fixture/schema drift and disable emails during import. + load_rc = run_loaddata_no_email(filtered_path, load_log_path) + # Treat missing "Installed ..." lines as a failed import. + installed_lines = [ + line for line in load_log_path.read_text().splitlines() if line.startswith("Installed ") + ] + if load_rc != 0 or not installed_lines: + print(f"loaddata failed. Last lines from {load_log_path}:", file=sys.stderr) + print_tail(load_log_path, 60) + return 1 + + for line in installed_lines: + print(line) + + print("[6/6] Running migrate") + # Ensure schema is fully current and post_migrate hooks run for regenerated metadata. + migrate_rc = run_migrate_no_email(migrate_log_path) + if migrate_rc != 0: + print(f"migrate failed. Last lines from {migrate_log_path}:", file=sys.stderr) + print_tail(migrate_log_path, 60) + return 1 + + print_tail(migrate_log_path, 8) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 35898529127094328babc76aae0fa3f025ae6d43 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Mon, 18 May 2026 16:01:26 +0100 Subject: [PATCH 15/15] Update settings --- Vagrantfile | 29 +++++++++++++++++++++-------- group_vars/deployed/vars.yml | 5 +++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 5a63514..311839d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,24 +7,37 @@ Vagrant.configure("2") do |config| config.vm.box = "ubuntu/bionic64" # Needs plugin vagrant-disksize: `vagrant plugin install vagrant-disksize` - config.disksize.size = '100GB' + config.disksize.size = "100GB" + + config.vm.network "private_network", type: "dhcp" # NOTE: This will enable public access to the opened ports. + # Vagrant cannot forward to host ports < 1024 config.vm.network "forwarded_port", guest: 80, host: 8088 # Django front-end - config.vm.network "forwarded_port", guest: 8080, host: 8089 # Experiment runner + #config.vm.network "forwarded_port", guest: 8080, host: 8089 # Don't expose experiment runner + + config.vm.network "forwarded_port", guest: 443, host: 8443 # https for Django front-end # NOTE: This restricts access to the opened ports via 127.0.0.1 only. # config.vm.network "forwarded_port", guest: 80, host: 8088, host_ip: "127.0.0.1" # config.vm.network "forwarded_port", guest: 8080, host: 8089, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 22, host: 2222, id: "ssh", host_ip: "127.0.0.1" + # This needs to look real enough for git to set a default identity - config.vm.hostname = "weblab.local" + config.vm.hostname = "weblab.vagrant.local" + + # Make default SSH configuration more secure + config.ssh.insert_key = true + # config.ssh.username = "weblabsupport" #(if changing default user, add this after provisioning) + + # config.vm.synced_folder ".", "/vagrant", disabled: true #(uncomment this after provisioning) # Provider-specific configuration for VirtualBox config.vm.provider "virtualbox" do |vb| - vb.name = "WebLab18" - vb.memory = "4096" - vb.cpus = "4" + vb.name = "WebLab23" + vb.memory = "8192" + vb.cpus = "8" end # Install the Web Lab using Ansible @@ -33,9 +46,9 @@ Vagrant.configure("2") do |config| ansible.install = true ansible.playbook = "site.yml" - ansible.inventory_path = "inventories/dev" + ansible.inventory_path = "inventories/production" ansible.limit = "localhost" - ansible.raw_arguments = ['--vault-id', 'dev@dev-vault-pw'] + ansible.raw_arguments = ['--vault-id', 'main@vault-main'] ansible.extra_vars = { django_git_branch: 'master', diff --git a/group_vars/deployed/vars.yml b/group_vars/deployed/vars.yml index 30b60ce..e31b2d0 100644 --- a/group_vars/deployed/vars.yml +++ b/group_vars/deployed/vars.yml @@ -3,8 +3,9 @@ # Email addresses to use as admin contacts for Django django_admin_emails: - - ['Jonathan', 'j.p.cooper@ucl.ac.uk'] -django_from_email: 'Cardiac Web Lab ' + - ['Gary', 'gary.mirams@nottingham.ac.uk'] + - ['Kwabena', 'kwabena.amponsah1@nottingham.ac.uk'] +django_from_email: 'Cardiac Web Lab ' # How to send emails from the server