From a0fd0f504c966d6f99e5eda5e0108a3320386c97 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 23 Oct 2025 13:58:16 +0100 Subject: [PATCH 01/10] Add ansible playbook fixes --- roles/celery/meta/main.yml | 2 +- roles/fc-backend/defaults/main.yml | 2 +- roles/fc-backend/meta/main.yml | 2 +- roles/fc-backend/tasks/main.yml | 24 ++++++++++++++++++++++++ roles/python/tasks/main.yml | 6 +++--- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/roles/celery/meta/main.yml b/roles/celery/meta/main.yml index e352f63..bebd29f 100644 --- a/roles/celery/meta/main.yml +++ b/roles/celery/meta/main.yml @@ -10,7 +10,7 @@ dependencies: - role: prep_venv user: '{{ celery_user }}' virtualenv: '{{ celery_virtualenv }}' - virtualenv_python: python3.7 + virtualenv_python: python3.8 # Then we need the FC backend available for the celery user, # since our tasks import modules from it - role: fc-backend diff --git a/roles/fc-backend/defaults/main.yml b/roles/fc-backend/defaults/main.yml index 38eed72..b7e33f4 100644 --- a/roles/fc-backend/defaults/main.yml +++ b/roles/fc-backend/defaults/main.yml @@ -7,7 +7,7 @@ fc_user: celery legacy_fc_virtualenv: "{{ user_meta[fc_user].home }}/python2_venv" # Chaste backend configuration -chaste_repo: https://chaste.cs.ox.ac.uk/git/chaste.git +chaste_repo: https://github.com/Chaste/Chaste.git fc_repo: https://chaste.cs.ox.ac.uk/svn/chaste/projects/FunctionalCuration chaste_root: "{{ user_meta[fc_user].home }}/eclipse/workspace/Chaste" fc_project_name: FunctionalCuration diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 2078f23..6c141fb 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -14,5 +14,5 @@ dependencies: - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' - virtualenv_python: python3.7 + virtualenv_python: python3.8 virtualenv_site_packages: no diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 2167cd0..4e387d4 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -68,6 +68,12 @@ version: 2.7.3 virtualenv: '{{ legacy_fc_virtualenv }}' + - name: FC | Install legacy cma that works on Python 2.7 + pip: + name: cma + version: ">=2,<3" + virtualenv: '{{ legacy_fc_virtualenv }}' + - name: FC | Install Python packages pip: name: ['numpy', 'scipy', 'cython', 'tables', 'matplotlib<2', 'lxml'] @@ -125,6 +131,24 @@ update: yes force: yes + - name: weblab-fc | Install setuptools + pip: + name: setuptools + version: ">=0,<3" + virtualenv: '{{ weblab_fc_virtualenv }}' + + - name: weblab-fc | Install Cython + pip: + name: cython + version: ">=0,<3" + virtualenv: '{{ weblab_fc_virtualenv }}' + + - name: weblab-fc | Install numpy + pip: + name: numpy + version: ">=1,<2" + virtualenv: '{{ weblab_fc_virtualenv }}' + - name: weblab-fc | Install packages needed by setup.py pip: requirements: '{{ weblab_fc_root }}/requirements/setup.txt' diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index 2d1271f..e316b7b 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -18,13 +18,13 @@ name: pyOpenSSL state: latest -- name: Install Python 3.7 +- name: Install Python 3.8 become: yes block: - name: Enable deadsnakes ppa apt_repository: repo: ppa:deadsnakes/ppa - - name: Install Python 3.7 + - name: Install Python 3.8 apt: - name: ['python3.7', 'python3.7-dev', 'python3.7-venv'] + name: ['python3.8', 'python3.8-dev', 'python3.8-venv'] state: present From 0893b8cda4383e6f3e78ea8a2ca7599e67694f71 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 23 Oct 2025 14:36:16 +0100 Subject: [PATCH 02/10] Update fc_backend playbook --- roles/fc-backend/tasks/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 4e387d4..7c8b38f 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -131,12 +131,6 @@ update: yes force: yes - - name: weblab-fc | Install setuptools - pip: - name: setuptools - version: ">=0,<3" - virtualenv: '{{ weblab_fc_virtualenv }}' - - name: weblab-fc | Install Cython pip: name: cython From cfba3007a3779e51468de2cfc7249798eea2ec8f Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 26 Nov 2025 16:45:29 +0000 Subject: [PATCH 03/10] Downgrade pip version --- roles/prep_venv/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/prep_venv/tasks/main.yml b/roles/prep_venv/tasks/main.yml index d23bb48..95834e5 100644 --- a/roles/prep_venv/tasks/main.yml +++ b/roles/prep_venv/tasks/main.yml @@ -11,7 +11,8 @@ - name: Upgrade pip in virtualenv pip: name: pip - state: latest + state: present + version: ">=20,<24.1" virtualenv: '{{ virtualenv }}' virtualenv_command: '{{ virtualenv_command | default(omit) }}' virtualenv_python: '{{ virtualenv_python | default(omit) }}' From c8dc223ae06120e635569b4005bcb53b69e04264 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 27 Nov 2025 14:29:25 +0000 Subject: [PATCH 04/10] Add Dockerfile --- Dockerfile | 104 ++++++++++++++++++++++++++++++++++++++ patches/fc/pyproject.toml | 3 ++ 2 files changed, 107 insertions(+) create mode 100755 Dockerfile create mode 100644 patches/fc/pyproject.toml diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..5c62a93 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,104 @@ +# Build the image: +# docker build -t weblab:test . + +# Run the container in interactive mode: +# docker run --init -it weblab:test /bin/bash + +FROM ubuntu:bionic + +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] + +USER root + +ENV DEFAULT_USER="celery" \ + DEFAULT_HOME="/home/celery" \ + CELERY_DIR="/opt/celery" \ + WEBLAB_FC_DIR="/opt/weblab-fc" \ + CHASTE_ROOT="/home/celery/eclipse/workspace/Chaste" + +# Add celery user and create necessary directories +RUN useradd -r -m -d ${DEFAULT_HOME} -s /bin/bash ${DEFAULT_USER} && \ + mkdir -p ${CELERY_DIR} && \ + mkdir -p ${WEBLAB_FC_DIR} && \ + chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${CELERY_DIR} && \ + chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${WEBLAB_FC_DIR} + +# Install dependencies +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + apt-utils \ + build-essential \ + ca-certificates \ + curl \ + git \ + gnupg \ + gnupg1 \ + gnupg2 \ + openssh-server \ + python-dev \ + python-pip \ + python-virtualenv \ + python3.8-dev \ + python3-pip \ + python3.8-venv \ + python-psycopg2 \ + rabbitmq-server \ + software-properties-common \ + sudo \ + ufw \ + wget && \ + echo 'deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/' > /etc/apt/sources.list.d/chaste.list && \ + apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 422C4D99 && \ + apt-get update && \ + apt-get install -y chaste-dependencies && \ + python -m pip install --upgrade pip && \ + python3.8 -m pip install --upgrade pip && \ + /tmp/tmp-runner/bin/installdependencies.sh && \ + apt-get -y clean && \ + rm -rf /var/cache/apt && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/tmp/* && \ + rm -rf /tmp/* + +# Clone repositories +USER ${DEFAULT_USER}:${DEFAULT_USER} +RUN git clone -b master --depth 1 https://github.com/ModellingWebLab/fc-runner.git ${CELERY_DIR}/repo && \ + git clone -b weblab --depth 1 https://github.com/Chaste/Chaste.git ${CHASTE_ROOT} && \ + git clone -b master --depth 1 https://github.com/ModellingWebLab/chaste-project-fitting-pints.git ${CHASTE_ROOT}/projects/AidanDaly && \ + git clone -b master --depth 1 https://github.com/ModellingWebLab/weblab-fc.git ${WEBLAB_FC_DIR}/repo + +# Apply patches +COPY patches/fc/pyproject.toml ${WEBLAB_FC_DIR}/repo/ + +# Install WebLab +RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ + source ${CELERY_DIR}/py3_venv/bin/activate && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "pip>=20,<24.1" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "numpy>=1,<2" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "cython>=0,<3" && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${CELERY_DIR}/repo/requirements/base.txt && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${WEBLAB_FC_DIR}/repo/requirements/setup.txt && \ + ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ + deactivate + +# Run ansible workflows +USER root +RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ + python3.8 -m venv ~/deploy_env && \ + source ~/deploy_env/bin/activate && \ + python3.8 -m pip install --upgrade pip && \ + python3.8 -m pip install ansible && \ + cd ~/deployment && \ + ansible-playbook -i inventories/dev site.yml \ + -e 'django_git_branch=master' \ + -e 'django_superuser_email="my.email@domain"' \ + -e 'django_superuser_full_name="My Full Name"' \ + -e 'django_superuser_institution="My Institution"' && \ + deactivate && \ + rm -rf ~/deploy_env && \ + rm -rf ~/deployment + +USER ${DEFAULT_USER}:${DEFAULT_USER} +WORKDIR ${DEFAULT_HOME} diff --git a/patches/fc/pyproject.toml b/patches/fc/pyproject.toml new file mode 100644 index 0000000..106dbc8 --- /dev/null +++ b/patches/fc/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] +build-backend = "setuptools.build_meta" From 7d164754ae2688239f4c895b625627d3d96f714b Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Thu, 4 Dec 2025 11:54:20 +0000 Subject: [PATCH 05/10] Install Chaste manually in Docker --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5c62a93..3f83276 100755 --- a/Dockerfile +++ b/Dockerfile @@ -83,6 +83,14 @@ RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ deactivate +# Install Chaste +RUN python -m venv ${CELERY_DIR}/venv && \ + source ${CELERY_DIR}/venv/bin/activate && \ + cd ${CHASTE_ROOT} && \ + scons -j$(nproc) b=GccOpt co=1 cl=1 projects/FunctionalCuration && \ + scons -j$(nproc) b=GccOpt co=1 cl=1 exe=1 projects/FunctionalCuration/apps && \ + deactivate + # Run ansible workflows USER root RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ From 6c613ce60ea77cf11aedb8c85b4cbc30d845dd95 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Sun, 21 Dec 2025 14:36:33 +0000 Subject: [PATCH 06/10] Update Ansible scripts --- Vagrantfile | 19 +- broker.yml | 2 + roles/add_user/tasks/main.yml | 4 +- roles/celery/tasks/main.yml | 20 +-- roles/core/tasks/main.yml | 8 +- roles/django/meta/main.yml | 6 +- roles/django/tasks/main.yml | 31 ++-- roles/fc-backend/meta/main.yml | 4 +- roles/fc-backend/tasks/main.yml | 167 ++++++++++-------- roles/fc-web-service/tasks/main.yml | 2 +- roles/fc-web-service/vars/main.yml | 2 +- roles/geerlingguy.nginx/README.md | 6 +- roles/geerlingguy.nginx/handlers/main.yml | 4 +- roles/geerlingguy.nginx/tasks/main.yml | 21 ++- .../geerlingguy.nginx/tasks/setup-Ubuntu.yml | 2 +- roles/geerlingguy.nginx/tasks/vhosts.yml | 2 +- roles/git/tasks/main.yml | 2 +- roles/nginx/tasks/main.yml | 39 ++-- roles/nginx/vars/main.yml | 4 +- roles/ontology/tasks/main.yml | 16 +- roles/prep_venv/tasks/main.yml | 4 +- roles/python/tasks/main.yml | 57 ++++-- roles/rabbitmq/tasks/main.yml | 4 +- roles/uwsgi/tasks/main.yml | 20 +-- site.yml | 23 ++- 25 files changed, 269 insertions(+), 200 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7b7b4a7..3e66fe2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,30 +9,32 @@ Vagrant.configure("2") do |config| # Needs plugin vagrant-disksize: `vagrant plugin install vagrant-disksize` config.disksize.size = '40GB' - config.vm.network :forwarded_port, guest: 80, host: 8088 # Django front-end - config.vm.network :forwarded_port, guest: 8080, host: 8089 # Expt runner + # NOTE: This will enable public access to the opened ports. + config.vm.network "forwarded_port", guest: 80, host: 8088 # Django front-end + config.vm.network "forwarded_port", guest: 8080, host: 8089 # Experiment runner + + # NOTE: This restricts access to the opened ports via 127.0.0.1 only. + # config.vm.network "forwarded_port", guest: 80, host: 8088, host_ip: "127.0.0.1" + # config.vm.network "forwarded_port", guest: 8080, host: 8089, host_ip: "127.0.0.1" # This needs to look real enough for git to set a default identity config.vm.hostname = "weblab.local" + # Provider-specific configuration for VirtualBox config.vm.provider "virtualbox" do |vb| vb.name = "WebLab18" vb.memory = "4096" + vb.cpus = "4" end # Install the Web Lab using Ansible config.vm.provision "ansible_local" do |ansible| - # Install a specific Ansible version with pip + ansible.compatibility_mode = "2.0" ansible.install = true - ansible.install_mode = "pip" - ansible.pip_install_cmd = "sudo apt-get install -y python-pip" - # The line above is because otherwise Vagrant tries to use pip3 and breaks because OS is Python 2 - ansible.version = "2.8.0" ansible.playbook = "site.yml" ansible.inventory_path = "inventories/dev" ansible.limit = "localhost" - ansible.raw_arguments = ['--vault-id', 'dev@dev-vault-pw'] ansible.extra_vars = { django_git_branch: 'master', @@ -43,4 +45,3 @@ Vagrant.configure("2") do |config| ansible.verbose = true end end - diff --git a/broker.yml b/broker.yml index 15ba6ac..dae2ff4 100644 --- a/broker.yml +++ b/broker.yml @@ -2,5 +2,7 @@ - hosts: broker roles: + - django + - { role: uwsgi, become: yes } - core - rabbitmq diff --git a/roles/add_user/tasks/main.yml b/roles/add_user/tasks/main.yml index bd8d336..3bbcb92 100644 --- a/roles/add_user/tasks/main.yml +++ b/roles/add_user/tasks/main.yml @@ -1,13 +1,13 @@ # Create a new user account, and record meta information - name: Create user's group - become: yes + become: true group: name: '{{ user_group }}' state: present - name: Create user - become: yes + become: true user: name: '{{ user_name }}' group: '{{ user_group }}' diff --git a/roles/celery/tasks/main.yml b/roles/celery/tasks/main.yml index 857e25c..d718a55 100644 --- a/roles/celery/tasks/main.yml +++ b/roles/celery/tasks/main.yml @@ -10,31 +10,31 @@ - '{{ celery_dir }}' - '{{ celery_state_dir }}' - '{{ celery_log_dir }}' - become: yes + become: true - name: Celery | List celery state - become: yes + become: true find: paths: '{{ celery_state_dir }}' patterns: '*' register: celery_state_files - name: Celery | Checkout repository - become: yes + become: true become_user: '{{ celery_user }}' git: - clone: yes + clone: true dest: '{{ celery_checkout }}' repo: '{{ celery_git_repo }}' version: '{{ celery_git_branch }}' - update: yes - force: yes + update: true + force: true notify: - clean celery state - restart experiment runner - name: Celery | Install python packages - become: yes + become: true become_user: '{{ celery_user }}' pip: requirements: '{{ celery_checkout }}/requirements/base.txt' @@ -46,7 +46,7 @@ - restart experiment runner - name: Celery | FCWS configuration - become: yes # Because the celery_user may not be in the nginx_group + become: true # Because the celery_user may not be in the nginx_group template: src: '{{ item }}.j2' dest: '{{ celery_checkout }}/fcws/{{ item }}' @@ -59,7 +59,7 @@ - restart experiment runner - name: Install Celery service - become: yes + become: true block: - name: Celery | Systemd service definition template: @@ -83,5 +83,5 @@ # TODO: Make this an always-run notify of 'start celery' handler? service: name: '{{ celery_service_name }}' - enabled: yes + enabled: true state: started diff --git a/roles/core/tasks/main.yml b/roles/core/tasks/main.yml index 01db4f4..63d07dd 100644 --- a/roles/core/tasks/main.yml +++ b/roles/core/tasks/main.yml @@ -1,7 +1,13 @@ # Setup common to all hosts +- name: Install ufw + apt: + name: ["ufw"] + state: present + become: true + - name: Open SSH port in firewall - become: yes + become: true ufw: name: OpenSSH rule: allow diff --git a/roles/django/meta/main.yml b/roles/django/meta/main.yml index 51f9e11..2fb07c8 100644 --- a/roles/django/meta/main.yml +++ b/roles/django/meta/main.yml @@ -3,13 +3,13 @@ dependencies: - role: add_user user_name: "{{ django_user }}" user_comment: Web Lab Django user account - - { role: geerlingguy.postgresql, become: yes } + - { role: geerlingguy.postgresql, become: true } # And we need a virtualenv - role: prep_venv user: '{{ django_user }}' virtualenv: '{{ django_virtualenv }}' - virtualenv_command: pyvenv + virtualenv_python: python3.8 # Ensure node is new enough (and installed) - - { role: geerlingguy.nodejs, become: yes } + - { role: geerlingguy.nodejs, become: true } diff --git a/roles/django/tasks/main.yml b/roles/django/tasks/main.yml index 9f50237..582ccb1 100644 --- a/roles/django/tasks/main.yml +++ b/roles/django/tasks/main.yml @@ -3,7 +3,7 @@ - name: Install postfix for Django email sending if needed import_role: name: postfix - become: yes + become: true when: email_smtp_host == 'localhost' - name: Ensure the Django dirs exist with correct permissions @@ -17,10 +17,10 @@ - '{{ django_data_path }}' - '{{ django_experiments_path }}' - '{{ django_datasets_path }}' - become: yes + become: true - name: Raise limit on max open files for Django - become: yes + become: true template: src: ulimit.j2 dest: /etc/security/limits.d/weblab.conf @@ -29,27 +29,26 @@ # Install Django & dependencies - name: Install Django - become: yes + become: true become_user: "{{ django_user }}" become_method: sudo block: - name: Django | Checkout repository git: - clone: yes + clone: true dest: "{{ django_checkout }}" repo: "{{ django_git_repo }}" version: "{{ django_git_branch }}" - update: yes - force: yes + update: true + force: true notify: - restart uwsgi - - name: Django | Build psycopg2 from source + - name: Django | Install psycopg2 pip: name: psycopg2 - version: 2.7.7 + version: 2.8.6 state: present - extra_args: '--no-binary psycopg2' virtualenv: '{{ django_virtualenv }}' virtualenv_command: pyvenv become_flags: "-H" @@ -122,7 +121,7 @@ command: npm install args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - block: @@ -130,7 +129,7 @@ command: ./node_modules/gulp/bin/gulp.js args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" rescue: # Errors are often due to a node upgrade @@ -138,13 +137,13 @@ command: npm rebuild args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - name: gulp | Rebuild static assets command: ./node_modules/gulp/bin/gulp.js args: chdir: "{{ django_static_path }}" - become: yes + become: true become_user: "{{ django_user }}" - name: Django | Collect static files @@ -153,7 +152,7 @@ app_path: "{{ django_app_path }}" settings: config.settings.deployed virtualenv: '{{ django_virtualenv }}' - become: yes + become: true become_user: "{{ django_user }}" - name: Django | Copy ontology definition @@ -174,4 +173,4 @@ weekday: "*" job: systemctl restart uwsgi state: present - become: yes + become: true diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 6c141fb..92fb265 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -10,9 +10,9 @@ dependencies: user: '{{ fc_user }}' virtualenv: '{{ legacy_fc_virtualenv }}' virtualenv_python: python2 - virtualenv_site_packages: yes + virtualenv_site_packages: true - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' virtualenv_python: python3.8 - virtualenv_site_packages: no + virtualenv_site_packages: false diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index 7c8b38f..dcdbf2a 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -1,7 +1,7 @@ # Set up the (WL1) functional curation experiment runner backend - name: Install Chaste dependencies - become: yes + become: true block: - name: Chaste | Add our apt key apt_key: @@ -13,144 +13,163 @@ repo: deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/ filename: chaste state: present - update_cache: yes + update_cache: true - name: Chaste | Install the dependencies packages apt: - name: ['chaste-dependencies', 'scons', 'subversion', 'python-numpy'] + name: ["chaste-dependencies", "scons", "subversion", "python-numpy"] state: present - install_recommends: yes + install_recommends: true - name: Install Chaste & FunctionalCuration - become: yes - become_user: '{{ fc_user }}' + become: true + become_user: "{{ fc_user }}" block: - name: Checkout Chaste git: - clone: yes - dest: '{{ chaste_root }}' - repo: '{{ chaste_repo }}' - version: '{{ chaste_version }}' - update: yes - force: yes + clone: true + dest: "{{ chaste_root }}" + repo: "{{ chaste_repo }}" + version: "{{ chaste_version }}" + update: true + force: true register: code_chaste - name: Checkout FunctionalCuration subversion: - checkout: yes + checkout: true username: anonymous password: ansible@weblab.org - dest: '{{ fc_root }}' - repo: '{{ fc_repo }}' - update: yes - force: yes + dest: "{{ fc_root }}" + repo: "{{ fc_repo }}" + update: true + force: true register: code_fc notify: restart celery - name: Checkout Fitting prototype git: - clone: yes - dest: '{{ fitting_prototype_root }}' - repo: '{{ fitting_prototype_repo }}' - version: 'master' - update: yes - force: yes + clone: true + dest: "{{ fitting_prototype_root }}" + repo: "{{ fitting_prototype_repo }}" + version: "master" + update: true + force: true register: code_fitting_prototype - name: Default to ubuntu hostconfig for Chaste copy: - dest: '{{ chaste_root }}/python/hostconfig/local.py' - src: '{{ chaste_root }}/python/hostconfig/ubuntu.py' - remote_src: yes + dest: "{{ chaste_root }}/python/hostconfig/local.py" + src: "{{ chaste_root }}/python/hostconfig/ubuntu.py" + remote_src: true - name: FC | Install legacy numexpr that works on Python 2.7 pip: name: numexpr version: 2.7.3 - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install legacy cma that works on Python 2.7 pip: name: cma version: ">=2,<3" - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install Python packages pip: - name: ['numpy', 'scipy', 'cython', 'tables', 'matplotlib<2', 'lxml'] + name: + [ + "numpy>=1,<2", + "scipy", + "cython>=0,<3", + "tables", + "matplotlib<2", + "lxml", + ] state: present - virtualenv: '{{ legacy_fc_virtualenv }}' + virtualenv: "{{ legacy_fc_virtualenv }}" - name: FC | Install legacy PINTS that works on Python 2.7 pip: - name: 'git+https://github.com/pints-team/pints.git@46092377397dc2a81461854a21c27df823333d7d#egg=pints' - virtualenv: '{{ legacy_fc_virtualenv }}' + name: "git+https://github.com/pints-team/pints.git@46092377397dc2a81461854a21c27df823333d7d#egg=pints" + virtualenv: "{{ legacy_fc_virtualenv }}" - name: Stat FC exe stat: - path: '{{ chaste_fc_exe }}' + path: "{{ chaste_fc_exe }}" register: fc_exe - name: Install Chaste build script template: src: build_chaste.j2 - dest: '{{ legacy_fc_virtualenv }}/bin/build_chaste' + dest: "{{ legacy_fc_virtualenv }}/bin/build_chaste" mode: 0755 register: build_script - name: Ensure Chaste is rebuilt at this point, if needed # We can't use a handler, as it's not possible to force a handler to run before the next role's tasks - command: '{{ legacy_fc_virtualenv }}/bin/build_chaste' - when: code_chaste.changed or code_fc.changed or build_script.changed or fc_exe.stat.exists == False + command: "{{ legacy_fc_virtualenv }}/bin/build_chaste" + when: + code_chaste.changed or code_fc.changed or build_script.changed or not + fc_exe.stat.exists - name: FC | Copy ontology definition import_role: name: ontology vars: - onto_destination: '{{ chaste_root }}/python/pycml/oxford-metadata.rdf' + onto_destination: "{{ chaste_root }}/python/pycml/oxford-metadata.rdf" # Moving towards WL2, we need the weblab-fc backend, although it isn't being used to run experiments yet - name: weblab-fc | Ensure parent folder exists file: - path: '{{ weblab_fc_root | dirname }}' + path: "{{ weblab_fc_root | dirname }}" state: directory - owner: '{{ fc_user }}' + owner: "{{ fc_user }}" mode: 0755 - become: yes + become: true - name: weblab-fc - become: yes - become_user: '{{ fc_user }}' + become: true + become_user: "{{ fc_user }}" block: - - name: weblab-fc | Clone repository - git: - clone: yes - dest: '{{ weblab_fc_root }}' - repo: '{{ weblab_fc_repo }}' - version: '{{ weblab_fc_branch }}' - update: yes - force: yes - - - name: weblab-fc | Install Cython - pip: - name: cython - version: ">=0,<3" - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install numpy - pip: - name: numpy - version: ">=1,<2" - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install packages needed by setup.py - pip: - requirements: '{{ weblab_fc_root }}/requirements/setup.txt' - state: present - virtualenv: '{{ weblab_fc_virtualenv }}' - - - name: weblab-fc | Install packages - pip: - name: ['{{ weblab_fc_root }}'] - state: latest - virtualenv: '{{ weblab_fc_virtualenv }}' + - name: weblab-fc | Clone repository + git: + clone: true + dest: "{{ weblab_fc_root }}" + repo: "{{ weblab_fc_repo }}" + version: "{{ weblab_fc_branch }}" + update: true + force: true + + - name: weblab-fc | Add pyproject.toml to fix build issues + copy: + dest: "{{ weblab_fc_root }}/pyproject.toml" + content: | + [build-system] + requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] + build-backend = "setuptools.build_meta" + force: true + + - name: weblab-fc | Install Cython + pip: + name: cython + version: ">=0,<3" + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install numpy + pip: + name: numpy + version: ">=1,<2" + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install packages needed by setup.py + pip: + requirements: "{{ weblab_fc_root }}/requirements/setup.txt" + state: present + virtualenv: "{{ weblab_fc_virtualenv }}" + + - name: weblab-fc | Install packages + pip: + name: ["{{ weblab_fc_root }}"] + state: latest + virtualenv: "{{ weblab_fc_virtualenv }}" diff --git a/roles/fc-web-service/tasks/main.yml b/roles/fc-web-service/tasks/main.yml index 8cba595..88d5d00 100644 --- a/roles/fc-web-service/tasks/main.yml +++ b/roles/fc-web-service/tasks/main.yml @@ -25,7 +25,7 @@ notify: restart fcgiwrap - name: fc-web-service | Configure nginx - become: yes + become: true import_role: name: geerlingguy.nginx diff --git a/roles/fc-web-service/vars/main.yml b/roles/fc-web-service/vars/main.yml index 02628c4..cc1357d 100644 --- a/roles/fc-web-service/vars/main.yml +++ b/roles/fc-web-service/vars/main.yml @@ -1,6 +1,6 @@ # Configure geerlingguy.nginx -nginx_remove_default_vhost: yes +nginx_remove_default_vhost: true nginx_vhosts: # Our main host configuration diff --git a/roles/geerlingguy.nginx/README.md b/roles/geerlingguy.nginx/README.md index 04223bf..8101112 100644 --- a/roles/geerlingguy.nginx/README.md +++ b/roles/geerlingguy.nginx/README.md @@ -56,7 +56,7 @@ An example of a secondary vhost which will redirect to the one shown above. *Note: The `filename` defaults to the first domain in `server_name`, if you have two vhosts with the same domain, eg. a redirect, you need to manually set the `filename` so the second one doesn't override the first one* - nginx_remove_default_vhost: false + nginx_remove_default_vhost: no Whether to remove the 'default' virtualhost configuration supplied by Nginx. Useful if you want the base `/` URL to be directed at one of your own virtual hosts configured in a separate .conf file. @@ -139,12 +139,12 @@ Configures Nginx's [`log_format`](http://nginx.org/en/docs/http/ngx_http_log_mod (For Debian/Ubuntu only) Allows you to set a different repository for the installation of Nginx. As an example, if you are running Debian's wheezy release, and want to get a newer version of Nginx, you can install the `wheezy-backports` repository and set that value here, and Ansible will use that as the `-t` option while installing Nginx. - nginx_ppa_use: false + nginx_ppa_use: no nginx_ppa_version: stable (For Ubuntu only) Allows you to use the official Nginx PPA instead of the system's package. You can set the version to `stable` or `development`. - nginx_yum_repo_enabled: true + nginx_yum_repo_enabled: yes (For RedHat/CentOS only) Set this to `false` to disable the installation of the `nginx` yum repository. This could be necessary if you want the default OS stable packages, or if you use Satellite. diff --git a/roles/geerlingguy.nginx/handlers/main.yml b/roles/geerlingguy.nginx/handlers/main.yml index 23b5afd..d09454e 100644 --- a/roles/geerlingguy.nginx/handlers/main.yml +++ b/roles/geerlingguy.nginx/handlers/main.yml @@ -1,7 +1,7 @@ --- - name: restart nginx service: name=nginx state=restarted - become: yes + become: true become_user: root - name: validate nginx configuration @@ -10,5 +10,5 @@ - name: reload nginx service: name=nginx state=reloaded - become: yes + become: true become_user: root diff --git a/roles/geerlingguy.nginx/tasks/main.yml b/roles/geerlingguy.nginx/tasks/main.yml index 4ee901a..2850eff 100644 --- a/roles/geerlingguy.nginx/tasks/main.yml +++ b/roles/geerlingguy.nginx/tasks/main.yml @@ -9,26 +9,33 @@ when: nginx_user is not defined # Setup/install tasks. -- include: setup-RedHat.yml +- name: Include RedHat setup tasks + include_tasks: setup-RedHat.yml when: ansible_os_family == 'RedHat' -- include: setup-Ubuntu.yml +- name: Include Ubuntu setup tasks + include_tasks: setup-Ubuntu.yml when: ansible_distribution == 'Ubuntu' -- include: setup-Debian.yml +- name: Include Debian setup tasks + include_tasks: setup-Debian.yml when: ansible_os_family == 'Debian' -- include: setup-FreeBSD.yml +- name: Include FreeBSD setup tasks + include_tasks: setup-FreeBSD.yml when: ansible_os_family == 'FreeBSD' -- include: setup-OpenBSD.yml +- name: Include OpenBSD setup tasks + include_tasks: setup-OpenBSD.yml when: ansible_os_family == 'OpenBSD' -- include: setup-Archlinux.yml +- name: Include Archlinux setup tasks + include_tasks: setup-Archlinux.yml when: ansible_os_family == 'Archlinux' # Vhost configuration. -- include: vhosts.yml +- name: Include vhost configuration tasks. + include_tasks: vhosts.yml # Nginx setup. - name: Copy nginx configuration in place. diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index 920dc0b..e198062 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -3,7 +3,7 @@ apt_repository: repo: 'ppa:nginx/{{ nginx_ppa_version }}' state: present - update_cache: yes + update_cache: true register: nginx_ppa_added when: nginx_ppa_use diff --git a/roles/geerlingguy.nginx/tasks/vhosts.yml b/roles/geerlingguy.nginx/tasks/vhosts.yml index 05af037..a148dec 100644 --- a/roles/geerlingguy.nginx/tasks/vhosts.yml +++ b/roles/geerlingguy.nginx/tasks/vhosts.yml @@ -16,7 +16,7 @@ template: src: "{{ item.template|default(nginx_vhost_template) }}" dest: "{{ nginx_vhost_path }}/{{ item.filename|default(item.server_name.split(' ')[0] ~ '.conf') }}" - force: yes + force: true owner: root group: root mode: 0644 diff --git a/roles/git/tasks/main.yml b/roles/git/tasks/main.yml index 4498c60..4a565f0 100644 --- a/roles/git/tasks/main.yml +++ b/roles/git/tasks/main.yml @@ -2,4 +2,4 @@ apt: name: 'git' state: present - become: yes + become: true diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 316e9af..bed14c6 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -1,28 +1,28 @@ # Our specific tasks for configuring nginx # Most is done by the imported geerlingguy.nginx role -- name: certbot | Install deps - become: yes +- name: Certbot | Install deps + become: true apt: - name: ['software-properties-common', 'python3-apt'] + name: ["software-properties-common", "python3-apt"] state: present when: inventory == 'cloud' -- name: certbot | Install ppa - become: yes +- name: Certbot | Install ppa + become: true apt_repository: repo: ppa:certbot/certbot state: present when: inventory == 'cloud' -- name: certbot | Install packages - become: yes +- name: Certbot | Install packages + become: true apt: - name: 'python-certbot-nginx' + name: "python-certbot-nginx" state: present when: inventory == 'cloud' -- name: nginx | Copy SSL certificate +- name: Nginx | Copy SSL certificate copy: src: weblab.crt dest: "{{ ssl_certificate_folder }}" @@ -33,7 +33,7 @@ - reload nginx when: inventory == 'production' -- name: nginx | Copy SSL private key +- name: Nginx | Copy SSL private key copy: src: weblab.key dest: "{{ ssl_key_folder }}" @@ -44,7 +44,7 @@ - reload nginx when: inventory == 'production' -- name: nginx | Copy DH params +- name: Nginx | Copy DH params copy: src: dhparams.pem dest: "{{ dh_param_path }}" @@ -55,22 +55,23 @@ - reload nginx when: inventory == 'production' -- name: nginx | Configure nginx - become: yes +- name: Nginx | Configure nginx + become: true import_role: name: geerlingguy.nginx -- name: nginx | Open http(s) ports +- name: Nginx | Open http(s) ports ufw: name: Nginx Full rule: allow state: enabled -- name: certbot | Configure nginx - become: yes - shell: 'certbot --non-interactive --nginx --agree-tos --email {{ certbot_email }} --no-eff-email - --keep-until-expiring - --domains {{ webserver_fqdn.split(" ") | join(",") }}' +- name: Certbot | Configure nginx + become: true + shell: + 'certbot --non-interactive --nginx --agree-tos --email {{ certbot_email }} + --no-eff-email --keep-until-expiring --domains {{ webserver_fqdn.split(" ") + | join(",") }}' register: certbot_result when: inventory == 'cloud' changed_when: '"Keeping" not in certbot_result.stdout' diff --git a/roles/nginx/vars/main.yml b/roles/nginx/vars/main.yml index a05976e..e5380ab 100644 --- a/roles/nginx/vars/main.yml +++ b/roles/nginx/vars/main.yml @@ -1,7 +1,7 @@ # Configure geerlingguy.nginx -nginx_ppa_use: yes -nginx_remove_default_vhost: yes +nginx_ppa_use: true +nginx_remove_default_vhost: true # nginx_extra_http_options: | # proxy_buffering off; # proxy_set_header X-Real-IP $remote_addr; diff --git a/roles/ontology/tasks/main.yml b/roles/ontology/tasks/main.yml index e079db7..bd4fb86 100644 --- a/roles/ontology/tasks/main.yml +++ b/roles/ontology/tasks/main.yml @@ -1,7 +1,7 @@ # Tasks to clone the Web Lab ontology definitions and install in the appropriate locations - name: Ensure folders exist with correct permissions - become: yes + become: true file: path: '{{ item }}' state: directory @@ -11,22 +11,22 @@ - '{{ onto_root }}' - name: Checkout repository - become: yes + become: true become_user: '{{ onto_user }}' become_method: sudo git: - clone: yes + clone: true dest: '{{ onto_root }}' repo: '{{ onto_repo }}' version: '{{ onto_branch }}' - update: yes - force: yes + update: true + force: true - name: Copy ontology file to destination - become: yes + become: true copy: src: '{{ onto_rdf_xml }}' - remote_src: yes + remote_src: true dest: '{{ onto_destination }}' - force: yes + force: true mode: 0644 diff --git a/roles/prep_venv/tasks/main.yml b/roles/prep_venv/tasks/main.yml index 95834e5..87a17db 100644 --- a/roles/prep_venv/tasks/main.yml +++ b/roles/prep_venv/tasks/main.yml @@ -6,7 +6,7 @@ state: directory owner: '{{ user }}' mode: 0755 - become: yes + become: true - name: Upgrade pip in virtualenv pip: @@ -17,6 +17,6 @@ virtualenv_command: '{{ virtualenv_command | default(omit) }}' virtualenv_python: '{{ virtualenv_python | default(omit) }}' virtualenv_site_packages: '{{ virtualenv_site_packages | default(omit) }}' - become: yes + become: true become_user: '{{ user }}' become_flags: "-H" diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index e316b7b..97a4a55 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -1,30 +1,49 @@ # Make sure we have our common Python tools installed -- name: Install Python +- name: Install Python 2.7 via apt apt: - name: ['python3', 'python3-pip', 'python3-venv', 'python-pip', 'python-virtualenv', 'python-psycopg2'] + name: ["python2.7-dev", "python-pip", "python-virtualenv"] state: present - become: yes + become: true -- name: Upgrade system pip - become: yes +- name: Install Python 3.8 via apt + apt: + name: ["python3.8-dev", "python3-pip", "python3.8-venv"] + state: present + become: true + +- name: Install additional Python tools via apt + apt: + name: ["python-psycopg2", "virtualenv"] + state: present + become: true + +# Set pip to versions known to work with Web Lab components +- name: Upgrade Python 2 system pip + become: true pip: name: pip - state: latest + version: ">=20,<24.1" + executable: pip2 -- name: Upgrade pyOpenSSL to avoid python-openssl breaking pip - become: yes +- name: Upgrade Python 3 system pip + become: true + pip: + name: pip + version: ">=20,<24.1" + executable: pip3 + +# Upgrade pyOpenSSL to avoid python-openssl breaking pip +- name: Upgrade Python 2 system pyOpenSSL + become: true pip: name: pyOpenSSL - state: latest + version: ">=21" + executable: pip2 -- name: Install Python 3.8 - become: yes - block: - - name: Enable deadsnakes ppa - apt_repository: - repo: ppa:deadsnakes/ppa - - name: Install Python 3.8 - apt: - name: ['python3.8', 'python3.8-dev', 'python3.8-venv'] - state: present +- name: Upgrade Python 3 system pyOpenSSL + become: true + pip: + name: pyOpenSSL + version: ">=21" + executable: pip3 diff --git a/roles/rabbitmq/tasks/main.yml b/roles/rabbitmq/tasks/main.yml index 3b55f4f..18f14fa 100644 --- a/roles/rabbitmq/tasks/main.yml +++ b/roles/rabbitmq/tasks/main.yml @@ -1,7 +1,7 @@ -# Make sure we have our common Python tools installed +# Install message broker - name: Install RabbitMQ apt: name: 'rabbitmq-server' state: present - become: yes + become: true diff --git a/roles/uwsgi/tasks/main.yml b/roles/uwsgi/tasks/main.yml index 40209bc..88ef11f 100644 --- a/roles/uwsgi/tasks/main.yml +++ b/roles/uwsgi/tasks/main.yml @@ -2,35 +2,35 @@ - name: uWSGI | Install uWSGI with Python plugin apt: - name: ['uwsgi', 'uwsgi-plugin-python3'] + name: ["uwsgi", "uwsgi-plugin-python3"] state: present - name: uWSGI | Create socket folder file: - path: '{{ uwsgi_socket_dir }}' + path: "{{ uwsgi_socket_dir }}" state: directory - owner: '{{ django_user }}' - group: '{{ nginx_group }}' + owner: "{{ django_user }}" + group: "{{ nginx_group }}" mode: 0770 - name: uWSGI | Configure logging template: src: 20-weblab.conf - dest: '/etc/rsyslog.d/20-weblab.conf' + dest: "/etc/rsyslog.d/20-weblab.conf" notify: - restart rsyslog - name: uWSGI | Configure app template: - src: django-uwsgi.ini - dest: '/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini' + src: django-uwsgi.ini + dest: "/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini" notify: - restart uwsgi - name: uWSGI | Enable app file: - dest: '/etc/uwsgi/apps-enabled/{{ uwsgi_app }}.ini' - src: '/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini' - state: link + dest: "/etc/uwsgi/apps-enabled/{{ uwsgi_app }}.ini" + src: "/etc/uwsgi/apps-available/{{ uwsgi_app }}.ini" + state: link notify: - restart uwsgi diff --git a/site.yml b/site.yml index a9a60b0..429c290 100644 --- a/site.yml +++ b/site.yml @@ -1,6 +1,21 @@ # Main playbook -- import_playbook: broker.yml -- import_playbook: task_queue.yml -- import_playbook: workers.yml -- import_playbook: webservers.yml +- hosts: localhost + gather_facts: false + tasks: + - name: Check Ansible version + debug: + msg: "{{ ansible_version.full }}" + run_once: true + +- name: Run RabbitMQ Playbook + import_playbook: broker.yml + +- name: Run Celery Task Queue Playbook + import_playbook: task_queue.yml + +- name: Run Celery Workers Playbook + import_playbook: workers.yml + +- name: Run Django Playbook + import_playbook: webservers.yml From f710d8eae1b843091103660a2e54a624a2ee16d0 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Mon, 5 Jan 2026 22:09:36 +0000 Subject: [PATCH 07/10] Reset broker tasks --- broker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/broker.yml b/broker.yml index dae2ff4..15ba6ac 100644 --- a/broker.yml +++ b/broker.yml @@ -2,7 +2,5 @@ - hosts: broker roles: - - django - - { role: uwsgi, become: yes } - core - rabbitmq From dac485e1561c6236ef418f342fb9c0e0897ceb3a Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 3 Mar 2026 14:29:05 +0000 Subject: [PATCH 08/10] Remove Dockerfile --- Dockerfile | 112 ----------------------------------------------------- 1 file changed, 112 deletions(-) delete mode 100755 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100755 index 3f83276..0000000 --- a/Dockerfile +++ /dev/null @@ -1,112 +0,0 @@ -# Build the image: -# docker build -t weblab:test . - -# Run the container in interactive mode: -# docker run --init -it weblab:test /bin/bash - -FROM ubuntu:bionic - -SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] - -USER root - -ENV DEFAULT_USER="celery" \ - DEFAULT_HOME="/home/celery" \ - CELERY_DIR="/opt/celery" \ - WEBLAB_FC_DIR="/opt/weblab-fc" \ - CHASTE_ROOT="/home/celery/eclipse/workspace/Chaste" - -# Add celery user and create necessary directories -RUN useradd -r -m -d ${DEFAULT_HOME} -s /bin/bash ${DEFAULT_USER} && \ - mkdir -p ${CELERY_DIR} && \ - mkdir -p ${WEBLAB_FC_DIR} && \ - chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${CELERY_DIR} && \ - chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${WEBLAB_FC_DIR} - -# Install dependencies -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y \ - apt-utils \ - build-essential \ - ca-certificates \ - curl \ - git \ - gnupg \ - gnupg1 \ - gnupg2 \ - openssh-server \ - python-dev \ - python-pip \ - python-virtualenv \ - python3.8-dev \ - python3-pip \ - python3.8-venv \ - python-psycopg2 \ - rabbitmq-server \ - software-properties-common \ - sudo \ - ufw \ - wget && \ - echo 'deb http://www.cs.ox.ac.uk/chaste/ubuntu bionic/' > /etc/apt/sources.list.d/chaste.list && \ - apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 422C4D99 && \ - apt-get update && \ - apt-get install -y chaste-dependencies && \ - python -m pip install --upgrade pip && \ - python3.8 -m pip install --upgrade pip && \ - /tmp/tmp-runner/bin/installdependencies.sh && \ - apt-get -y clean && \ - rm -rf /var/cache/apt && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /var/tmp/* && \ - rm -rf /tmp/* - -# Clone repositories -USER ${DEFAULT_USER}:${DEFAULT_USER} -RUN git clone -b master --depth 1 https://github.com/ModellingWebLab/fc-runner.git ${CELERY_DIR}/repo && \ - git clone -b weblab --depth 1 https://github.com/Chaste/Chaste.git ${CHASTE_ROOT} && \ - git clone -b master --depth 1 https://github.com/ModellingWebLab/chaste-project-fitting-pints.git ${CHASTE_ROOT}/projects/AidanDaly && \ - git clone -b master --depth 1 https://github.com/ModellingWebLab/weblab-fc.git ${WEBLAB_FC_DIR}/repo - -# Apply patches -COPY patches/fc/pyproject.toml ${WEBLAB_FC_DIR}/repo/ - -# Install WebLab -RUN python3.8 -m venv ${CELERY_DIR}/py3_venv && \ - source ${CELERY_DIR}/py3_venv/bin/activate && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "pip>=20,<24.1" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "numpy>=1,<2" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -U "cython>=0,<3" && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${CELERY_DIR}/repo/requirements/base.txt && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install -r ${WEBLAB_FC_DIR}/repo/requirements/setup.txt && \ - ${CELERY_DIR}/py3_venv/bin/python3 -m pip install ${WEBLAB_FC_DIR}/repo && \ - deactivate - -# Install Chaste -RUN python -m venv ${CELERY_DIR}/venv && \ - source ${CELERY_DIR}/venv/bin/activate && \ - cd ${CHASTE_ROOT} && \ - scons -j$(nproc) b=GccOpt co=1 cl=1 projects/FunctionalCuration && \ - scons -j$(nproc) b=GccOpt co=1 cl=1 exe=1 projects/FunctionalCuration/apps && \ - deactivate - -# Run ansible workflows -USER root -RUN git clone -b docker --depth 1 --recursive https://github.com/ModellingWebLab/deployment.git ~/deployment && \ - python3.8 -m venv ~/deploy_env && \ - source ~/deploy_env/bin/activate && \ - python3.8 -m pip install --upgrade pip && \ - python3.8 -m pip install ansible && \ - cd ~/deployment && \ - ansible-playbook -i inventories/dev site.yml \ - -e 'django_git_branch=master' \ - -e 'django_superuser_email="my.email@domain"' \ - -e 'django_superuser_full_name="My Full Name"' \ - -e 'django_superuser_institution="My Institution"' && \ - deactivate && \ - rm -rf ~/deploy_env && \ - rm -rf ~/deployment - -USER ${DEFAULT_USER}:${DEFAULT_USER} -WORKDIR ${DEFAULT_HOME} From 00545b1f9a943c7c84d272c4a142fe11489d75b6 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 4 Mar 2026 11:35:51 +0000 Subject: [PATCH 09/10] Fix nginx setup --- README.md | 2 +- Vagrantfile | 1 + roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml | 9 ++++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 676179e..9ed1134 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Note that you will need to set up the appropriate proxying *outside* the Web Lab ### Using Vagrant -Version 2.2 of Vagrant is required for this. We recommend using the VirtualBox provider with it. +Version 2.4 of Vagrant is required for this. We recommend using the VirtualBox provider with it. You may need to edit some options in the `Vagrantfile` depending on how you have configured your local variables. For instance, remove the `raw_arguments` if you're not encrypting any secrets. diff --git a/Vagrantfile b/Vagrantfile index 3e66fe2..33dcbd2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,6 +35,7 @@ Vagrant.configure("2") do |config| ansible.playbook = "site.yml" ansible.inventory_path = "inventories/dev" ansible.limit = "localhost" + ansible.raw_arguments = ['--vault-id', 'dev@dev-vault-pw'] ansible.extra_vars = { django_git_branch: 'master', diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index e198062..6a8611d 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -1,9 +1,16 @@ --- +# This step is a workaround for the next step below which fails because +# it tries to add a key from an hkp:// address instead of an hkps:// +- name: Add PPA for Nginx manually. + command: add-apt-repository ppa:nginx/{{ nginx_ppa_version }} + when: nginx_ppa_use + - name: Add PPA for Nginx. apt_repository: - repo: 'ppa:nginx/{{ nginx_ppa_version }}' + repo: "ppa:nginx/{{ nginx_ppa_version }}" state: present update_cache: true + validate_certs: true register: nginx_ppa_added when: nginx_ppa_use From 6533e7a959d5709dca87b1ad12a7d5b697b99d79 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 10 Mar 2026 15:24:37 +0000 Subject: [PATCH 10/10] Use Python 3.6 --- patches/fc/pyproject.toml | 2 +- roles/celery/meta/main.yml | 2 +- roles/django/meta/main.yml | 2 +- roles/django/tasks/main.yml | 6 ++++++ roles/fc-backend/meta/main.yml | 2 +- roles/fc-backend/tasks/main.yml | 2 +- roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml | 2 +- roles/python/tasks/main.yml | 4 ++-- 8 files changed, 14 insertions(+), 8 deletions(-) diff --git a/patches/fc/pyproject.toml b/patches/fc/pyproject.toml index 106dbc8..5c5771b 100644 --- a/patches/fc/pyproject.toml +++ b/patches/fc/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] +requires = ["setuptools", "wheel", "numpy>=1,<2", "cython>=0,<3"] build-backend = "setuptools.build_meta" diff --git a/roles/celery/meta/main.yml b/roles/celery/meta/main.yml index bebd29f..32cb12c 100644 --- a/roles/celery/meta/main.yml +++ b/roles/celery/meta/main.yml @@ -10,7 +10,7 @@ dependencies: - role: prep_venv user: '{{ celery_user }}' virtualenv: '{{ celery_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 # Then we need the FC backend available for the celery user, # since our tasks import modules from it - role: fc-backend diff --git a/roles/django/meta/main.yml b/roles/django/meta/main.yml index 2fb07c8..e9efab2 100644 --- a/roles/django/meta/main.yml +++ b/roles/django/meta/main.yml @@ -9,7 +9,7 @@ dependencies: - role: prep_venv user: '{{ django_user }}' virtualenv: '{{ django_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 # Ensure node is new enough (and installed) - { role: geerlingguy.nodejs, become: true } diff --git a/roles/django/tasks/main.yml b/roles/django/tasks/main.yml index 582ccb1..edff083 100644 --- a/roles/django/tasks/main.yml +++ b/roles/django/tasks/main.yml @@ -44,6 +44,12 @@ notify: - restart uwsgi + - name: Django | fix git-python dependency + lineinfile: + path: '{{ django_checkout }}/requirements/base.txt' + regexp: '^gitpython==' + line: gitpython==3.1.18 + - name: Django | Install psycopg2 pip: name: psycopg2 diff --git a/roles/fc-backend/meta/main.yml b/roles/fc-backend/meta/main.yml index 92fb265..b7a45e3 100644 --- a/roles/fc-backend/meta/main.yml +++ b/roles/fc-backend/meta/main.yml @@ -14,5 +14,5 @@ dependencies: - role: prep_venv user: '{{ fc_user }}' virtualenv: '{{ weblab_fc_virtualenv }}' - virtualenv_python: python3.8 + virtualenv_python: python3.6 virtualenv_site_packages: false diff --git a/roles/fc-backend/tasks/main.yml b/roles/fc-backend/tasks/main.yml index dcdbf2a..8b85ff8 100644 --- a/roles/fc-backend/tasks/main.yml +++ b/roles/fc-backend/tasks/main.yml @@ -146,7 +146,7 @@ dest: "{{ weblab_fc_root }}/pyproject.toml" content: | [build-system] - requires = ["setuptools>=64", "wheel", "numpy>=1,<2", "cython>=0,<3"] + requires = ["setuptools", "wheel", "numpy>=1,<2", "cython>=0,<3"] build-backend = "setuptools.build_meta" force: true diff --git a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml index 6a8611d..e829d65 100644 --- a/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml +++ b/roles/geerlingguy.nginx/tasks/setup-Ubuntu.yml @@ -2,7 +2,7 @@ # This step is a workaround for the next step below which fails because # it tries to add a key from an hkp:// address instead of an hkps:// - name: Add PPA for Nginx manually. - command: add-apt-repository ppa:nginx/{{ nginx_ppa_version }} + command: add-apt-repository -y ppa:nginx/{{ nginx_ppa_version }} when: nginx_ppa_use - name: Add PPA for Nginx. diff --git a/roles/python/tasks/main.yml b/roles/python/tasks/main.yml index 97a4a55..92ba00f 100644 --- a/roles/python/tasks/main.yml +++ b/roles/python/tasks/main.yml @@ -6,9 +6,9 @@ state: present become: true -- name: Install Python 3.8 via apt +- name: Install Python 3.6 via apt apt: - name: ["python3.8-dev", "python3-pip", "python3.8-venv"] + name: ["python3.6-dev", "python3-pip", "python3.6-venv"] state: present become: true