adds latest changes

Author: martindemko

galaxy.yml

@@ -46,75 +46,6 @@
         state: absent
       when: ansible_os_family == 'Debian'
 
-    - name: Ensure the python command exists
-      ansible.builtin.shell: 
-        cmd: which python||exit 0
-      register: python_path
-      tags: py
-    - name: Register python3 path
-      ansible.builtin.command: which python3
-      register: python3_path
-      when: python_path.stdout == ""
-      tags: py
-    - name: Link python to python3 if missing
-      ansible.builtin.file:
-        src: "{{ python3_path.stdout }}"
-        dest: "{{ python3_path.stdout[:-1] }}"
-        state: link
-        force: yes
-      when: python_path.stdout == ""
-      tags: py
-
-    - git:
-        repo: 'https://github.com/usegalaxy-eu/libraries-training-repo'
-        dest: /libraries/
-      when: "'usegalaxy' in inventory_hostname"
-#    - name: Create data storage for UMSA
-#      file:
-#        owner: "{{ galaxy_user_name }}"
-#        group: "{{ galaxy_user_group_name }}"
-#        path:  "{{ galaxy_data_dir }}"
-#        state: directory
-#        mode:  "0755"
-#      when: "'umsa' in inventory_hostname"
-    - name: Create the RBD storage directory
-      file:
-        owner: "{{ galaxy_user_name }}"
-        group: "{{ galaxy_user_group_name }}"
-        path: /rbd/data
-        state: directory
-        mode: '0755'
-      when: "'usegalaxy' in inventory_hostname"
-
-    - name: Ensure galaxy temp dir mount
-      mount:
-        path: "{{ galaxy_temp_dir }}"
-        src: "/dev/rbd0"
-        fstype: "ext4"
-        opts: "defaults,nodev,nosuid,noexec,noatime,_netdev"
-        passno: "2"
-        state: mounted
-    - name: Create the galaxy temp directory under root
-      file:
-        owner: root
-        group: root
-        path: "{{ galaxy_temp_dir }}"
-        state: directory
-        mode: '1777'
-    - name: Create the nginx upload job files store
-      file:
-        owner: www-data
-        group: "{{ galaxy_user_group_name }}"
-        path: "{{ galaxy_temp_dir }}/_upload_job_files"
-        state: directory
-        mode: '0775'
-    - name: Create the client body temp path   
-      file:
-        owner: www-data
-        group: "{{ galaxy_user_group_name }}"
-        path: "{{ galaxy_temp_dir }}/nginx_upload_store"
-        state: directory
-        mode: '0775'
       ## Part responsible for setting up NBD mount (specific for galaxy-re)
     - name: Check if /dev/nbd0 exists
       stat:
@@ -153,116 +84,13 @@
         state: mounted
       when: "'galaxy-re' in inventory_hostname or 'galaxy-re' in ansible_hostname"
 
-# recover from WAL based on https://training.galaxyproject.org/training-material/topics/admin/tutorials/backup-cleanup/tutorial.html#restoration
-    ## Part responsible for recovering Galaxy DB from backup if exists (it should be in post_tasks of dbservers but NDB access needs to be set up first)
-    - name: Register psql data directory
-      ansible.builtin.shell:
-        cmd: psql -Atc 'show data_directory;'
-      register: psql_data_dir
-      become: true
-      become_user: postgres
-      tags: gal_db
-    - name: Check if Galaxy DB exists
-      ansible.builtin.shell:
-        cmd: "psql -XAtc '\\l' |cut -f 1 -d '|' |grep -qw {{ galaxy_db_name }}"
-      register: galaxy_db_in_psql
-      ignore_errors: true
-      become: true
-      become_user: postgres
-      tags: gal_db
-    - name: Restore Galaxy DB backup - stop postgresql
-      ansible.builtin.systemd:
-        name: postgresql
-        state: stopped
-      when: galaxy_db_in_psql.rc == 1
-      tags: gal_db
-    - name: Restore Galaxy DB backup - backup current postgresql
-      ansible.builtin.shell:
-        cmd: "mv {{ psql_data_dir.stdout }} {{ psql_data_dir.stdout+'.backup_'+ansible_date_time.iso8601 }}"
-      ignore_errors: true
-      when: galaxy_db_in_psql.rc == 1 and psql_data_dir.stdout != ''
-      become: true
-      become_user: postgres
-      tags: gal_db
-
-    - name: Restore Galaxy DB backup - pick proper backup version
-      ansible.builtin.shell:
-        cmd: "if [ '{{ galaxy_db_restore_version }}' = 'latest' ]; then ls -dt {{ postgresql_backup_dir }}/20*Z | head -1; else ls -d {{ postgresql_backup_dir }}/{{ galaxy_db_restore_version }}; fi "
-      register: galaxy_db_restore_dir
-      when: galaxy_db_in_psql.rc == 1
-      become: true
-      become_user: postgres
-      tags: gal_db
-    - name: Restore Galaxy DB backup - copy proper backup version
-      ansible.builtin.shell:
-        cmd: "rsync -a {{ galaxy_db_restore_dir.stdout }}/ {{ psql_data_dir.stdout }}/ && chmod -R 0750 {{ psql_data_dir.stdout }} "
-      when: galaxy_db_in_psql.rc == 1 and galaxy_db_restore_dir.stdout != '' and psql_data_dir.stdout != ''
-      become: true
-      become_user: postgres
-      tags: gal_db
-#    - name: Restore Galaxy DB backup - pick the latest backup version
-#      ansible.builtin.shell:
-#        cmd: "ls -dt {{ postgresql_backup_dir }}/20*Z | head -1 "
-#      register: galaxy_db_restore_dir_latest
-#      ignore_errors: true
-#      when: galaxy_db_in_psql.rc == 1 and galaxy_db_restore_version == 'latest'
-#      become: true
-#      become_user: postgres
-#      tags: gal_db
-#    - name: Restore Galaxy DB backup - pick the user-defined backup version
-#      ansible.builtin.shell:
-#        cmd: "ls -d {{ postgresql_backup_dir }}/{{ galaxy_db_restore_version }} "
-#      register: galaxy_db_restore_dir
-#      ignore_errors: true
-#      when: galaxy_db_in_psql.rc == 1 and galaxy_db_restore_version != 'latest'
-#      become: true
-#      become_user: postgres
-#      tags: gal_db
-#    - name: Restore Galaxy DB backup - copy latest backup version
-#      ansible.builtin.shell:
-#        cmd: "rsync -a {{ galaxy_db_restore_dir_latest.stdout }}/ {{ psql_data_dir.stdout }}/ && chmod -R 0750 {{ psql_data_dir.stdout }} "
-#      when: galaxy_db_in_psql.rc == 1 and galaxy_db_restore_dir_latest.stdout != '' and psql_data_dir.stdout != ''
-#      become: true
-#      become_user: postgres
-#      tags: gal_db
-#    - name: Restore Galaxy DB backup - copy user-defined backup version
-#      ansible.builtin.shell:
-#        cmd: "rsync -a {{ galaxy_db_restore_dir.stdout }}/ {{ psql_data_dir.stdout }}/ && chmod -R 0750 {{ psql_data_dir.stdout }} "
-#      when: galaxy_db_in_psql.rc == 1 and galaxy_db_restore_dir.stdout != '' and psql_data_dir.stdout != ''
-#      become: true
-#      become_user: postgres
-#      tags: gal_db
-    - name: Restore Galaxy DB backup - fill postgresql.auto.conf
-      ansible.builtin.lineinfile:
-        path: "{{ psql_data_dir.stdout }}/postgresql.auto.conf"
-        insertafter: EOF
-        line: "restore_command = 'cp \"{{ postgresql_backup_dir }}/wal_archive/%f\" \"%p\"'"
-        state: present
-        create: true
-      when: galaxy_db_in_psql.rc == 1 and psql_data_dir.stdout != ''
-      become: true
-      become_user: postgres
-      tags: gal_db
-    - name: Restore Galaxy DB backup - touch the signal file
-      ansible.builtin.shell:
-        cmd: "touch {{ psql_data_dir.stdout }}/recovery.signal "
-      when: galaxy_db_in_psql.rc == 1 and psql_data_dir.stdout != ''
-      become: true
-      become_user: postgres
-      tags: gal_db
-    - name: Restore Galaxy DB backup - start postgresql
-      ansible.builtin.systemd:
-        name: postgresql
-        state: started
-      tags: gal_db      
-
   roles:
     - galaxyproject.tusd
     - galaxyproject.galaxy
     - usegalaxy_eu.tpv_auto_lint
-    - role: galaxyproject.miniconda
-      become: true
-      become_user: "{{ galaxy_user_name }}"
+#    - role: galaxyproject.miniconda
+#      become: true
+#      become_user: "{{ galaxy_user_name }}"
     - geerlingguy.redis
     - galaxyproject.nginx
     - usegalaxy_eu.flower
@@ -289,40 +117,40 @@
         minute: "2"
         hour: "*" 
         #DEMON: zvazit ci nepouzit find /rbd/data/datasets/*/*/*/dataset_*_files -type f ! -perm /g+rw -exec chmod g+rw '{}' ';'
-        job: "/usr/bin/find {{ galaxy_config.galaxy.file_path }}/*/*/*/dataset_*_files -type d -mmin -180 -maxdepth 0 -exec chmod -R g+rw '{}' ';'"
+        job: "/usr/bin/find {{ galaxy_config.galaxy.file_path }}/*/*/*/dataset_*_files -maxdepth 0 -type d -mmin -240 -exec chmod -R g+rw '{}' ';'"
       become: yes
       become_user: "{{ galaxy_user_name }}"
     - name: Creates entry in crontab "tmpreaper call in tus upload dir"
       ansible.builtin.cron:
         name: "tmpreaper in tus upload dir"
         minute: "5"
         hour: "*" 
-        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 4h {{ galaxy_tus_upload_store }} >> {{ galaxy_data_dir }}/cleaning_tus_upload_dir.log"
+        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_tus_upload_store }} >> {{ galaxy_data_dir }}/cleaning_tus_upload_dir.log"
       become: yes
       become_user: "{{ galaxy_user_name }}"
     - name: Creates entry in crontab "tmpreaper call in galaxy tmp dir"
       ansible.builtin.cron:
         name: "tmpreaper in galaxy temp dir"
         minute: "7"
         hour: "*" 
-        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 4h {{ galaxy_temp_dir }}/tmp >> {{ galaxy_data_dir }}/cleaning_galaxy_temp_dir.log"
+        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_temp_dir }}/tmp >> {{ galaxy_data_dir }}/cleaning_galaxy_temp_dir.log"
       become: yes
       become_user: "{{ galaxy_user_name }}"
+#    - name: Creates entry in crontab "tmpreaper call in galaxy jobs dir"
+#      ansible.builtin.cron:
+#        name: "tmpreaper in galaxy jobs dir"
+#        minute: "9"
+#        hour: "*" 
+#        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 40d {{ galaxy_temp_dir }}/jobs >> {{ galaxy_data_dir }}/cleaning_galaxy_jobs_dir.log"
+#      become: yes
+#      become_user: "{{ galaxy_user_name }}"
     - name: Setup gxadmin cleanup task
       ansible.builtin.cron:
         name: "Cleanup Old User Data"
         user: "{{ galaxy_user_name }}" # Run as the Galaxy user
         minute: "0"
         hour: "0"
-        job: "GALAXY_LOG_DIR=/tmp/gxadmin/ GALAXY_ROOT={{ galaxy_root }}/server /usr/local/bin/gxadmin galaxy cleanup 60"
-    - name: Setup user - group association rutine
-      ansible.builtin.cron:
-        name: "Add E-infra users to E-infra group"
-        user: postgres # Run as the postgres user
-        minute: "0"
-        hour: "*"
-        job: "python3 {{ playbook_dir }}/add_group_users.py {{ inventory_hostname }} {{ api_key }}"
-      when: "'usegalaxy' in inventory_hostname"
+        job: "GALAXY_LOG_DIR=/tmp/gxadmin/ GALAXY_ROOT={{ galaxy_root }}/server /usr/local/bin/gxadmin galaxy cleanup 60 >> {{ galaxy_data_dir }}/cleaning_gxadmin_cleanup.log"
 #    - name: RE Galaxy instance patch in lib/galaxy/webapps/galaxy/api/job_files.py
 #      replace:
 #        path: "{{ galaxy_root }}/server/lib/galaxy/webapps/galaxy/api/job_files.py"

group_vars/all.yml

@@ -9,7 +9,8 @@ galaxy_user_group_name: galaxyelixir
 galaxy_user_group_gid: 10145
 galaxy_db_name: galaxyelixir
 galaxy_data_dir: "/mnt/galaxy-re"
-galaxy_temp_dir: "/tmp_galaxy"
+#galaxy_temp_dir: "/tmp_galaxy"
+galaxy_temp_dir: "{{ galaxy_data_dir }}"
 galaxy_prefix: "/galaxy"
 
 # CVMFS vars

group_vars/galaxyservers.yml

@@ -23,7 +23,7 @@ galaxy_commit_id: 72070dbd0a6d0f418d0b8f914825ea2b8e6a0a88
 galaxy_systemd_nofile_limit: 150000
 
 miniconda_prefix: "{{ galaxy_tool_dependency_dir }}/conda"
-miniconda_version: 'latest' # 23.5.2 # 23.5.0 # 4.12.0
+miniconda_version: 23.5.2 # 23.5.0 # 4.12.0
 miniconda_channels: ['conda-forge', 'bioconda', 'iuc', 'r', 'anaconda', 'petrnovak', 'defaults']
 miniconda_base_env_packages: ['mamba']
 
@@ -85,7 +85,7 @@ galaxy_job_config:
       environment: local
   limits:
     - type: registered_user_concurrent_jobs
-      value: 5
+      value: 4
     - type: anonymous_user_concurrent_jobs
       value: 1
     - type: output_size
@@ -105,6 +105,7 @@ galaxy_config:
     - kavonrtep@repeatexplorer-elixir.cerit-sc.cz
     - salvet@repeatexplorer-elixir.cerit-sc.cz
     - jimacas@repeatexplorer-elixir.cerit-sc.cz
+    - martincech@repeatexplorer-elixir.cerit-sc.cz
     database_connection: "postgresql:///{{ galaxy_db_name }}?host=/var/run/postgresql"
     #umask: 022
     file_path: "{{ galaxy_data_dir }}/datasets"
@@ -121,13 +122,14 @@ galaxy_config:
     - "{{ galaxy_root }}/server/config/tool_conf.xml.sample"
     tool_sheds_config_file: "{{ galaxy_config_dir }}/tool_sheds_conf.xml"
     sanitize_allowlist_file: "{{ galaxy_mutable_config_dir }}/sanitize_allowlist.txt"
+    cache_dir: "{{ galaxy_data_dir }}/cache"
     # This allows to do custom ordering of tools in sections, default was true, but Petr Novak wants specific orderring of tools
     toolbox_auto_sort: false
     # This hide specified tools for non-admin users
     tool_filters: 'examples:restrict_testing_tools_to_admins'
     smtp_server: "rs.cesnet.cz:25"
     error_email_to: "regalaxy@rt.cesnet.cz"
-    email_from: "regalaxy@rt.cesnet.cz"
+#    email_from: "regalaxy@rt.cesnet.cz"
     # SQL Performance
     slow_query_log_threshold: 5
     enable_per_request_sql_debugging: true
@@ -226,7 +228,7 @@ galaxy_config:
       queues: celery,galaxy.internal,galaxy.external
       pool: threads
       memory_limit: 4
-      concurrency: 2
+      concurrency: 8
       loglevel: DEBUG
     tusd:
       enable: true
@@ -326,6 +328,7 @@ certbot_post_renewal: |
 certbot_domains:
  - "{{ inventory_hostname }}"
  - galaxy-re.grid.cesnet.cz
+ - galaxyelixir.vm.cesnet.cz
 certbot_expand: true
 certbot_agree_tos: --agree-tos