diff --git a/files/usr/local/bin/move_old_logs.sh b/files/usr/local/bin/move_old_logs.sh index cdf2e8d0cfc86ee1b153d186c40fb59f6a4a3048..ee6eb55e308aea77207a511dd8c4ac70e3c7ade1 100644 --- a/files/usr/local/bin/move_old_logs.sh +++ b/files/usr/local/bin/move_old_logs.sh @@ -8,8 +8,8 @@ START_YEAR="2015" CURRENT_YEAR="$( date +%Y )" PREVIOUS_YEAR="$(( CURRENT_YEAR - 1 ))" -for APP in subapp disapp; do - cd "/var/log/$APP/${HOSTNAME}/" || exit 1 +for APP in disapp subapp subapp_ws; do + cd "/var/log/$APP/" || exit 1 # create directories for old logfiles for YEAR in $( seq ${START_YEAR} ${PREVIOUS_YEAR} ); do diff --git a/tasks/configure_bash.yml b/tasks/configure_bash.yml index f26060f2fa7a3c8991683889d07d28f93419fec0..4debb330c3344668f2a28b48546f1021ff794b86 100644 --- a/tasks/configure_bash.yml +++ b/tasks/configure_bash.yml @@ -17,4 +17,3 @@ mode: "0644" owner: "root" group: "root" - diff --git a/tasks/configure_nfs_mounts.yml b/tasks/configure_nfs_mounts.yml index a3df2f2286bc7c5b3466927dbc9409857e760066..ae07eb539d91b85cbabbdedbc396a9ef842ce35b 100644 --- a/tasks/configure_nfs_mounts.yml +++ b/tasks/configure_nfs_mounts.yml @@ -13,7 +13,9 @@ - "{{ nfs_mounts_subapp.hosts[ansible_hostname]['import']['path'] | default('/mnt/import') }}" - "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_upload']['path'] | default('/home/import/upload') }}" - "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_download']['path'] | default('/home/import/download') }}" - - "/var/log/rosetta/{{ ansible_hostname }}/" + - "{{ paths.log_disapp.mountpoint }}" + - "{{ paths.log_subapp.mountpoint }}" + - "{{ paths.log_subapp_ws.mountpoint }}" # - "/var/log/rosetta/{{ ansible_hostname }}/disapp/" # - "/var/log/rosetta/{{ ansible_hostname }}/subapp/" # - "/var/log/rosetta/{{ ansible_hostname }}/legacy/" @@ -51,25 +53,25 @@ path: "{{ item.path }}" src: "{{ item.src | default(omit) }}" state: "{{ item.state | default('mounted') }}" - fstype: nfs + fstype: "{{ item.fstype | default('nfs') }}" opts: "{{ item.opts | default( nfs_opts.v3 ) }}" loop: # common Log - use this once the migration to the separated dis-/subapp is finished - - path: "/var/log/rosetta/{{ ansible_hostname }}/" - src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_disapp']['nfs_share'] }}{{ ansible_hostname }}/" - opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_disapp']['nfs_opts'] }}" + - path: "/mnt/logs/" + src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log']['nfs_share'] }}{{ ansible_hostname }}/" + opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log']['nfs_opts'] }}" # DisApp - - path: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['access']['path'] | default('/mnt/' + ansible_hostname + '_access') }}" + - path: "{{ paths.access.mountpoint }}" src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['access']['nfs_share'] }}" opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['access']['nfs_opts'] }}" # SubApp - - path: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['ingest']['path'] | default('/mnt/' + ansible_hostname + '_ingest') }}" + - path: "{{ paths.ingest.mountpoint }}" src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['ingest']['nfs_share'] }}" opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['ingest']['nfs_opts'] }}" - - path: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['import']['path'] | default('/mnt/import') }}" + - path: "{{ paths.import.mountpoint }}" src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['import']['nfs_share'] }}" opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['import']['nfs_opts'] }}" - - path: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_upload']['path'] | default('/home/import/upload') }}" + - path: "{{ paths.sftp_upload.mountpoint }}" src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_upload']['nfs_share'] }}" opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_upload']['nfs_opts'] }}" tags: [notest] @@ -81,27 +83,35 @@ loop: - path: "{{ paths.access.mountpoint }}/consumer_dir" mode: "0770" - - path: "/var/log/rosetta/{{ ansible_hostname }}/disapp/" - - path: "/var/log/rosetta/{{ ansible_hostname }}/subapp/" + - path: "/mnt/logs/disapp/" + - path: "/mnt/logs/subapp/" + - path: "/mnt/logs/subapp_ws/" # tags: [notest] - name: ... and now mount them ansible.posix.mount: path: "{{ item.path }}" src: "{{ item.src }}" state: "{{ item.state | default('mounted') }}" - fstype: nfs + fstype: "{{ item.fstype | default('nfs') }}" opts: "{{ item.opts | default( nfs_opts.v3 ) }}" loop: # DisApp - - path: "/var/log/rosetta/{{ ansible_hostname }}/disapp/" - src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_disapp']['nfs_share'] }}{{ ansible_hostname }}/disapp" - opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_disapp']['nfs_opts'] }}" + - path: "{{ paths.log_disapp.mountpoint }}" + src: "/mnt/logs/disapp/" + opts: "bind" + fstype: none # SubApp - - path: "/var/log/rosetta/{{ ansible_hostname }}/subapp" - src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_subapp']['nfs_share'] }}{{ ansible_hostname }}/subapp" - opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['log_subapp']['nfs_opts'] }}" + - path: "{{ paths.log_subapp.mountpoint }}" + src: "/mnt/logs/subapp/" + opts: "bind" + fstype: none + # SubApp-Webservice + - path: "{{ paths.log_subapp_ws.mountpoint }}" + src: "/mnt/logs/subapp_ws/" + opts: "bind" + fstype: none # SFTP - - path: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_download']['path'] | default('/home/import/download') }}" + - path: "{{ paths.sftp_download.mountpoint }}" src: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_download']['nfs_share'] }}" opts: "{{ nfs_mounts_subapp.hosts[ansible_hostname]['sftp_download']['nfs_opts'] }}" tags: [notest] @@ -123,6 +133,10 @@ owner: "{{ paths.log_subapp.owner }}" group: "{{ paths.log_subapp.group }}" mode: "{{ paths.log_subapp.mode }}" + - path: "{{ paths.log_subapp_ws.mountpoint }}" + owner: "{{ paths.log_subapp_ws.owner }}" + group: "{{ paths.log_subapp_ws.group }}" + mode: "{{ paths.log_subapp_ws.mode }}" - path: "{{ paths.access.mountpoint }}" owner: "{{ paths.access.owner }}" group: "{{ paths.access.group }}" diff --git a/tasks/configure_processing_user.yml b/tasks/configure_processing_user.yml index 62b5af23e26b863f31b24255a9dfce29eda3d97e..3400f4dffda5279125f10c9bc833b05dec6463b3 100644 --- a/tasks/configure_processing_user.yml +++ b/tasks/configure_processing_user.yml @@ -43,7 +43,7 @@ if [[ $1 != "" ]]; then YEAR="$1" else YEAR="$( date +%Y )" fi - LOG="/var/log/subapp/$(hostname)/staff_mails.log" + LOG="/var/log/subapp/staff_mails.log" ( if [ -e ${LOG} ]; then cat ${LOG} fi @@ -79,7 +79,7 @@ block: | # custom aliases alias disapp='/usr/bin/perl -I /usr/local/perl/ /usr/local/bin/disapp_rosetta.pl --config-file {{ vault_subapp_vars.files.disapp.path }}' # DisApp Alias - alias disapp_log='tail -f /var/log/rosetta/${HOSTNAME}/disapp.log' # show last log entries in disapp.log + alias disapp_log='tail -f /var/log/disapp/disapp.log' # show last log entries in disapp.log alias disapp_monitor='curl localhost:9003' - name: Add aliases for SubApp user @@ -96,8 +96,8 @@ # custom aliases - user processing alias subapp='/usr/bin/perl -I /usr/local/perl/ /usr/local/bin/subapp_bagit.pl --config-file {{ vault_subapp_vars.files.subapp.path }}' # SubApp alias alias sips_number='find /mnt/import/ -maxdepth 1 -type d | wc -l' # count directories in /mnt/import/ - alias sips_log='tail -f /var/log/subapp/${HOSTNAME}/sips.log | grep "what:.*,"' # show last log entries in sips.log - alias subapp_log='tail -f /var/log/subapp/${HOSTNAME}/subapp.log' # show last log entries in subapp.log + alias sips_log='tail -f /var/log/subapp//sips.log | grep "what:.*,"' # show last log entries in sips.log + alias subapp_log='tail -f /var/log/subapp/subapp.log' # show last log entries in subapp.log alias subapp_monitor='curl localhost:9001' alias verify_bag='perl -e '\''use Archive::BagIt; my $bag = Archive::BagIt->new(".")->verify_bag( {report_all_errors => 1} );'\''; echo $?' alias block_subapp='echo "SubApp must not run, startup is blocked manually by an administrator. This file is here for a reason! Only remove it after consultation!" > ~/.subapp/BLOCKFILE' @@ -106,4 +106,4 @@ alias list_quarantine_full='for sip in $(ls -t ~/.subapp/quarantine); do echo -e "$sip# ($( stat -c %y ~/.subapp/quarantine/$sip/sip.ERROR ))# => $(cat ~/.subapp/quarantine/$sip/sip.ERROR)\n===\n"; done | column -s "#" -t' alias quarantine_summary='echo -e "$( ls ~/.subapp/quarantine/ | wc -l ) SIPs in quarantine.\n"; for sip in $(ls -t ~/.subapp/quarantine/); do echo "$( du -sh /mnt/import/${sip} ),# $( stat -c %y ~/.subapp/quarantine/$sip/sip.ERROR ),# $( find /mnt/import/${sip}/ -type f | wc -l) files"; done | column -s "#" -t' alias quarantine_size='sum=0; for dir in $(find ~/.subapp/quarantine/ -name "sip.bagit"); do if [[ -d $(realpath ${dir}) ]]; then sum=$((sum+$(du -s $(realpath ${dir}) | cut -f1))); else echo "broken Q-Link: ${dir}"; fi; done; echo "$((sum/1024)) MiB in quarantine."' - alias next='/usr/bin/perl -I /usr/local/perl/ /usr/local/bin/subapp_bagit.pl --config-file /home/processing/.subapp/subapp.cfg --single_run; echo "################# LOG #####################"; echo "last 10 lines..."; tail -n 10 /var/log/subapp/${HOSTNAME}/sips.log; echo "################# PROBLEMS #####################"; START=$( grep -n "starting..." /var/log/subapp/${HOSTNAME}/subapp.log | tail -1 | cut -f1 -d: ); ALL_LINES=$( wc -l < /var/log/subapp/${HOSTNAME}/subapp.log ); let LINES=$ALL_LINES-$START; echo "checking last $LINES log lines..."; tail -n $LINES /var/log/subapp/${HOSTNAME}/subapp.log | grep --color -e "\[ERROR\]" -e "\[FATAL\]" -e "\[WARN\]"' + alias next='/usr/bin/perl -I /usr/local/perl/ /usr/local/bin/subapp_bagit.pl --config-file /home/processing/.subapp/subapp.cfg --single_run; echo "################# LOG #####################"; echo "last 10 lines..."; tail -n 10 /var/log/subapp/sips.log; echo "################# PROBLEMS #####################"; START=$( grep -n "starting..." /var/log/subapp/subapp.log | tail -1 | cut -f1 -d: ); ALL_LINES=$( wc -l < /var/log/subapp/subapp.log ); let LINES=$ALL_LINES-$START; echo "checking last $LINES log lines..."; tail -n $LINES /var/log/subapp/subapp.log | grep --color -e "\[ERROR\]" -e "\[FATAL\]" -e "\[WARN\]"' diff --git a/tasks/install_subapp.yml b/tasks/install_subapp.yml index fd72a4611ef9a64aa3079bb1b43bb03dd9eff8ae..16553000d383c96bb4af915e75bcc90dcdbc17a0 100644 --- a/tasks/install_subapp.yml +++ b/tasks/install_subapp.yml @@ -186,24 +186,29 @@ owner: "{{ item.owner }}" group: "{{ item.group }}" loop: - # SubApp - - src: "{{ vault_subapp_vars.files.subapp.path }}" - dest: "/home/{{ vault_subapp_user }}/.subapp.cfg" - owner: "{{ vault_subapp_user }}" - group: "{{ vault_subapp_group }}" - - src: "/var/log/rosetta/{{ ansible_hostname }}/subapp/" - dest: "/home/{{ vault_subapp_user }}/.subapp/{{ ansible_hostname }}" - owner: "{{ vault_subapp_user }}" - group: "{{ vault_subapp_group }}" # DisApp - src: "{{ vault_subapp_vars.files.disapp.path }}" dest: "/home/{{ vault_disapp_user }}/.disapp.cfg" owner: "{{ vault_disapp_user }}" group: "{{ vault_disapp_group }}" - - src: "/var/log/rosetta/{{ ansible_hostname }}/disapp/" - dest: "/home/{{ vault_disapp_user }}/.disapp/{{ ansible_hostname }}" + - src: "/var/log/disapp/" + dest: "/home/{{ vault_disapp_user }}/.disapp/disapp_logs" owner: "{{ vault_disapp_user }}" group: "{{ vault_disapp_group }}" + # SubApp + - src: "{{ vault_subapp_vars.files.subapp.path }}" + dest: "/home/{{ vault_subapp_user }}/.subapp.cfg" + owner: "{{ vault_subapp_user }}" + group: "{{ vault_subapp_group }}" + - src: "/var/log/subapp/" + dest: "/home/{{ vault_subapp_user }}/.subapp/subapp_logs" + owner: "{{ vault_subapp_user }}" + group: "{{ vault_subapp_group }}" + # Webservice + - src: "/var/log/subapp_ws/" + dest: "/home/{{ vault_subapp_user }}/.subapp/subapp_ws_logs" + owner: "{{ vault_subapp_user }}" + group: "{{ vault_subapp_group }}" # Bash-Completion funktioniert ab 2020.2 anders, s. Abschnitt AUTOCOMPLETION in perldoc bin/subapp_rosetta.pl und bin/disapp_rosetta.pl # https://ansible-lint.readthedocs.io/en/latest/usage/#false-positives-skipping-rules diff --git a/vars/nfs-mounts.vault.example b/vars/nfs-mounts.vault.example index 1b5c106cbfd9d9be1c275a94f17fe08df0e4c59e..eb6f7e170e92e9e361ca6f6d5a9012d9a82aaae7 100644 --- a/vars/nfs-mounts.vault.example +++ b/vars/nfs-mounts.vault.example @@ -19,7 +19,7 @@ nfs_opts: paths: log: nfs_share: "/vol/linux_server_logs/{{ ansible_hostname }}" - mountpoint: "/var/log/subapp/{{ ansible_hostname }}" + mountpoint: "/var/log/subapp/" owner: "USERNAME_HERE" group: "GROUPNAME_HERE" mode: "0xxx"