diff --git a/tasks/configure_iptables_externalusers.yml b/tasks/configure_iptables_externalusers.yml new file mode 100644 index 0000000000000000000000000000000000000000..f979f95cca3042b64b65c31d02a8325f7f1f851f --- /dev/null +++ b/tasks/configure_iptables_externalusers.yml @@ -0,0 +1,20 @@ +--- +- name: iptables-Regeln setzen (ZIH der TU Dresden) + ansible.builtin.iptables: + action: insert + chain: INPUT + comment: "{{ item.comment }}" + destination: "{{ item.dest | default(omit) }}" + destination_port: "{{ item.dest_port | default(omit) }}" + jump: ACCEPT + # limit: 100/s + # limit_burst: 1000/s + protocol: tcp + # rule_num: 1 + source: "{{ item.src | default(omit) }}" + source_port: "{{ item.src_port | default(omit) }}" + state: "{{ item.state | default('present') }}" + table: filter + loop: "{{ vault_iptables | flatten(levels=1) }}" + notify: + - save iptables rules diff --git a/tasks/install_validation_tools.yml b/tasks/install_validation_tools.yml new file mode 100644 index 0000000000000000000000000000000000000000..5760961ed10cd5bce50e17768a4cb96a02e87981 --- /dev/null +++ b/tasks/install_validation_tools.yml @@ -0,0 +1,150 @@ +--- +- name: install validation dependencies from SLUB Debian repo + ansible.builtin.package: + name: [ + "git", + "inotify-tools", + "checkit-tiff-current", + "checkit-tiff-upcoming", + "libzen0v5", + "libmediainfo0v5", + "iccmax", # Icc-Tools, installed from SLUBs private package repo + "mediaconch", + "xsltproc", + ] + # CAUTION: + # Do NOT use "state: latest", because the repair tools might need specific + # versions of these tools! + state: present + +- name: remove existing MediaConch-Profile repo directory (otherwise, we can't clone it in the next task -.-) + ansible.builtin.file: + path: "/tmp/mediaconch-profile/" + state: absent + changed_when: false # there's no other way to get this task idempotent + +- name: checkout MediaConch-Profile repo + ansible.builtin.git: + repo: "https://git.slub-dresden.de/digital-preservation/mediaconch-profile.git" + dest: "/tmp/mediaconch-profile/" + register: mc_profile_repo_cloned + changed_when: false # there's no other way to get this task idempotent + +- name: compile MediaConch profile + ansible.builtin.command: + cmd: "bash ./build_all.sh" + chdir: "/tmp/mediaconch-profile/" + when: mc_profile_repo_cloned.before != mc_profile_repo_cloned.after + register: mc_profile_repo_built + changed_when: false # there's no other way to get this task idempotent + +- name: deploy MediaConch profile to Rosetta + ansible.builtin.copy: + src: "/tmp/mediaconch-profile/build/SLUB_mediaconch_policy_all.xml" + dest: "/usr/local/etc/SLUB_mediaconch_policy_all.xml" + remote_src: true + mode: "0644" + +# According to FHS 3.0 (https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch04s09.html), +# use "/usr/local/etc/" for "Host-specific system configuration for local binaries". +# As our config files are shareable according to https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch02.html, +# we place them below "/usr/" instead of "/etc/". +- name: create symlinks for validation tool profiles + ansible.builtin.file: + src: "{{ item.s }}" # file to link to + dest: "/usr/local/etc/{{ item.d }}" # symlink file path + state: link + loop: + # TIFF 6 + - s: "/usr/share/checkit_tiff_current/example_configs/cit_tiff6_baseline_SLUB.cfg" + d: "cit_tiff6_baseline_SLUB_current.cfg" + - s: "/usr/share/checkit_tiff_upcoming/example_configs/cit_tiff6_baseline_SLUB.cfg" + d: "cit_tiff6_baseline_SLUB_upcoming.cfg" + # GeoTIFF 6 + - s: "/usr/share/checkit_tiff_current/example_configs/cit_tiff6_geotiff_SLUB.cfg" + d: "cit_tiff6_geotiff_SLUB_current.cfg" + - s: "/usr/share/checkit_tiff_upcoming/example_configs/cit_tiff6_geotiff_SLUB.cfg" + d: "cit_tiff6_geotiff_SLUB_upcoming.cfg" + # Retrofotos Fotothek + - s: "/usr/share/checkit_tiff_current/example_configs/cit_tiff_retrofotos_SLUB.cfg" + d: "cit_tiff_retrofotos_SLUB_current.cfg" + - s: "/usr/share/checkit_tiff_upcoming/example_configs/cit_tiff_retrofotos_SLUB.cfg" + d: "cit_tiff_retrofotos_SLUB_upcoming.cfg" + +- name: clone validation tool repo + ansible.builtin.git: + repo: "https://git.slub-dresden.de/digital-preservation/validate_workflows.git" + dest: "/tmp/validate_workflows/" + # bare: true + depth: 1 + recursive: false + single_branch: true + +- name: install validation tool + ansible.builtin.copy: + src: "/tmp/validate_workflows/validate_workflow.sh" + dest: "/usr/local/bin/" + mode: "0755" + remote_src: true + +- name: create validation directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0777" # it's created in a mixed mode NFS/CIFS share, so mode will always be 0777, no matter what we set + owner: "lza" + group: "aw_lza_repair" + loop: + - "/mnt/lza_repair_ddz/validate/unchecked/" + - "/mnt/lza_repair_ddz/validate/results/" + - "/mnt/lza_repair_digas/validate/unchecked/" + - "/mnt/lza_repair_digas/validate/results/" + - "/mnt/lza_repair_fotothek/validate/unchecked/" + - "/mnt/lza_repair_fotothek/validate/results/" + - "/mnt/lza_repair_mediathek/validate/unchecked/" + - "/mnt/lza_repair_mediathek/validate/results/" + - "/mnt/lza_repair_save/validate/unchecked/" + - "/mnt/lza_repair_save/validate/results/" + +# man 5 systemd.unit, table "Unit File Load Path" +- name: create Systemd unit directory in Unit File Load Path + ansible.builtin.file: + path: "/usr/local/lib/systemd/system/" + state: directory + mode: "0755" + +- name: install validation service units and sockets + ansible.builtin.template: + src: "usr/local/lib/systemd/system/{{ item }}.j2" + dest: "/usr/local/lib/systemd/system/{{ item }}" + mode: "0640" + owner: "root" + group: "root" + loop: + - "validation_daemon_any@.service" + - "validation_daemon_any.socket" + - "validation_daemon_@.service" # Daemon service + +- name: enable & start validation daemon + ansible.builtin.systemd: + name: "validation_daemon_@{{ item }}.service" + daemon_reload: true + enabled: true + state: restarted # this can never be idempotent + loop: + - "ddz" + - "digas" + - "fotothek" + - "mediathek" + - "save" + changed_when: false + +- name: enable & start validation Webservice + ansible.builtin.systemd: + name: "validation_daemon_any.{{ item }}" + daemon_reload: true + enabled: true + state: restarted # this can never be idempotent + loop: + - "socket" + changed_when: false diff --git a/tasks/main.yml b/tasks/main.yml index 17ffe15acc0f05bbbe05af0e66748a6dd4decdf1..0cf285965ecefe5f043048ecfcd11595b4baeda5 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: include Ansible Vaults - include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" + ansible.builtin.include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" loop: - "nfs_mounts.vault" - "repos.vault" @@ -28,6 +28,10 @@ import_tasks: install-repair-tools.yml tags: [repairtools, ci] +- name: install validation tools + ansible.builtin.import_tasks: "install_validation_tools.yml" + tags: [validationtools] + ### INSTALLATION SYSTEMD SERVICES - name: install SystemD-Services import_tasks: configure-systemd-services.yml diff --git a/templates/usr/local/lib/systemd/system/validation_daemon_@.service.j2 b/templates/usr/local/lib/systemd/system/validation_daemon_@.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..bc9cf8534f0f56ddde9913cc42b9381c93da0d22 --- /dev/null +++ b/templates/usr/local/lib/systemd/system/validation_daemon_@.service.j2 @@ -0,0 +1,42 @@ +[Unit] +Description=Validation Tool Daemon (%i) +After=network.target + +[Service] +ExecStart=/usr/local/bin/validate_workflow.sh \ + --watch-folder "/mnt/lza_repair_%i/validate/unchecked/" \ + --result-folder "/mnt/lza_repair_%i/validate/results/" \ + --mode "%i" \ + --files-mode delete \ + --stage current \ + --daemon +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +User={{ vault_service_user }} +Group={{ vault_service_group }} +Type=simple + +### Security features +# documented at https://www.freedesktop.org/software/systemd/man/systemd.exec.html +#ProtectSystem=strict +#ProtectHome=read-only +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#LockPersonality=true +#MemoryDenyWriteExecute=true +#RestrictRealtime=true +#RestrictSUIDSGID=true +## RemoveIPC=true +## PrivateMounts=true +## MountFlags= +## SystemCallFilter is a Whitelist!!! +#SystemCallFilter=@aio,@basic-io,@debug,@file-system,@network-io +#SystemCallErrorNumber=1337 + +[Install] +WantedBy=multi-user.target diff --git a/templates/usr/local/lib/systemd/system/validation_daemon_any.socket.j2 b/templates/usr/local/lib/systemd/system/validation_daemon_any.socket.j2 new file mode 100644 index 0000000000000000000000000000000000000000..f2c02fdf2d3a201f50a2c3da1d54fd3f5acbe045 --- /dev/null +++ b/templates/usr/local/lib/systemd/system/validation_daemon_any.socket.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Validation Tool Daemon (Socket Activated Service) +After=network.target + +[Socket] +# https://www.linux.com/training-tutorials/end-road-systemds-socket-units/ +ListenStream=1234 +Accept=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/usr/local/lib/systemd/system/validation_daemon_any@.service.j2 b/templates/usr/local/lib/systemd/system/validation_daemon_any@.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..8a765ff52a45032fb9e2cc9888a3b8b86812f09d --- /dev/null +++ b/templates/usr/local/lib/systemd/system/validation_daemon_any@.service.j2 @@ -0,0 +1,41 @@ +[Unit] +Description=Validation Tool Daemon (Socket Activated Service) +After=network.target + +[Service] +# https://www.linux.com/training-tutorials/end-road-systemds-socket-units/ +ExecStart=/usr/local/bin/validate_workflow.sh \ + --pipe \ + --mode auto \ + --stage current +# https://www.freedesktop.org/software/systemd/man/systemd.exec.html#StandardOutput= +StandardInput=socket +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +User={{ vault_service_user }} +Group={{ vault_service_group }} +Type=simple + +### Security features +# documented at https://www.freedesktop.org/software/systemd/man/systemd.exec.html +#ProtectSystem=strict +#ProtectHome=read-only +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#LockPersonality=true +#MemoryDenyWriteExecute=true +#RestrictRealtime=true +#RestrictSUIDSGID=true +## RemoveIPC=true +## PrivateMounts=true +## MountFlags= +## SystemCallFilter is a Whitelist!!! +#SystemCallFilter=@aio,@basic-io,@debug,@file-system,@network-io +#SystemCallErrorNumber=1337 + +[Install] +WantedBy=multi-user.target