diff --git a/.ansible-lint b/.ansible-lint index 9c9323e8df825eddaf10a9188e0fd26583c5dde0..f18a6472898d714ef85310cdbe01e39500977528 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,7 +6,7 @@ # and not relative to the CWD of execution. CLI arguments passed to the --exclude # option will be parsed relative to the CWD of execution. exclude_paths: - - .cache/ # implicit unless exclude_paths is defined in config + - .cache/ # implicit unless exclude_paths is defined in config - .git/ - .githooks/ - backups/ @@ -15,24 +15,19 @@ exclude_paths: # verbosity: 1 # Mock modules or roles in order to pass ansible-playbook --syntax-check -#mock_modules: -# - zuul_return -# # note the foo.bar is invalid as being neither a module or a collection -# - fake_namespace.fake_collection.fake_module -# - fake_namespace.fake_collection.fake_module.fake_submodule -#mock_roles: -# - mocked_role -# - author.role_name # old standalone galaxy role -# - fake_namespace.fake_collection.fake_role # role within a collection +# mock_modules: +# - zuul_return +# # note the foo.bar is invalid as being neither a module or a collection +# - fake_namespace.fake_collection.fake_module +# - fake_namespace.fake_collection.fake_module.fake_submodule +# mock_roles: +# - mocked_role +# - author.role_name # old standalone galaxy role +# - fake_namespace.fake_collection.fake_role # role within a collection # Enable checking of loop variable prefixes in roles loop_var_prefix: "{role}_" -# Enforce variable names to follow pattern below, in addition to Ansible own -# requirements, like avoiding python identifiers. To disable add `var-naming` -# to skip_list. -var_naming_pattern: "^[a-z_][a-z0-9_]*$" - use_default_rules: true # Load custom rules from this specific folder # rulesdir: @@ -46,9 +41,9 @@ skip_list: # Any rule that has the 'opt-in' tag will not be loaded unless its 'id' is # mentioned in the enable_list: enable_list: - - empty-string-compare # opt-in - - no-log-password # opt-in - - no-same-owner # opt-in + - empty-string-compare # opt-in + - no-log-password # opt-in + - no-same-owner # opt-in # add yaml here if you want to avoid ignoring yaml checks when yamllint # library is missing. Normally its absence just skips using that rule. - yaml @@ -60,19 +55,19 @@ enable_list: warn_list: - skip_this_tag - git-latest - - experimental # experimental is included in the implicit list + - experimental # experimental is included in the implicit list # - role-name # Offline mode disables installation of requirements.yml offline: false # Define required Ansible's variables to satisfy syntax check -#extra_vars: -# foo: bar -# multiline_string_variable: | -# line1 -# line2 -# complex_variable: ":{;\t$()" +# extra_vars: +# foo: bar +# multiline_string_variable: | +# line1 +# line2 +# complex_variable: ":{;\t$()" # Uncomment to enforce action validation with tasks, usually is not # needed as Ansible syntax check also covers it. diff --git a/.config/molecule/config.yml b/.config/molecule/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..ece7ff6dd671a6a55ec8f7e9e498c0310e52d4cc --- /dev/null +++ b/.config/molecule/config.yml @@ -0,0 +1,2 @@ +--- +prerun: false diff --git a/.gitignore b/.gitignore index 7ccd9c109dcf4281907fc0dd21706cb1bc62da05..ed783329d62a4c1e481a467fe674eb420a2b81dd 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,8 @@ Thumbs.db *.retry *.vault +inventory.* +inv.* # Vim # ####### @@ -73,13 +75,14 @@ tags .vagrant/ *.box -# Build Environments # -###################### +# Temporary/Build/Backup # +########################## +backups/ build/ -# Backups # -########### +# CONFIDENTIAL # +################ + +ssh_host_* -backups/ -backup/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ea9f615433973e9b5b0e08687105da13cb3de66..036df263edb76aad459181f7df8789168c4a1ee5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,52 +1,35 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - -# This is a sample GitLab CI/CD configuration file that should run without any modifications. -# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, -# it uses echo commands to simulate the pipeline execution. -# +--- # A pipeline is composed of independent jobs that run scripts, grouped into stages. # Stages run in sequential order, but jobs within stages run in parallel. # # For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages stages: # List of stages for jobs, and their order of execution - - build - test -image: - name: debian:stable-slim - -cache: - key: build - untracked: true - policy: pull-push - -build-job: - stage: build - script: - - echo "Installing environment..." - - apt-get -y update - - apt-get -y install python3 virtualenv python3-setuptools - - virtualenv -p python3 ansible - - source ansible/bin/activate - - pip install ansible +default: + before_script: + - source /opt/molecule/bin/activate + - ansible --version + - molecule --version + after_script: + - source /opt/molecule/bin/activate + - molecule destroy --scenario-name default -unit-test-job: # This job runs in the test stage. - stage: test # It only starts when the job in the build stage completes successfully. +test-job: + stage: test + tags: + - "shell" script: - - echo "Running unit tests..." - - apt-get -y update - - apt-get -y install python3 git - - source ansible/bin/activate - - ansible-galaxy install -r requirements.yml - - echo $vault_lza_install_common > ${CI_BUILDS_DIR}lza_install_common.pass - - echo $vault_lza_server_hardening > ${CI_BUILDS_DIR}/lza_server_hardening.pass - - echo $vault_slub_osquery > ${CI_BUILDS_DIR}/slub_osquery.pass - - echo $vault_lza_sanitytest > ${CI_BUILDS_DIR}/lza_sanitytest.pass - - ansible-playbook site.yml -b -l localhost --vault-password-file ${CI_BUILDS_DIR}lza_install_common.pass ${CI_BUILDS_DIR}/lza_server_hardening.pass ${CI_BUILDS_DIR}/slub_osquery.pass ${CI_BUILDS_DIR}/lza_sanitytest.pass - variables: - ANSIBLE_ROLES_PATH: "${CI_BUILDS_DIR}/${CI_PROJECT_PATH}/" + # make sure that Ansible Vaults are present and can be decrypted + - echo "${VAULT_INSTALL_COMMON}" > ../lza_install_common.pass + - export ANSIBLE_VAULT_IDENTITY_LIST="../lza_install_common.pass, ../passfile2" + - rm -rf ../ansible_vaults/ + - git clone https://gitlab+deploy-token-25:${VAULT_ACCESS_TOKEN}@git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git ../ansible_vaults/; \ + # run Molecule tests + - molecule syntax --scenario-name default + - molecule lint --scenario-name default + - molecule create --scenario-name default + - molecule converge --scenario-name default + - molecule idempotence --scenario-name default + # - molecule verify --scenario-name default diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md new file mode 100644 index 0000000000000000000000000000000000000000..acc60bf704eb7d16e00ff44ad58e5b18f8cc0d07 --- /dev/null +++ b/.gitlab/issue_templates/Default.md @@ -0,0 +1,16 @@ +## Expected Behavior + + +## Actual Behavior + + +## Steps to Reproduce the Problem + + 1. + 1. + 1. + +## Specifications + + - Version/Commit: + - Platform: diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000000000000000000000000000000000000..4030f6fe7d82d4a78fe30cfa9c959eedce2249ed --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,7 @@ +Fixes # + +## Proposed Changes + + - + - + - diff --git a/.yamllint b/.yamllint index 7c0f15cf782b192e2b80d4eadb9dcbf92b4553bf..88276760562cb58bb9bc47fa890e8a3df1125b9b 100644 --- a/.yamllint +++ b/.yamllint @@ -1,7 +1,5 @@ --- -# based on documentation available at -# https://yamllint.readthedocs.io/en/stable/rules.html - +# Based on ansible-lint config extends: default rules: @@ -11,13 +9,25 @@ rules: brackets: max-spaces-inside: 1 level: error - comments: - min-spaces-from-content: 4 + colons: + max-spaces-after: -1 + level: error + commas: + max-spaces-after: -1 + level: error + comments: disable comments-indentation: disable - document-end: disable - document-start: - level: warning - octal-values: - forbid-explicit-octal: false + document-start: disable + empty-lines: + max: 3 + level: error + hyphens: + level: error + indentation: disable + key-duplicates: enable line-length: disable - truthy: enable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + truthy: disable diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..19d2d20a7ef607c068c5a56a45e0eabef5efba31 --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +Just be excellent to each other. That's it. EoC. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/README.md b/README.md index 55144681212a4f28f07c618f25cc637390376c8e..2250485ed60c682ccf7b2624fc0e9723bd7133fc 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # Ansible-Role "ansible_lza_sanitytest" -## Description +## What does it do? This role provides all necessary tasks to setup a sanitytest server for SLUBArchiv. -## Prerequisites +## What do I need? + +### Prerequisites To use this role, the following software must be installed on your workstation: * ansible @@ -13,29 +15,32 @@ To deploy this role to a managed host, the following software must be installed * Python3 * SSHd +### Dependencies + Other roles required by this role can be easily installed using `ansible-galaxy` if necessary. As the roles reside in SLUB's local Git server instance, you might need a password for certain repositories despite most of them being publicly available. This command will install the required roles in the correct location so they are found in the search path: -``` - ansible-galaxy install -r requirements.yml -p ../ +```bash +ansible-galaxy install -r requirements.yml -p ../ ``` -It is recommended to use Debian VMs as deployed by SLUB's UDA tool with this role. +### Infrastructure -## Quick Start +It is recommended to use Debian VMs as deployed by SLUB's GUBS tool with this role. Otherwise you will not have access to the software packages that are located in SLUB's private Debian package repository. + +## Can i have a Quick Start? Most options already have sensible defaults in `ansible.cfg`. However, you can override these defaults using CLI options/flags if you want to. The flags can be combined if necessary, and most of them have long versions as well. Get more information using `ansible-playbook --help`. To simply run the playbook, just call the `site.yml` playbook like this: +```bash +ansible-playbook site.yml -u <username> ``` - ansible-playbook site.yml -u <username> -``` - If you want to limit the execution to a subset of all hosts that are listed in the inventory, use the `-l` or `--limit` option like this: -``` - ansible-playbook site.yml -l <hostna*> - ansible-playbook site.yml -l <hostname> - ansible-playbook site.yml -l <hostname1>:<hostname2>:... - ansible-playbook site.yml -l <inventory_group> - ansible-playbook site.yml --limit=<hostna*> +```bash +ansible-playbook site.yml -l <hostna*> +ansible-playbook site.yml -l <hostname> +ansible-playbook site.yml -l <hostname1>:<hostname2>:... +ansible-playbook site.yml -l <inventory_group> +ansible-playbook site.yml --limit=<hostna*> ``` If you do not have Vault password files in the directory above the role direcory, you have to give the Vault password before execution: @@ -43,6 +48,12 @@ If you do not have Vault password files in the directory above the role direcory ansible-playbook site.yml --ask-vault-pass ``` +You can use your own inventory file by adding the `-i` or `--inventory=INVENTORY` option: +``` + ansible-playbook site.yml -i inventory.yml + ansible-playbook site.yml --inventory=inventory.yml +``` + Tasks in this role have been tagged to enable users to only run subsets of tasks. This can be leveraged to decrease run times or run only certain tasks after small changes. To list all available tags, use: ``` @@ -54,20 +65,62 @@ You can then run only certain tagged tasks by using the `--tags` option: ansible-playbook site.yml --tags=tag1,tag2,...,tagN ``` -## Testing the role +## How can I run tests? Tests have been implemented using the Molecule framework. The details on using the test suite are described below `molecule/`. -## Variables +To run some quick tests, you can do: +```bash +# pure syntax check +molecule syntax +# run yamllint and ansible-lint +molecule lint +# list available test scenarios, e.g. based on different OS images or platforms +molecule list +# create the test environment for a specific scenario (can be left out for "default") +molecule create [-s scenario] +# run your tasks against the test env +molecule converge [-s scenario] # multiple times if needed +# run idempotence checks to se if any of the tasks keeps changing (subsequent runs shouldn't trigger changes) +molecule idempotence [-s scenario] +# cleanup test env (remove VM/container) +molecule destroy [-s scenario] +``` + +We recommend running those tests before pushing any code to the Git server. + +On every `git push`, the GitLab-CI pipeline will run a similar set of tests to ensure that all changes are working. Find the details in the `.gitlab-ci.yml` file located at the project root directory. + +## What can be configured? + +### Ansible Role Many variables have been "hidden" in encrypted Ansible Vaults. For security reasons, these Vaults are maintained in a separate private internal repository of SLUB's Git. However, in order to better understand the data within the vaults, you can find `\*.vault.example` files below the `vars/` directory. If you work outside of SLUBArchive and have no access to the vault repository, make sure to put the necessary vaults in the expected paths at `../ansible_vaults/<ROLENAME>/`. -## git configuration +Variable defaults have been set in `defaults/main.yml`. You can overwrite them with your own values by setting them in `vars/main.yml`. + +### Git configuration Just run the `setup_gitconfig.sh` script that comes with the repo to correctly setup all necessary local Git configurations. -## Author Information +## What changes have been made lately? + +All changes can be found in the `CHANGELOG` file located at the project root directory. Alternatively, you can have a look at the commit log to get a detailed view. + +## Who is maintaining this project? + +All authors/maintainers are listed in the `kudos.txt` file located at the project root directory. + +## How can I contribute? + +If you have any comments or find bugs, please contact langzeitarchiv@slub-dresden.de, create an issue or send us a pull request. + +Details on how to contribute to this project can be found at the `CONTRIBUTING.md` file located at the project root directory. + +If you have commited to the project yourself, you can leave a note in the `kudos.txt` file located at the project root directory. Be assured of our eternal gratitude. + +## Is there a Code of Conduct? -If you have any comments or find bugs, please contact langzeitarchiv@slub-dresden.de or issue a pull request. +Yes there is. You can find it in the `CODE-OF-CONDUCT.md` file located at the project root directory. It's kept very brief by design. diff --git a/handlers/main.yml b/handlers/main.yml index 26f5fa9b530814f0b2dd37815af90610b9912771..e272215d8147edfd45813495a1542f7a9bb66791 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,13 +1,13 @@ --- - name: restart postgres - systemd: + ansible.builtin.systemd: name: "postgresql" state: restarted - name: save iptables rules (Debian) block: - name: Ordner für iptables-Config erstellen - file: + ansible.builtin.file: path: "/etc/iptables" state: directory owner: "root" @@ -15,11 +15,12 @@ mode: 0755 listen: "save iptables rules" - name: install netfilter-persistent to be able to save iptables rules - apt: + ansible.builtin.apt: name: netfilter-persistent state: present listen: "save iptables rules" - name: save iptables rules - command: 'netfilter-persistent save' + ansible.builtin.command: 'netfilter-persistent save' listen: "save iptables rules" + changed_when: false when: ansible_os_family == "Debian" diff --git a/kudos.txt b/kudos.txt new file mode 100644 index 0000000000000000000000000000000000000000..a63a1c9ff1ca3883794dd53818c031f92d6f5077 --- /dev/null +++ b/kudos.txt @@ -0,0 +1,48 @@ +# kudos.txt - Express gratitude to your contributors. +# https://github.com/kudos-txt + +project: + - name: ansible_lza_sanitytest + site: https://git.slub-dresden.de/digital-preservation/ansible_lza_sanitytest +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> + +contributor: +# - name: <name or nick of the contributor> +# role: <role in the project> +# site: <website url> +# blog: <blog url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# home: <country>, <region>, <zip>, <city>, <address> +# work: <company>, <job title>, <job description> +# note: <free text notes> + - name: Jörg Sachse + role: Maintainer + mail: Joerg.Sachse@slub-dresden.de + home: Germany, Saxony, 01059, Dresden, Zellescher Weg 18 + work: SLUB Dresden, Digital Preservationist &IT Administrator & Installation Wizard + +#partner: +# - name: <name of the partner> +# site: <website url> +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> + +#software: +# - name: <name of the software> +# site: <website url> +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> diff --git a/meta/main.yml b/meta/main.yml index cb159002a60fac94e3bd5ae5a73f95cb4f37dbfe..f41d21ca7037d5d0fd1e2c9ae1e3b093f3fa35cf 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,16 +1,30 @@ --- galaxy_info: author: Jörg Sachse - description: role to install a post-upgrade sanity testing server for the SLUBarchiv digital preservation repository company: SLUB Dresden - # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker Some suggested licenses: - BSD - # (default) - MIT - GPLv2 - GPLv3 - Apache - CC-BY - license: public domain - min_ansible_version: 2.4 - # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo - # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only - # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch: - # + description: role to install a post-upgrade sanity testing server for the SLUBarchiv digital preservation repository + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you + # add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + # issue_tracker_url: "https://example.com/" + # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: GPL-3.0-or-later + + min_ansible_version: "2.5" + # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo + # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only + # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch: # Provide a list of supported platforms, and for each platform a list of versions. If you don't wish to enumerate all versions for a particular platform, use 'all'. To view available # platforms and versions (or releases), visit: https://galaxy.ansible.com/api/v1/platforms/ # @@ -19,14 +33,6 @@ galaxy_info: platforms: - name: Debian versions: - - 9 - - 10 - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you - # add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. -dependencies: -# - ansible_lza_install_common -# - ansible_lza_server_hardening + - "buster" + - "bullseye" +dependencies: [] diff --git a/molecule/README.md b/molecule/README.md index 33a7eb56b8f65d94165fbc812cf391d629d8fd2a..fdb41fe522e28da231b23e705509e2d0fd254317 100644 --- a/molecule/README.md +++ b/molecule/README.md @@ -4,39 +4,53 @@ In order to be able to use the tests, you need to have some software packages installed. You may need sudo privileges for some of these operations. - ### install VirtualBox - # do NOT use distribution packages - # process documented at https://www.virtualbox.org/wiki/Linux_Downloads - # - # add repository URL - sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list - # add GPG key - wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - - # update sources - sudo apt update - # install VirtualBox - sudo apt-get install virtualbox-6.1 - - ### install Vagrant - # do NOT use distribution packages - # - # download Debian package from Hashicorp - wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb - # install package - sudo dpkg -i vagrant_2.2.9_x86_64.deb - - ### install Molecule et. al. - # prepare directories - mkdir ~/python-envs/ && cd ~/python-env/ - # create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!) - virtualenv -p python3 molecule-env - # enter the Virtual Environment in your current shell (other shells will be unaffected) - source molecule-env/bin/activate - # install packages - pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker - - # leave the Virtual Environment only when you're done - deactivate +There are two ways to accomplish this. + +### Option 1: use `molecule-skel` (SLUB staff only) + +1. Clone the project at [molecule-skel](https://git.slub-dresden.de/slub-referat-2-3/molecule-skel): `git clone https://git.slub-dresden.de/slub-referat-2-3/molecule-skel.git` +2. Switch to the `molecule-skel` repository directory and run `./install_testenv.sh`. +3. PROFIT! + + Check the `README.md` over at [molecule-skel](https://git.slub-dresden.de/slub-referat-2-3/molecule-skel) for details on how to use that. It's easy, I promise. + +### Option 2: manual installation + +Run these steps on your terminal. + +```bash +### install VirtualBox +# do NOT use distribution packages, as they may be too old! +# process documented at https://www.virtualbox.org/wiki/Linux_Downloads +# +# add GPG key +wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - +# add repository URL +sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list +# update sources and install VirtualBox +sudo apt update; sudo apt-get install virtualbox-6.1 + +### install Vagrant +# do NOT use distribution packages, as they may be too old! +# +# download Debian package from Hashicorp +wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb +# install package +sudo dpkg -i vagrant_2.2.9_x86_64.deb + +### install Molecule et. al. +# prepare directories +mkdir ~/python-envs/ && cd ~/python-env/ +# create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!) +virtualenv -p python3 molecule-env +# enter the Virtual Environment in your current shell (other shells will be unaffected) +source molecule-env/bin/activate +# install packages +pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker + +# leave the Virtual Environment only when you're done +deactivate +``` You can find suitable documentation at the respective vendors' websites. * [Vagrant Installation Guide](https://www.vagrantup.com/docs/installation/) @@ -46,10 +60,14 @@ You can find suitable documentation at the respective vendors' websites. ## Initialising a new Molecule scenario If you have already created a role without the Molecule test framework or you want to add test scenarios, you can use: - molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra] +```bash +molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra] +``` If you need any help with the options, please use: - molecule init role --help +```bash +molecule init role --help +``` ## Running Tests @@ -60,26 +78,31 @@ Various test environments are separated into so-called "scenarios" that can be b In the simplest configuration, the `molecule/` directory only contains one `default/` directory that contains the default scenario. This scenario is run if no other scenario is chosen using the `-s` CLI option. This is the basic usage of Molecule: - # create test infrastructure - cd <role_directory> - molecule create - # run playbooks against test infrastructure - molecule converge - # run Testinfra tests - molecule verify - # remove test infrastructure - molecule destroy - - # run all steps at once: - molecule test - -It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed. - # First, get UUID of the VM - vagrant global-status - # Then, create the snapshot - vagrant snapshot save <uuid> <snapshot_name> - # To restore the snapshot, use - vagrant snapshot restore <uuid> <snapshot_name> - # And to remove the snapshot, use - vagrant snapshot delete <uuid> <snapshot_name> - +```bash +# create test infrastructure +cd <role_directory> +molecule create +# run playbooks against test infrastructure +molecule converge +# run idempotence tests +molecule idempotence +# run tests, if they exist +molecule verify +# remove test infrastructure +molecule destroy + +# run all steps at once: +molecule test +``` + +It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed. Just like this: +```bash +# First, get UUID of the VM +vagrant global-status +# Then, create the snapshot +vagrant snapshot save <uuid> <snapshot_name> +# To restore the snapshot, use +vagrant snapshot restore <uuid> <snapshot_name> +# And to remove the snapshot, use +vagrant snapshot delete <uuid> <snapshot_name> +``` diff --git a/molecule/default b/molecule/default new file mode 120000 index 0000000000000000000000000000000000000000..3841ab1f6fbdfc4b16f9491b776a826e19fa583c --- /dev/null +++ b/molecule/default @@ -0,0 +1 @@ +./virtualbox \ No newline at end of file diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml deleted file mode 100644 index d7e83aceb6692118aa19d004239929d316977d89..0000000000000000000000000000000000000000 --- a/molecule/default/converge.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Converge - hosts: all - roles: - - {role: "ansible_lza_sanitytest", become: true} diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml deleted file mode 100644 index 1ade3a012e09a904c0fc3c815d964280a7a0a130..0000000000000000000000000000000000000000 --- a/molecule/default/molecule.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: vagrant - provider: - name: virtualbox -lint: | - set -e - yamllint . - ansible-lint -x formatting - flake8 --ignore=E501 -platforms: - - name: molecule-test-sanity - box: debian/buster64 - memory: 512 - cpus: 1 -provisioner: - name: ansible - log: true - lint: | - set -e - ansible-lint - config_options: - defaults: - vault_identity_list: "@$HOME/.ansible/roles/molecule_prepare.pass, @$HOME/.ansible/roles/lza_install_common.pass, @$HOME/.ansible/roles/lza_server_hardening.pass, @$HOME/.ansible/roles/lza_sanitytest.pass" - vvv: false -verifier: - name: testinfra - env: - PYTHONWARNINGS: "ignore:.*U.*mode is deprecated:DeprecationWarning" - lint: | - set -e - flake8 - options: - v: 1 diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml deleted file mode 100644 index d903f567ea8ed2309c8359152b80adcf44855d1b..0000000000000000000000000000000000000000 --- a/molecule/default/prepare.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: true - pre_tasks: - - name: include vars - include_vars: "../../../ansible_vaults/molecule_prepare/{{ item }}" - loop: - - "prepare.vault" - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - become: true - changed_when: false - - name: create users (as deployed in production) - user: - name: "{{ item.name }}" - uid: "{{ item.uid }}" - create_home: "yes" - shell: "/bin/bash" - loop: "{{ vault_molecule_users | flatten(levels=1) }}" - become: true - - name: add nonfree repos - apt_repository: - repo: "deb http://ftp2.de.debian.org/debian/ buster main non-free contrib" - state: present - update-cache: "yes" - become: true - - name: Install required packages - apt: - name: [ - 'aptitude', - 'gpg', - 'less', - 'libuser' - ] - state: present - become: true - roles: - - {role: ansible_lza_install_common, become: true} - - {role: ansible_lza_server_hardening, become: true} diff --git a/molecule/default/tests/conftest.py b/molecule/default/tests/conftest.py deleted file mode 100644 index ba0f1e81239226ea76950de710f5ec996831b7bc..0000000000000000000000000000000000000000 --- a/molecule/default/tests/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""PyTest Fixtures.""" -from __future__ import absolute_import -import os -import pytest - - -def pytest_runtest_setup(item): - """Run tests only when under molecule with testinfra installed.""" - try: - import testinfra - except ImportError: - pytest.skip("Test requires testinfra", allow_module_level=True) - if "MOLECULE_INVENTORY_FILE" in os.environ: - pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ["MOLECULE_INVENTORY_FILE"] - ).get_hosts("all") - else: - pytest.skip( - "Test should run only from inside molecule.", allow_module_level=True - ) diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py deleted file mode 100644 index 39da80ab2f779c2ea1dc20daba64d1c3b2999201..0000000000000000000000000000000000000000 --- a/molecule/default/tests/test_default.py +++ /dev/null @@ -1,6 +0,0 @@ -def test_hosts_file(host): - f = host.file('/etc/hosts') - - assert f.exists - assert f.user == 'root' - assert f.group == 'root' diff --git a/molecule/default/INSTALL.rst b/molecule/resources/playbooks/INSTALL.rst similarity index 84% rename from molecule/default/INSTALL.rst rename to molecule/resources/playbooks/INSTALL.rst index 4f44b6745beb4e4a1ca19e93e42ccc8e98919d41..0c4bf5c7eb43b1b428b1824a62f8fb8a213f3600 100644 --- a/molecule/default/INSTALL.rst +++ b/molecule/resources/playbooks/INSTALL.rst @@ -1,6 +1,6 @@ -******* +********************************* Vagrant driver installation guide -******* +********************************* Requirements ============ @@ -20,4 +20,4 @@ widely recommended `'--user' flag`_ when invoking ``pip``. .. code-block:: bash - $ pip install 'molecule[vagrant]' + $ pip install 'molecule_vagrant' diff --git a/molecule/resources/playbooks/README.md b/molecule/resources/playbooks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0c91883011699e3b0ec08bad9bcc19d201ef6331 --- /dev/null +++ b/molecule/resources/playbooks/README.md @@ -0,0 +1,3 @@ +This drectory contains shared playbooks and a shared Dockerfile. + +Visit https://molecule.readthedocs.io/en/latest/examples.html#sharing-across-scenarios for details on sharing playbooks, tests etc. across multiple scenarios. diff --git a/molecule/resources/playbooks/converge.yml b/molecule/resources/playbooks/converge.yml new file mode 100644 index 0000000000000000000000000000000000000000..91a077811934e19de127f5f65dba95114f4f9c7e --- /dev/null +++ b/molecule/resources/playbooks/converge.yml @@ -0,0 +1,17 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: update apt cache + ansible.builtin.apt: + update_cache: true + upgrade: dist + become: true + when: ansible_os_family == "Debian" + - name: update yum cache + ansible.builtin.yum: + update_cache: true + become: true + when: ansible_os_family == "RedHat" + roles: + - {role: "ansible_lza_sanitytest", become: true} diff --git a/molecule/resources/playbooks/prepare.yml b/molecule/resources/playbooks/prepare.yml new file mode 100644 index 0000000000000000000000000000000000000000..d3dbe45d733859e8f37a17606ce6fe8641ab9355 --- /dev/null +++ b/molecule/resources/playbooks/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare + hosts: "*" + pre_tasks: + - name: configure additional package repositories for Debian + block: + - name: install GPG + ansible.builtin.apt: + name: "gnupg" + state: latest + update_cache: true + become: true + - name: add GPG key for SLUB Debian repository + ansible.builtin.apt_key: + url: "https://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key" + state: present + become: true + - name: add repo URL to sources.list + ansible.builtin.apt_repository: + repo: "deb https://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main" + state: present + update_cache: true + mode: "0644" + become: true + when: ansible_os_family == "Debian" + + - name: configure additional package repositories for RedHat + block: + - name: add custom repositories + ansible.builtin.yum_repository: + name: "{{ item.name }}" + description: "{{ item.description }}" + baseurl: "{{ item.baseurl }}" + gpgcheck: "{{ item.gpgcheck | default('true') }}" + gpgkey: "{{ item.gpgkey | default(omit) }}" + loop: + - name: "epel" + description: EPEL YUM repo + baseurl: "https://download.fedoraproject.org/pub/epel/{{ ansible_distribution_major_version }}/x86_64/" + gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" + - name: "slub" + description: SLUB YUM repo + baseurl: "http://sdvrhelrepo.slub-dresden.de/" + gpgcheck: "false" + - name: remove legacy repo configuration to avoid double configuration for SLUB repo + ansible.builtin.file: + path: "/etc/yum.repos.d/SLUB.repo" + state: absent + when: ansible_os_family == "RedHat" diff --git a/molecule/resources/playbooks/verify.yml b/molecule/resources/playbooks/verify.yml new file mode 100644 index 0000000000000000000000000000000000000000..e707420ab5c87edfa59c7805ce4534ff1b387177 --- /dev/null +++ b/molecule/resources/playbooks/verify.yml @@ -0,0 +1,10 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/molecule/virtualbox/molecule.yml b/molecule/virtualbox/molecule.yml new file mode 100644 index 0000000000000000000000000000000000000000..d1182bc75e2f73d29e299a5848805c06b93bbe49 --- /dev/null +++ b/molecule/virtualbox/molecule.yml @@ -0,0 +1,40 @@ +--- +dependency: + name: galaxy + enabled: false +driver: + name: vagrant +lint: | + set -e + yamllint . + ansible-lint -x no-loop-var-prefix,command-instead-of-module,package-latest +platforms: + # Check out the documentation at + # https://github.com/ansible-community/molecule-vagrant#documentation + # for more platform parameters. + - name: mol-samity + box: debian/bullseye64 + memory: 1024 + # List of raw Vagrant `config` options. + # provider_raw_config_args: + # - "customize [ 'modifyvm', :id, '--natdnshostresolver1', 'on' ]" + # Dictionary of `config` options. + config_options: + ssh.keep_alive: yes + ssh.remote_user: "'lza'" +provisioner: + name: ansible + log: true + config_options: + defaults: + # https://stackoverflow.com/questions/57435811/ansible-molecule-pass-multiple-vault-ids + vault_identity_list: "../lza_sanitytest.pass" + vvv: false + playbooks: + # create: ../resources/playbooks/create.yml + # destroy: ../resources/playbooks/destroy.yml + converge: ../resources/playbooks/converge.yml + prepare: ../resources/playbooks/prepare.yml + verify: ../resources/playbooks/verify.yml +verifier: + name: ansible diff --git a/requirements.yml b/requirements.yml deleted file mode 100644 index 58de2735f5b6827e12b1562b10af1fa3d6cd0566..0000000000000000000000000000000000000000 --- a/requirements.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-multiple-roles-from-a-file -- src: git+https://git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_lza_install_common.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_lza_server_hardening.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_slub_osquery.git - scm: git -#- src: git+ -# scm: git diff --git a/requirements.yml.example b/requirements.yml.example new file mode 100644 index 0000000000000000000000000000000000000000..97bfe9c0c57a4ee9c76049915a074e8d946b1f73 --- /dev/null +++ b/requirements.yml.example @@ -0,0 +1,4 @@ +--- +# https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-multiple-roles-from-a-file +- src: git+https://git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git + scm: git diff --git a/site.yml b/site.yml index 26f3577a34a0e5851f8c643b19c6b49be9d8e4ff..0393e748867460240d4ca9db9a1f95de25433e17 100644 --- a/site.yml +++ b/site.yml @@ -24,7 +24,7 @@ strategy: linear roles: - - { role: ansible_lza_install_common, become: true } - - { role: ansible_lza_server_hardening, become: true } - - { role: ansible_slub_osquery, become: true } +# - { role: ansible_lza_install_common, become: true } +# - { role: ansible_lza_server_hardening, become: true } +# - { role: ansible_slub_osquery, become: true } - { role: ansible_lza_sanitytest, become: true } diff --git a/tasks/configure_nfs_mounts.yml b/tasks/configure_nfs_mounts.yml index 08122edaa6d3bd17fcbc0aef64ba8687649fe872..99f3cd0c3b34517f37a72bc5c4f11808e4f6b4cf 100644 --- a/tasks/configure_nfs_mounts.yml +++ b/tasks/configure_nfs_mounts.yml @@ -1,8 +1,9 @@ --- - name: Mountpoint für Logging anlegen - file: + ansible.builtin.file: path: "{{ vault_log_nfs_mounts.log.mountpoint }}{{ ansible_hostname }}" state: directory + mode: "0400" - name: NFS-Shares für Logging mounten (/var/log/rossanity/) ansible.posix.mount: @@ -15,9 +16,10 @@ ### MOUNTPOINTS PERMANENT ERSTELLEN ### - name: Mountpoints für Permanent Storage anlegen - file: + ansible.builtin.file: path: "{{ item.mountpoint }}" state: directory + mode: "0400" loop: "{{ vault_permanent_nfs_mounts }}" # - "{{ vault_nfs_mounts.permanent_dev.mountpoint }}" # - "{{ vault_nfs_mounts.permanent_test.mountpoint }}" diff --git a/tasks/configure_ssh_keys.yml b/tasks/configure_ssh_keys.yml index 30f03371b86e26245841c013b88573605b13c550..6ba5d95122f0f0e7bea669fcbcb7f1563324dc24 100644 --- a/tasks/configure_ssh_keys.yml +++ b/tasks/configure_ssh_keys.yml @@ -1,19 +1,25 @@ --- - name: deploy SSH key for management user - authorized_key: + ansible.builtin.authorized_key: user: "{{ item.user }}" state: present key: "{{ item.ssh_key }}" comment: "{{ item.ssh_comment }}" loop: "{{ vault_ssh_access }}" +- name: create ~/.ssh directory + ansible.builtin.file: + path: "~/.ssh/" + state: directory + mode: "0700" + - name: copy SSH key files to managed servers - copy: + ansible.builtin.copy: src: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" dest: "~/.ssh/{{ item }}" owner: "root" group: "root" - mode: 0400 + mode: "0400" loop: - "id_ed25519_deploykey" - "id_ed25519_deploykey.pub" diff --git a/tasks/install_debug_symbols.yml b/tasks/install_debug_symbols.yml index 85666bdb71a3ae6c6785765f34d7dfc8fc3fada7..51cd03aff2df25b5057d1660666be0348f51aaf9 100644 --- a/tasks/install_debug_symbols.yml +++ b/tasks/install_debug_symbols.yml @@ -2,14 +2,14 @@ # based on https://wiki.debian.org/HowToGetABacktrace - name: add package sources for debug symbols - apt-repository: + ansible.builtin.apt-repository: repo: "{{ item }}" loop: - "deb http://deb.debian.org/debian-debug/ buster-debug main" - "deb http://deb.debian.org/debian-debug/ buster-proposed-updates-debug main" - name: install debug symbols and further tools - apt: + ansible.builtin.apt: name: [ 'linux-image-amd64-dbg', 'gdb', diff --git a/tasks/install_packages.yml b/tasks/install_packages.yml index 1ce00a52625bb82ada314d6bd65b4046ffe2f078..d990291b81598cd69da4f6a260563c8db7c18b58 100644 --- a/tasks/install_packages.yml +++ b/tasks/install_packages.yml @@ -1,7 +1,12 @@ --- ### INSTALL PACKAGES ### +- name: purge old Debian 10 packages + ansible.builtin.apt: + state: absent + name: "perl-modules-5.28" + - name: Install/Update Software Packages - apt: + ansible.builtin.apt: state: present name: [ 'cpanminus', @@ -9,7 +14,7 @@ 'make', 'perl', 'perl-base', - 'perl-modules', + 'perl-modules-5.32', 'rsync', 'sqlite3', 'tar', @@ -18,7 +23,7 @@ update_cache: "yes" - name: Install/Update Perl-Pakages for Exit - apt: + ansible.builtin.apt: state: present name: [ 'libdbd-sqlite3-perl', @@ -30,7 +35,7 @@ ] - name: install Perl packages from CPAN - cpanm: + ansible.builtin.cpanm: name: "{{ item }}" loop: # these are part of the "perl-base" Debian 10 package, so we get them for free: diff --git a/tasks/install_test_scripts.yml b/tasks/install_test_scripts.yml index c45fbb7de7417e86317b1bc6c8c2b9ae70d36d26..41596880151360efd6931813f73933a28cda2896 100644 --- a/tasks/install_test_scripts.yml +++ b/tasks/install_test_scripts.yml @@ -2,7 +2,7 @@ ### INSTALL TEST SCRIPTS ### - name: install prerequisites - apt: + ansible.builtin.apt: update_cache: true name: [ 'libdata-printer-perl', @@ -11,7 +11,7 @@ ] state: latest - name: install prerequisites for deep_fixity - apt: + ansible.builtin.apt: update_cache: true name: [ 'libbloom-filter-perl', @@ -22,7 +22,7 @@ state: latest - name: deploy wrapper script for exit strategy - template: + ansible.builtin.template: src: "{{ item }}.j2" dest: "/usr/local/bin/{{ item }}" owner: "root" @@ -52,7 +52,7 @@ repo: "git@{{ vault_slub_git_repo_fqdn }}:digital-preservation/rosettadeepfixity.git" - name: Perl syntax check for exit strategy script - command: "perl -c /usr/local/bin/rosettaExitStrategy/perl/exit_strategy.pl" + ansible.builtin.command: "perl -c /usr/local/bin/rosettaExitStrategy/perl/exit_strategy.pl" register: depcheck_exit failed_when: "'syntax OK' not in depcheck_exit.stderr" changed_when: false @@ -65,7 +65,7 @@ success_msg: "exit strategy script has all required prerequisites installed" - name: Perl syntax check for deep_fixity script - command: "perl -c /usr/local/bin/rosettaDeepFixity/deep_fixitycheck.pl" + ansible.builtin.command: "perl -c /usr/local/bin/rosettaDeepFixity/deep_fixitycheck.pl" register: depcheck_deep_fixity failed_when: "'syntax OK' not in depcheck_deep_fixity.stderr" changed_when: false diff --git a/tasks/main.yml b/tasks/main.yml index ee5909ee6b0c40d985f8ab37a40b7ff903753038..f530a2166a8d5dddc0e22414dbd914d9f9b67bdd 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,35 +1,35 @@ --- - name: include Ansible Vaults - include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" + ansible.builtin.include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" loop: - "sanitytest.vault" tags: [always] - name: install packages - import_tasks: "install_packages.yml" + ansible.builtin.import_tasks: "install_packages.yml" tags: [apt, packages, cpan, cpanm] - name: create users and groups - import_tasks: "create_users_groups.yml" + ansible.builtin.import_tasks: "create_users_groups.yml" tags: [users] - name: configure SSH keys - import_tasks: "configure_ssh_keys.yml" + ansible.builtin.import_tasks: "configure_ssh_keys.yml" tags: [ssh] - name: configure nfs mounts - import_tasks: "configure_nfs_mounts.yml" + ansible.builtin.import_tasks: "configure_nfs_mounts.yml" tags: [nfs] - name: install test scripts - import_tasks: "install_test_scripts.yml" + ansible.builtin.import_tasks: "install_test_scripts.yml" tags: [testscripts] - name: configure iptables - import_tasks: "configure_iptables_external.yml" + ansible.builtin.import_tasks: "configure_iptables_external.yml" tags: [firewall, iptables] # only use if needed, this doesn't need to run everytime # - name: install kernel debug symbols -# import_tasks: "install_debug_symbols.yml" +# ansible.builtin.import_tasks: "install_debug_symbols.yml" # tags: [kernel, linux, debug, apt]