From e956507314aa599a75203291e5400adef06b63ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Sachse?= <joerg.sachse@slub-dresden.de> Date: Thu, 8 Sep 2022 16:29:10 +0200 Subject: [PATCH] test: implement GitLab-CI --- .ansible-lint | 45 +++--- .config/molecule/config.yml | 2 + .gitignore | 15 +- .gitlab-ci.yml | 67 ++++----- .gitlab/issue_templates/Default.md | 16 ++ .gitlab/merge_request_templates/Default.md | 7 + .yamllint | 32 ++-- CODE-OF-CONDUCT.md | 3 + CONTRIBUTING.md | 0 README.md | 93 +++++++++--- handlers/main.yml | 9 +- kudos.txt | 48 ++++++ meta/main.yml | 46 +++--- molecule/README.md | 139 ++++++++++-------- molecule/default | 1 + molecule/default/converge.yml | 5 - molecule/default/molecule.yml | 36 ----- molecule/default/prepare.yml | 40 ----- molecule/default/tests/conftest.py | 20 --- molecule/default/tests/test_default.py | 6 - .../playbooks}/INSTALL.rst | 6 +- molecule/resources/playbooks/README.md | 3 + molecule/resources/playbooks/converge.yml | 17 +++ molecule/resources/playbooks/prepare.yml | 49 ++++++ molecule/resources/playbooks/verify.yml | 10 ++ molecule/virtualbox/molecule.yml | 40 +++++ requirements.yml | 12 -- requirements.yml.example | 4 + site.yml | 6 +- tasks/configure_nfs_mounts.yml | 6 +- tasks/configure_ssh_keys.yml | 12 +- tasks/install_debug_symbols.yml | 4 +- tasks/install_packages.yml | 13 +- tasks/install_test_scripts.yml | 10 +- tasks/main.yml | 16 +- 35 files changed, 503 insertions(+), 335 deletions(-) create mode 100644 .config/molecule/config.yml create mode 100644 .gitlab/issue_templates/Default.md create mode 100644 .gitlab/merge_request_templates/Default.md create mode 100644 CODE-OF-CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 kudos.txt create mode 120000 molecule/default delete mode 100644 molecule/default/converge.yml delete mode 100644 molecule/default/molecule.yml delete mode 100644 molecule/default/prepare.yml delete mode 100644 molecule/default/tests/conftest.py delete mode 100644 molecule/default/tests/test_default.py rename molecule/{default => resources/playbooks}/INSTALL.rst (84%) create mode 100644 molecule/resources/playbooks/README.md create mode 100644 molecule/resources/playbooks/converge.yml create mode 100644 molecule/resources/playbooks/prepare.yml create mode 100644 molecule/resources/playbooks/verify.yml create mode 100644 molecule/virtualbox/molecule.yml delete mode 100644 requirements.yml create mode 100644 requirements.yml.example diff --git a/.ansible-lint b/.ansible-lint index 9c9323e..f18a647 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,7 +6,7 @@ # and not relative to the CWD of execution. CLI arguments passed to the --exclude # option will be parsed relative to the CWD of execution. exclude_paths: - - .cache/ # implicit unless exclude_paths is defined in config + - .cache/ # implicit unless exclude_paths is defined in config - .git/ - .githooks/ - backups/ @@ -15,24 +15,19 @@ exclude_paths: # verbosity: 1 # Mock modules or roles in order to pass ansible-playbook --syntax-check -#mock_modules: -# - zuul_return -# # note the foo.bar is invalid as being neither a module or a collection -# - fake_namespace.fake_collection.fake_module -# - fake_namespace.fake_collection.fake_module.fake_submodule -#mock_roles: -# - mocked_role -# - author.role_name # old standalone galaxy role -# - fake_namespace.fake_collection.fake_role # role within a collection +# mock_modules: +# - zuul_return +# # note the foo.bar is invalid as being neither a module or a collection +# - fake_namespace.fake_collection.fake_module +# - fake_namespace.fake_collection.fake_module.fake_submodule +# mock_roles: +# - mocked_role +# - author.role_name # old standalone galaxy role +# - fake_namespace.fake_collection.fake_role # role within a collection # Enable checking of loop variable prefixes in roles loop_var_prefix: "{role}_" -# Enforce variable names to follow pattern below, in addition to Ansible own -# requirements, like avoiding python identifiers. To disable add `var-naming` -# to skip_list. -var_naming_pattern: "^[a-z_][a-z0-9_]*$" - use_default_rules: true # Load custom rules from this specific folder # rulesdir: @@ -46,9 +41,9 @@ skip_list: # Any rule that has the 'opt-in' tag will not be loaded unless its 'id' is # mentioned in the enable_list: enable_list: - - empty-string-compare # opt-in - - no-log-password # opt-in - - no-same-owner # opt-in + - empty-string-compare # opt-in + - no-log-password # opt-in + - no-same-owner # opt-in # add yaml here if you want to avoid ignoring yaml checks when yamllint # library is missing. Normally its absence just skips using that rule. - yaml @@ -60,19 +55,19 @@ enable_list: warn_list: - skip_this_tag - git-latest - - experimental # experimental is included in the implicit list + - experimental # experimental is included in the implicit list # - role-name # Offline mode disables installation of requirements.yml offline: false # Define required Ansible's variables to satisfy syntax check -#extra_vars: -# foo: bar -# multiline_string_variable: | -# line1 -# line2 -# complex_variable: ":{;\t$()" +# extra_vars: +# foo: bar +# multiline_string_variable: | +# line1 +# line2 +# complex_variable: ":{;\t$()" # Uncomment to enforce action validation with tasks, usually is not # needed as Ansible syntax check also covers it. diff --git a/.config/molecule/config.yml b/.config/molecule/config.yml new file mode 100644 index 0000000..ece7ff6 --- /dev/null +++ b/.config/molecule/config.yml @@ -0,0 +1,2 @@ +--- +prerun: false diff --git a/.gitignore b/.gitignore index 7ccd9c1..ed78332 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,8 @@ Thumbs.db *.retry *.vault +inventory.* +inv.* # Vim # ####### @@ -73,13 +75,14 @@ tags .vagrant/ *.box -# Build Environments # -###################### +# Temporary/Build/Backup # +########################## +backups/ build/ -# Backups # -########### +# CONFIDENTIAL # +################ + +ssh_host_* -backups/ -backup/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ea9f61..036df26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,52 +1,35 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - -# This is a sample GitLab CI/CD configuration file that should run without any modifications. -# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, -# it uses echo commands to simulate the pipeline execution. -# +--- # A pipeline is composed of independent jobs that run scripts, grouped into stages. # Stages run in sequential order, but jobs within stages run in parallel. # # For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages stages: # List of stages for jobs, and their order of execution - - build - test -image: - name: debian:stable-slim - -cache: - key: build - untracked: true - policy: pull-push - -build-job: - stage: build - script: - - echo "Installing environment..." - - apt-get -y update - - apt-get -y install python3 virtualenv python3-setuptools - - virtualenv -p python3 ansible - - source ansible/bin/activate - - pip install ansible +default: + before_script: + - source /opt/molecule/bin/activate + - ansible --version + - molecule --version + after_script: + - source /opt/molecule/bin/activate + - molecule destroy --scenario-name default -unit-test-job: # This job runs in the test stage. - stage: test # It only starts when the job in the build stage completes successfully. +test-job: + stage: test + tags: + - "shell" script: - - echo "Running unit tests..." - - apt-get -y update - - apt-get -y install python3 git - - source ansible/bin/activate - - ansible-galaxy install -r requirements.yml - - echo $vault_lza_install_common > ${CI_BUILDS_DIR}lza_install_common.pass - - echo $vault_lza_server_hardening > ${CI_BUILDS_DIR}/lza_server_hardening.pass - - echo $vault_slub_osquery > ${CI_BUILDS_DIR}/slub_osquery.pass - - echo $vault_lza_sanitytest > ${CI_BUILDS_DIR}/lza_sanitytest.pass - - ansible-playbook site.yml -b -l localhost --vault-password-file ${CI_BUILDS_DIR}lza_install_common.pass ${CI_BUILDS_DIR}/lza_server_hardening.pass ${CI_BUILDS_DIR}/slub_osquery.pass ${CI_BUILDS_DIR}/lza_sanitytest.pass - variables: - ANSIBLE_ROLES_PATH: "${CI_BUILDS_DIR}/${CI_PROJECT_PATH}/" + # make sure that Ansible Vaults are present and can be decrypted + - echo "${VAULT_INSTALL_COMMON}" > ../lza_install_common.pass + - export ANSIBLE_VAULT_IDENTITY_LIST="../lza_install_common.pass, ../passfile2" + - rm -rf ../ansible_vaults/ + - git clone https://gitlab+deploy-token-25:${VAULT_ACCESS_TOKEN}@git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git ../ansible_vaults/; \ + # run Molecule tests + - molecule syntax --scenario-name default + - molecule lint --scenario-name default + - molecule create --scenario-name default + - molecule converge --scenario-name default + - molecule idempotence --scenario-name default + # - molecule verify --scenario-name default diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md new file mode 100644 index 0000000..acc60bf --- /dev/null +++ b/.gitlab/issue_templates/Default.md @@ -0,0 +1,16 @@ +## Expected Behavior + + +## Actual Behavior + + +## Steps to Reproduce the Problem + + 1. + 1. + 1. + +## Specifications + + - Version/Commit: + - Platform: diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000..4030f6f --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,7 @@ +Fixes # + +## Proposed Changes + + - + - + - diff --git a/.yamllint b/.yamllint index 7c0f15c..8827676 100644 --- a/.yamllint +++ b/.yamllint @@ -1,7 +1,5 @@ --- -# based on documentation available at -# https://yamllint.readthedocs.io/en/stable/rules.html - +# Based on ansible-lint config extends: default rules: @@ -11,13 +9,25 @@ rules: brackets: max-spaces-inside: 1 level: error - comments: - min-spaces-from-content: 4 + colons: + max-spaces-after: -1 + level: error + commas: + max-spaces-after: -1 + level: error + comments: disable comments-indentation: disable - document-end: disable - document-start: - level: warning - octal-values: - forbid-explicit-octal: false + document-start: disable + empty-lines: + max: 3 + level: error + hyphens: + level: error + indentation: disable + key-duplicates: enable line-length: disable - truthy: enable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + truthy: disable diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000..19d2d20 --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +Just be excellent to each other. That's it. EoC. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index 5514468..2250485 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # Ansible-Role "ansible_lza_sanitytest" -## Description +## What does it do? This role provides all necessary tasks to setup a sanitytest server for SLUBArchiv. -## Prerequisites +## What do I need? + +### Prerequisites To use this role, the following software must be installed on your workstation: * ansible @@ -13,29 +15,32 @@ To deploy this role to a managed host, the following software must be installed * Python3 * SSHd +### Dependencies + Other roles required by this role can be easily installed using `ansible-galaxy` if necessary. As the roles reside in SLUB's local Git server instance, you might need a password for certain repositories despite most of them being publicly available. This command will install the required roles in the correct location so they are found in the search path: -``` - ansible-galaxy install -r requirements.yml -p ../ +```bash +ansible-galaxy install -r requirements.yml -p ../ ``` -It is recommended to use Debian VMs as deployed by SLUB's UDA tool with this role. +### Infrastructure -## Quick Start +It is recommended to use Debian VMs as deployed by SLUB's GUBS tool with this role. Otherwise you will not have access to the software packages that are located in SLUB's private Debian package repository. + +## Can i have a Quick Start? Most options already have sensible defaults in `ansible.cfg`. However, you can override these defaults using CLI options/flags if you want to. The flags can be combined if necessary, and most of them have long versions as well. Get more information using `ansible-playbook --help`. To simply run the playbook, just call the `site.yml` playbook like this: +```bash +ansible-playbook site.yml -u <username> ``` - ansible-playbook site.yml -u <username> -``` - If you want to limit the execution to a subset of all hosts that are listed in the inventory, use the `-l` or `--limit` option like this: -``` - ansible-playbook site.yml -l <hostna*> - ansible-playbook site.yml -l <hostname> - ansible-playbook site.yml -l <hostname1>:<hostname2>:... - ansible-playbook site.yml -l <inventory_group> - ansible-playbook site.yml --limit=<hostna*> +```bash +ansible-playbook site.yml -l <hostna*> +ansible-playbook site.yml -l <hostname> +ansible-playbook site.yml -l <hostname1>:<hostname2>:... +ansible-playbook site.yml -l <inventory_group> +ansible-playbook site.yml --limit=<hostna*> ``` If you do not have Vault password files in the directory above the role direcory, you have to give the Vault password before execution: @@ -43,6 +48,12 @@ If you do not have Vault password files in the directory above the role direcory ansible-playbook site.yml --ask-vault-pass ``` +You can use your own inventory file by adding the `-i` or `--inventory=INVENTORY` option: +``` + ansible-playbook site.yml -i inventory.yml + ansible-playbook site.yml --inventory=inventory.yml +``` + Tasks in this role have been tagged to enable users to only run subsets of tasks. This can be leveraged to decrease run times or run only certain tasks after small changes. To list all available tags, use: ``` @@ -54,20 +65,62 @@ You can then run only certain tagged tasks by using the `--tags` option: ansible-playbook site.yml --tags=tag1,tag2,...,tagN ``` -## Testing the role +## How can I run tests? Tests have been implemented using the Molecule framework. The details on using the test suite are described below `molecule/`. -## Variables +To run some quick tests, you can do: +```bash +# pure syntax check +molecule syntax +# run yamllint and ansible-lint +molecule lint +# list available test scenarios, e.g. based on different OS images or platforms +molecule list +# create the test environment for a specific scenario (can be left out for "default") +molecule create [-s scenario] +# run your tasks against the test env +molecule converge [-s scenario] # multiple times if needed +# run idempotence checks to se if any of the tasks keeps changing (subsequent runs shouldn't trigger changes) +molecule idempotence [-s scenario] +# cleanup test env (remove VM/container) +molecule destroy [-s scenario] +``` + +We recommend running those tests before pushing any code to the Git server. + +On every `git push`, the GitLab-CI pipeline will run a similar set of tests to ensure that all changes are working. Find the details in the `.gitlab-ci.yml` file located at the project root directory. + +## What can be configured? + +### Ansible Role Many variables have been "hidden" in encrypted Ansible Vaults. For security reasons, these Vaults are maintained in a separate private internal repository of SLUB's Git. However, in order to better understand the data within the vaults, you can find `\*.vault.example` files below the `vars/` directory. If you work outside of SLUBArchive and have no access to the vault repository, make sure to put the necessary vaults in the expected paths at `../ansible_vaults/<ROLENAME>/`. -## git configuration +Variable defaults have been set in `defaults/main.yml`. You can overwrite them with your own values by setting them in `vars/main.yml`. + +### Git configuration Just run the `setup_gitconfig.sh` script that comes with the repo to correctly setup all necessary local Git configurations. -## Author Information +## What changes have been made lately? + +All changes can be found in the `CHANGELOG` file located at the project root directory. Alternatively, you can have a look at the commit log to get a detailed view. + +## Who is maintaining this project? + +All authors/maintainers are listed in the `kudos.txt` file located at the project root directory. + +## How can I contribute? + +If you have any comments or find bugs, please contact langzeitarchiv@slub-dresden.de, create an issue or send us a pull request. + +Details on how to contribute to this project can be found at the `CONTRIBUTING.md` file located at the project root directory. + +If you have commited to the project yourself, you can leave a note in the `kudos.txt` file located at the project root directory. Be assured of our eternal gratitude. + +## Is there a Code of Conduct? -If you have any comments or find bugs, please contact langzeitarchiv@slub-dresden.de or issue a pull request. +Yes there is. You can find it in the `CODE-OF-CONDUCT.md` file located at the project root directory. It's kept very brief by design. diff --git a/handlers/main.yml b/handlers/main.yml index 26f5fa9..e272215 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,13 +1,13 @@ --- - name: restart postgres - systemd: + ansible.builtin.systemd: name: "postgresql" state: restarted - name: save iptables rules (Debian) block: - name: Ordner für iptables-Config erstellen - file: + ansible.builtin.file: path: "/etc/iptables" state: directory owner: "root" @@ -15,11 +15,12 @@ mode: 0755 listen: "save iptables rules" - name: install netfilter-persistent to be able to save iptables rules - apt: + ansible.builtin.apt: name: netfilter-persistent state: present listen: "save iptables rules" - name: save iptables rules - command: 'netfilter-persistent save' + ansible.builtin.command: 'netfilter-persistent save' listen: "save iptables rules" + changed_when: false when: ansible_os_family == "Debian" diff --git a/kudos.txt b/kudos.txt new file mode 100644 index 0000000..a63a1c9 --- /dev/null +++ b/kudos.txt @@ -0,0 +1,48 @@ +# kudos.txt - Express gratitude to your contributors. +# https://github.com/kudos-txt + +project: + - name: ansible_lza_sanitytest + site: https://git.slub-dresden.de/digital-preservation/ansible_lza_sanitytest +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> + +contributor: +# - name: <name or nick of the contributor> +# role: <role in the project> +# site: <website url> +# blog: <blog url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# home: <country>, <region>, <zip>, <city>, <address> +# work: <company>, <job title>, <job description> +# note: <free text notes> + - name: Jörg Sachse + role: Maintainer + mail: Joerg.Sachse@slub-dresden.de + home: Germany, Saxony, 01059, Dresden, Zellescher Weg 18 + work: SLUB Dresden, Digital Preservationist &IT Administrator & Installation Wizard + +#partner: +# - name: <name of the partner> +# site: <website url> +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> + +#software: +# - name: <name of the software> +# site: <website url> +# blog: <blog url> +# help: <support url> +# news: <news url>, <news feed url> +# mail: <mail@domain.tld> +# chat: <service:nick>, <service:channel>, <url> +# note: <free text notes> diff --git a/meta/main.yml b/meta/main.yml index cb15900..f41d21c 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,16 +1,30 @@ --- galaxy_info: author: Jörg Sachse - description: role to install a post-upgrade sanity testing server for the SLUBarchiv digital preservation repository company: SLUB Dresden - # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker Some suggested licenses: - BSD - # (default) - MIT - GPLv2 - GPLv3 - Apache - CC-BY - license: public domain - min_ansible_version: 2.4 - # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo - # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only - # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch: - # + description: role to install a post-upgrade sanity testing server for the SLUBarchiv digital preservation repository + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you + # add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + # issue_tracker_url: "https://example.com/" + # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: GPL-3.0-or-later + + min_ansible_version: "2.5" + # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo + # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only + # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch: # Provide a list of supported platforms, and for each platform a list of versions. If you don't wish to enumerate all versions for a particular platform, use 'all'. To view available # platforms and versions (or releases), visit: https://galaxy.ansible.com/api/v1/platforms/ # @@ -19,14 +33,6 @@ galaxy_info: platforms: - name: Debian versions: - - 9 - - 10 - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you - # add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. -dependencies: -# - ansible_lza_install_common -# - ansible_lza_server_hardening + - "buster" + - "bullseye" +dependencies: [] diff --git a/molecule/README.md b/molecule/README.md index 33a7eb5..fdb41fe 100644 --- a/molecule/README.md +++ b/molecule/README.md @@ -4,39 +4,53 @@ In order to be able to use the tests, you need to have some software packages installed. You may need sudo privileges for some of these operations. - ### install VirtualBox - # do NOT use distribution packages - # process documented at https://www.virtualbox.org/wiki/Linux_Downloads - # - # add repository URL - sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list - # add GPG key - wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - - # update sources - sudo apt update - # install VirtualBox - sudo apt-get install virtualbox-6.1 - - ### install Vagrant - # do NOT use distribution packages - # - # download Debian package from Hashicorp - wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb - # install package - sudo dpkg -i vagrant_2.2.9_x86_64.deb - - ### install Molecule et. al. - # prepare directories - mkdir ~/python-envs/ && cd ~/python-env/ - # create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!) - virtualenv -p python3 molecule-env - # enter the Virtual Environment in your current shell (other shells will be unaffected) - source molecule-env/bin/activate - # install packages - pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker - - # leave the Virtual Environment only when you're done - deactivate +There are two ways to accomplish this. + +### Option 1: use `molecule-skel` (SLUB staff only) + +1. Clone the project at [molecule-skel](https://git.slub-dresden.de/slub-referat-2-3/molecule-skel): `git clone https://git.slub-dresden.de/slub-referat-2-3/molecule-skel.git` +2. Switch to the `molecule-skel` repository directory and run `./install_testenv.sh`. +3. PROFIT! + + Check the `README.md` over at [molecule-skel](https://git.slub-dresden.de/slub-referat-2-3/molecule-skel) for details on how to use that. It's easy, I promise. + +### Option 2: manual installation + +Run these steps on your terminal. + +```bash +### install VirtualBox +# do NOT use distribution packages, as they may be too old! +# process documented at https://www.virtualbox.org/wiki/Linux_Downloads +# +# add GPG key +wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - +# add repository URL +sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list +# update sources and install VirtualBox +sudo apt update; sudo apt-get install virtualbox-6.1 + +### install Vagrant +# do NOT use distribution packages, as they may be too old! +# +# download Debian package from Hashicorp +wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb +# install package +sudo dpkg -i vagrant_2.2.9_x86_64.deb + +### install Molecule et. al. +# prepare directories +mkdir ~/python-envs/ && cd ~/python-env/ +# create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!) +virtualenv -p python3 molecule-env +# enter the Virtual Environment in your current shell (other shells will be unaffected) +source molecule-env/bin/activate +# install packages +pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker + +# leave the Virtual Environment only when you're done +deactivate +``` You can find suitable documentation at the respective vendors' websites. * [Vagrant Installation Guide](https://www.vagrantup.com/docs/installation/) @@ -46,10 +60,14 @@ You can find suitable documentation at the respective vendors' websites. ## Initialising a new Molecule scenario If you have already created a role without the Molecule test framework or you want to add test scenarios, you can use: - molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra] +```bash +molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra] +``` If you need any help with the options, please use: - molecule init role --help +```bash +molecule init role --help +``` ## Running Tests @@ -60,26 +78,31 @@ Various test environments are separated into so-called "scenarios" that can be b In the simplest configuration, the `molecule/` directory only contains one `default/` directory that contains the default scenario. This scenario is run if no other scenario is chosen using the `-s` CLI option. This is the basic usage of Molecule: - # create test infrastructure - cd <role_directory> - molecule create - # run playbooks against test infrastructure - molecule converge - # run Testinfra tests - molecule verify - # remove test infrastructure - molecule destroy - - # run all steps at once: - molecule test - -It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed. - # First, get UUID of the VM - vagrant global-status - # Then, create the snapshot - vagrant snapshot save <uuid> <snapshot_name> - # To restore the snapshot, use - vagrant snapshot restore <uuid> <snapshot_name> - # And to remove the snapshot, use - vagrant snapshot delete <uuid> <snapshot_name> - +```bash +# create test infrastructure +cd <role_directory> +molecule create +# run playbooks against test infrastructure +molecule converge +# run idempotence tests +molecule idempotence +# run tests, if they exist +molecule verify +# remove test infrastructure +molecule destroy + +# run all steps at once: +molecule test +``` + +It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed. Just like this: +```bash +# First, get UUID of the VM +vagrant global-status +# Then, create the snapshot +vagrant snapshot save <uuid> <snapshot_name> +# To restore the snapshot, use +vagrant snapshot restore <uuid> <snapshot_name> +# And to remove the snapshot, use +vagrant snapshot delete <uuid> <snapshot_name> +``` diff --git a/molecule/default b/molecule/default new file mode 120000 index 0000000..3841ab1 --- /dev/null +++ b/molecule/default @@ -0,0 +1 @@ +./virtualbox \ No newline at end of file diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml deleted file mode 100644 index d7e83ac..0000000 --- a/molecule/default/converge.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Converge - hosts: all - roles: - - {role: "ansible_lza_sanitytest", become: true} diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml deleted file mode 100644 index 1ade3a0..0000000 --- a/molecule/default/molecule.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: vagrant - provider: - name: virtualbox -lint: | - set -e - yamllint . - ansible-lint -x formatting - flake8 --ignore=E501 -platforms: - - name: molecule-test-sanity - box: debian/buster64 - memory: 512 - cpus: 1 -provisioner: - name: ansible - log: true - lint: | - set -e - ansible-lint - config_options: - defaults: - vault_identity_list: "@$HOME/.ansible/roles/molecule_prepare.pass, @$HOME/.ansible/roles/lza_install_common.pass, @$HOME/.ansible/roles/lza_server_hardening.pass, @$HOME/.ansible/roles/lza_sanitytest.pass" - vvv: false -verifier: - name: testinfra - env: - PYTHONWARNINGS: "ignore:.*U.*mode is deprecated:DeprecationWarning" - lint: | - set -e - flake8 - options: - v: 1 diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml deleted file mode 100644 index d903f56..0000000 --- a/molecule/default/prepare.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: true - pre_tasks: - - name: include vars - include_vars: "../../../ansible_vaults/molecule_prepare/{{ item }}" - loop: - - "prepare.vault" - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - become: true - changed_when: false - - name: create users (as deployed in production) - user: - name: "{{ item.name }}" - uid: "{{ item.uid }}" - create_home: "yes" - shell: "/bin/bash" - loop: "{{ vault_molecule_users | flatten(levels=1) }}" - become: true - - name: add nonfree repos - apt_repository: - repo: "deb http://ftp2.de.debian.org/debian/ buster main non-free contrib" - state: present - update-cache: "yes" - become: true - - name: Install required packages - apt: - name: [ - 'aptitude', - 'gpg', - 'less', - 'libuser' - ] - state: present - become: true - roles: - - {role: ansible_lza_install_common, become: true} - - {role: ansible_lza_server_hardening, become: true} diff --git a/molecule/default/tests/conftest.py b/molecule/default/tests/conftest.py deleted file mode 100644 index ba0f1e8..0000000 --- a/molecule/default/tests/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""PyTest Fixtures.""" -from __future__ import absolute_import -import os -import pytest - - -def pytest_runtest_setup(item): - """Run tests only when under molecule with testinfra installed.""" - try: - import testinfra - except ImportError: - pytest.skip("Test requires testinfra", allow_module_level=True) - if "MOLECULE_INVENTORY_FILE" in os.environ: - pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ["MOLECULE_INVENTORY_FILE"] - ).get_hosts("all") - else: - pytest.skip( - "Test should run only from inside molecule.", allow_module_level=True - ) diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py deleted file mode 100644 index 39da80a..0000000 --- a/molecule/default/tests/test_default.py +++ /dev/null @@ -1,6 +0,0 @@ -def test_hosts_file(host): - f = host.file('/etc/hosts') - - assert f.exists - assert f.user == 'root' - assert f.group == 'root' diff --git a/molecule/default/INSTALL.rst b/molecule/resources/playbooks/INSTALL.rst similarity index 84% rename from molecule/default/INSTALL.rst rename to molecule/resources/playbooks/INSTALL.rst index 4f44b67..0c4bf5c 100644 --- a/molecule/default/INSTALL.rst +++ b/molecule/resources/playbooks/INSTALL.rst @@ -1,6 +1,6 @@ -******* +********************************* Vagrant driver installation guide -******* +********************************* Requirements ============ @@ -20,4 +20,4 @@ widely recommended `'--user' flag`_ when invoking ``pip``. .. code-block:: bash - $ pip install 'molecule[vagrant]' + $ pip install 'molecule_vagrant' diff --git a/molecule/resources/playbooks/README.md b/molecule/resources/playbooks/README.md new file mode 100644 index 0000000..0c91883 --- /dev/null +++ b/molecule/resources/playbooks/README.md @@ -0,0 +1,3 @@ +This drectory contains shared playbooks and a shared Dockerfile. + +Visit https://molecule.readthedocs.io/en/latest/examples.html#sharing-across-scenarios for details on sharing playbooks, tests etc. across multiple scenarios. diff --git a/molecule/resources/playbooks/converge.yml b/molecule/resources/playbooks/converge.yml new file mode 100644 index 0000000..91a0778 --- /dev/null +++ b/molecule/resources/playbooks/converge.yml @@ -0,0 +1,17 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: update apt cache + ansible.builtin.apt: + update_cache: true + upgrade: dist + become: true + when: ansible_os_family == "Debian" + - name: update yum cache + ansible.builtin.yum: + update_cache: true + become: true + when: ansible_os_family == "RedHat" + roles: + - {role: "ansible_lza_sanitytest", become: true} diff --git a/molecule/resources/playbooks/prepare.yml b/molecule/resources/playbooks/prepare.yml new file mode 100644 index 0000000..d3dbe45 --- /dev/null +++ b/molecule/resources/playbooks/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare + hosts: "*" + pre_tasks: + - name: configure additional package repositories for Debian + block: + - name: install GPG + ansible.builtin.apt: + name: "gnupg" + state: latest + update_cache: true + become: true + - name: add GPG key for SLUB Debian repository + ansible.builtin.apt_key: + url: "https://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key" + state: present + become: true + - name: add repo URL to sources.list + ansible.builtin.apt_repository: + repo: "deb https://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main" + state: present + update_cache: true + mode: "0644" + become: true + when: ansible_os_family == "Debian" + + - name: configure additional package repositories for RedHat + block: + - name: add custom repositories + ansible.builtin.yum_repository: + name: "{{ item.name }}" + description: "{{ item.description }}" + baseurl: "{{ item.baseurl }}" + gpgcheck: "{{ item.gpgcheck | default('true') }}" + gpgkey: "{{ item.gpgkey | default(omit) }}" + loop: + - name: "epel" + description: EPEL YUM repo + baseurl: "https://download.fedoraproject.org/pub/epel/{{ ansible_distribution_major_version }}/x86_64/" + gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" + - name: "slub" + description: SLUB YUM repo + baseurl: "http://sdvrhelrepo.slub-dresden.de/" + gpgcheck: "false" + - name: remove legacy repo configuration to avoid double configuration for SLUB repo + ansible.builtin.file: + path: "/etc/yum.repos.d/SLUB.repo" + state: absent + when: ansible_os_family == "RedHat" diff --git a/molecule/resources/playbooks/verify.yml b/molecule/resources/playbooks/verify.yml new file mode 100644 index 0000000..e707420 --- /dev/null +++ b/molecule/resources/playbooks/verify.yml @@ -0,0 +1,10 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/molecule/virtualbox/molecule.yml b/molecule/virtualbox/molecule.yml new file mode 100644 index 0000000..d1182bc --- /dev/null +++ b/molecule/virtualbox/molecule.yml @@ -0,0 +1,40 @@ +--- +dependency: + name: galaxy + enabled: false +driver: + name: vagrant +lint: | + set -e + yamllint . + ansible-lint -x no-loop-var-prefix,command-instead-of-module,package-latest +platforms: + # Check out the documentation at + # https://github.com/ansible-community/molecule-vagrant#documentation + # for more platform parameters. + - name: mol-samity + box: debian/bullseye64 + memory: 1024 + # List of raw Vagrant `config` options. + # provider_raw_config_args: + # - "customize [ 'modifyvm', :id, '--natdnshostresolver1', 'on' ]" + # Dictionary of `config` options. + config_options: + ssh.keep_alive: yes + ssh.remote_user: "'lza'" +provisioner: + name: ansible + log: true + config_options: + defaults: + # https://stackoverflow.com/questions/57435811/ansible-molecule-pass-multiple-vault-ids + vault_identity_list: "../lza_sanitytest.pass" + vvv: false + playbooks: + # create: ../resources/playbooks/create.yml + # destroy: ../resources/playbooks/destroy.yml + converge: ../resources/playbooks/converge.yml + prepare: ../resources/playbooks/prepare.yml + verify: ../resources/playbooks/verify.yml +verifier: + name: ansible diff --git a/requirements.yml b/requirements.yml deleted file mode 100644 index 58de273..0000000 --- a/requirements.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-multiple-roles-from-a-file -- src: git+https://git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_lza_install_common.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_lza_server_hardening.git - scm: git -- src: git+https://git.slub-dresden.de/digital-preservation/ansible_slub_osquery.git - scm: git -#- src: git+ -# scm: git diff --git a/requirements.yml.example b/requirements.yml.example new file mode 100644 index 0000000..97bfe9c --- /dev/null +++ b/requirements.yml.example @@ -0,0 +1,4 @@ +--- +# https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-multiple-roles-from-a-file +- src: git+https://git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git + scm: git diff --git a/site.yml b/site.yml index 26f3577..0393e74 100644 --- a/site.yml +++ b/site.yml @@ -24,7 +24,7 @@ strategy: linear roles: - - { role: ansible_lza_install_common, become: true } - - { role: ansible_lza_server_hardening, become: true } - - { role: ansible_slub_osquery, become: true } +# - { role: ansible_lza_install_common, become: true } +# - { role: ansible_lza_server_hardening, become: true } +# - { role: ansible_slub_osquery, become: true } - { role: ansible_lza_sanitytest, become: true } diff --git a/tasks/configure_nfs_mounts.yml b/tasks/configure_nfs_mounts.yml index 08122ed..99f3cd0 100644 --- a/tasks/configure_nfs_mounts.yml +++ b/tasks/configure_nfs_mounts.yml @@ -1,8 +1,9 @@ --- - name: Mountpoint für Logging anlegen - file: + ansible.builtin.file: path: "{{ vault_log_nfs_mounts.log.mountpoint }}{{ ansible_hostname }}" state: directory + mode: "0400" - name: NFS-Shares für Logging mounten (/var/log/rossanity/) ansible.posix.mount: @@ -15,9 +16,10 @@ ### MOUNTPOINTS PERMANENT ERSTELLEN ### - name: Mountpoints für Permanent Storage anlegen - file: + ansible.builtin.file: path: "{{ item.mountpoint }}" state: directory + mode: "0400" loop: "{{ vault_permanent_nfs_mounts }}" # - "{{ vault_nfs_mounts.permanent_dev.mountpoint }}" # - "{{ vault_nfs_mounts.permanent_test.mountpoint }}" diff --git a/tasks/configure_ssh_keys.yml b/tasks/configure_ssh_keys.yml index 30f0337..6ba5d95 100644 --- a/tasks/configure_ssh_keys.yml +++ b/tasks/configure_ssh_keys.yml @@ -1,19 +1,25 @@ --- - name: deploy SSH key for management user - authorized_key: + ansible.builtin.authorized_key: user: "{{ item.user }}" state: present key: "{{ item.ssh_key }}" comment: "{{ item.ssh_comment }}" loop: "{{ vault_ssh_access }}" +- name: create ~/.ssh directory + ansible.builtin.file: + path: "~/.ssh/" + state: directory + mode: "0700" + - name: copy SSH key files to managed servers - copy: + ansible.builtin.copy: src: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" dest: "~/.ssh/{{ item }}" owner: "root" group: "root" - mode: 0400 + mode: "0400" loop: - "id_ed25519_deploykey" - "id_ed25519_deploykey.pub" diff --git a/tasks/install_debug_symbols.yml b/tasks/install_debug_symbols.yml index 85666bd..51cd03a 100644 --- a/tasks/install_debug_symbols.yml +++ b/tasks/install_debug_symbols.yml @@ -2,14 +2,14 @@ # based on https://wiki.debian.org/HowToGetABacktrace - name: add package sources for debug symbols - apt-repository: + ansible.builtin.apt-repository: repo: "{{ item }}" loop: - "deb http://deb.debian.org/debian-debug/ buster-debug main" - "deb http://deb.debian.org/debian-debug/ buster-proposed-updates-debug main" - name: install debug symbols and further tools - apt: + ansible.builtin.apt: name: [ 'linux-image-amd64-dbg', 'gdb', diff --git a/tasks/install_packages.yml b/tasks/install_packages.yml index 1ce00a5..d990291 100644 --- a/tasks/install_packages.yml +++ b/tasks/install_packages.yml @@ -1,7 +1,12 @@ --- ### INSTALL PACKAGES ### +- name: purge old Debian 10 packages + ansible.builtin.apt: + state: absent + name: "perl-modules-5.28" + - name: Install/Update Software Packages - apt: + ansible.builtin.apt: state: present name: [ 'cpanminus', @@ -9,7 +14,7 @@ 'make', 'perl', 'perl-base', - 'perl-modules', + 'perl-modules-5.32', 'rsync', 'sqlite3', 'tar', @@ -18,7 +23,7 @@ update_cache: "yes" - name: Install/Update Perl-Pakages for Exit - apt: + ansible.builtin.apt: state: present name: [ 'libdbd-sqlite3-perl', @@ -30,7 +35,7 @@ ] - name: install Perl packages from CPAN - cpanm: + ansible.builtin.cpanm: name: "{{ item }}" loop: # these are part of the "perl-base" Debian 10 package, so we get them for free: diff --git a/tasks/install_test_scripts.yml b/tasks/install_test_scripts.yml index c45fbb7..4159688 100644 --- a/tasks/install_test_scripts.yml +++ b/tasks/install_test_scripts.yml @@ -2,7 +2,7 @@ ### INSTALL TEST SCRIPTS ### - name: install prerequisites - apt: + ansible.builtin.apt: update_cache: true name: [ 'libdata-printer-perl', @@ -11,7 +11,7 @@ ] state: latest - name: install prerequisites for deep_fixity - apt: + ansible.builtin.apt: update_cache: true name: [ 'libbloom-filter-perl', @@ -22,7 +22,7 @@ state: latest - name: deploy wrapper script for exit strategy - template: + ansible.builtin.template: src: "{{ item }}.j2" dest: "/usr/local/bin/{{ item }}" owner: "root" @@ -52,7 +52,7 @@ repo: "git@{{ vault_slub_git_repo_fqdn }}:digital-preservation/rosettadeepfixity.git" - name: Perl syntax check for exit strategy script - command: "perl -c /usr/local/bin/rosettaExitStrategy/perl/exit_strategy.pl" + ansible.builtin.command: "perl -c /usr/local/bin/rosettaExitStrategy/perl/exit_strategy.pl" register: depcheck_exit failed_when: "'syntax OK' not in depcheck_exit.stderr" changed_when: false @@ -65,7 +65,7 @@ success_msg: "exit strategy script has all required prerequisites installed" - name: Perl syntax check for deep_fixity script - command: "perl -c /usr/local/bin/rosettaDeepFixity/deep_fixitycheck.pl" + ansible.builtin.command: "perl -c /usr/local/bin/rosettaDeepFixity/deep_fixitycheck.pl" register: depcheck_deep_fixity failed_when: "'syntax OK' not in depcheck_deep_fixity.stderr" changed_when: false diff --git a/tasks/main.yml b/tasks/main.yml index ee5909e..f530a21 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,35 +1,35 @@ --- - name: include Ansible Vaults - include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" + ansible.builtin.include_vars: "{{ role_path }}/../ansible_vaults/{{ role_name }}/{{ item }}" loop: - "sanitytest.vault" tags: [always] - name: install packages - import_tasks: "install_packages.yml" + ansible.builtin.import_tasks: "install_packages.yml" tags: [apt, packages, cpan, cpanm] - name: create users and groups - import_tasks: "create_users_groups.yml" + ansible.builtin.import_tasks: "create_users_groups.yml" tags: [users] - name: configure SSH keys - import_tasks: "configure_ssh_keys.yml" + ansible.builtin.import_tasks: "configure_ssh_keys.yml" tags: [ssh] - name: configure nfs mounts - import_tasks: "configure_nfs_mounts.yml" + ansible.builtin.import_tasks: "configure_nfs_mounts.yml" tags: [nfs] - name: install test scripts - import_tasks: "install_test_scripts.yml" + ansible.builtin.import_tasks: "install_test_scripts.yml" tags: [testscripts] - name: configure iptables - import_tasks: "configure_iptables_external.yml" + ansible.builtin.import_tasks: "configure_iptables_external.yml" tags: [firewall, iptables] # only use if needed, this doesn't need to run everytime # - name: install kernel debug symbols -# import_tasks: "install_debug_symbols.yml" +# ansible.builtin.import_tasks: "install_debug_symbols.yml" # tags: [kernel, linux, debug, apt] -- GitLab