Skip to content
Snippets Groups Projects
Commit a430a0c8 authored by Jörg Sachse's avatar Jörg Sachse
Browse files

test: implement all necessary changes for using GitLab-CI, including Linter...

test: implement all necessary changes for using GitLab-CI, including Linter recommendations and idempotency changes
parent b072ca20
No related branches found
No related tags found
No related merge requests found
Showing
with 194 additions and 268 deletions
...@@ -28,11 +28,6 @@ exclude_paths: ...@@ -28,11 +28,6 @@ exclude_paths:
# Enable checking of loop variable prefixes in roles # Enable checking of loop variable prefixes in roles
loop_var_prefix: "{role}_" loop_var_prefix: "{role}_"
# Enforce variable names to follow pattern below, in addition to Ansible own
# requirements, like avoiding python identifiers. To disable add `var-naming`
# to skip_list.
var_naming_pattern: "^[a-z_][a-z0-9_]*$"
use_default_rules: true use_default_rules: true
# Load custom rules from this specific folder # Load custom rules from this specific folder
# rulesdir: # rulesdir:
......
---
prerun: false
...@@ -51,6 +51,8 @@ Thumbs.db ...@@ -51,6 +51,8 @@ Thumbs.db
*.retry *.retry
*.vault *.vault
inventory.*
inv.*
# Vim # # Vim #
####### #######
...@@ -73,7 +75,14 @@ tags ...@@ -73,7 +75,14 @@ tags
.vagrant/ .vagrant/
*.box *.box
# Backups # # Temporary/Build/Backup #
########### ##########################
backups/ backups/
build/
# CONFIDENTIAL #
################
ssh_host_*
---
# A pipeline is composed of independent jobs that run scripts, grouped into stages.
# Stages run in sequential order, but jobs within stages run in parallel.
#
# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
stages: # List of stages for jobs, and their order of execution
- test
default:
before_script:
- source /opt/molecule/bin/activate
- ansible --version
- molecule --version
test-job:
stage: test
tags:
- "shell"
script:
# make sure that Ansible Vaults are present and can be decrypted
- echo "${VAULT_SERVER_HARDENING}" > ../lza_server_hardening.pass
- export ANSIBLE_VAULT_PASSWORD_FILE=../lza_server_hardening.pass
- rm -rf ../ansible_vaults/
- git clone https://gitlab+deploy-token-25:${VAULT_ACCESS_TOKEN}@git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git ../ansible_vaults/; \
# run Molecule tests
- molecule syntax --scenario-name default
- molecule lint --scenario-name default
- molecule create --scenario-name default
- molecule converge --scenario-name default
- molecule idempotence --scenario-name default
# - molecule verify --scenario-name default
- molecule destroy --scenario-name default
--- ---
# based on documentation available at # Based on ansible-lint config
# https://yamllint.readthedocs.io/en/stable/rules.html
extends: default extends: default
rules: rules:
...@@ -11,13 +9,25 @@ rules: ...@@ -11,13 +9,25 @@ rules:
brackets: brackets:
max-spaces-inside: 1 max-spaces-inside: 1
level: error level: error
comments: colons:
min-spaces-from-content: 4 max-spaces-after: -1
level: error
commas:
max-spaces-after: -1
level: error
comments: disable
comments-indentation: disable comments-indentation: disable
document-end: disable document-start: disable
document-start: empty-lines:
level: warning max: 3
octal-values: level: error
forbid-explicit-octal: false hyphens:
level: error
indentation: disable
key-duplicates: enable
line-length: disable line-length: disable
truthy: enable new-line-at-end-of-file: disable
new-lines:
type: unix
trailing-spaces: disable
truthy: disable
[defaults] [defaults]
# If set, configures the path to the Vault password file as an alternative to # If set, configures the path to the Vault password file as an alternative to
# specifying --vault-password-file on the command line. # specifying --vault-password-file on the command line.
vault_identity_list = ../lza_install_common.pass, ../lza_server_hardening.pass, ../slub_osquery.pass # vault_identity_list = ../lza_install_common.pass, ../lza_server_hardening.pass, ../slub_osquery.pass
vault_identity_list = ../lza_server_hardening.pass
# Path to default inventory file # Path to default inventory file
# Administrators can override this by using the "-i <inventoryfile>" CLI # Administrators can override this by using the "-i <inventoryfile>" CLI
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
- name: save iptables rules (Debian) - name: save iptables rules (Debian)
block: block:
- name: Ordner für iptables-Config erstellen - name: Ordner für iptables-Config erstellen
file: ansible.builtin.file:
path: "/etc/iptables" path: "/etc/iptables"
state: directory state: directory
owner: "root" owner: "root"
...@@ -10,19 +10,20 @@ ...@@ -10,19 +10,20 @@
mode: 0755 mode: 0755
listen: "save iptables rules" listen: "save iptables rules"
- name: install netfilter-persistent to be able to save iptables rules - name: install netfilter-persistent to be able to save iptables rules
apt: ansible.builtin.apt:
name: netfilter-persistent name: netfilter-persistent
state: present state: present
listen: "save iptables rules" listen: "save iptables rules"
- name: save iptables rules - name: save iptables rules
command: 'netfilter-persistent save' ansible.builtin.command: 'netfilter-persistent save'
listen: "save iptables rules" listen: "save iptables rules"
changed_when: false
when: ansible_os_family == "Debian" when: ansible_os_family == "Debian"
- name: save iptables rules (RedHat) - name: save iptables rules (RedHat)
block: block:
- name: make sure iptables config file exists - name: make sure iptables config file exists
file: ansible.builtin.file:
path: "/etc/sysconfig/iptables" path: "/etc/sysconfig/iptables"
state: touch state: touch
owner: "root" owner: "root"
...@@ -30,33 +31,34 @@ ...@@ -30,33 +31,34 @@
mode: 0600 mode: 0600
listen: "save iptables rules" listen: "save iptables rules"
- name: save rules - name: save rules
command: /usr/sbin/iptables-save # noqa 303 ansible.builtin.command: /usr/sbin/iptables-save # noqa 303
listen: "save iptables rules" listen: "save iptables rules"
changed_when: false
when: ansible_os_family == "RedHat" when: ansible_os_family == "RedHat"
- name: activate kernel parameter changes - name: activate kernel parameter changes
command: sysctl -p ansible.builtin.command: sysctl -p
ignore_errors: true changed_when: false
- name: restart fail2ban.service - name: restart fail2ban.service
service: ansible.builtin.service:
name: "fail2ban" name: "fail2ban"
state: restarted state: restarted
- name: restart sshd - name: restart sshd
service: ansible.builtin.service:
name: "sshd" name: "sshd"
state: restarted state: restarted
- name: restart auditd.service - name: restart auditd.service
service: ansible.builtin.service:
name: "auditd" name: "auditd"
state: restarted state: restarted
when: ansible_os_family == "Debian" when: ansible_os_family == "Debian"
listen: restart auditd.service listen: restart auditd.service
- name: restart auditd.service - name: restart auditd.service
service: ansible.builtin.service:
name: "auditd" name: "auditd"
state: restarted state: restarted
use: "service" use: "service"
...@@ -64,13 +66,13 @@ ...@@ -64,13 +66,13 @@
listen: restart auditd.service listen: restart auditd.service
- name: restart clamav-daemon service - name: restart clamav-daemon service
service: ansible.builtin.service:
name: "clamav-daemon" name: "clamav-daemon"
state: restarted state: restarted
when: ansible_os_family == "Debian" when: ansible_os_family == "Debian"
- name: restart clamd service - name: restart clamd service
service: ansible.builtin.service:
name: "clamd@{{ ansible_hostname }}.service" name: "clamd@{{ ansible_hostname }}.service"
state: restarted state: restarted
when: ansible_os_family == "RedHat" when: ansible_os_family == "RedHat"
--- ---
galaxy_info: galaxy_info:
author: Jörg Sachse author: Jörg Sachse <Joerg.Sachse@slub-dresden.de>
description: role to deploy a hardened install of Debian for use in the SLUBarchiv digital preservation repository
company: SLUB Dresden company: SLUB Dresden
# If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker Some suggested licenses: - BSD description: role to deploy a hardened install of Debian for use in the SLUBarchiv digital preservation repository
galaxy_tags: []
# List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you
# add tags to this list.
#
# NOTE: A tag is limited to a single word comprised of alphanumeric characters.
# Maximum 20 tags per role.
# issue_tracker_url: "https://example.com/"
# If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker
license: GPLv3
# Some suggested licenses: - BSD
# (default) - MIT - GPLv2 - GPLv3 - Apache - CC-BY # (default) - MIT - GPLv2 - GPLv3 - Apache - CC-BY
license: public domain min_ansible_version: "2.5"
min_ansible_version: 2.4
# If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo
# for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only
# notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch: # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch:
# namespace: "slub"
# Provide a list of supported platforms, and for each platform a list of versions. If you don't wish to enumerate all versions for a particular platform, use 'all'. To view available # Provide a list of supported platforms, and for each platform a list of versions. If you don't wish to enumerate all versions for a particular platform, use 'all'. To view available
# platforms and versions (or releases), visit: https://galaxy.ansible.com/api/v1/platforms/ # platforms and versions (or releases), visit: https://galaxy.ansible.com/api/v1/platforms/
# #
...@@ -19,16 +27,10 @@ galaxy_info: ...@@ -19,16 +27,10 @@ galaxy_info:
platforms: platforms:
- name: Debian - name: Debian
versions: versions:
- 9 - "buster"
- 10 - "bullseye"
- name: RedHat - name: EL
versions: versions:
- 6 - "7"
- 7 - "8"
galaxy_tags: [] dependencies: []
# List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you
# add tags to this list.
#
# NOTE: A tag is limited to a single word comprised of alphanumeric characters.
# Maximum 20 tags per role.
# dependencies: []
# Testing with Molecule
## Prerequisites
In order to be able to use the tests, you need to have some software packages installed. You may need sudo privileges for some of these operations.
### install VirtualBox
# do NOT use distribution packages
# process documented at https://www.virtualbox.org/wiki/Linux_Downloads
#
# add repository URL
sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list
# add GPG key
wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add -
# update sources
sudo apt update
# install VirtualBox
sudo apt-get install virtualbox-6.1
### install Vagrant
# do NOT use distribution packages
#
# download Debian package from Hashicorp
wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb
# install package
sudo dpkg -i vagrant_2.2.9_x86_64.deb
### install Molecule et. al.
# prepare directories
mkdir ~/python-envs/ && cd ~/python-env/
# create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!)
virtualenv -p python3 molecule-env
# enter the Virtual Environment in your current shell (other shells will be unaffected)
source molecule-env/bin/activate
# install packages
pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker
# leave the Virtual Environment only when you're done
deactivate
You can find suitable documentation at the respective vendors' websites.
* [Vagrant Installation Guide](https://www.vagrantup.com/docs/installation/)
* [VirtualBox Installation Guide](https://www.virtualbox.org/wiki/Downloads)
* [Molecule Installation Guide](https://molecule.readthedocs.io/en/stable/installation.html)
## Initialising a new Molecule scenario
If you have already created a role without the Molecule test framework or you want to add test scenarios, you can use:
molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra]
If you need any help with the options, please use:
molecule init role --help
## Running Tests
Molecule helps with creating a test infrastructure, running tests against it and removing the test infrastructure afterwards.
Various test environments are separated into so-called "scenarios" that can be based on different OSses, drivers, verifiers or might just differ in a minor detail.
In the simplest configuration, the `molecule/` directory only contains one `default/` directory that contains the default scenario. This scenario is run if no other scenario is chosen using the `-s` CLI option.
This is the basic usage of Molecule:
# create test infrastructure
cd <role_directory>
molecule create
# run playbooks against test infrastructure
molecule converge
# run Testinfra tests
molecule verify
# remove test infrastructure
molecule destroy
# run all steps at once:
molecule test
It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed.
# First, get UUID of the VM
vagrant global-status
# Then, create the snapshot
vagrant snapshot save <uuid> <snapshot_name>
# To restore the snapshot, use
vagrant snapshot restore <uuid> <snapshot_name>
# And to remove the snapshot, use
vagrant snapshot delete <uuid> <snapshot_name>
./virtualbox
\ No newline at end of file
---
- name: Converge
hosts: all
roles:
- {role: "ansible_lza_server_hardening", become: true}
---
dependency:
name: galaxy
driver:
name: vagrant
provider:
name: virtualbox
lint: |
set -e
yamllint .
ansible-lint -x formatting
flake8 --ignore=E501
platforms:
- name: molecule-server-hardening-debian
box: debian/buster64
memory: 512
cpus: 1
provisioner:
name: ansible
log: true
config_options:
defaults:
vault_identity_list: "@$HOME/.ansible/roles/molecule_prepare.pass, @$HOME/.ansible/roles/lza_install_common.pass, @$HOME/.ansible/roles/lza_server_hardening.pass"
lint: |
set -e
ansible-lint
vvv: false
verifier:
name: testinfra
env:
PYTHONWARNINGS: "ignore:.*U.*mode is deprecated:DeprecationWarning"
lint: |
set -e
flake8
options:
v: 1
---
- name: Prepare
hosts: all
gather_facts: true
pre_tasks:
- name: include vars
include_vars: "../../../ansible_vaults/molecule_prepare/{{ item }}"
loop:
- "prepare.vault"
- name: Install python for Ansible
raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
become: true
changed_when: false
- name: create users (as deployed in production)
user:
name: "{{ item.name }}"
uid: "{{ item.uid }}"
create_home: "yes"
shell: "/bin/bash"
loop: "{{ vault_molecule_users | flatten(levels=1) }}"
become: true
- name: add nonfree repos
apt_repository:
repo: "deb http://ftp2.de.debian.org/debian/ buster main non-free contrib"
state: present
update-cache: "yes"
become: true
- name: Install required packages
apt:
name: [
'aptitude',
'gpg',
'less',
'libuser'
]
state: present
become: true
roles:
- {role: ansible_lza_install_common, become: true}
"""PyTest Fixtures."""
from __future__ import absolute_import
import os
import pytest
def pytest_runtest_setup(item):
"""Run tests only when under molecule with testinfra installed."""
try:
import testinfra
except ImportError:
pytest.skip("Test requires testinfra", allow_module_level=True)
if "MOLECULE_INVENTORY_FILE" in os.environ:
pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ["MOLECULE_INVENTORY_FILE"]
).get_hosts("all")
else:
pytest.skip(
"Test should run only from inside molecule.", allow_module_level=True
)
def test_hosts_file(host):
f = host.file('/etc/hosts')
assert f.exists
assert f.user == 'root'
assert f.group == 'root'
FROM debian:stable-slim
RUN adduser lza;
### configure SLUB Debian Repository
RUN apt-get update; \
apt-get install -y --no-install-recommends gnupg wget git python3 ansible sudo; \
wget -O - http://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key | apt-key add - ; \
echo "deb http://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main" > /etc/apt/sources.list.d/slub.list; \
apt-get update;
#apt-get -y --no-install-recommends install python3-pip python3-virtualenv;
RUN echo "lza ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/lza-user
#RUN mkdir /opt/venv/ && cd /opt/venv/; \
# virtualenv -p python3 molecule; \
# . /opt/venv/molecule/bin/activate; \
# pip3 install ansible molecule molecule-docker;
USER lza
******* *********************************
Vagrant driver installation guide Vagrant driver installation guide
******* *********************************
Requirements Requirements
============ ============
...@@ -20,4 +20,4 @@ widely recommended `'--user' flag`_ when invoking ``pip``. ...@@ -20,4 +20,4 @@ widely recommended `'--user' flag`_ when invoking ``pip``.
.. code-block:: bash .. code-block:: bash
$ pip install 'molecule[vagrant]' $ pip install 'molecule_vagrant'
This drectory contains shared playbooks and a shared Dockerfile.
Visit https://molecule.readthedocs.io/en/latest/examples.html#sharing-across-scenarios for details on sharing playbooks, tests etc. across multiple scenarios.
---
- name: Converge
hosts: all
pre_tasks:
- name: update apt cache
ansible.builtin.apt:
update_cache: true
upgrade: dist
become: true
when: ansible_os_family == "Debian"
- name: update yum cache
ansible.builtin.yum:
update_cache: true
become: true
when: ansible_os_family == "RedHat"
roles:
- {role: "ansible_lza_server_hardening", become: true}
---
- name: Prepare
hosts: "*"
tasks:
- name: install GPG
ansible.builtin.apt:
name: "gnupg"
state: latest
update_cache: true
become: true
- name: add GPG key for SLUB Debian repository
ansible.builtin.apt_key:
url: "https://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key"
state: present
become: true
- name: add repo URL to sources.list
ansible.builtin.apt_repository:
repo: "deb https://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main"
state: present
update_cache: true
mode: "0644"
become: true
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment