test: implement all necessary changes for using GitLab-CI, including Linter...

test: implement all necessary changes for using GitLab-CI, including Linter recommendations and idempotency changes

.ansible-lint

@@ -28,11 +28,6 @@ exclude_paths:
 # Enable checking of loop variable prefixes in roles
 loop_var_prefix: "{role}_"
 
-# Enforce variable names to follow pattern below, in addition to Ansible own
-# requirements, like avoiding python identifiers. To disable add `var-naming`
-# to skip_list.
-var_naming_pattern: "^[a-z_][a-z0-9_]*$"
-
 use_default_rules: true
 # Load custom rules from this specific folder
 # rulesdir:

.config/molecule/config.yml

+---
+prerun: false

.gitignore

@@ -51,6 +51,8 @@ Thumbs.db
 
 *.retry
 *.vault
+inventory.*
+inv.*
 
 # Vim #
 #######
@@ -73,7 +75,14 @@ tags
 .vagrant/
 *.box
 
-# Backups #
-###########
+# Temporary/Build/Backup #
+##########################
 
 backups/
+build/
+
+# CONFIDENTIAL #
+################
+
+ssh_host_*
+

.gitlab-ci.yml

+---
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
+
+stages:          # List of stages for jobs, and their order of execution
+  - test
+
+default:
+  before_script:
+    - source /opt/molecule/bin/activate
+    - ansible --version
+    - molecule --version
+
+test-job:
+  stage: test
+  tags:
+    - "shell"
+  script:
+    # make sure that Ansible Vaults are present and can be decrypted
+    - echo "${VAULT_SERVER_HARDENING}" > ../lza_server_hardening.pass
+    - export ANSIBLE_VAULT_PASSWORD_FILE=../lza_server_hardening.pass
+    - rm -rf ../ansible_vaults/
+    - git clone https://gitlab+deploy-token-25:${VAULT_ACCESS_TOKEN}@git.slub-dresden.de/slub-referat-2-3/ansible_vaults.git ../ansible_vaults/; \
+    # run Molecule tests
+    - molecule syntax --scenario-name default
+    - molecule lint --scenario-name default
+    - molecule create --scenario-name default
+    - molecule converge --scenario-name default
+    - molecule idempotence --scenario-name default
+    # - molecule verify --scenario-name default
+    - molecule destroy --scenario-name default

.yamllint

 ---
-# based on documentation available at
-# https://yamllint.readthedocs.io/en/stable/rules.html
-
+# Based on ansible-lint config
 extends: default
 
 rules:
@@ -11,13 +9,25 @@ rules:
   brackets:
     max-spaces-inside: 1
     level: error
-  comments:
-    min-spaces-from-content: 4
+  colons:
+    max-spaces-after: -1
+    level: error
+  commas:
+    max-spaces-after: -1
+    level: error
+  comments: disable
   comments-indentation: disable
-  document-end: disable
-  document-start:
-    level: warning
-  octal-values:
-    forbid-explicit-octal: false
+  document-start: disable
+  empty-lines:
+    max: 3
+    level: error
+  hyphens:
+    level: error
+  indentation: disable
+  key-duplicates: enable
   line-length: disable
-  truthy: enable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable
+  truthy: disable

ansible.cfg

 [defaults]
 # If set, configures the path to the Vault password file as an alternative to
 # specifying --vault-password-file on the command line.
-vault_identity_list = ../lza_install_common.pass, ../lza_server_hardening.pass, ../slub_osquery.pass
+# vault_identity_list = ../lza_install_common.pass, ../lza_server_hardening.pass, ../slub_osquery.pass
+vault_identity_list = ../lza_server_hardening.pass
 
 # Path to default inventory file
 # Administrators can override this by using the "-i <inventoryfile>" CLI

handlers/main.yml

@@ -2,7 +2,7 @@
 - name: save iptables rules (Debian)
   block:
     - name: Ordner für iptables-Config erstellen
-      file:
+      ansible.builtin.file:
         path: "/etc/iptables"
         state: directory
         owner: "root"
@@ -10,19 +10,20 @@
         mode: 0755
       listen: "save iptables rules"
     - name: install netfilter-persistent to be able to save iptables rules
-      apt:
+      ansible.builtin.apt:
         name: netfilter-persistent
         state: present
       listen: "save iptables rules"
     - name: save iptables rules
-      command: 'netfilter-persistent save'
+      ansible.builtin.command: 'netfilter-persistent save'
       listen: "save iptables rules"
+      changed_when: false
   when: ansible_os_family == "Debian"
 
 - name: save iptables rules (RedHat)
   block:
     - name: make sure iptables config file exists
-      file:
+      ansible.builtin.file:
         path: "/etc/sysconfig/iptables"
         state: touch
         owner: "root"
@@ -30,33 +31,34 @@
         mode: 0600
       listen: "save iptables rules"
     - name: save rules
-      command: /usr/sbin/iptables-save        # noqa 303
+      ansible.builtin.command: /usr/sbin/iptables-save        # noqa 303
       listen: "save iptables rules"
+      changed_when: false
   when: ansible_os_family == "RedHat"
 
 - name: activate kernel parameter changes
-  command: sysctl -p
-  ignore_errors: true
+  ansible.builtin.command: sysctl -p
+  changed_when: false
 
 - name: restart fail2ban.service
-  service:
+  ansible.builtin.service:
     name: "fail2ban"
     state: restarted
 
 - name: restart sshd
-  service:
+  ansible.builtin.service:
     name: "sshd"
     state: restarted
 
 - name: restart auditd.service
-  service:
+  ansible.builtin.service:
     name: "auditd"
     state: restarted
   when: ansible_os_family == "Debian"
   listen: restart auditd.service
 
 - name: restart auditd.service
-  service:
+  ansible.builtin.service:
     name: "auditd"
     state: restarted
     use: "service"
@@ -64,13 +66,13 @@
   listen: restart auditd.service
 
 - name: restart clamav-daemon service
-  service:
+  ansible.builtin.service:
     name: "clamav-daemon"
     state: restarted
   when: ansible_os_family == "Debian"
 
 - name: restart clamd service
-  service:
+  ansible.builtin.service:
     name: "clamd@{{ ansible_hostname }}.service"
     state: restarted
   when: ansible_os_family == "RedHat"

meta/main.yml

 ---
 galaxy_info:
-  author: Jörg Sachse
-  description: role to deploy a hardened install of Debian for use in the SLUBarchiv digital preservation repository
+  author: Jörg Sachse <Joerg.Sachse@slub-dresden.de>
   company: SLUB Dresden
-  # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker Some suggested licenses: - BSD
+  description: role to deploy a hardened install of Debian for use in the SLUBarchiv digital preservation repository
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you
+    # add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+  # issue_tracker_url: "https://example.com/"
+    # If the issue tracker for your role is not on github, uncomment the next line and provide a value issue_tracker_url: http://example.com/issue/tracker
+  license: GPLv3
+    # Some suggested licenses: - BSD
     # (default) - MIT - GPLv2 - GPLv3 - Apache - CC-BY
-  license: public domain
-  min_ansible_version: 2.4
+  min_ansible_version: "2.5"
     # If this a Container Enabled role, provide the minimum Ansible Container version. min_ansible_container_version: Optionally specify the branch Galaxy will use when accessing the GitHub repo
     # for this role. During role install, if no tags are available, Galaxy will use this branch. During import Galaxy will access files on this branch. If Travis integration is configured, only
     # notifications for this branch will be accepted. Otherwise, in all cases, the repo's default branch (usually master) will be used. github_branch:
-  #
+  namespace: "slub"
   # Provide a list of supported platforms, and for each platform a list of versions. If you don't wish to enumerate all versions for a particular platform, use 'all'. To view available
   # platforms and versions (or releases), visit: https://galaxy.ansible.com/api/v1/platforms/
   #
@@ -19,16 +27,10 @@ galaxy_info:
   platforms:
     - name: Debian
       versions:
-        - 9
-        - 10
-    - name: RedHat
+        - "buster"
+        - "bullseye"
+    - name: EL
       versions:
-        - 6
-        - 7
-  galaxy_tags: []
-  # List tags for your role here, one per line. A tag is a keyword that describes and categorizes the role. Users find roles by searching for tags. Be sure to remove the '[]' above, if you
-  # add tags to this list.
-  #
-  # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
-  #       Maximum 20 tags per role.
-# dependencies: []
+        - "7"
+        - "8"
+dependencies: []

molecule/README.md

-# Testing with Molecule
-
-## Prerequisites
-
-In order to be able to use the tests, you need to have some software packages installed. You may need sudo privileges for some of these operations.
-
-        ### install VirtualBox
-        # do NOT use distribution packages
-        # process documented at https://www.virtualbox.org/wiki/Linux_Downloads
-        #
-        # add repository URL
-        sudo echo "deb [arch=amd64] https://download.virtualbox.org/virtualbox/debian stretch contrib" > /etc/apt/sources.d/virtualbox.list
-        # add GPG key
-        wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add -
-        # update sources
-        sudo apt update
-        # install VirtualBox
-        sudo apt-get install virtualbox-6.1
-
-        ### install Vagrant
-        # do NOT use distribution packages
-        # 
-        # download Debian package from Hashicorp
-        wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb
-        # install package
-        sudo dpkg -i vagrant_2.2.9_x86_64.deb
-
-        ### install Molecule et. al.
-        # prepare directories
-        mkdir ~/python-envs/ && cd ~/python-env/
-        # create Python Virtual Environment with Python3 interpreter (Python2 is deprecated!)
-        virtualenv -p python3 molecule-env
-        # enter the Virtual Environment in your current shell (other shells will be unaffected)
-        source molecule-env/bin/activate
-        # install packages
-        pip3 install molecule ansible testinfra ansible-lint molecule-vagrant molecule-docker
-
-        # leave the Virtual Environment only when you're done
-        deactivate
-
-You can find suitable documentation at the respective vendors' websites.
-* [Vagrant Installation Guide](https://www.vagrantup.com/docs/installation/)
-* [VirtualBox Installation Guide](https://www.virtualbox.org/wiki/Downloads)
-* [Molecule Installation Guide](https://molecule.readthedocs.io/en/stable/installation.html)
-
-## Initialising a new Molecule scenario
-
-If you have already created a role without the Molecule test framework or you want to add test scenarios, you can use:
-	molecule init scenario --scenario-name <my_scenario> --driver [azure|delegated|docker|ec2|gce|linode|lxc|lxd|openstack|vagrant] --verifier-name [ansible|testinfra]
-
-If you need any help with the options, please use:
-	molecule init role --help
-
-## Running Tests
-
-Molecule helps with creating a test infrastructure, running tests against it and removing the test infrastructure afterwards.
-
-Various test environments are separated into so-called "scenarios" that can be based on different OSses, drivers, verifiers or might just differ in a minor detail. 
-
-In the simplest configuration, the `molecule/` directory only contains one `default/` directory that contains the default scenario. This scenario is run if no other scenario is chosen using the `-s` CLI option.
-
-This is the basic usage of Molecule:
-	# create test infrastructure
-	cd <role_directory>
-	molecule create
-	# run playbooks against test infrastructure
-	molecule converge
-	# run Testinfra tests
-	molecule verify
-	# remove test infrastructure
-	molecule destroy
-
-	# run all steps at once:
-	molecule test
-
-It has proven helpful to use Vagrant to create a snapshot of the VM after the creation phase has completed.
-	# First, get UUID of the VM
-	vagrant global-status
-	# Then, create the snapshot
-	vagrant snapshot save <uuid> <snapshot_name>
-	# To restore the snapshot, use
-	vagrant snapshot restore <uuid> <snapshot_name>
-	# And to remove the snapshot, use
-	vagrant snapshot delete <uuid> <snapshot_name>
-

molecule/default

+./virtualbox
\ No newline at end of file

molecule/default/converge.yml

----
-- name: Converge
-  hosts: all
-  roles:
-    - {role: "ansible_lza_server_hardening", become: true}

molecule/default/molecule.yml

----
-dependency:
-  name: galaxy
-driver:
-  name: vagrant
-  provider:
-    name: virtualbox
-lint: |
-  set -e
-  yamllint .
-  ansible-lint -x formatting
-  flake8 --ignore=E501
-platforms:
-  - name: molecule-server-hardening-debian
-    box: debian/buster64
-    memory: 512
-    cpus: 1
-provisioner:
-  name: ansible
-  log: true
-  config_options:
-    defaults:
-      vault_identity_list: "@$HOME/.ansible/roles/molecule_prepare.pass, @$HOME/.ansible/roles/lza_install_common.pass, @$HOME/.ansible/roles/lza_server_hardening.pass"
-  lint: |
-    set -e
-    ansible-lint
-  vvv: false
-verifier:
-  name: testinfra
-  env:
-    PYTHONWARNINGS: "ignore:.*U.*mode is deprecated:DeprecationWarning"
-  lint: |
-    set -e
-    flake8
-  options:
-    v: 1

molecule/default/prepare.yml

----
-- name: Prepare
-  hosts: all
-  gather_facts: true
-  pre_tasks:
-    - name: include vars
-      include_vars: "../../../ansible_vaults/molecule_prepare/{{ item }}"
-      loop:
-        - "prepare.vault"
-    - name: Install python for Ansible
-      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
-      become: true
-      changed_when: false
-    - name: create users (as deployed in production)
-      user:
-        name: "{{ item.name }}"
-        uid: "{{ item.uid }}"
-        create_home: "yes"
-        shell: "/bin/bash"
-      loop: "{{ vault_molecule_users | flatten(levels=1) }}"
-      become: true
-    - name: add nonfree repos
-      apt_repository:
-        repo: "deb http://ftp2.de.debian.org/debian/ buster main non-free contrib"
-        state: present
-        update-cache: "yes"
-      become: true
-    - name: Install required packages
-      apt:
-        name: [
-          'aptitude',
-          'gpg',
-          'less',
-          'libuser'
-        ]
-        state: present
-      become: true
-  roles:
-    - {role: ansible_lza_install_common, become: true}

molecule/default/tests/conftest.py

-"""PyTest Fixtures."""
-from __future__ import absolute_import
-import os
-import pytest
-
-
-def pytest_runtest_setup(item):
-    """Run tests only when under molecule with testinfra installed."""
-    try:
-        import testinfra
-    except ImportError:
-        pytest.skip("Test requires testinfra", allow_module_level=True)
-    if "MOLECULE_INVENTORY_FILE" in os.environ:
-        pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-            os.environ["MOLECULE_INVENTORY_FILE"]
-        ).get_hosts("all")
-    else:
-        pytest.skip(
-            "Test should run only from inside molecule.", allow_module_level=True
-        )

molecule/default/tests/test_default.py

-def test_hosts_file(host):
-    f = host.file('/etc/hosts')
-
-    assert f.exists
-    assert f.user == 'root'
-    assert f.group == 'root'

molecule/resources/playbooks/Dockerfile

+FROM debian:stable-slim
+
+RUN adduser lza;
+
+### configure SLUB Debian Repository
+RUN apt-get update; \
+    apt-get install -y --no-install-recommends gnupg wget git python3 ansible sudo; \
+    wget -O - http://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key | apt-key add - ; \
+    echo "deb http://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main" > /etc/apt/sources.list.d/slub.list; \
+    apt-get update;
+    #apt-get -y --no-install-recommends install python3-pip python3-virtualenv;
+
+RUN echo "lza     ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/lza-user
+
+#RUN mkdir /opt/venv/ && cd /opt/venv/; \
+#    virtualenv -p python3 molecule; \
+#    . /opt/venv/molecule/bin/activate; \
+#    pip3 install ansible molecule molecule-docker;
+
+USER lza

molecule/default/INSTALL.rst → molecule/resources/playbooks/INSTALL.rst

-*******
+*********************************
 Vagrant driver installation guide
-*******
+*********************************
 
 Requirements
 ============
@@ -20,4 +20,4 @@ widely recommended `'--user' flag`_ when invoking ``pip``.
 
 .. code-block:: bash
 
-    $ pip install 'molecule[vagrant]'
+    $ pip install 'molecule_vagrant'

molecule/resources/playbooks/README.md

+This drectory contains shared playbooks and a shared Dockerfile.
+
+Visit https://molecule.readthedocs.io/en/latest/examples.html#sharing-across-scenarios for details on sharing playbooks, tests etc. across multiple scenarios.

molecule/resources/playbooks/converge.yml

+---
+- name: Converge
+  hosts: all
+  pre_tasks:
+    - name: update apt cache
+      ansible.builtin.apt:
+        update_cache: true
+        upgrade: dist
+      become: true
+      when: ansible_os_family == "Debian"
+    - name: update yum cache
+      ansible.builtin.yum:
+        update_cache: true
+      become: true
+      when: ansible_os_family == "RedHat"
+  roles:
+    - {role: "ansible_lza_server_hardening", become: true}

molecule/resources/playbooks/prepare.yml

+---
+- name: Prepare
+  hosts: "*"
+  tasks:
+    - name: install GPG
+      ansible.builtin.apt:
+        name: "gnupg"
+        state: latest
+        update_cache: true
+      become: true
+    - name: add GPG key for SLUB Debian repository
+      ansible.builtin.apt_key:
+        url: "https://sdvdebianrepo.slub-dresden.de/deb-repository/pub.gpg.key"
+        state: present
+      become: true
+    - name: add repo URL to sources.list
+      ansible.builtin.apt_repository:
+        repo: "deb https://sdvdebianrepo.slub-dresden.de/deb-repository bullseye main"
+        state: present
+        update_cache: true
+        mode: "0644"
+      become: true