upgrade to release 8.0.0 (#44)

- Upgrade to release 8 (reinstall openstack) - Install Scaphandre and Promentheus - Add preinstall/postinstall validation scrips - Specify minimal kernel version - Unsuccessfuly k3s experiments - Create a systemd unit to establish NFT rules (add logger commands for better analytics) - Minimal SSHD hardening - Install HWE kernel - A minimal helper script for running all OSISM validations --------- Signed-off-by: Marc Schöchlin <[email protected]>
SovereignCloudStack · Sep 25, 2024 · cb8bfa8 · cb8bfa8
1 parent 78cd3fe
commit cb8bfa8
Show file tree

Hide file tree

Showing 23 changed files with 1,693 additions and 1,541 deletions.
diff --git a/documentation/System_Deployment.md b/documentation/System_Deployment.md
@@ -34,8 +34,8 @@
      ```
      ssh st01-mgmt-r01-u30
      sudo -u dragon -i
-     osism apply configuration
-     osism reconciler sync
+     osism sync configuration
+     osism sync inventory
      osism apply facts
      ```
 3. Install Manager Infrastructure from manager
@@ -118,7 +118,7 @@ Please just add issues to this project with hints or directly [contact me](https
   ```
 * Install the installation infrastructure
   ```
-  osism apply scs_infra 
+  osism apply scs_infra
   ```
 * Execute the [bootstrap procedure](https://osism.tech/de/docs/guides/deploy-guide/bootstrap)
 * Run Basic customizations
@@ -131,6 +131,11 @@ Please just add issues to this project with hints or directly [contact me](https
   osism apply scs_check_preinstall
   osism validate ceph-connectivity
   ```
+* Reboot all hosts
+  ```
+  osism apply reboot -l 'all:manager' -e ireallymeanit=yes -e reboot_wait=true
+  osism apply reboot -l 'all:!manager' -e ireallymeanit=yes -e reboot_wait=true
+  ```
 
 ## Deploy the infratructure services
 
@@ -148,7 +153,11 @@ The OVN database is deployed to the first 3 compute nodes because the ATOM CPUs
 
 ### Step 3: Logging & Monitoring
 
-TODO
+1. Follow the [Logging & Monitoring deployment](https://osism.tech/docs/guides/deploy-guide/services/logging-monitoring)
+2. Deploy Scaphandre
+   ```
+   osism apply scaphandre
+   ```
 
 ### Step 4: Ceph
 
@@ -168,3 +177,14 @@ For the steps described in the osd configurtion there are the following exceptio
    git push
    ```
 
+### Step 5: Validate the Installation
+
+* Run the Postinstallation validation
+  ```
+  osism apply scs_check_postinstall
+  ```
+* Run the OSISM validations
+  ```
+  /opt/configuration/misc/run_validations.sh
+  ```
+
diff --git a/environments/configuration.yml b/environments/configuration.yml
@@ -3,6 +3,7 @@
 ##########################################################
 # Global
 traefik_enable: true
+host_facts: generic
 
 ##########################################################
 # proxy/client
@@ -45,7 +46,8 @@ squid_tag: edge
 ##########################
 # system settings
 motd_show_ssh: true
-network_interface: dummy0
+network_interface: "dummy0"
+kernel_version_expected: 6.8
 
 #osism_setup_gather_subset:
 # - "!network_switches"
@@ -74,12 +76,16 @@ cilium_mode: "native"
 
 k3s_interface: "dummy0"
 #kube_vip_iface: "dummy0"
-#calico_iface: "dummy0"
-#flannel_iface: "dummy0"
-#cilium_iface: "dummy0"
+cilium_iface: "dummy0"
+
+# Because the FRR instance on the node itself is responsible for propagate routes
+# to the metallb endpoints, there is no need to have a extra BGP routing instance in K3s.
+kube_vip_bgp: true
+
+# ARP Broadcasts are not available in Layer3 Underlay Networks, therefore we disabled it
+# to make leader election functional.
+kube_vip_arp: false
 
-#metal_lb_type: "frr"
-#metal_lb_mode: "bgp"
 
 ##########################
 # docker
@@ -243,6 +249,11 @@ hosts_additional_entries:
 ##########################
 # ceph
 
+# TODO: can be removed when the follwing PR is released: 
+monitor_interface: "dummy0"
+radosgw_interface: "dummy0"
+rgw_multisite: false
+
 ceph_cluster_fsid: 44bc2d19-fd83-4b92-a6c4-eed358e35672
 # BEGIN: TLS
 certificates_ca:

diff --git a/environments/custom/playbook-scs_check_postinstall.yml b/environments/custom/playbook-scs_check_postinstall.yml
@@ -10,3 +10,10 @@
       ansible.builtin.command: /usr/local/scripts/scs_check_network_connectivity.sh
     - name: Check services - /usr/local/scripts/scs_check_services.sh
       ansible.builtin.command: /usr/local/scripts/scs_check_services.sh
+
+- name: Check the state of all openstack after installation
+  hosts: manager
+  gather_facts: no
+  tasks:
+    - name: Check openstack - /usr/local/scripts/scs_check_openstack.sh
+      ansible.builtin.command: /usr/local/scripts/scs_check_openstack.sh
diff --git a/environments/custom/playbook-scs_infra.yml b/environments/custom/playbook-scs_infra.yml
@@ -18,3 +18,27 @@
     - scs-mgr-httpd
     - scs-mgr-samba
     - scs-mgr-tinyproxy
+
+- name: Configure network rules on system boot
+  hosts: manager_infra
+  become: true
+  tasks:
+    - ansible.builtin.copy:
+        dest: /etc/rc.local
+        mode: 0755
+        owner: root
+        group: root
+        content: |
+          [Unit]
+          Description=Add nftables rules on system boot
+          After=network.target
+
+          [Service]
+          Type=simple
+          ExecStart=/usr/local/scripts/scs_add_nftables_rules.sh
+          User=root
+
+          [Install]
+          WantedBy=default.target
+    - name: Run Systemd Reload for NFT rules
+      shell: "systemctl daemon-reload"
diff --git a/environments/custom/roles/scs-landscape-nodes/files/scripts/scs_add_nftables_rules.sh b/environments/custom/roles/scs-landscape-nodes/files/scripts/scs_add_nftables_rules.sh
@@ -9,13 +9,17 @@ if [ "$1" = "log" ];then
 fi
 
 mkdir -p /var/backups/nft
-nft list ruleset > /var/backups/nft/nft-`date --date="today" "+%Y-%m-%d_%H-%M-%S"`.nft
+BACKUP_FILE="/var/backups/nft/nft-`date --date="today" "+%Y-%m-%d_%H-%M-%S"`.nft"
+
+echo "create backup of old rules: $BACKUP_FILE"|logger -s -t scs_add_nftables_rules
+nft list ruleset > $BACKUP_FILE
 find /var/backups/nft -mtime +30 -type f -delete
 
 ZONE1_IPV4="$(ip -json addr ls | jq -r '.[] | .addr_info[] | select(.local | startswith("10.10.21")) | .local')"
 MGMT_IPV4="$(ip -json addr ls | jq -r '.[] | .addr_info[] | select(.local | startswith("10.10.23")) | .local')"
 MGMTP2P_IPV4="$(ip -json addr ls | jq -r '.[] | .addr_info[] | select(.local | startswith("10.10.22")) | .local')"
 
+echo "add standard rules"|logger -s -t scs_add_nftables_rules
 set -x
 nft "insert rule ip nat POSTROUTING position 0 $LOG ip saddr 172.31.100.0/23 ip daddr 10.10.21.0/24 snat to $ZONE1_IPV4 comment SCS-MASQ"
 nft "insert rule ip nat POSTROUTING position 0 $LOG ip saddr 10.10.1.0/24 ip daddr 10.10.21.0/24 snat to $ZONE1_IPV4 comment SCS-MASQ"
@@ -26,13 +30,14 @@ set +x
 
 if [ -n "$MGMTP2P_IPV4" ];then
 	set -x
+   echo "add MGMTP2P_IPV4 rules"|logger -s -t scs_add_nftables_rules
 	nft "insert rule ip nat POSTROUTING position 0 $LOG ip saddr 10.10.1.0/24 ip daddr 10.10.22.0/24 snat to $MGMTP2P_IPV4 comment SCS-MASQ"
 	nft "insert rule ip nat POSTROUTING position 0 $LOG ip saddr 172.31.100.0/23 ip daddr 10.10.22.0/24 snat $MGMTP2P_IPV4 comment SCS-MASQ"
 	set +x
 fi
 
 for rule in $OLD_RULES; do
-   echo "remove rule with handle $rule"
+   echo "remove rule with handle $rule"|logger -s -t scs_add_nftables_rules
    nft delete rule nat POSTROUTING handle $rule
 done
 
diff --git a/environments/custom/roles/scs-landscape-nodes/files/scripts/scs_check_openstack.sh b/environments/custom/roles/scs-landscape-nodes/files/scripts/scs_check_openstack.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+FAILED=""
+
+if [ $(openstack compute service list --os-cloud admin -f json|jq '.[] | select(.Status != "enabled" or .State != "up")'|wc -l) -gt 0 ];then
+	FAILED="NODES-DOWN"
+	openstack compute service list --os-cloud admin
+fi	
+
+if [ -z "FAILED" ];then
+	echo "FAILED: $FAILED"
+	exit 1
+fi
diff --git a/environments/custom/roles/scs-landscape-nodes/handlers/main.yml b/environments/custom/roles/scs-landscape-nodes/handlers/main.yml
@@ -1,2 +1,6 @@
 ---
-# handlers file for scs-landscape
+- name: Reload ssh service
+  become: true
+  ansible.builtin.service:
+    name: ssh
+    state: reloaded
diff --git a/environments/custom/roles/scs-landscape-nodes/tasks/main.yml b/environments/custom/roles/scs-landscape-nodes/tasks/main.yml
@@ -1,6 +1,7 @@
 - name: Setup Node
   include_tasks: "{{ item }}"
   loop:
+    - sshd.yml
     - lvm.yml
     - scripts.yml
     - vim.yml

diff --git a/environments/custom/roles/scs-landscape-nodes/tasks/packages.yml b/environments/custom/roles/scs-landscape-nodes/tasks/packages.yml
@@ -12,6 +12,9 @@
       - numactl
       - tcpdump
       - linux-tools-common
+      - "linux-generic-hwe-{{ ansible_distribution_version}}"
+      - "linux-headers-generic-hwe-{{ ansible_distribution_version}}"
+      - "linux-tools-generic-hwe-{{ ansible_distribution_version}}"
       - linux-tools-{{ ansible_kernel }}
       - bpfcc-tools
       - bpftrace

diff --git a/environments/custom/roles/scs-landscape-nodes/tasks/sshd.yml b/environments/custom/roles/scs-landscape-nodes/tasks/sshd.yml
@@ -0,0 +1,21 @@
+---
+- name: Hardening Settings for SSHD
+  ansible.builtin.copy:
+    dest: /etc/ssh/sshd_config.d/99-scs-hardening.conf
+    content: |
+      PermitRootLogin no
+
+      PasswordAuthentication no
+      PermitEmptyPasswords no
+
+      ChallengeResponseAuthentication no
+      KerberosAuthentication no
+      GSSAPIAuthentication no
+
+      AllowGroups {{ operator_group }}
+  notify: Reload ssh service
+- name: Ensure that SSH passwordless login from cloud-init is removed
+  ansible.builtin.file:
+    path: /etc/ssh/sshd_config.d/50-cloud-init.conf
+    state: absent
+  notify: Reload ssh service
diff --git a/environments/custom/roles/scs-mgr-httpd/templates/docker-compose.yml.j2 b/environments/custom/roles/scs-mgr-httpd/templates/docker-compose.yml.j2
@@ -5,7 +5,7 @@ services:
     restart: unless-stopped
     image: "{{ httpd_image }}"
     ports:
-      - "18080:80"
+      - "28080:80"
     volumes:
       - "{{ httpd_configuration_directory }}/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro"
       - "{{ httpd_media_directory }}/:/srv/:ro" 

diff --git a/environments/kolla/files/overlays/prometheus/prometheus.yml.d/50-scaphandre.yml b/environments/kolla/files/overlays/prometheus/prometheus.yml.d/50-scaphandre.yml
@@ -0,0 +1,17 @@
+---
+scrape_configs:
+  - job_name: scaphandre
+    static_configs:
+      - targets:
+{% for host in groups['scaphandre'] %}
+        - "{{ host }}:9155"
+{% endfor %}
+    relabel_configs:
+      # remove port from instance name
+      - source_labels:
+          - __address__
+        regex: '(.*):.*'
+        replacement: $1
+        target_label: instance
+    scrape_interval: 1m
+