Fail early if any locally attached physical disks are not ok

Flash memory storage can also fail, typically when erase cycles have
exceeded a threshold.

In this commit we add a fixture that checks all locally attached
physical disks across all nodes, via smartctl, and fails
if the self-assessment result is not OK.

This check can be skipped through the FIXTURES env var.

Relates: elastic#153

night_rally.sh

@@ -32,7 +32,7 @@ if [[ -n $VAULT_SECRET_ID && -n $VAULT_ROLE_ID ]]; then
 fi
 export RALLY_METRICS_STORE_CREDENTIAL_PATH=${RALLY_METRICS_STORE_CREDENTIAL_PATH:-"/secret/rally/cloud/nightly-rally-metrics"}
 
-ANSIBLE_ALL_TAGS=(encryption-at-rest initialize-data-disk trim drop-caches)
+ANSIBLE_ALL_TAGS=(check-drive-health encryption-at-rest initialize-data-disk trim drop-caches)
 ANSIBLE_SKIP_TAGS=( )
 ANSIBLE_SKIP_TAGS_STRING=""
 # Don't update night-rally by default, unless specified by env var
@@ -206,6 +206,7 @@ then
 
         cd ${NIGHT_RALLY_HOME}/night_rally/fixtures/ansible
         ansible-playbook -i inventory/production -u rally playbooks/update-rally.yml --extra-vars="rally_environment=${RALLY_ENVIRONMENT} in_vagrant=${IN_VAGRANT} skip_rally_update=${SKIP_RALLY_UPDATE}"
+        ansible-playbook -i inventory/production -u rally playbooks/check-drive-health.yml ${ANSIBLE_SKIP_TAGS_STRING} --extra-vars="in_vagrant=${IN_VAGRANT}"
         ansible-playbook -i inventory/production -u rally playbooks/setup.yml ${ANSIBLE_SKIP_TAGS_STRING}
 
         popd >/dev/null 2>&1

night_rally/fixtures/ansible/playbooks/check-drive-health.yml

@@ -0,0 +1,9 @@
+---
+
+# NOTE: This is also a fixture, but needs to target all hosts, so doesn't live under setup.yml
+- name: check disk health on all hosts
+# ====================================
+  hosts: all
+  gather_facts: true
+  roles:
+    - { role: check-drive-health, tags: check-drive-health }

night_rally/fixtures/ansible/roles/check-drive-health/README.md

@@ -0,0 +1 @@
+Role to check health of locally attached physical disks via smartctl

night_rally/fixtures/ansible/roles/check-drive-health/tasks/common/check_health.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Fail if any locally attached disks don't pass the smartctl health check
+  command: smartctl -a /dev/{{ item.name }}
+  register: smartctloutput
+  failed_when: "'SMART overall-health self-assessment test result: PASSED' not in smartctloutput.stdout"
+  changed_when: false
+  loop: "{{ (phys_disks.stdout | from_json).blockdevices | flatten(levels=1) }}"
+  when: not in_vagrant | default("false") | bool

night_rally/fixtures/ansible/roles/check-drive-health/tasks/debian/main.yml

@@ -0,0 +1,13 @@
+---
+
+- name: Install system packages for lsblk and smartctl
+  package:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - util-linux
+    - smartmontools
+
+- name: Retrieve list of locally attached physical disks
+  shell: lsblk --json --nodeps
+  register: phys_disks

night_rally/fixtures/ansible/roles/check-drive-health/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+
+- set_fact:
+    system_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_os_family | lower }}/main.yml"
+        - unsupported.yml
+
+- block:
+  - include_tasks: "{{ system_tasks }}"
+  - include_tasks: "common/check_health.yml"
+  become: true