Add role to deploy cASO

cASO is an OpenStack accounting extractor. For more detail see:
https://github.com/IFCA/caso

By default, cASO is configured to output to Fluentd via TCP. The
accounting information can then be shipped off to ElasticSearch.

(cherry picked from commit d8ab00f)
(cherry picked from commit 67ee60e)
Change-Id: Ib81ec443bbb14d494d7c81dff617b371a1b68c52

ansible/group_vars/all.yml

@@ -284,6 +284,8 @@ barbican_api_listen_port: "{{ barbican_api_port }}"
 
 blazar_api_port: "1234"
 
+caso_tcp_output_port: "24224"
+
 ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}"
 ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}"
 ceph_rgw_port: "6780"
@@ -608,6 +610,7 @@ enable_haproxy_memcached: "no"
 enable_aodh: "no"
 enable_barbican: "no"
 enable_blazar: "no"
+enable_caso: "no"
 enable_ceilometer: "no"
 enable_ceilometer_ipmi: "no"
 enable_cells: "no"

ansible/inventory/all-in-one

@@ -18,6 +18,10 @@ localhost       ansible_connection=local
 [deployment]
 localhost       ansible_connection=local
 
+# Caso
+[caso:children]
+monitoring
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 

ansible/inventory/multinode

@@ -42,6 +42,10 @@ monitoring
 [tls-backend:children]
 control
 
+# Caso
+[caso:children]
+monitoring
+
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
 

ansible/roles/caso/defaults/main.yml

@@ -0,0 +1,48 @@
+---
+caso_services:
+  caso:
+    container_name: caso
+    group: caso
+    enabled: true
+    image: "{{ caso_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/caso/:{{ container_config_directory }}/"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "caso_spool:/var/lib/caso"
+      - "caso_ssm_outgoing:/var/spool/apel/outgoing/openstack"
+      - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ caso_dimensions }}"
+
+####################
+# caso
+####################
+caso_site_name: "kolla_caso"
+caso_logging_debug: "{{ openstack_logging_debug }}"
+caso_log_dir: "/var/log/kolla/caso"
+caso_cron_table: "10 * * * *"
+caso_messengers:
+  - caso.messenger.logstash.LogstashMessenger
+
+####################
+# OpenStack
+####################
+caso_openstack_auth: "{{ openstack_auth }}"
+caso_keystone_user: "caso"
+caso_projects: []
+caso_ks_users_tmpl: >
+  {%- for project in caso_projects -%}
+  - project: "{{ project }}"
+    user: "{{ caso_keystone_user }}"
+    password: "{{ caso_keystone_password }}"
+    role: "admin"
+  {% endfor %}
+caso_ks_users: "{{ caso_ks_users_tmpl | from_yaml if caso_projects else [] }}"
+
+####################
+# Docker
+####################
+caso_install_type: "{{ kolla_install_type }}"
+caso_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ caso_install_type }}-caso"
+caso_tag: "{{ openstack_tag }}"
+caso_image_full: "{{ caso_image }}:{{ caso_tag }}"
+caso_dimensions: "{{ default_container_dimensions }}"

ansible/roles/caso/handlers/main.yml

@@ -0,0 +1,24 @@
+---
+- name: Restart caso container
+  vars:
+    service_name: "caso"
+    service: "{{ caso_services[service_name] }}"
+    config_json: "{{ caso_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    caso_container: "{{ check_caso_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or caso_conf.changed | bool
+      or caso_vom_conf.changed | bool
+      or caso_crontab.changed | bool
+      or caso_container.changed | bool

ansible/roles/caso/tasks/check.yml

@@ -0,0 +1 @@
+---

ansible/roles/caso/tasks/config.yml

@@ -0,0 +1,90 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    mode: "0660"
+  become: true
+  register: caso_config_jsons
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+  notify:
+    - Restart caso container
+
+- name: Copying over caso config
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/caso.conf.j2"
+      - "{{ node_custom_config }}//caso/caso.conf"
+      - "{{ node_custom_config }}/{{ item.key }}/{{ inventory_hostname }}/caso.conf"
+    dest: "{{ node_config_directory }}/{{ item.key }}/caso.conf"
+    mode: "0660"
+  become: true
+  register: caso_conf
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+  notify:
+    - Restart caso container
+
+- name: Copying over caso crontab
+  template:
+    src: "{{ role_path }}/templates/caso.crontab.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/caso.crontab"
+    mode: "0660"
+  become: true
+  register: caso_crontab
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+  notify:
+    - Restart caso container
+
+- name: Copying over caso voms file
+  template:
+    src: "{{ role_path }}/templates/voms.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/voms.json"
+    mode: "0660"
+  become: true
+  register: caso_vom_conf
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+  notify:
+    - Restart caso container
+
+- name: Check caso containers
+  become: true
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes }}"
+    dimensions: "{{ item.value.dimensions }}"
+  register: check_caso_containers
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"
+  notify:
+    - Restart caso container

ansible/roles/caso/tasks/deploy.yml

@@ -0,0 +1,12 @@
+---
+- include_tasks: register.yml
+  when: inventory_hostname in groups['caso']
+
+- include_tasks: config.yml
+  when: inventory_hostname in groups['caso']
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- include_tasks: check.yml
+  when: inventory_hostname in groups['caso']

ansible/roles/caso/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/caso/tasks/precheck.yml

@@ -0,0 +1 @@
+---

ansible/roles/caso/tasks/pull.yml

@@ -0,0 +1,11 @@
+---
+- name: Pulling caso images
+  become: true
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ caso_services }}"

ansible/roles/caso/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: deploy.yml

ansible/roles/caso/tasks/register.yml

@@ -0,0 +1,7 @@
+---
+- import_role:
+    name: service-ks-register
+  vars:
+    service_ks_register_auth: "{{ caso_openstack_auth }}"
+    service_ks_register_users: "{{ caso_ks_users }}"
+  tags: always

ansible/roles/caso/tasks/upgrade.yml

@@ -0,0 +1,5 @@
+---
+- include_tasks: config.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/caso/templates/caso.conf.j2

@@ -0,0 +1,23 @@
+[DEFAULT]
+messengers = {{ caso_messengers|join(', ') }}
+site_name = {{ caso_site_name }}
+projects = {{ caso_projects|join(', ') }}
+debug = {{ caso_logging_debug }}
+log_file = caso.log
+log_dir = {{ caso_log_dir }}
+log_rotation_type = none
+spooldir = /var/lib/caso
+
+[keystone_auth]
+auth_type = password
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+project_domain_id = {{ default_project_domain_id }}
+username = {{ caso_keystone_user }}
+user_domain_id = {{ default_user_domain_id }}
+password = {{ caso_keystone_password }}
+
+[logstash]
+port = {{ caso_tcp_output_port }}
+
+[ssm]
+output_path = /var/spool/apel/outgoing/openstack

ansible/roles/caso/templates/caso.crontab.j2

@@ -0,0 +1 @@
+{{ caso_cron_table }} caso-extract --config-file /etc/caso/caso.conf

ansible/roles/caso/templates/caso.json.j2

@@ -0,0 +1,41 @@
+{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
+{
+    "command": "{{ cron_cmd }}",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/caso.crontab",
+            "dest": "/var/spool/cron/caso",
+            "owner": "caso",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/caso.conf",
+            "dest": "/etc/caso/caso.conf",
+            "owner": "caso",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/voms.json",
+            "dest": "/etc/caso/voms.json",
+            "owner": "caso",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/caso",
+            "owner": "caso:caso",
+            "recurse": true
+        },
+        {
+            "path": "/var/spool/apel/outgoing/openstack",
+            "owner": "caso:caso",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/caso",
+            "owner": "caso:caso",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/caso/templates/voms.json.j2

@@ -0,0 +1,9 @@
+{
+        "VO FQAN": {
+            "projects": ["local tenant 1", "local tenant 2"]
+        },
+        "VO NAME": {
+            "projects": ["local tenant 3"]
+        }
+}
+

ansible/roles/common/tasks/config.yml

@@ -119,6 +119,8 @@
     default_input_files:
       - name: "conf/input/00-global.conf.j2"
         enabled: true
+      - name: "conf/input/99-caso.conf.j2"
+        enabled: true
       - name: "conf/input/01-syslog.conf.j2"
         enabled: true
       - name: "conf/input/02-mariadb.conf.j2"
@@ -185,6 +187,7 @@
       - { name: "barbican", enabled: "{{ enable_barbican | bool }}" }
       - { name: "blazar", enabled: "{{ enable_blazar | bool }}" }
       - { name: "ceilometer", enabled: "{{ enable_ceilometer | bool }}" }
+      - { name: "caso", enabled: "{{ enable_caso | bool }}" }
       - { name: "chrony", enabled: "{{ enable_chrony | bool }}" }
       - { name: "cinder", enabled: "{{ enable_cinder | bool }}" }
       - { name: "cloudkitty", enabled: "{{ enable_cloudkitty | bool }}" }

ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2

@@ -73,3 +73,12 @@
   </record>
 </filter>
 {% endif %}
+
+{% if enable_caso | bool and inventory_hostname in groups['caso'] %}
+<filter apel.events>
+    @type parser
+    format json
+    key_name Payload
+    reserve_data true
+</filter>
+{% endif %}

ansible/roles/common/templates/conf/input/99-caso.conf.j2

@@ -0,0 +1,8 @@
+<source>
+  @type tcp
+  tag apel.events
+  port {{ caso_tcp_output_port }}
+  bind 127.0.0.1
+  format /^(?<Payload>.*)$/
+  emit_unmatched_lines true
+</source>

ansible/roles/common/templates/conf/output/01-es.conf.j2

@@ -1,3 +1,19 @@
+{% if enable_caso | bool and inventory_hostname in groups['caso'] %}
+<match apel.events>
+    @type copy
+    <store>
+       @type elasticsearch
+       host {% raw %}{{ elasticsearch_address }}
+{% endraw %}
+       port {% raw %}{{ elasticsearch_port }}
+{% endraw %}
+       logstash_format true
+       logstash_prefix apel
+       flush_interval 15s
+    </store>
+</match>
+{% endif %}
+
 <match *.**>
     @type copy
     <store>

ansible/roles/common/templates/cron-logrotate-caso.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/caso/*.log"
+{
+}