Celery service

compute-build.yaml

@@ -0,0 +1,5 @@
+---
+- hosts: compute
+  roles:
+    - compute_build_image
+    - install_ww_bin

group_vars/all

@@ -7,8 +7,8 @@
 # Headnode Info
   public_interface: "eth0" # NIC that allows access to the public internet
   private_interface: "eth1" #NIC that allows access to compute nodes
-  headnode_private_ip: "10.1.1.1"
-  build_kernel_ver: '3.10.0-957.1.3.el7.x86_64' # `uname -r` at build time... for wwbootstrap
+  headnode_private_ip: "10.1.1.10"
+  build_kernel_ver: '3.10.0-957.12.2.el7.x86_64' # `uname -r` at build time... for wwbootstrap
 
 #Private network Info
   private_network: "10.1.1.0"  
@@ -24,7 +24,7 @@
 
 # sacct user list
   cluster_users:
-    - vagrant   # include each username on separate line as a list
+    - centos   # include each username on separate line as a list
 
 #Stateful compute or not?
   stateful_nodes: false
@@ -56,7 +56,7 @@
   login_chroot: centos7-login 
 
 # Node Inventory method - automatic, or manual
-  node_inventory_auto: true
+  node_inventory_auto: false
 
 #Node naming variables - no need to change
   compute_node_prefix: "c"
@@ -69,7 +69,7 @@
 #OpenOnDemand
   ood_nodename: "ood"
   ood_version: 1.5
-  ood_ip_addr: 10.1.1.254
+  ood_private_ip: "10.1.1.11"
   ood_rpm_repo: "https://yum.osc.edu/ondemand/{{ ood_version }}/ondemand-release-web-{{ ood_version }}-1.el7.noarch.rpm"
 
 #Node Inventory - not in the Ansible inventory sense! Just for WW and Slurm config.
@@ -153,7 +153,7 @@
   enable_shib: false
 
 # User Registration
-  enable_user_reg: false
+  enable_user_reg: true
   user_register_app: "flask_user_reg"
   user_register_app_path: "/var/www/ood/register/{{ user_register_app }}"
   user_register_app_repo: "https://gitlab.rc.uab.edu/mmoo97/flask_user_reg.git"
@@ -163,8 +163,12 @@
   RegUser_app_user_passwd: "qweasd"
 
 # User Create Scripts
-  enable_user_create_scripts: false
+  enable_user_create_scripts: true
   user_create_scripts: "ohpc_user_create"
   user_create_scripts_path: "/opt/{{ user_create_scripts }}"
   user_create_script_repo: "https://gitlab.rc.uab.edu/tr27p/ohpc_user_create.git"
 
+# RabbitMQ
+  rabbitmq_provision: false
+  rabbitmq_user: "reggie"
+  rabbitmq_user_password: ""

hosts

@@ -11,7 +11,7 @@ ood
 sshgroup=ood
 
 [compute]
-c1
+c[0:9]
 
 [compute:vars]
 sshgroup=compute

ohpc-build.yaml

@@ -0,0 +1,15 @@
+---
+- hosts: headnode
+  roles:
+    - { name: 'pre_ohpc', tags: 'pre_ohpc' }
+    - { name: 'ohpc_install', tags: 'ohpc_install' }
+    - { name: 'ohpc_config_build', tags: 'ohpc_config' }
+    - { name: 'ohpc_add_easybuild', tags: 'ohpc_add_easybuild' }
+    - { name: 'ohpc_jupyter', tags: 'ohpc_jupyter', when: jupyter_provision}
+    - { name: 'ohpc_matlab', tags: 'ohpc_matlab', when: matlab_provision }
+    - { name: 'ohpc_sas', tags: 'ohpc_sas', when: sas_provision }
+    - { name: 'ohpc_rabbitmq', tags: 'ohpc_rabbitmq', when: rabbitmq_provision }
+    - { name: 'ohpc_add_rstudio', tags: 'ohpc_add_rstudio', when: rstudio_provision }
+    - { name: 'ohpc_user_reg', tags: 'ohpc_user_reg', when: enable_user_reg }
+    - { name: 'reg_user_create_scripts', tags: 'reg_user_create_scripts', when: enable_user_create_scripts }
+    - { name: 'ohpc_add_rabbitmq_agents', tags: 'ohpc_add_rabbitmq_agents', when: rabbitmq_provision }

ohpc-ops.yaml

@@ -0,0 +1,6 @@
+---
+- hosts: headnode
+  roles:
+    - { name: 'ohpc_install_cloud'}
+    - { name: 'compute_build_nodes_cloud', tags: 'compute_build_nodes_cloud' }
+    - { name: 'nodes_vivify_cloud', tags: 'nodes_vivify_cloud' }

ohpc.yaml

@@ -1,5 +1,5 @@
 ---
-- hosts: ohpc
+- hosts: headnode
   roles:
     - { name: 'pre_ohpc', tags: 'pre_ohpc' }
     - { name: 'ohpc_install', tags: 'ohpc_install' }

ood-build.yaml

@@ -0,0 +1,16 @@
+---
+- hosts: ood
+  roles:
+    - { name: 'prep_ood_cloud', tags: 'prep_ood' }
+    - { name: 'ood', tags: 'ood' }
+    - { name: 'ood_jupyter', tags: 'ood_jupyter', when: jupyter_provision}
+    - { name: 'ood_vnc_form', tags: 'ood_vnc_form' }
+    - { name: 'ood_add_rstudio', tags: 'ood_add_rstudio', when: rstudio_provision }
+    - { name: 'ood_matlab', tags: 'ood_matlab', when: matlab_provision }
+    - { name: 'ood_sas', tags: 'ood_sas', when: sas_provision }
+    - { name: 'install_ww_bin', tags: 'install_ww_bin' }
+    - { name: 'ood_firewall_and_services', tags: 'ood_firewall_and_services' }
+    - { name: 'ohpc_firewall_and_services', tags: 'ohpc_firewall_and_services' }
+    - { name: 'ood_user_reg_cloud', tags: 'ood_user_reg_cloud' }
+    - { name: 'reg_user_create_scripts', tags: 'reg_user_create_scripts', when: enable_user_create_scripts }
+    - { name: 'ood_add_rabbitmq_agents', tags: 'ood_add_rabbitmq_agents', when: rabbitmq_provision }

ood-ops.yaml

@@ -0,0 +1,4 @@
+---
+- hosts: ood
+  roles:
+    - { name: 'ood_user_reg_ops', tags: 'ood_user_reg_ops' }

ood.yaml

@@ -13,4 +13,5 @@
     - { name: 'ohpc_firewall_and_services', tags: 'ohpc_firewall_and_services' }
     - { name: 'ood_shib_sso', tags: 'ood_shib_sso', when: enable_shib }
     - { name: 'ood_user_reg', tags: 'ood_user_reg', when: enable_user_reg }
+    - { name: 'warewulf_sync', tags: 'warewulf_sync' }
     - { name: 'reg_user_create_scripts', tags: 'reg_user_create_scripts', when: enable_user_create_scripts }

roles/compute_build_image/tasks/main.yml

@@ -0,0 +1,106 @@
+---
+- name: Get OpenHPC Repo
+  yum:
+    name: https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm
+    state: present
+
+- name: yum install
+  yum:
+    state: present
+    name:
+      - autofs
+      - chrony
+      - lmod-ohpc
+      - grub2
+      - freeipmi
+      - ipmitool
+      - ohpc-slurm-client
+      - ohpc-base-compute
+      - tmux
+      - ruby
+      - turbojpeg
+      - nc
+      - '@X Window System'
+      - '@Xfce'
+
+- name: Install TurboVNC via rpm
+  yum:
+    name: https://sourceforge.net/projects/turbovnc/files/2.2/turbovnc-2.2.x86_64.rpm
+    state: present
+
+- name: Download and extract Websockify source code
+  unarchive:
+    src: https://github.com/novnc/websockify/archive/v0.8.0.tar.gz
+    dest: /tmp
+    remote_src: yes
+
+- name: Install Websockify
+  command: python setup.py install
+  args:
+    chdir: /tmp/websockify-0.8.0
+
+# Change to autofs
+- name: add autofs maps
+  lineinfile:
+    path: "/etc/auto.master.d/uab-dev.autofs"
+    line: "{{ item.path }} /etc/auto.{{ item.name }}"
+    create: yes
+  with_items:
+    - { name: "home", path: "/home"}
+    - { name: "opt-ohpc-pub", path: "/opt/ohpc/pub"}
+    - { name: "export", path: "/export"}
+
+- lineinfile:
+    path: "/etc/auto.{{ item.name }}"
+    line: "* -{{ item.opt }} {{ headnode_private_ip }}:{{ item.path }}/&"
+    create: yes
+  with_items:
+    - { name: "home", opt: "rw,soft,intr,_netdev", path: "/home"}
+    - { name: "opt-ohpc-pub", opt: "ro", path: "/opt/ohpc/pub"}
+    - { name: "export", opt: "ro", path: "/export"}
+
+- name: Put SELinux in permissive mode, logging actions that would be blocked.
+  selinux:
+    policy: targeted
+    state: permissive
+
+- name: Create folder for mounting
+  file:
+    state: directory
+    path: /export
+    owner: root
+    group: root
+    mode: 0777
+
+- name: Enable autofs on compute image
+  service:
+    name: autofs
+    enabled: yes
+
+# After we installed Xfce, the compute node is set to bootup in graphical mode.
+# This task is to unset that back to multi-user mode.
+- name: Set compute node to boot with multi-user mode
+  command: systemctl set-default multi-user.target
+
+- name: Disable firewalld on compute image
+  service:
+    name: firewalld
+    state: stopped
+    enabled: no
+
+- name: add headnode to compute chrony.conf
+  lineinfile:
+    line: "server {{ headnode_private_ip }}"
+    dest: /etc/chrony.conf
+    state: present
+
+- name: Enable chronyd on compute image
+  service:
+    name: chronyd
+    enabled: yes
+
+- name: Enable slurmd on compute image
+  service:
+    name: slurmd
+    enabled: yes
+

roles/compute_build_nodes_cloud/tasks/main.yml

@@ -0,0 +1,66 @@
+---
+
+#   - name: print single node info
+#     debug:
+#       var: item.mac
+#     with_items: "{{ compute_nodes }}"
+#   - name: print single node info
+#     debug:
+#       var: item.vnfs
+#     with_items: "{{ compute_nodes }}"
+#
+#   - fail:
+#       msg: "Quick fail for test!"
+
+   - block:
+
+     - name: Check if node is already in warewulf database
+       command: sudo wwsh node list
+       register: wwsh_node_list
+       ignore_errors: yes
+
+     - name: Add node(s) into warewulf database
+       command: wwsh -y node new {{ item.name }} --ipaddr={{ item.ip }} --hwaddr={{ item.mac }} -D {{ compute_private_nic }}
+       when: "'{{ item.name }}' not in wwsh_node_list.stdout"
+       with_items: "{{ compute_nodes }}"
+
+     - name: Register node IP and MAC address in warewulf database
+       command: sudo wwsh node set {{ item.name }} -y -I {{ item.ip }} --hwadd {{ item.mac }}
+       when: "'{{ item.name }}' in wwsh_node_list.stdout"
+       with_items: "{{ compute_nodes }}"
+
+     - name: set files to provision
+       command: wwsh -y provision set {{ item.name }} --vnfs={{ item.vnfs }} --bootstrap={{ build_kernel_ver }} --files=passwd,group,shadow,munge.key,slurm.conf,dynamic_hosts,network --bootlocal=exit
+       with_items: "{{ compute_nodes }}"
+
+     - name: remove node from slurm.conf if it exists already # to avoid duplication!
+       lineinfile:
+         dest: /etc/slurm/slurm.conf
+         regexp: "^NodeName={{ item.name }}"
+         state: absent
+       with_items: "{{ compute_nodes }}"
+
+     - name: add node to slurm.conf
+       lineinfile:
+         dest: /etc/slurm/slurm.conf
+         line: "NodeName={{ item.name }} Sockets={{ item.sockets }} CoresPerSocket={{ item.corespersocket }} State=UNKNOWN"
+         insertbefore: "^# PARTITIONS"
+         state: present
+       with_items: "{{ compute_nodes }}"
+
+     when: node_inventory_auto == false
+
+   - name: sync files #also generates dynamic hosts on headnode!
+     command: wwsh file sync
+
+#   - name: add compute nodes to ansible inventory for wait
+#     add_host: name={{ node_glob }} group="compute nodes"
+
+#   - name: wait for compute nodes to boot
+#     local_action: wait_for host={{ last_node }} state=started delay=30 timeout=600
+
+
+#   vars:
+#    - compute_node_glob: "{{ compute_node_prefix }}[0-{{ num_compute_nodes|int - 1}}]"
+#    - node_glob_bash: "{{ compute_node_prefix }}{0..{{ num_compute_nodes|int - 1}}}"
+#    - last_node: "{{ node_prefix }}{{ num_nodes|int - 1 }}"