Merge pull request #9 from netmanagers/master

Various fixes for crunch-dispatch-local and tests

arvados/api/package/clean.sls

@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.api.gem.name %}
+{%- for gm in arvados.api.gem.name %}
 arvados-api-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-api-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-api-package-clean-gems-deps-pkg-removed:
   pkg.removed:

arvados/api/package/install.sls

@@ -21,7 +21,7 @@ arvados-api-package-install-gems-deps-pkg-installed:
     - pkgs: {{ arvados.ruby.gems_deps | unique | json }}
     - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true"
 
-{% for gm in arvados.api.gem.name | unique %}
+{%- for gm in arvados.api.gem.name | unique %}
 arvados-api-package-install-gem-{{ gm }}-installed:
   gem.installed:
     - name: {{ gm }}
@@ -32,7 +32,7 @@ arvados-api-package-install-gem-{{ gm }}-installed:
       {%- endif %}
     - require_in:
       - pkg: arvados-api-package-install-pkg-installed
-{% endfor %}
+{%- endfor %}
 
 arvados-api-package-install-pkg-installed:
   pkg.installed:

arvados/dispatcher/service/file.sls

@@ -25,6 +25,23 @@ arvados-dispatcher-service-file-file-managed-crunch-run-sh:
     - user: root
     - group: root
     - makedirs: True
+    - context:
+        arvados: {{ arvados | json }}
+    - require:
+      - pkg: arvados-dispatcher-package-install-pkg-installed
+
+arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-credentials:
+  file.managed:
+    - name: /etc/arvados/crunch-dispatch-local-credentials
+    - source: {{ files_switch(['crunch-dispatch-local-credentials.tmpl'],
+                              lookup='arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-credentials',
+                              use_subpath=True
+                 )
+              }}
+    - mode: '0640'
+    - user: root
+    - group: root
+    - makedirs: True
     - template: jinja
     - context:
         arvados: {{ arvados | json }}
@@ -48,6 +65,7 @@ arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-service:
         arvados: {{ arvados | json }}
     - require:
       - file: arvados-dispatcher-service-file-file-managed-crunch-run-sh
+      - file: arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-credentials
       - pkg: arvados-dispatcher-package-install-pkg-installed
   cmd.run:
     - name: systemctl daemon-reload

arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl

@@ -0,0 +1,7 @@
+########################################################################
+# File managed by Salt at <{{ source }}>.
+# Your changes will be overwritten.
+########################################################################
+ARVADOS_API_HOST={{ arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) }}
+ARVADOS_API_HOST_INSECURE={{ '1' if arvados.cluster.tls.insecure | default('0') }}
+ARVADOS_API_TOKEN={{ arvados.cluster.tokens.system_root }}

arvados/dispatcher/service/files/default/crunch-dispatch-local-service.tmpl

@@ -15,7 +15,7 @@ StartLimitIntervalSec=0
 
 [Service]
 Type=simple
-EnvironmentFile=-/etc/arvados/environment
+EnvironmentFile=-/etc/arvados/crunch-dispatch-local-credentials
 ExecStart=/usr/bin/crunch-dispatch-local -poll-interval=1 -crunch-run-command=/usr/local/bin/crunch-run.sh
 # Set a reasonable default for the open file limit
 LimitNOFILE=65536

arvados/dispatcher/service/files/default/crunch-run-sh.tmpl

@@ -1,6 +1,6 @@
+#!/bin/sh
 ########################################################################
 # File managed by Salt at <{{ source }}>.
 # Your changes will be overwritten.
 ########################################################################
-#!/bin/sh
-exec /usr/bin/crunch-run -container-enable-networking=default -container-network-mode=host $@
+exec /usr/bin/crunch-run -container-enable-networking=default -container-network-mode=host ${@}

arvados/repo/clean.sls

@@ -5,8 +5,8 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% if arvados.use_upstream_repo -%}
-  {% if grains.get('os_family') == 'Debian' -%}
+{%- if arvados.use_upstream_repo %}
+  {%- if grains.get('os_family') == 'Debian' %}
 arvados-repo-clean-repo-absent:
   pkgrepo.absent:
     - file: {{ arvados.repo.file }}

arvados/repo/install.sls

@@ -5,8 +5,8 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{%- if arvados.use_upstream_repo -%}
-  {%- if grains.get('os_family') == 'Debian' -%}
+{%- if arvados.use_upstream_repo %}
+  {%- if grains.get('os_family') == 'Debian' %}
     {%- if arvados.release == 'testing' %}
       {%- set release = grains.get('lsb_distrib_codename') ~ '-testing' %}
     {%- elif arvados.release == 'development' %}

arvados/ruby/package/clean.sls

@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-shell-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-shell-package-clean-gems-deps-pkg-removed:
   pkg.removed:

arvados/shell/package/clean.sls

@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-shell-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-shell-package-clean-gems-deps-pkg-removed:
   pkg.removed:

arvados/shell/package/install.sls

@@ -37,7 +37,7 @@ arvados-shell-package-install-gems-deps-pkg-installed:
     - pkgs: {{ arvados.ruby.gems_deps | json }}
     - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true"
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-install-gem-{{ gm }}-installed:
   gem.installed:
     - name: {{ gm }}
@@ -46,4 +46,4 @@ arvados-shell-package-install-gem-{{ gm }}-installed:
       - {{ ruby_dep }}: arvados-ruby-package-install-ruby-{{ ruby_dep }}-installed
       {%- endif %}
       - pkg: arvados-shell-package-install-gems-deps-pkg-installed
-{% endfor %}
+{%- endfor %}

kitchen.yml

@@ -104,6 +104,7 @@ suites:
       state_top:
         base:
           '*':
+            - example_single_host_host_entries
             - example_add_snakeoil_certs
             - locale
             - nginx.passenger
@@ -145,6 +146,8 @@ suites:
         example_nginx_controller.sls: test/salt/pillar/examples/nginx_controller_configuration.sls
         # yamllint enable rule:line-length
       dependencies:
+        - name: example_single_host_host_entries
+          path: test/salt/states
         - name: example_add_snakeoil_certs
           path: test/salt/states
         - name: locale
@@ -172,6 +175,7 @@ suites:
       state_top:
         base:
           '*':
+            - example_single_host_host_entries
             - example_add_snakeoil_certs
             - nginx.passenger
             - arvados.repo
@@ -193,6 +197,8 @@ suites:
         example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls
         # yamllint enable rule:line-length
       dependencies:
+        - name: example_single_host_host_entries
+          path: test/salt/states
         - name: example_add_snakeoil_certs
           path: test/salt/states
         - name: nginx

pillar.example

@@ -82,19 +82,22 @@ arvados:
 
     ### TOKENS
     tokens:
-      system_root: changeme_system_root_token
-      management: changeme_management_token
-      rails_secret: changeme_rails_secret_token
-      anonymous_user: changeme_anonymous_user_token
+      # SystemRootToken has to be alphanumeric, it does not accept underscores
+      # or special characters. See
+      # https://dev.arvados.org/issues/17150
+      system_root: changemesystemroottoken
+      management: changememanagementtoken
+      rails_secret: changemerailssecrettoken
+      anonymous_user: changemeanonymoususertoken
 
     ### KEYS
     secrets:
-      blob_signing_key: changeme_blob_signing_key
-      workbench_secret_key: changeme_workbench_secret_key
-      dispatcher_access_key: changeme_dispatcher_access_key
-      dispatcher_secret_key: changeme_dispatcher_secret_key
-      keep_access_key: changeme_keep_access_key
-      keep_secret_key: changeme_keep_secret_key
+      blob_signing_key: changemeblobsigningkey
+      workbench_secret_key: changemeworkbenchsecretkey
+      dispatcher_access_key: changemedispatcheraccesskey
+      dispatcher_secret_key: changemedispatchersecretkey
+      keep_access_key: changemekeepaccesskey
+      keep_secret_key: changemekeepsecretkey
 
     AuditLogs:
       Section_to_ignore:

test/integration/api/controls/config_spec.rb

@@ -2,13 +2,13 @@
 
 api_stanza = <<-API_STANZA
     API:
-      RailsSessionSecretToken: "changeme_rails_secret_token"
+      RailsSessionSecretToken: "changemerailssecrettoken"
 API_STANZA
 
 rails_stanza = <<-RAILS_STANZA
       RailsAPI:
         InternalURLs:
-          http://127.0.0.2:8004: {}
+          http://api.internal:8004: {}
 RAILS_STANZA
 
 group = case os[:name]

test/integration/keepweb/controls/config_spec.rb

@@ -4,7 +4,7 @@
       WebDAV:
         ExternalURL: https://collections.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:9002: {}
+          http://collections.internal:9002: {}
       WebDAVDownload:
         ExternalURL: https://download.fixme.example.net
 KEEPWEB_STANZA

test/integration/websocket/controls/config_spec.rb

@@ -4,7 +4,7 @@
       Websocket:
         ExternalURL: wss://ws.fixme.example.net/websocket
         InternalURLs:
-          http://127.0.0.2:8005: {}
+          http://ws.internal:8005: {}
 WEBSOCKET_STANZA
 
 group = case os[:name]

test/integration/workbench/controls/config_spec.rb

@@ -2,7 +2,7 @@
 
 workbench_config = <<-WORKBENCH_STANZA
     Workbench:
-      SecretKeyBase: "changeme_workbench_secret_key"
+      SecretKeyBase: "changemeworkbenchsecretkey"
       SiteName: FIXME
 WORKBENCH_STANZA
 

test/salt/pillar/arvados.sls

@@ -58,19 +58,19 @@ arvados:
 
     ### TOKENS
     tokens:
-      system_root: changeme_system_root_token
-      management: changeme_management_token
-      rails_secret: changeme_rails_secret_token
-      anonymous_user: changeme_anonymous_user_token
+      system_root: changemesystemroottoken
+      management: changememanagementtoken
+      rails_secret: changemerailssecrettoken
+      anonymous_user: changemeanonymoususertoken
 
     ### KEYS
     secrets:
-      blob_signing_key: changeme_blob_signing_key
-      workbench_secret_key: changeme_workbench_secret_key
-      dispatcher_access_key: changeme_dispatcher_access_key
-      dispatcher_secret_key: changeme_dispatcher_secret_key
-      keep_access_key: changeme_keep_access_key
-      keep_secret_key: changeme_keep_secret_key
+      blob_signing_key: changemeblobsigningkey
+      workbench_secret_key: changemeworkbenchsecretkey
+      dispatcher_access_key: changemedispatcheraccesskey
+      dispatcher_secret_key: changemedispatchersecretkey
+      keep_access_key: changemekeepaccesskey
+      keep_secret_key: changemekeepsecretkey
 
     AuditLogs:
       Section_to_ignore:
@@ -100,7 +100,7 @@ arvados:
       Controller:
         ExternalURL: https://fixme.example.net
         InternalURLs:
-          http://127.0.0.2:8003: {}
+          http://controller.internal:8003: {}
       DispatchCloud:
         InternalURLs:
           http://fixme.example.net:9006: {}
@@ -110,25 +110,25 @@ arvados:
       Keepproxy:
         ExternalURL: https://keep.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:25100: {}
+          http://keep.internal:25100: {}
       Keepstore:
         InternalURLs:
           http://keep0.fixme.example.net:25107: {}
       RailsAPI:
         InternalURLs:
-          http://127.0.0.2:8004: {}
+          http://api.internal:8004: {}
       WebDAV:
         ExternalURL: https://collections.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:9002: {}
+          http://collections.internal:9002: {}
       WebDAVDownload:
         ExternalURL: https://download.fixme.example.net
       WebShell:
         ExternalURL: https://webshell.fixme.example.net
       Websocket:
         ExternalURL: wss://ws.fixme.example.net/websocket
         InternalURLs:
-          http://127.0.0.2:8005: {}
+          http://ws.internal:8005: {}
       Workbench1:
         ExternalURL: https://workbench.fixme.example.net
       Workbench2: