Update playbook for dealing with NFS out of space and use kustomize. (k…

…ubeflow#583)

* Instead of using ksonnet to manage the PV and PVC for NFS switch
  to using kustomize and update the playbook.

Fix kubeflow#582

playbook.md

@@ -166,15 +166,14 @@ kubectl config set-context $(kubectl config current-context) --namespace=kubeflo
 
 1. Set the IP address in the PV
 
-   ```
-   cd test-infra/ks_app
-   ks param set --env=kubeflow-ci nfs-external nfsServer <NFS-IP-address>
-   ```
+   * Edit `test-infra/base/pvc.yaml`
+   * Change the server address of the persistent volume to the new PV   
 
 1. Recreate the PV and PVC
 
    ```
-   ks apply kubeflow-ci -c nfs-external
+   cd test-infra/
+   kustomize build base | kubectl apply -f -
    ```
 
 1. Make sure the `debug-worker-0` pod is able to successfully mount the PV

test-infra/base/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow-test-infra  
+resources:
+- pvc.yaml
+- statefulset.yaml

test-infra/base/nfs-external.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+  name: nfs-external
+  namespace: kubeflow-test-infra  
+spec:
+  accessModes:
+  - ReadWriteMany
+  dataSource: null
+  resources:
+    requests:
+      storage: 500Mi
+  storageClassName: gcfs-storage
+  volumeMode: Filesystem
+  volumeName: gcfs

test-infra/base/pvc.yaml

@@ -0,0 +1,33 @@
+apiVersion: "v1"
+kind: "PersistentVolume"
+metadata: 
+  name: gcfs
+spec: 
+  accessModes:
+  - "ReadWriteMany"  
+  capacity:
+    storage: "5Gi"
+  nfs:
+    path: "/kubeflow"
+    server: "10.86.7.242"  
+  persistentVolumeReclaimPolicy: "Retain"
+  storageClassName: gcfs-storage
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nfs-external
+  annotations:
+    "volume.beta.kubernetes.io/storage-class": gcfs-storage
+spec:
+  accessModes:
+  - ReadWriteMany
+  dataSource: null
+  resources:
+    requests:
+      storage: 500Mi
+  storageClassName: gcfs-storage
+  volumeMode: Filesystem
+  volumeName: gcfs

test-infra/base/statefulset.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: debug-worker
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: debug-worker
+  serviceName: ""
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: debug-worker
+    spec:
+      containers:
+      - command:
+        - tail
+        - -f
+        - /dev/null
+        env:
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /secret/gcp-credentials/key.json
+        image: gcr.io/kubeflow-ci/test-worker:latest
+        imagePullPolicy: Always
+        name: test-container
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /mnt/test-data-volume
+          name: nfs-external
+        - mountPath: /secret/gcp-credentials
+          name: gcp-credentials
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: nfs-external
+        persistentVolumeClaim:
+          claimName: nfs-external
+      - name: gcp-credentials
+        secret:
+          defaultMode: 420
+          secretName: kubeflow-testing-credentials
+  updateStrategy:
+    type: RollingUpdate