New pipeline data validation (#1488)

Azure · Aug 2, 2024 · 9fb7645 · 9fb7645
1 parent 2c1781e
commit 9fb7645
Show file tree

Hide file tree

Showing 2 changed files with 141 additions and 0 deletions.
diff --git a/azure-pipeline-templates/blobfuse2-data-validation.yml b/azure-pipeline-templates/blobfuse2-data-validation.yml
@@ -0,0 +1,67 @@
+parameters:
+  - name: conf_template
+    type: string
+  - name: config_file
+    type: string
+  - name: container
+    type: string
+  - name: temp_dir
+    type: string
+  - name: mount_dir
+    type: string    
+  - name: idstring
+    type: string
+  - name: adls
+    type: boolean
+  - name: account_name
+    type: string
+  - name: account_key
+    type: string
+  - name: account_type
+    type: string
+  - name: account_endpoint
+  - name: distro_name
+    type: string
+  - name: quick_test
+    type: boolean
+    default: true
+  - name: verbose_log
+    type: boolean
+    default: false
+  - name: clone
+    type: boolean
+    default: false
+  - name: kversion
+    type: string
+
+steps:    
+  - template: 'mount.yml'
+    parameters:
+      working_dir: $(WORK_DIR)
+      mount_dir: ${{ parameters.mount_dir }}
+      temp_dir: ${{ parameters.temp_dir }}
+      prefix: ${{ parameters.idstring }}
+      mountStep: 
+        script: |
+          $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) 
+
+  - script: |
+      cd ${{ parameters.mount_dir }}
+      wget https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${{ parameters.kversion }}.tar.xz  
+    displayName: 'Get kernel tarfile'
+
+  - script: |
+      tar -xvf {{ parameters.mount_dir }}/linux-${{ parameters.kversion }}.tar.xz
+    displayName: 'Untar kernel'
+
+  - script: |
+      cd {{ parameters.mount_dir }}/linux-${{ parameters.kversion }}
+      make defconfig
+      make
+    displayName: 'Run MAKE on the kernel'
+
+  - template: 'cleanup.yml'
+    parameters:
+      working_dir: $(WORK_DIR)
+      mount_dir: ${{ parameters.mount_dir }}
+      temp_dir: ${{ parameters.temp_dir }}
diff --git a/blobfuse2-data-validation.yaml b/blobfuse2-data-validation.yaml
@@ -0,0 +1,74 @@
+stages:
+  - stage: KernelBuild 
+    jobs:
+      # Ubuntu Tests
+      - job: Set_1
+        timeoutInMinutes: 360
+        strategy:
+          matrix:
+            Ubuntu-22:
+              AgentName: 'blobfuse-benchmark-ubn22'
+              containerName: 'test-cnt-ubn-22'
+        pool:
+          name: "blobfuse-perf-pool"
+          demands:
+            - ImageOverride -equals $(AgentName)
+
+        variables:
+          - group: NightlyBlobFuse
+          - name: ROOT_DIR
+            value: "/usr/pipeline/workv2"
+          - name: WORK_DIR
+            value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
+          - name: MOUNT_DIR
+            value: "/usr/pipeline/workv2/blob_mnt"
+          - name: TEMP_DIR
+            value: "/usr/pipeline/workv2/temp"
+          - name: BLOBFUSE2_CFG
+            value: "/usr/pipeline/workv2/blobfuse2.yaml"
+          - name: GOPATH
+            value: "/usr/pipeline/workv2/go"
+
+        steps:
+          - template: 'azure-pipeline-templates/setup.yml'
+            parameters:
+              tags: $(tags)
+              installStep:
+                script: |
+                  sudo apt-get update --fix-missing
+                  sudo apt update
+                  sudo apt-get install cmake gcc libfuse3-dev git parallel -y
+                  sudo apt-get install fuse3 -y
+                displayName: 'Install fuse'
+
+          - script: |
+              sudo apt-get install git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison -y
+            displayName: 'Install kernel build dependencies'
+
+          - script: |
+              cd $(WORK_DIR)
+              $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_block_perf.yaml --container-name=$(containerName) --output-file=$(BLOBFUSE2_CFG)
+            displayName: "Create Config File"
+            env:
+                NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME)
+                NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY)
+                ACCOUNT_TYPE: 'block'
+                ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net'
+                VERBOSE_LOG: ${{ parameters.verbose_log }}
+            continueOnError: false
+              
+          - script: |
+              cat $(BLOBFUSE2_CFG)
+            displayName: 'Print config file'
+
+          - template: 'azure-pipeline-templates/blobfuse2-data-validation.yml'
+            parameters:
+              working_dir: $(WORK_DIR)
+              mount_dir: $(MOUNT_DIR)
+              temp_dir: $(TEMP_DIR)
+              prefix: 'ubn-22'
+              kversion: "6.10.2"
+
+
+
+