Merge branch 'main' into random_meta_test

juicedata · Jan 9, 2024 · 25cb44e · 25cb44e
2 parents 631bc8d + 5d644fe
commit 25cb44e
Show file tree

Hide file tree

Showing 138 changed files with 3,497 additions and 786 deletions.
diff --git a/.github/scripts/chaos/workflow.yaml b/.github/scripts/chaos/workflow.yaml
@@ -65,7 +65,7 @@ spec:
           labelSelectors:
             app: minio-server
         volumePath: /data
-        delay: '1s'
+        delay: '50ms'
     # minio 内存压力
     - name: minio-memory
       templateType: StressChaos

diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh
@@ -25,7 +25,7 @@ test_dump_load_with_iflag(){
     ./juicefs mount -d $META_URL /jfs --enable-ioctl
     echo "hello" > /jfs/hello.txt
     chattr +i /jfs/hello.txt
-    ./juicefs dump $META_URL dump.json
+    ./juicefs dump $META_URL dump.json --fast
     umount_jfs /jfs $META_URL
     python3 .github/scripts/flush_meta.py $META_URL
     ./juicefs load $META_URL dump.json
@@ -40,7 +40,7 @@ test_dump_with_keep_secret()
 {
     prepare_test
     ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
-    ./juicefs dump --keep-secret-key $META_URL dump.json
+    ./juicefs dump --keep-secret-key $META_URL dump.json --fast
     python3 .github/scripts/flush_meta.py $META_URL
     ./juicefs load $META_URL dump.json
     ./juicefs mount -d $META_URL /jfs
@@ -52,7 +52,7 @@ test_dump_without_keep_secret()
 {
     prepare_test
     ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
-    ./juicefs dump $META_URL dump.json
+    ./juicefs dump $META_URL dump.json --fast
     python3 .github/scripts/flush_meta.py $META_URL
     ./juicefs load $META_URL dump.json
     ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true
@@ -110,7 +110,7 @@ do_dump_load(){
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
     python3 .github/scripts/fsrand.py -c 1000 /jfs/fsrand -v -a
-    ./juicefs dump  $META_URL $dump_file
+    ./juicefs dump  $META_URL $dump_file --fast
 
     ./juicefs load  sqlite3://test2.db $dump_file   
     ./juicefs mount -d sqlite3://test2.db /jfs2

diff --git a/.github/scripts/command/load_dump_bench.sh b/.github/scripts/command/load_dump_bench.sh
@@ -25,7 +25,7 @@ test_load_dump_with_small_dir(){
   python3 .github/scripts/db.py --name load_small_dir --result $runtime --version $version --meta $META --storage file
   echo "load cost $runtime seconds"
   start=`date +%s`
-  ./juicefs dump $META_URL dump.json
+  ./juicefs dump $META_URL dump.json --fast
   end=`date +%s`
   runtime=$((end-start))
   echo "dump cost $runtime seconds"
@@ -63,9 +63,9 @@ do_load_dump_with_big_dir(){
   python3 .github/scripts/db.py --name load_big_dir --result $runtime --version $version --meta $META --storage file
   start=`date +%s`
   if [ "$with_subdir" = true ] ; then
-    ./juicefs dump $META_URL dump.json --subdir test
+    ./juicefs dump $META_URL dump.json --subdir test --fast
   else
-    ./juicefs dump $META_URL dump.json
+    ./juicefs dump $META_URL dump.json --fast
   fi
   end=`date +%s`
   runtime=$((end-start))

diff --git a/.github/scripts/command/quota.sh b/.github/scripts/command/quota.sh
@@ -55,7 +55,7 @@ test_total_inodes(){
     sleep $((HEARTBEAT_INTERVAL+1))
     set +x
     for i in {1001..2000}; do
-        echo $i | tee /jfs/test$i > /dev/null || (echo "df -i /jfs" && exit 1)
+        echo $i | tee /jfs/test$i > /dev/null || (df -i /jfs && ls /jfs/ -l | wc-l  && exit 1)
     done
     set -x
     sleep $VOLUME_QUOTA_FLUSH_INTERVAL
@@ -224,7 +224,7 @@ test_dump_load(){
     mkdir -p /jfs/d
     ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
     sleep $((HEARTBEAT_INTERVAL+1))
-    ./juicefs dump --log-level error $META_URL > dump.json 
+    ./juicefs dump --log-level error $META_URL --fast > dump.json
     umount_jfs /jfs $META_URL
     python3 .github/scripts/flush_meta.py $META_URL
     ./juicefs load $META_URL dump.json

diff --git a/.github/scripts/command/sync_cluster.sh b/.github/scripts/command/sync_cluster.sh
@@ -129,7 +129,7 @@ check_sync_log(){
     [ -z "$count3" ] && count3=0
     count1=$((file_count - count2 - count3))
     echo "count1, $count1, count2, $count2, count3, $count3"
-    min_count=$((file_count / 10))
+    min_count=$((file_count / 20))
     # check if count1 is less than min_count
     if [ "$count1" -lt "$min_count" ] || [ "$count2" -lt "$min_count" ] || [ "$count3" -lt "$min_count" ]; then
         echo "count is less than min_count, $count1, $count2, $count3, $min_count"

diff --git a/.github/scripts/command/sync_minio.sh b/.github/scripts/command/sync_minio.sh
@@ -113,7 +113,7 @@ prepare_test(){
     (./mc rb myminio/myjfs > /dev/null 2>&1 --force || true) && ./mc mb myminio/myjfs
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
-    lsof -i :9005 | awk 'NR!=1 {print $2}' | xargs -r kill -9
+    lsof -i :9005 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
     MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin ./juicefs gateway $META_URL localhost:9005 &
     ./mc alias set juicegw http://localhost:9005 minioadmin minioadmin --api S3v4
 }