Skip to content

Commit

Permalink
add mysql encoding test (#1838)
Browse files Browse the repository at this point in the history 
* add mysql encoding test

* add mysql encoding test

* fix bug
  • Loading branch information
@zhoucheng361
zhoucheng361 authored Apr 20, 2022
1 parent d16a03b commit 179de1f
Showing 1 changed file with 300 additions and 0 deletions.
300 changes: 300 additions & 0 deletions .github/workflows/encoding_mysql.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
name: "mysql-encoding-test"

on:
push:
branches:
- release-**
paths-ignore:
- 'docs/**'
- '**.md'
pull_request:
#The branches below must be a subset of the branches above
branches:
- release-**
paths-ignore:
- 'docs/**'
- '**.md'
workflow_dispatch:

jobs:
encoding-test:
runs-on: [self-hosted, daily-build]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 1

- name: Set Variable
id: vars
run: |
echo ::set-output name=META_URL1::mysql://root:${{secrets.MYSQL_PASSWORD}}@(172.27.0.1)/mysql_encoding1
echo ::set-output name=META_URL2::mysql://root:${{secrets.MYSQL_PASSWORD}}@(172.27.0.1)/mysql_encoding2
echo ::set-output name=MOUNT_POINT1::/tmp/juicefs-mysql-encoding-test1
echo ::set-output name=MOUNT_POINT2::/tmp/juicefs-mysql-encoding-test2
echo ::set-output name=BUCKET::/var/jfs/
echo ::set-output name=NAME::mysql-encoding-test
- name: Build linux target
run: |
export GOPATH=/usr/local/go
export HOME=/root
make juicefs
wget https://github.com/juicedata/juicefs/releases/download/v1.0.0-beta2/juicefs-1.0.0-beta2-linux-amd64.tar.gz
tar zxf https://github.com/juicedata/juicefs/releases/download/v1.0.0-beta2/juicefs-1.0.0-beta2-linux-amd64.tar.gz
mv juicefs juicefs_old
- name: Init
run: |
if [ -d ${{ steps.vars.outputs.MOUNT_POINT1 }} ]; then
juicefs umount ${{ steps.vars.outputs.MOUNT_POINT1 }} || true
fi
UUID=$(juicefs status ${{ steps.vars.outputs.META_URL1 }} | grep UUID | cut -d '"' -f 4)
if [ -n "$UUID" ];then
echo UUID is: $UUID
sudo juicefs destroy --force ${{ steps.vars.outputs.META_URL }} $UUID
fi
db_name=$(echo "${{steps.vars.outputs.META_URL1}}" | awk -F/ 'print $4')
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "drop database $db_name; create database $db_name;"
if [ -d ${{ steps.vars.outputs.MOUNT_POINT2 }} ]; then
juicefs umount ${{ steps.vars.outputs.MOUNT_POINT2 }} || true
fi
UUID=$(juicefs status ${{ steps.vars.outputs.META_URL2 }} | grep UUID | cut -d '"' -f 4)
if [ -n "$UUID" ];then
echo UUID is: $UUID
sudo juicefs destroy --force ${{ steps.vars.outputs.META_URL2 }} $UUID
fi
db_name=$(echo "${{steps.vars.outputs.META_URL2}}" | awk -F/ 'print $4')
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "drop database $db_name; create database $db_name;"
if [ -f ${{ steps.vars.outputs.BUCKET}}/${{ steps.vars.outputs.NAME}} ]; then
rm ${{ steps.vars.outputs.BUCKET}}/${{ steps.vars.outputs.NAME}} -f
fi
- name: Test mysql gbk support with new client
run: |
filename="file_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
dirname="dir_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
attr_key=user.k
attr_value_set="attr_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
data_write="data_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
soft_link="softlink_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
juicefs format --bucket ${{ steps.vars.outputs.BUCKET}} ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.NAME}}
juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.MOUNT_POINT1}} -d
echo $data_write > `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`
mkdir `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"`
setfattr -n $attr_key -v $attr_value_set `printf "/tmp/juicefs-encoding-test/$filename"`
ln -s `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"` `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link"`
juicefs dump ${{steps.vars.outputs.META_URL1}} backup.dump
juicefs load ${{steps.vars.outputs.META_URL2}} backup.dump
juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL2}} ${{steps.vars.outputs.MOUNT_POINT2}} -d
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$filename"` ]; then
echo "Fatal: file does not exist: ", ${{steps.vars.outputs.MOUNT_POINT2}}/$filename
exit 1
fi
data_read=$(cat `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$filename"`)
if [ "$data_write" != "$data_read" ]; then
echo "Fatal: data read is not the same as data write:", $data_write, "data read:", $data_read
exit 1
fi
if [ ! -d `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$dirname"` ]; then
echo "Fatal: directory does not exist", ${{steps.vars.outputs.MOUNT_POINT2}}/$dirname
exit 1
fi
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$soft_link"` ]; then
echo "Fatal: soft link does not exist: ", ${{steps.vars.outputs.MOUNT_POINT2}}/$soft_link
exit 1
fi
attr_get=$(getfattr -e text -n $attr_key `printf "/tmp/juicefs-encoding-test2/$filename"` | grep "$attr_key" | awk -F= '{print $2}' | sed 's/^.\(.*\).$/\1/' | sed 's/\\\\/\\/g' )
attr_value_set=$(echo $attr_value_set | sed 's/\\\\/\\/g')
if [ "$attr_get" != "$attr_value_set" ]; then
echo "Fatal: attr get: ":, $attr_get, " is not equal to attr set:" $attr_value_set
exit 1
fi
- name: Test mount with old client and alter db to binary
run: |
filename="file_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
dirname="dir_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
attr_key=user.k
attr_value_set="attr_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
data_write="data_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
soft_link="softlink_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
./juicefs_old format --bucket ${{ steps.vars.outputs.BUCKET}} ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.NAME}}
./juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.MOUNT_POINT1}} -d
db_name=$(echo "${{steps.vars.outputs.META_URL1}}" | awk -F/ 'print $4')
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "use $db_name; alter table jfs_edge modify name varbinary(255) not null;"
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "use $db_name; alter table jfs_symlink modify target varbinary(4096) not null;"
echo $data_write > `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`
mkdir `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"`
setfattr -n $attr_key -v $attr_value_set `printf "/tmp/juicefs-encoding-test/$filename"`
ln -s `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"` `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link"`
./juicefs_old dump ${{steps.vars.outputs.META_URL1}} backup.dump
juicefs load ${{steps.vars.outputs.META_URL2}} backup.dump
juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL2}} ${{steps.vars.outputs.MOUNT_POINT2}} -d
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$filename"` ]; then
echo "Fatal: file does not exist: ", ${{steps.vars.outputs.MOUNT_POINT2}}/$filename
exit 1
fi
data_read=$(cat `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$filename"`)
if [ "$data_write" != "$data_read" ]; then
echo "Fatal: data read is not the same as data write:", $data_write, "data read:", $data_read
exit 1
fi
if [ ! -d `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$dirname"` ]; then
echo "Fatal: directory does not exist", ${{steps.vars.outputs.MOUNT_POINT2}}/$dirname
exit 1
fi
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT2}}/$soft_link"` ]; then
echo "Fatal: soft link does not exist: ", ${{steps.vars.outputs.MOUNT_POINT2}}/$soft_link
exit 1
fi
attr_get=$(getfattr -e text -n $attr_key `printf "/tmp/juicefs-encoding-test2/$filename"` | grep "$attr_key" | awk -F= '{print $2}' | sed 's/^.\(.*\).$/\1/' | sed 's/\\\\/\\/g' )
attr_value_set=$(echo $attr_value_set | sed 's/\\\\/\\/g')
if [ "$attr_get" != "$attr_value_set" ]; then
echo "Fatal: attr get: ":, $attr_get, " is not equal to attr set:" $attr_value_set
exit 1
fi
- name: Test remount with new client after alter db to binary
run: |
filename="file_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
dirname="dir_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
attr_key=user.k
attr_value_set="attr_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
data_write="data_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
soft_link="softlink_\xb9\xfb\xd6\xad\xca\xfd\xbe\xdd\\\"%%\\\\"
./juicefs_old format --bucket ${{ steps.vars.outputs.BUCKET}} ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.NAME}}
./juicefs_old mount --enable-xattr ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.MOUNT_POINT1}} -d
db_name=$(echo "${{steps.vars.outputs.META_URL1}}" | awk -F/ 'print $4')
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "use $db_name; alter table jfs_edge modify name varbinary(255) not null;"
mysql -h 172.27.0.1 -u root -p${{secrets.MYSQL_PASSWORD}} -e "use $db_name; alter table jfs_symlink modify target varbinary(4096) not null;"
echo $data_write > `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`
mkdir `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"`
setfattr -n $attr_key -v $attr_value_set `printf "/tmp/juicefs-encoding-test/$filename"`
./juicefs_old umount ${{steps.vars.outputs.MOUNT_POINT1}} --force
juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.MOUNT_POINT1}} -d
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"` ]; then
echo "Fatal: file does not exist: ", ${{steps.vars.outputs.MOUNT_POINT1}}/$filename
exit 1
fi
data_read=$(cat `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`)
if [ "$data_write" != "$data_read" ]; then
echo "Fatal: data read is not the same as data write:", $data_write, "data read:", $data_read
exit 1
fi
if [ ! -d `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"` ]; then
echo "Fatal: directory does not exist", ${{steps.vars.outputs.MOUNT_POINT1}}/$dirname
exit 1
fi
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link"` ]; then
echo "Fatal: soft link does not exist: ", ${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link
exit 1
fi
attr_get=$(getfattr -e text -n $attr_key `printf "/tmp/juicefs-encoding-test2/$filename"` | grep "$attr_key" | awk -F= '{print $2}' | sed 's/^.\(.*\).$/\1/' | sed 's/\\\\/\\/g' )
attr_value_set=$(echo $attr_value_set | sed 's/\\\\/\\/g')
if [ "$attr_get" != "$attr_value_set" ]; then
echo "Fatal: attr get: ":, $attr_get, " is not equal to attr set:" $attr_value_set
exit 1
fi
- name: Test mount old meta using new client without db change
run: |
filename="file1"
dirname="dir1"
attr_key=user.k
attr_value_set="attr1"
data_write="data1"
soft_link="softlink1"
./juicefs_old format --bucket ${{ steps.vars.outputs.BUCKET}} ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.NAME}}
juicefs mount --enable-xattr ${{steps.vars.outputs.META_URL1}} ${{steps.vars.outputs.MOUNT_POINT1}} -d
echo $data_write > `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`
mkdir `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"`
setfattr -n $attr_key -v $attr_value_set `printf "/tmp/juicefs-encoding-test/$filename"`
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"` ]; then
echo "Fatal: file does not exist: ", ${{steps.vars.outputs.MOUNT_POINT1}}/$filename
exit 1
fi
data_read=$(cat `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$filename"`)
if [ "$data_write" != "$data_read" ]; then
echo "Fatal: data read is not the same as data write:", $data_write, "data read:", $data_read
exit 1
fi
if [ ! -d `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$dirname"` ]; then
echo "Fatal: directory does not exist", ${{steps.vars.outputs.MOUNT_POINT1}}/$dirname
exit 1
fi
if [ ! -f `printf "${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link"` ]; then
echo "Fatal: soft link does not exist: ", ${{steps.vars.outputs.MOUNT_POINT1}}/$soft_link
exit 1
fi
attr_get=$(getfattr -e text -n $attr_key `printf "/tmp/juicefs-encoding-test2/$filename"` | grep "$attr_key" | awk -F= '{print $2}' | sed 's/^.\(.*\).$/\1/' | sed 's/\\\\/\\/g' )
attr_value_set=$(echo $attr_value_set | sed 's/\\\\/\\/g')
if [ "$attr_get" != "$attr_value_set" ]; then
echo "Fatal: attr get: ":, $attr_get, " is not equal to attr set:" $attr_value_set
exit 1
fi
- name: Clean
if: always()
run: |
if [ -d ${{ steps.vars.outputs.MOUNT_POINT1 }} ]; then
juicefs umount ${{ steps.vars.outputs.MOUNT_POINT1 }} || true
fi
UUID=$(juicefs status ${{ steps.vars.outputs.META_URL1 }} | grep UUID | cut -d '"' -f 4)
if [ -n "$UUID" ];then
echo UUID is: $UUID
sudo juicefs destroy --force ${{ steps.vars.outputs.META_URL }} $UUID
fi
if [ -d ${{ steps.vars.outputs.MOUNT_POINT2 }} ]; then
juicefs umount ${{ steps.vars.outputs.MOUNT_POINT2 }} || true
fi
UUID=$(juicefs status ${{ steps.vars.outputs.META_URL2 }} | grep UUID | cut -d '"' -f 4)
if [ -n "$UUID" ];then
echo UUID is: $UUID
sudo juicefs destroy --force ${{ steps.vars.outputs.META_URL2 }} $UUID
fi
if [ -f ${{ steps.vars.outputs.BUCKET}}/${{ steps.vars.outputs.NAME}} ]; then
rm ${{ steps.vars.outputs.BUCKET}}/${{ steps.vars.outputs.NAME}} -f
fi
- name: Send Slack Notification
if: ${{ failure() }}
uses: juicedata/slack-notify-action@main
with:
channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"

0 comments on commit 179de1f

Please sign in to comment.