remake ftextcvt, improve log format and optimize code

YuriSizuku · Feb 28, 2024 · d8002f4 · d8002f4
1 parent 9ce82df
commit d8002f4
Show file tree

Hide file tree

Showing 19 changed files with 959 additions and 871 deletions.
diff --git a/.github/workflows/build_pyexe.yml b/.github/workflows/build_pyexe.yml
@@ -25,8 +25,7 @@ jobs:
 
     - name: make python enviroment
       run: |
-        python -m pip install nuitka zstandard
-        python -m pip install python-docx
+        python -m pip install -r .\project\pyexe_${{ matrix.pyexe_name }}\requirements.txt
     
     - name: build single pyexe by nuitka 
       run: |

diff --git a/.github/workflows/build_pysrc.yml b/.github/workflows/build_pysrc.yml
@@ -0,0 +1,55 @@
+name: build_pysrc
+on:
+  push: {tags:  ['v*'] } # Push events to matching v*, i.e. v1.0, v20.15.10
+  pull_request: 
+
+permissions:
+  contents: write
+
+jobs:
+  test_pysrc:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: pull and init
+      uses: actions/checkout@v3
+      with: {submodules: true}
+
+    - name: make enviroment
+      run: |
+        pip install python-docx pillow numpy numba
+
+    - name: test pysrc
+      run: |
+        cd project/pysrc_all && chmod +x *.sh && ./test_pysrc.sh
+
+    - name: test pycli
+      run: |
+        cd project/pysrc_all && chmod +x *.sh && ./test_pycli.sh
+
+  build_pysrc:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: pull and init
+      uses: actions/checkout@v3
+      with: {submodules: true}
+
+    - name: make enviroment
+      run: |
+        sudo apt-get update && sudo apt-get install p7zip-full
+        pip install python-docx pillow numpy numba
+    
+    - name: build pysrc
+      run: |
+        cd project/pysrc_all 
+        chmod +x *.sh && bash ./build_pysrc.sh
+        archive_name=LocalizationPytool_${{ github.ref_name }}
+        7z a ${archive_name}.7z build
+        7z rn ${archive_name}.7z build ${archive_name}
+
+    - name: create a release
+      uses: ncipollo/release-action@v1
+      if: github.event_name == 'push'
+      with:
+        artifacts: "project/pysrc_all/*.7z"
+        allowUpdates: "true"
+        token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/README.md b/README.md
@@ -25,20 +25,56 @@ See also, [GalgameReverse](https://github.com/YuriSizuku/GalgameReverse) for spe
 
 ## CLI Example
 
+Using ">" to load or save files in zip, such as `path1/file1.zip>path2/file2`
+For these examples, you need `mkdir -p project/pyexe_bintext/build` before.  
+
 ### bintext
 
 You can also replace `python src/libtext.py` with `cbintext.exe` in command line.  
 
 ```shell
+
+# insert ftext (save direct or in gz file)
 python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pyexe_bintext/build/COM001_rebuild.bin --log_level info --bytes_padding "2020" --bytes_fallback "815A" --insert_shorter --insert_longer  --text_replace "季" "季季季" --text_replace "煌びやかな光" "你你你你你" 
+python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pyexe_bintext/build/COM001_rebuild.bin.gz --log_level info
+
+# extract ftext from bin file (save direct or in zip file)
 python src/libtext.py extract project/pyexe_bintext/build/COM001_rebuild.bin -o project/pyexe_bintext/build/COM001_rebuild.txt --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+python src/libtext.py extract project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild.txt" --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+
+# check ftext (direct or in zip file)
 python src/libtext.py check project/pyexe_bintext/build/COM001_rebuild.txt --refer project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001_rebuild_check.txt" --log_level info -e sjis
+python src/libtext.py check "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild.txt" --refer project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild_check.txt" --log_level info -e sjis
 ```
 
 ### ftextpack
 
 ```shell
-python src/ftextpack.py test/sample/COM001 test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.fp01 -t test/sample/COM001.tbl --pack_compact
+# pack both of origin and new text in fp01 file
+python src/ftextpack.py test/sample/COM001 test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.fp01 -t test/sample/COM001.tbl --pack_org
+
+# pack compact mode in zip file
+python src/ftextpack.py test/sample/COM001 test/sample/COM001.txt -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001.fp01" -t test/sample/COM001.tbl --pack_compact
+```
+
+### ftextcvt
+
+``` shell
+# json convert
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.json
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.json -o project/pyexe_bintext/build/COM001.json.txt
+
+# csv convert
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.csv
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.csv -o project/pyexe_bintext/build/COM001.csv.txt
+
+# docx convert
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.docx
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.docx -o project/pyexe_bintext/build/COM001.docx.txt
+
+# pretty ftext format
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.json.txt -o project/pyexe_bintext/build/COM001.json.txt
+
 ```
 
 ## File Formats
@@ -133,3 +169,11 @@ v0.1, initial version with data.fp01
 v0.1.1, add allow_compat for smaller memory use
 v0.2, remake according to libtext v0.6
 ```
+
+* `ftextcvt.py`
+
+```shell
+v0.1, initial version with formatftext, docx2ftext, ftext2docx
+v0.2, add support for csv and json, compatiable with paratranz.cn
+v0.3, remake according to libtext v0.6
+```
diff --git a/project/pyexe_bintext/build_nuitka.bat b/project/pyexe_bintext/build_nuitka.bat
@@ -1,5 +1,4 @@
 @echo off
 call %~dp0_env.bat
 
-python -m pip install nuitka
 nuitka --standalone --onefile --full-compat --show-progress "%PYSRC_PATH%" --windows-icon-from-ico="%ICON_PATH%" --output-dir="%OUT_DIR%" -o "c%TARGET_NAME%.exe" --assume-yes-for-downloads
diff --git a/project/pyexe_bintext/build_pyinstaller.bat b/project/pyexe_bintext/build_pyinstaller.bat
@@ -1,4 +1,4 @@
-:: build by pyinstaller
+@echo off
 call %~dp0_env.bat
-python -m pip install pyinstaller
+
 pyinstaller -F "%PYSRC_PATH%" --name "%TARGET_NAME%.exe" --distpath="%OUT_DIR%" --workpath="%OUT_DIR%/obj/pyinstaller" --specpath="%OUT_DIR%/obj/pyinstaller" --icon="%ICON_PATH%" --exclude-module=numpy --exclude-module=PIL --console --clean --noupx -y
diff --git a/project/pyexe_bintext/build_viapyenv.bat b/project/pyexe_bintext/build_viapyenv.bat
diff --git a/project/pyexe_bintext/requirements.txt b/project/pyexe_bintext/requirements.txt
@@ -0,0 +1,2 @@
+zstandard
+nuitka
diff --git a/project/pyexe_ftextcvt/build_nuitka.bat b/project/pyexe_ftextcvt/build_nuitka.bat
@@ -1,4 +1,8 @@
-:: build single files
+@echo off
 call %~dp0_env.bat
-python -m pip install nuitka
-nuitka --standalone --onefile --full-compat --show-progress "%PYSRC_PATH%" --windows-icon-from-ico="%ICON_PATH%" --output-dir="%OUT_DIR%" -o "c%TARGET_NAME%.exe" --assume-yes-for-downloads
+
+for /f "tokens=* USEBACKQ" %%f IN (`python -c "import os, docx;print(os.path.dirname(docx.__file__))"`) DO (
+    set DOCX_DIR=%%f
+)
+
+nuitka --standalone --onefile --full-compat --show-progress "%PYSRC_PATH%" --windows-icon-from-ico="%ICON_PATH%" --output-dir="%OUT_DIR%" -o "c%TARGET_NAME%.exe" --include-data-dir="%DOCX_DIR%\templates=docx/templates" --assume-yes-for-downloads
diff --git a/project/pyexe_ftextcvt/build_pyinstaller.bat b/project/pyexe_ftextcvt/build_pyinstaller.bat
@@ -1,4 +1,4 @@
-:: build by pyinstaller
+@echo off
 call %~dp0_env.bat
-python -m pip install pyinstaller
+
 pyinstaller -F "%PYSRC_PATH%" --name "%TARGET_NAME%.exe" --distpath="%OUT_DIR%" --workpath="%OUT_DIR%/obj/pyinstaller" --specpath="%OUT_DIR%/obj/pyinstaller" --icon="%ICON_PATH%" --exclude-module=numpy --exclude-module=PIL --console --clean --noupx -y
diff --git a/project/pyexe_ftextcvt/build_viapyenv.bat b/project/pyexe_ftextcvt/build_viapyenv.bat
diff --git a/project/pyexe_ftextcvt/requirements.txt b/project/pyexe_ftextcvt/requirements.txt
@@ -0,0 +1,3 @@
+zstandard
+nuitka
+python-docx
diff --git a/project/pysrc_all/build_pysrc.sh b/project/pysrc_all/build_pysrc.sh
@@ -1,19 +1,19 @@
-build_pysrc()
-{
-    src_dir=$1
-    dst_dir=$2
-    module_name=$3
-    module_ver=$(python -c "import sys, os; sys.path.append(r'$src_dir'); import $module_name as l; print(l.__version__)")
-    echo build ${module_name}_v${module_ver}.py
-    cp -f $src_dir/${module_name}.py $dst_dir/${module_name}_v${module_ver}.py
-}
-
-if ! [ -d build ]; then mkdir build; fi
-
-build_pysrc ../../src ./build libutil
-build_pysrc ../../src ./build libtext
-# build_pysrc ../../src ./build libfont
-# build_pysrc ../../src ./build libimage
-# build_pysrc ../../src ./build libalg
-# build_pysrc ../../src ./build ftextcvt
-build_pysrc ../../src ./build ftextpack
+build_pysrc()
+{
+    src_dir=$1
+    dst_dir=$2
+    module_name=$3
+    module_ver=$(python -c "import sys, os; sys.path.append(r'$src_dir'); import $module_name as l; print(l.__version__)")
+    echo build ${module_name}_v${module_ver}.py
+    cp -f $src_dir/${module_name}.py $dst_dir/${module_name}_v${module_ver}.py
+}
+
+if ! [ -d build ]; then mkdir build; fi
+
+build_pysrc ../../src ./build libutil
+build_pysrc ../../src ./build libtext
+# build_pysrc ../../src ./build libfont
+# build_pysrc ../../src ./build libimage
+# build_pysrc ../../src ./build libalg
+build_pysrc ../../src ./build ftextcvt
+build_pysrc ../../src ./build ftextpack
diff --git a/project/pysrc_all/test_pycli.sh b/project/pysrc_all/test_pycli.sh
@@ -0,0 +1,30 @@
+pushd ../..
+mkdir -p project/pyexe_bintext/build
+rm -rf project/pyexe_bintext/build/COM001.zip
+
+echo "## test libtext insert"
+python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pyexe_bintext/build/COM001_rebuild.bin --log_level info --bytes_padding "2020" --bytes_fallback "815A" --insert_shorter --insert_longer  --text_replace "季" "季季季" --text_replace "煌びやかな光" "你你你你你" 
+python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pyexe_bintext/build/COM001_rebuild.bin.gz --log_level info
+
+echo "## test libtext extract"
+python src/libtext.py extract project/pyexe_bintext/build/COM001_rebuild.bin -o project/pyexe_bintext/build/COM001_rebuild.txt --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+python src/libtext.py extract project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild.txt" --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+
+echo "## test libtext check"
+python src/libtext.py check project/pyexe_bintext/build/COM001_rebuild.txt --refer project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001_rebuild_check.txt" --log_level info -e sjis
+python src/libtext.py check "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild.txt" --refer project/pyexe_bintext/build/COM001_rebuild.bin -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001_rebuild_check.txt" --log_level info -e sjis
+
+echo "## test ftextpack"
+python src/ftextpack.py test/sample/COM001 test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.fp01 -t test/sample/COM001.tbl --pack_org
+python src/ftextpack.py test/sample/COM001 test/sample/COM001.txt -o "project/pyexe_bintext/build/COM001.zip>COM001/COM001.fp01" -t test/sample/COM001.tbl --pack_compact
+
+echo "## test ftextcvt"
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.json
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.json -o project/pyexe_bintext/build/COM001.json.txt
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.csv
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.csv -o project/pyexe_bintext/build/COM001.csv.txt
+python src/ftextcvt.py test/sample/COM001.txt -o project/pyexe_bintext/build/COM001.docx
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.docx -o project/pyexe_bintext/build/COM001.docx.txt
+python src/ftextcvt.py project/pyexe_bintext/build/COM001.json.txt -o project/pyexe_bintext/build/COM001.json.txt
+
+popd