add batch operations to lower overhead

project/pysrc_all/History.md

@@ -26,6 +26,7 @@ v0.5.6, add typing hint and prepare read lines for pyscript in web
 v0.5.7, add repalced map in check method, fix -e in check 
 v0.5.8, add f_extension for {{}}, f_adjust in patch_text, and align for patch
 v0.6, remake to increase speed and simplify functions
+v0.6.1, add batch mode on extract, insert to optimize performance
 ```
 
 * `libfont.py`
@@ -59,6 +60,7 @@ v0.1.1, added BGR mode
 v0.2, add swizzle method
 v0.2.1, change cv2 to PIL.image
 v0.3, remake with libutil v0.6, accelerate by numba parallel
+v0.3.1, add batch mode to optimize performance
 ```
 
 * `libword.py`

project/pysrc_all/test_pycli.sh

@@ -3,14 +3,17 @@ test_libtext()
     echo "## test libtext insert"
     python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pysrc_all//build/COM001_rebuild.bin --log_level info --bytes_padding "2020" --bytes_fallback "815A" --insert_shorter --insert_longer  --text_replace "季" "季季季" --text_replace "煌びやかな光" "你你你你你" 
     python src/libtext.py insert test/sample/COM001 test/sample/COM001.txt --refer test/sample/COM001 -t test/sample/COM001.tbl -o project/pysrc_all//build/COM001_rebuild.bin.gz --log_level info
+    python src/libtext.py insert --batch "test/sample;COM001" "test/sample;COM001.txt" --refer "test/sample;COM001"  -t test/sample/COM001.tbl -o "project/pysrc_all/build;COM001_rebuild.bin" --bytes_padding "2020" --bytes_fallback "815A"
 
     echo "## test libtext extract"
-    python src/libtext.py extract project/pysrc_all//build/COM001_rebuild.bin -o project/pysrc_all//build/COM001_rebuild.txt --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
-    python src/libtext.py extract project/pysrc_all//build/COM001_rebuild.bin -o "project/pysrc_all//build/COM001.zip>COM001/COM001_rebuild.txt" --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+    python src/libtext.py extract project/pysrc_all//build/COM001_rebuild.bin -o project/pysrc_all/build/COM001_rebuild.txt --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+    python src/libtext.py extract project/pysrc_all//build/COM001_rebuild.bin -o "project/pysrc_all/build/COM001.zip>COM001/COM001_rebuild.txt" --log_level info -e sjis --has_cjk --min_len 4 --skip 0x16 --size 1024
+    python src/libtext.py extract --batch "project/pysrc_all//build;COM001_rebuild.bin" -o "project/pysrc_all//build;COM001_rebuild.txt" --log_level info -e sjis --has_cjk --min_len 4
 
     echo "## test libtext check"
     python src/libtext.py check project/pysrc_all//build/COM001_rebuild.txt --refer project/pysrc_all//build/COM001_rebuild.bin -o "project/pysrc_all//build/COM001_rebuild_check.txt" --log_level info -e sjis
     python src/libtext.py check "project/pysrc_all//build/COM001.zip>COM001/COM001_rebuild.txt" --refer project/pysrc_all//build/COM001_rebuild.bin -o "project/pysrc_all//build/COM001.zip>COM001/COM001_rebuild_check.txt" --log_level info -e sjis
+    python src/libtext.py check --batch "project/pysrc_all/build;COM001.zip>COM001/COM001_rebuild.txt" --refer "project/pysrc_all;build/COM001_rebuild.bin" -o "project/pysrc_all;build/COM001_rebuild_check.txt" --log_level info -e sjis
 }
 
 test_libfont()
@@ -32,8 +35,10 @@ test_libfont()
 test_libimage()
 {
     echo "## test_libimage"
-    python src/libimage.py decode --format tile "test/sample/it.bin" -o "project/pysrc_all/build/it_decode.png" --tilew 20 --tileh 18 --tilebpp 2 --tilesize 92 --palette "ff ff ff 00 ff ff ff 3f ff ff ff 8f ff ff ff ff" 
+    python src/libimage.py decode --format tile "test/sample/it.bin" -o "project/pysrc_all/build/it_decode.png" --tilew 20 --tileh 18 --tilebpp 2 --tilesize 92 --palette "ff ff ff 00 ff ff ff 3f ff ff ff 8f ff ff ff ff"
     python src/libimage.py encode --format tile "project/pysrc_all/build/it_decode.png" -o "project/pysrc_all/build/it_encode1.bin" --tilebpp 2 --palette "ff ff ff 00 ff ff ff 3f ff ff ff 8f ff ff ff ff"
+    python src/libimage.py decode --batch --format tile "test/sample;it.bin" -o "project/pysrc_all;build/it_decode.png" --tilew 20 --tileh 18 --tilebpp 2 --tilesize 92 --palette "ff ff ff 00 ff ff ff 3f ff ff ff 8f ff ff ff ff"
+    python src/libimage.py encode --batch --format tile "project/pysrc_all/build;it_decode.png" -o "project;pysrc_all/build/it_encode1.bin" --tilebpp 2 --palette "ff ff ff 00 ff ff ff 3f ff ff ff 8f ff ff ff ff"
 }
 
 test_libword()

src/ftextpack.py

@@ -167,8 +167,7 @@ def save_fpack(fpack: Fpack, outobj, pack_compact=False) -> None:
     return fpack
 
 def cli(cmdstr=None):
-    def cmd_pack(args):
-        logging.debug(repr(args))
+    def filter_paths(args):
         if args.batch:
             binpaths = load_batch(args.binpath)
             ftextpaths = load_batch(args.ftextpath)
@@ -177,12 +176,19 @@ def cmd_pack(args):
             binpaths = [args.binpath]
             ftextpaths = [args.ftextpath]
             outpaths = [args.outpath if args.outpath!="" else None]
-
-        print(args.tbl)
+        n = min(len(binpaths), len(ftextpaths), len(outpaths))
+        return binpaths, ftextpaths, outpaths, n
+
+    def filter_cfgs(args):
         tbl = load_tbl(args.tbl)
         text_replace = dict((t[0], t[1]) for t in  args.text_replace) if args.text_replace else None
         bytes_fallback = bytes.fromhex(args.bytes_fallback) if args.bytes_fallback else None
-        n = min(len(binpaths), len(ftextpaths), len(outpaths))
+        return tbl, text_replace, bytes_fallback
+
+    def cmd_pack(args):
+        logging.debug(repr(args))
+        binpaths, ftextpaths, outpaths, n = filter_paths(args)
+        tbl, text_replace, bytes_fallback = filter_cfgs(args)
         for i, (binpath, ftextpath, outpath) in enumerate(zip(binpaths, ftextpaths, outpaths)):
             if args.batch: logging.info(f"batch {i+1}/{n} [binpath={binpath} ftextpath={ftextpath} outpath={outpath}]")
             pack_ftexts(binpath, ftextpath, outpath, 

src/libimage.py

@@ -2,7 +2,7 @@
 __description__ = """
 A image tool (remake) for image encoding or decoding, 
 all the intermediate format is rgba, index in alpha channel
-    v0.3, develope by devseed
+    v0.3.1, develope by devseed
 """
 
 import math
@@ -17,11 +17,11 @@
 readonly = lambda dtype, dim: numba.types.Array(dtype, dim, "C", True)
 
 try:
-    from libutil import tile_t, writebytes, writeimage, filter_loadfiles, filter_loadimages, valid_tile
+    from libutil import tile_t, writebytes, writeimage, filter_loadfiles, filter_loadimages, load_batch, valid_tile
 except ImportError:
-    exec("from libutil_v600 import tile_t, writebytes, writeimage, filter_loadfiles, filter_loadimages, valid_tile")
+    exec("from libutil_v600 import tile_t, writebytes, writeimage, filter_loadfiles, filter_loadimages, load_batch, valid_tile")
 
-__version__ = 300
+__version__ = 310
 
 # methods for generate patterns
 def make_swizzle_pattern(tileorder) -> np.ndarray:
@@ -286,23 +286,37 @@ def decode_tile_image(binobj: Union[str, bytes, np.ndarray], tileinfo: tile_t, o
     return img
 
 def cli(cmdstr=None):
-    def cmd_decode(args):
-        logging.debug(repr(args))
+    def filter_paths(args):
+        if args.batch:
+            inpaths = load_batch(args.inpath)
+            outpaths = load_batch(args.outpath)
+        else: inpaths, outpaths = [args.inpath], [args.outpath]
+        n = min(len(inpaths), len(outpaths))
+        return inpaths, outpaths, n
+
+    def filter_cfgs(args):
         tileinfo = tile_t(args.tilew, args.tileh,args.tilebpp, args.tilesize)
         palette = bytes.fromhex(args.palette) if args.palette else None
         if palette: palette = np.frombuffer(palette, dtype=np.uint8).reshape((-1, 4))
-        if args.format == "tile":
-            decode_tile_image(args.inpath, tileinfo, args.outpath, 
-                palette=palette, n_tile=args.n_tile, n_row=args.n_row)
+        return tileinfo, palette
+
+    def cmd_decode(args):
+        logging.debug(repr(args))
+        inpaths, outpaths, n = filter_paths(args)
+        tileinfo, palette = filter_cfgs(args)
+        for i, (inpath, outpath) in enumerate(zip(inpaths, outpaths)):
+            if args.batch: logging.info(f"batch {i+1}/{n} [inpath={inpath} outpath={outpath}]")
+            if args.format == "tile":
+                decode_tile_image(inpath, tileinfo, outpath, palette=palette, n_tile=args.n_tile, n_row=args.n_row)
 
     def cmd_encode(args):
         logging.debug(repr(args))
-        tileinfo = tile_t(args.tilew, args.tileh,args.tilebpp, args.tilesize)
-        palette = bytes.fromhex(args.palette) if args.palette else None
-        if palette: palette = np.frombuffer(palette, dtype=np.uint8).reshape((-1, 4))
-        if args.format == "tile":
-            encode_tile_image(args.inpath, tileinfo, args.outpath, 
-                palette=palette, n_tile=args.n_tile)
+        inpaths, outpaths, n = filter_paths(args)
+        tileinfo, palette = filter_cfgs(args)
+        for i, (inpath, outpath) in enumerate(zip(inpaths, outpaths)):
+            if args.batch: logging.info(f"batch {i+1}/{n} [inpath={inpath} outpath={outpath}]")
+            if args.format == "tile":
+                encode_tile_image(inpath, tileinfo, outpath, palette=palette, n_tile=args.n_tile)
 
     p = argparse.ArgumentParser(description=__description__)
     p2 = p.add_subparsers(title="operations")
@@ -312,6 +326,7 @@ def cmd_encode(args):
         t.add_argument("-o", "--outpath", default="out")
         t.add_argument("--log_level", default="info", help="set log level", 
             choices=("none", "critical", "error", "warnning", "info", "debug"))
+        t.add_argument("--batch", action="store_true", help="batch mode on inpath, outpath")
         t.add_argument("--format", default="tile", choices=["tile"], help="output format")
         t.add_argument("--palette", type=str, default=None)
         t.add_argument("--tilew", type=int, default=0)
@@ -342,4 +357,5 @@ def cmd_encode(args):
 v0.2, add swizzle method
 v0.2.1, change cv2 to PIL.image
 v0.3, remake with libutil v0.6, accelerate by numba parallel
+v0.3.1, add batch mode to optimize performance
 """