From 3b9212c2bd4dbf41c05ac6e6eb2b49b63eaaea26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CJionghao=E2=80=9D?= <jionghah@andrew.cmu.ed>
Date: Tue, 30 Jan 2024 22:06:45 -0500
Subject: [PATCH 1/4] jsut song init

---
 egs/jsut_song/voc1/cmd.sh                     |  91 +++++++++
 .../voc1/conf/parallel_wavegan.v1.yaml        | 121 +++++++++++
 egs/jsut_song/voc1/conf/slurm.conf            |  12 ++
 egs/jsut_song/voc1/local/data_prep.sh         |  87 ++++++++
 egs/jsut_song/voc1/path.sh                    |  33 +++
 egs/jsut_song/voc1/run.sh                     | 188 ++++++++++++++++++
 egs/jsut_song/voc1/utils                      |   1 +
 7 files changed, 533 insertions(+)
 create mode 100644 egs/jsut_song/voc1/cmd.sh
 create mode 100644 egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
 create mode 100644 egs/jsut_song/voc1/conf/slurm.conf
 create mode 100755 egs/jsut_song/voc1/local/data_prep.sh
 create mode 100644 egs/jsut_song/voc1/path.sh
 create mode 100755 egs/jsut_song/voc1/run.sh
 create mode 120000 egs/jsut_song/voc1/utils

diff --git a/egs/jsut_song/voc1/cmd.sh b/egs/jsut_song/voc1/cmd.sh
new file mode 100644
index 00000000..19f34210
--- /dev/null
+++ b/egs/jsut_song/voc1/cmd.sh
@@ -0,0 +1,91 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentaion for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend="local"
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="utils/run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="utils/run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="utils/run.pl"
+
+# Local machine, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="utils/stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="utils/stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="utils/stdout.pl"
+
+# "qsub" (SGE, Torque, PBS, etc.)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="utils/queue.pl"
+    export cuda_cmd="utils/queue.pl"
+    export decode_cmd="utils/queue.pl"
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partion" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by defualt for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="utils/slurm.pl"
+    export cuda_cmd="utils/slurm.pl"
+    export decode_cmd="utils/slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="utils/ssh.pl"
+    export cuda_cmd="utils/ssh.pl"
+    export decode_cmd="utils/ssh.pl"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml b/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
new file mode 100644
index 00000000..29ff94d0
--- /dev/null
+++ b/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
@@ -0,0 +1,121 @@
+# This is the hyperparameter configuration file for Parallel WaveGAN.
+# This configuration requires 12 GB GPU memory and takes ~3 days on TITAN V.
+# You need to change the setting depneding on your dataset.
+
+###########################################################
+#                FEATURE EXTRACTION SETTING               #
+###########################################################
+sampling_rate: 22050     # Sampling rate.
+fft_size: 1024           # FFT size.
+hop_size: 256            # Hop size.
+win_length: null         # Window length.
+                         # If set to null, it will be the same as fft_size.
+window: "hann"           # Window function.
+num_mels: 80             # Number of mel basis.
+fmin: 80                 # Minimum freq in mel basis calculation.
+fmax: 7600               # Maximum frequency in mel basis calculation.
+global_gain_scale: 1.0   # Will be multiplied to all of waveform.
+trim_silence: false       # Whether to trim the start and end of silence.
+trim_threshold_in_db: 60 # Need to tune carefully if the recording is not good.
+trim_frame_size: 2048    # Frame size in trimming.
+trim_hop_size: 512       # Hop size in trimming.
+format: "hdf5"           # Feature file format. "npy" or "hdf5" is supported.
+
+###########################################################
+#         GENERATOR NETWORK ARCHITECTURE SETTING          #
+###########################################################
+generator_params:
+    in_channels: 1        # Number of input channels.
+    out_channels: 1       # Number of output channels.
+    kernel_size: 3        # Kernel size of dilated convolution.
+    layers: 30            # Number of residual block layers.
+    stacks: 3             # Number of stacks i.e., dilation cycles.
+    residual_channels: 64 # Number of channels in residual conv.
+    gate_channels: 128    # Number of channels in gated conv.
+    skip_channels: 64     # Number of channels in skip conv.
+    aux_channels: 80      # Number of channels for auxiliary feature conv.
+                          # Must be the same as num_mels.
+    aux_context_window: 2 # Context window size for auxiliary feature.
+                          # If set to 2, previous 2 and future 2 frames will be considered.
+    dropout: 0.0          # Dropout rate. 0.0 means no dropout applied.
+    use_weight_norm: true # Whether to use weight norm.
+                          # If set to true, it will be applied to all of the conv layers.
+    upsample_net: "ConvInUpsampleNetwork" # Upsampling network architecture.
+    upsample_params:                      # Upsampling network parameters.
+        upsample_scales: [4, 4, 4, 4]     # Upsampling scales. Prodcut of these must be the same as hop size.
+
+###########################################################
+#       DISCRIMINATOR NETWORK ARCHITECTURE SETTING        #
+###########################################################
+discriminator_params:
+    in_channels: 1        # Number of input channels.
+    out_channels: 1       # Number of output channels.
+    kernel_size: 3        # Number of output channels.
+    layers: 10            # Number of conv layers.
+    conv_channels: 64     # Number of chnn layers.
+    bias: true            # Whether to use bias parameter in conv.
+    use_weight_norm: true # Whether to use weight norm.
+                          # If set to true, it will be applied to all of the conv layers.
+    nonlinear_activation: "LeakyReLU" # Nonlinear function after each conv.
+    nonlinear_activation_params:      # Nonlinear function parameters
+        negative_slope: 0.2           # Alpha in LeakyReLU.
+
+###########################################################
+#                   STFT LOSS SETTING                     #
+###########################################################
+stft_loss_params:
+    fft_sizes: [1024, 2048, 512]  # List of FFT size for STFT-based loss.
+    hop_sizes: [120, 240, 50]     # List of hop size for STFT-based loss
+    win_lengths: [600, 1200, 240] # List of window length for STFT-based loss.
+    window: "hann_window"         # Window function for STFT-based loss
+
+###########################################################
+#               ADVERSARIAL LOSS SETTING                  #
+###########################################################
+lambda_adv: 4.0  # Loss balancing coefficient.
+
+###########################################################
+#                  DATA LOADER SETTING                    #
+###########################################################
+batch_size: 6              # Batch size.
+batch_max_steps: 25600     # Length of each audio in batch. Make sure dividable by hop_size.
+pin_memory: true           # Whether to pin memory in Pytorch DataLoader.
+num_workers: 2             # Number of workers in Pytorch DataLoader.
+remove_short_samples: true # Whether to remove samples the length of which are less than batch_max_steps.
+allow_cache: true          # Whether to allow cache in dataset. If true, it requires cpu memory.
+
+###########################################################
+#             OPTIMIZER & SCHEDULER SETTING               #
+###########################################################
+generator_optimizer_params:
+    lr: 0.0001             # Generator's learning rate.
+    eps: 1.0e-6            # Generator's epsilon.
+    weight_decay: 0.0      # Generator's weight decay coefficient.
+generator_scheduler_params:
+    step_size: 200000      # Generator's scheduler step size.
+    gamma: 0.5             # Generator's scheduler gamma.
+                           # At each step size, lr will be multiplied by this parameter.
+generator_grad_norm: 10    # Generator's gradient norm.
+discriminator_optimizer_params:
+    lr: 0.00005            # Discriminator's learning rate.
+    eps: 1.0e-6            # Discriminator's epsilon.
+    weight_decay: 0.0      # Discriminator's weight decay coefficient.
+discriminator_scheduler_params:
+    step_size: 200000      # Discriminator's scheduler step size.
+    gamma: 0.5             # Discriminator's scheduler gamma.
+                           # At each step size, lr will be multiplied by this parameter.
+discriminator_grad_norm: 1 # Discriminator's gradient norm.
+
+###########################################################
+#                    INTERVAL SETTING                     #
+###########################################################
+discriminator_train_start_steps: 100000 # Number of steps to start to train discriminator.
+train_max_steps: 400000                 # Number of training steps.
+save_interval_steps: 5000               # Interval steps to save checkpoint.
+eval_interval_steps: 1000               # Interval steps to evaluate the network.
+log_interval_steps: 100                 # Interval steps to record the training log.
+
+###########################################################
+#                     OTHER SETTING                       #
+###########################################################
+num_save_intermediate_results: 4  # Number of results to be saved as intermediate results.
diff --git a/egs/jsut_song/voc1/conf/slurm.conf b/egs/jsut_song/voc1/conf/slurm.conf
new file mode 100644
index 00000000..cefd21f0
--- /dev/null
+++ b/egs/jsut_song/voc1/conf/slurm.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command sbatch --export=PATH  --ntasks-per-node=1
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* --cpus-per-task $0 --ntasks-per-node=1
+option num_threads=1 --cpus-per-task 1  --ntasks-per-node=1 # Do not add anything to qsub_opts
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs/jsut_song/voc1/local/data_prep.sh b/egs/jsut_song/voc1/local/data_prep.sh
new file mode 100755
index 00000000..b2044d8f
--- /dev/null
+++ b/egs/jsut_song/voc1/local/data_prep.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Copyright 2020 Tomoki Hayashi
+#  MIT License (https://opensource.org/licenses/MIT)
+
+# shellcheck disable=SC1091
+. ./path.sh || exit 1;
+
+fs=22050
+num_dev=100
+num_eval=100
+train_set="train_nodev"
+dev_set="dev"
+eval_set="eval"
+shuffle=false
+
+# shellcheck disable=SC1091
+. utils/parse_options.sh || exit 1;
+
+db_root=$1
+data_dir=$2
+
+# check arguments
+if [ $# != 2 ]; then
+    echo "Usage: $0 [Options] <db_root> <data_dir>"
+    echo ""
+    echo "Options:"
+    echo "    --num_dev: number of development uttreances (default=250)."
+    echo "    --num_eval: number of evaluation uttreances (default=250)."
+    echo "    --train_set: name of train set (default=train_nodev)."
+    echo "    --dev_set: name of dev set (default=dev)."
+    echo "    --eval_set: name of eval set (default=eval)."
+    echo "    --shuffle: whether to perform shuffle in making dev / eval set (default=false)."
+    exit 1
+fi
+
+set -euo pipefail
+
+[ ! -e "${data_dir}/all" ] && mkdir -p "${data_dir}/all"
+
+# set filenames
+scp="${data_dir}/all/wav.scp"
+
+# check file existence
+[ -e "${scp}" ] && rm "${scp}"
+
+# make all scp
+find "${db_root}" -follow -name "*.wav" | sort | while read -r filename; do
+    id=$(basename "${filename}" | sed -e "s/\.[^\.]*$//g")
+    echo "${id} cat ${filename} | sox -t wav - -c 1 -b 16 -t wav - rate ${fs} |" >> "${scp}"
+done
+
+# split
+num_all=$(wc -l < "${scp}")
+num_deveval=$((num_dev + num_eval))
+num_train=$((num_all - num_deveval))
+if [ ${num_eval} -ne 0 ]; then
+    utils/split_data.sh \
+        --num_first "${num_train}" \
+        --num_second "${num_deveval}" \
+        --shuffle "${shuffle}" \
+        "${data_dir}/all" \
+        "${data_dir}/${train_set}" \
+        "${data_dir}/deveval"
+    utils/split_data.sh \
+        --num_first "${num_dev}" \
+        --num_second "${num_eval}" \
+        --shuffle "${shuffle}" \
+        "${data_dir}/deveval" \
+        "${data_dir}/${dev_set}" \
+        "${data_dir}/${eval_set}"
+else
+    utils/split_data.sh \
+        --num_first "${num_train}" \
+        --num_second "${num_deveval}" \
+        --shuffle "${shuffle}" \
+        "${data_dir}/all" \
+        "${data_dir}/${train_set}" \
+        "${data_dir}/${dev_set}"
+    cp -r "${data_dir}/${dev_set}" "${data_dir}/${eval_set}"
+fi
+
+# remove tmp directories
+rm -rf "${data_dir}/all"
+rm -rf "${data_dir}/deveval"
+
+echo "Successfully prepared data."
diff --git a/egs/jsut_song/voc1/path.sh b/egs/jsut_song/voc1/path.sh
new file mode 100644
index 00000000..dcc1c7d0
--- /dev/null
+++ b/egs/jsut_song/voc1/path.sh
@@ -0,0 +1,33 @@
+# cuda related
+export CUDA_HOME=/usr/local/cuda-11.3
+export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}"
+
+# path related
+export PRJ_ROOT="${PWD}/../../.."
+if [ -e "${PRJ_ROOT}/tools/venv/bin/activate" ]; then
+    # shellcheck disable=SC1090
+    . "${PRJ_ROOT}/tools/venv/bin/activate"
+fi
+
+# python related
+export OMP_NUM_THREADS=1
+export PYTHONIOENCODING=UTF-8
+export MPL_BACKEND=Agg
+
+# check installation
+if ! command -v parallel-wavegan-train > /dev/null; then
+    echo "Error: It seems setup is not finished." >&2
+    echo "Error: Please setup your environment by following README.md" >&2
+    return 1
+fi
+if ! command -v jq > /dev/null; then
+    echo "Error: It seems jq is not installed." >&2
+    echo "Error: Please install via \`sudo apt-get install jq\`." >&2
+    echo "Error: If you do not have sudo, please download from https://stedolan.github.io/jq/download/." >&2
+    return 1
+fi
+if ! command -v yq > /dev/null; then
+    echo "Error: It seems yq is not installed." >&2
+    echo "Error: Please install via \`pip install yq\`." >&2
+    return 1
+fi
diff --git a/egs/jsut_song/voc1/run.sh b/egs/jsut_song/voc1/run.sh
new file mode 100755
index 00000000..c3872e47
--- /dev/null
+++ b/egs/jsut_song/voc1/run.sh
@@ -0,0 +1,188 @@
+#!/bin/bash
+
+# Copyright 2020 Tomoki Hayashi
+#  MIT License (https://opensource.org/licenses/MIT)
+
+. ./cmd.sh || exit 1;
+. ./path.sh || exit 1;
+
+# basic settings
+stage=0        # stage to start
+stop_stage=100 # stage to stop
+verbose=1      # verbosity level (lower is less info)
+n_gpus=1       # number of gpus in training
+n_jobs=4       # number of parallel jobs in feature extraction
+
+# NOTE(kan-bayashi): renamed to conf to avoid conflict in parse_options.sh
+conf=conf/parallel_wavegan.v1.yaml
+
+# directory path setting
+db_root=/path/to/database # direcotry including wavfiles (MODIFY BY YOURSELF)
+                          # each wav filename in the directory should be unique
+                          # e.g.
+                          # /path/to/database
+                          # ├── utt_1.wav
+                          # ├── utt_2.wav
+                          # │   ...
+                          # └── utt_N.wav
+dumpdir=dump # directory to dump features
+
+# subset setting
+shuffle=false # whether to shuffle the data to create subset
+num_dev=100   # the number of development data
+num_eval=100  # the number of evaluation data
+              # (if set to 0, the same dev set is used as eval set)
+
+# training related setting
+tag=""     # tag for directory to save model
+resume=""  # checkpoint path to resume training
+           # (e.g. <path>/<to>/checkpoint-10000steps.pkl)
+pretrain="" # checkpoint path to load pretrained parameters
+            # (e.g. ../../jsut/<path>/<to>/checkpoint-400000steps.pkl)
+
+# decoding related setting
+checkpoint="" # checkpoint path to be used for decoding
+              # if not provided, the latest one will be used
+              # (e.g. <path>/<to>/checkpoint-400000steps.pkl)
+
+# shellcheck disable=SC1091
+. utils/parse_options.sh || exit 1;
+
+train_set="train_nodev" # name of training data directory
+dev_set="dev"           # name of development data direcotry
+eval_set="eval"         # name of evaluation data direcotry
+
+set -euo pipefail
+
+if [ "${stage}" -le 0 ] && [ "${stop_stage}" -ge 0 ]; then
+    echo "Stage 0: Data preparation"
+    local/data_prep.sh \
+        --fs "$(yq ".sampling_rate" "${conf}")" \
+        --num_dev "${num_dev}" \
+        --num_eval "${num_eval}" \
+        --train_set "${train_set}" \
+        --dev_set "${dev_set}" \
+        --eval_set "${eval_set}" \
+        --shuffle "${shuffle}" \
+        "${db_root}" data
+fi
+
+stats_ext=$(grep -q "hdf5" <(yq ".format" "${conf}") && echo "h5" || echo "npy")
+if [ "${stage}" -le 1 ] && [ "${stop_stage}" -ge 1 ]; then
+    echo "Stage 1: Feature extraction"
+    # extract raw features
+    pids=()
+    for name in "${train_set}" "${dev_set}" "${eval_set}"; do
+    (
+        [ ! -e "${dumpdir}/${name}/raw" ] && mkdir -p "${dumpdir}/${name}/raw"
+        echo "Feature extraction start. See the progress via ${dumpdir}/${name}/raw/preprocessing.*.log."
+        utils/make_subset_data.sh "data/${name}" "${n_jobs}" "${dumpdir}/${name}/raw"
+        ${train_cmd} JOB=1:${n_jobs} "${dumpdir}/${name}/raw/preprocessing.JOB.log" \
+            parallel-wavegan-preprocess \
+                --config "${conf}" \
+                --scp "${dumpdir}/${name}/raw/wav.JOB.scp" \
+                --dumpdir "${dumpdir}/${name}/raw/dump.JOB" \
+                --verbose "${verbose}"
+        echo "Successfully finished feature extraction of ${name} set."
+    ) &
+    pids+=($!)
+    done
+    i=0; for pid in "${pids[@]}"; do wait "${pid}" || ((++i)); done
+    [ "${i}" -gt 0 ] && echo "$0: ${i} background jobs are failed." && exit 1;
+    echo "Successfully finished feature extraction."
+
+    # calculate statistics for normalization
+    if [ -z "${pretrain}" ]; then
+        # calculate statistics for normalization
+        echo "Statistics computation start. See the progress via ${dumpdir}/${train_set}/compute_statistics.log."
+        ${train_cmd} "${dumpdir}/${train_set}/compute_statistics.log" \
+            parallel-wavegan-compute-statistics \
+                --config "${conf}" \
+                --rootdir "${dumpdir}/${train_set}/raw" \
+                --dumpdir "${dumpdir}/${train_set}" \
+                --verbose "${verbose}"
+        echo "Successfully finished calculation of statistics."
+    else
+        echo "Use statistics of pretrained model. Skip statistics computation."
+        cp "$(dirname "${pretrain}")/stats.${stats_ext}" "${dumpdir}/${train_set}"
+    fi
+
+    # normalize and dump them
+    pids=()
+    for name in "${train_set}" "${dev_set}" "${eval_set}"; do
+    (
+        [ ! -e "${dumpdir}/${name}/norm" ] && mkdir -p "${dumpdir}/${name}/norm"
+        echo "Nomalization start. See the progress via ${dumpdir}/${name}/norm/normalize.*.log."
+        ${train_cmd} JOB=1:${n_jobs} "${dumpdir}/${name}/norm/normalize.JOB.log" \
+            parallel-wavegan-normalize \
+                --config "${conf}" \
+                --stats "${dumpdir}/${train_set}/stats.${stats_ext}" \
+                --rootdir "${dumpdir}/${name}/raw/dump.JOB" \
+                --dumpdir "${dumpdir}/${name}/norm/dump.JOB" \
+                --verbose "${verbose}"
+        echo "Successfully finished normalization of ${name} set."
+    ) &
+    pids+=($!)
+    done
+    i=0; for pid in "${pids[@]}"; do wait "${pid}" || ((++i)); done
+    [ "${i}" -gt 0 ] && echo "$0: ${i} background jobs are failed." && exit 1;
+    echo "Successfully finished normalization."
+fi
+
+if [ -z "${tag}" ]; then
+    expdir="exp/${train_set}_$(basename "${conf}" .yaml)"
+    if [ -n "${pretrain}" ]; then
+        pretrain_tag=$(basename "$(dirname "${pretrain}")")
+        expdir+="_${pretrain_tag}"
+    fi
+else
+    expdir="exp/${train_set}_${tag}"
+fi
+if [ "${stage}" -le 2 ] && [ "${stop_stage}" -ge 2 ]; then
+    echo "Stage 2: Network training"
+    [ ! -e "${expdir}" ] && mkdir -p "${expdir}"
+    cp "${dumpdir}/${train_set}/stats.${stats_ext}" "${expdir}"
+    if [ "${n_gpus}" -gt 1 ]; then
+        train="python -m parallel_wavegan.distributed.launch --nproc_per_node ${n_gpus} -c parallel-wavegan-train"
+    else
+        train="parallel-wavegan-train"
+    fi
+    echo "Training start. See the progress via ${expdir}/train.log."
+    ${cuda_cmd} --gpu "${n_gpus}" "${expdir}/train.log" \
+        ${train} \
+            --config "${conf}" \
+            --train-dumpdir "${dumpdir}/${train_set}/norm" \
+            --dev-dumpdir "${dumpdir}/${dev_set}/norm" \
+            --outdir "${expdir}" \
+            --resume "${resume}" \
+            --pretrain "${pretrain}" \
+            --verbose "${verbose}"
+    echo "Successfully finished training."
+fi
+
+if [ "${stage}" -le 3 ] && [ "${stop_stage}" -ge 3 ]; then
+    echo "Stage 3: Network decoding"
+    # shellcheck disable=SC2012
+    [ -z "${checkpoint}" ] && checkpoint="$(ls -dt "${expdir}"/*.pkl | head -1 || true)"
+    outdir="${expdir}/wav/$(basename "${checkpoint}" .pkl)"
+    pids=()
+    for name in "${dev_set}" "${eval_set}"; do
+    (
+        [ ! -e "${outdir}/${name}" ] && mkdir -p "${outdir}/${name}"
+        [ "${n_gpus}" -gt 1 ] && n_gpus=1
+        echo "Decoding start. See the progress via ${outdir}/${name}/decode.log."
+        ${cuda_cmd} --gpu "${n_gpus}" "${outdir}/${name}/decode.log" \
+            parallel-wavegan-decode \
+                --dumpdir "${dumpdir}/${name}/norm" \
+                --checkpoint "${checkpoint}" \
+                --outdir "${outdir}/${name}" \
+                --verbose "${verbose}"
+        echo "Successfully finished decoding of ${name} set."
+    ) &
+    pids+=($!)
+    done
+    i=0; for pid in "${pids[@]}"; do wait "${pid}" || ((++i)); done
+    [ "${i}" -gt 0 ] && echo "$0: ${i} background jobs are failed." && exit 1;
+    echo "Successfully finished decoding."
+fi
+echo "Finished."
diff --git a/egs/jsut_song/voc1/utils b/egs/jsut_song/voc1/utils
new file mode 120000
index 00000000..973afe67
--- /dev/null
+++ b/egs/jsut_song/voc1/utils
@@ -0,0 +1 @@
+../../../utils
\ No newline at end of file

From 8ff456e0bcf61ebb342a864f98fbc24ab7b13c21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CJionghao=E2=80=9D?= <jionghah@andrew.cmu.ed>
Date: Tue, 30 Jan 2024 22:32:05 -0500
Subject: [PATCH 2/4] data download init from jsut/

---
 egs/jsut_song/voc1/local/data_download.sh | 39 +++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 egs/jsut_song/voc1/local/data_download.sh

diff --git a/egs/jsut_song/voc1/local/data_download.sh b/egs/jsut_song/voc1/local/data_download.sh
new file mode 100755
index 00000000..f5f251f2
--- /dev/null
+++ b/egs/jsut_song/voc1/local/data_download.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2019 Tomoki Hayashi
+#  MIT License (https://opensource.org/licenses/MIT)
+
+download_dir=$1
+
+# check arguments
+if [ $# != 1 ]; then
+    echo "Usage: $0 <download_dir>"
+    exit 1
+fi
+
+set -euo pipefail
+
+cwd=$(pwd)
+if [ ! -e "${download_dir}/jsut_ver1.1" ]; then
+    mkdir -p "${download_dir}"
+    cd "${download_dir}" || exit 1;
+    wget http://ss-takashi.sakura.ne.jp/corpus/jsut_ver1.1.zip
+    unzip -o ./*.zip
+    rm ./*.zip
+    cd "${cwd}" || exit 1;
+    echo "Successfully downloaded data."
+else
+    echo "Already exists. Skipped."
+fi
+
+if [ ! -e "${download_dir}/jsut_lab" ]; then
+    cd "${download_dir}" || exit 1;
+    git clone https://github.com/r9y9/jsut-lab
+    for name in loanword128 repeat500 voiceactress100 basic5000 onomatopee300 travel1000 countersuffix26 precedent130 utparaphrase512; do
+        cp -vr "jsut-lab/${name}" jsut_ver1.1/
+    done
+    cd - || exit 1;
+    echo "Successfully downloaded context label."
+else
+    echo "Already exists. Skipped."
+fi

From f8e605b3bbf197fe63e169a73d18a466ed04ce7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CJionghao=E2=80=9D?= <jionghah@andrew.cmu.ed>
Date: Fri, 2 Feb 2024 10:14:15 -0500
Subject: [PATCH 3/4] update scripts for jsut-song

---
 egs/jsut_song/voc1/conf/hifigan.v1.yaml       | 182 +++++++++++
 .../voc1/conf/parallel_wavegan.v1.yaml        | 121 -------
 egs/jsut_song/voc1/local/data_download.sh     |  39 ---
 egs/jsut_song/voc1/local/data_prep.py         | 294 ++++++++++++++++++
 egs/jsut_song/voc1/local/data_prep.sh         |  87 ------
 egs/jsut_song/voc1/run.sh                     |  62 ++--
 6 files changed, 516 insertions(+), 269 deletions(-)
 create mode 100644 egs/jsut_song/voc1/conf/hifigan.v1.yaml
 delete mode 100644 egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
 delete mode 100755 egs/jsut_song/voc1/local/data_download.sh
 create mode 100644 egs/jsut_song/voc1/local/data_prep.py
 delete mode 100755 egs/jsut_song/voc1/local/data_prep.sh

diff --git a/egs/jsut_song/voc1/conf/hifigan.v1.yaml b/egs/jsut_song/voc1/conf/hifigan.v1.yaml
new file mode 100644
index 00000000..1c9d4fb6
--- /dev/null
+++ b/egs/jsut_song/voc1/conf/hifigan.v1.yaml
@@ -0,0 +1,182 @@
+# Original Source: https://github.com/kan-bayashi/ParallelWaveGAN/blob/master/egs/libritts/voc1/conf/hifigan.v1.yaml
+
+# This is the configuration file for LibriTTS dataset.
+# This configuration is based on HiFiGAN V1, which is
+# an official configuration. But I found that the optimizer
+# setting does not work well with my implementation.
+# So I changed optimizer settings as follows:
+# - AdamW -> Adam
+# - betas: [0.8, 0.99] -> betas: [0.5, 0.9]
+# - Scheduler: ExponentialLR -> MultiStepLR
+# To match the shift size difference, the upsample scales
+# is also modified from the original 256 shift setting.
+
+###########################################################
+#                FEATURE EXTRACTION SETTING               #
+###########################################################
+sampling_rate: 24000     # Sampling rate.
+fft_size: 2048           # FFT size.
+hop_size: 300            # Hop size.
+win_length: 1200         # Window length.
+                         # If set to null, it will be the same as fft_size.
+window: "hann"           # Window function.
+num_mels: 80             # Number of mel basis.
+fmin: 80                 # Minimum freq in mel basis calculation.
+fmax: 7600               # Maximum frequency in mel basis calculation.
+global_gain_scale: 1.0   # Will be multiplied to all of waveform.
+trim_silence: false      # Whether to trim the start and end of silence.
+trim_threshold_in_db: 20 # Need to tune carefully if the recording is not good.
+trim_frame_size: 1024    # Frame size in trimming.
+trim_hop_size: 256       # Hop size in trimming.
+format: "hdf5"           # Feature file format. "npy" or "hdf5" is supported.
+
+###########################################################
+#         GENERATOR NETWORK ARCHITECTURE SETTING          #
+###########################################################
+generator_type: HiFiGANGenerator
+generator_params:
+    in_channels: 80                       # Number of input channels.
+    out_channels: 1                       # Number of output channels.
+    channels: 512                         # Number of initial channels.
+    kernel_size: 7                        # Kernel size of initial and final conv layers.
+    upsample_scales: [5, 5, 4, 3]         # Upsampling scales.
+    upsample_kernel_sizes: [10, 10, 8, 6] # Kernel size for upsampling layers.
+    resblock_kernel_sizes: [3, 7, 11]     # Kernel size for residual blocks.
+    resblock_dilations:                   # Dilations for residual blocks.
+        - [1, 3, 5]
+        - [1, 3, 5]
+        - [1, 3, 5]
+    use_additional_convs: true            # Whether to use additional conv layer in residual blocks.
+    bias: true                            # Whether to use bias parameter in conv.
+    nonlinear_activation: "LeakyReLU"     # Nonlinear activation type.
+    nonlinear_activation_params:          # Nonlinear activation paramters.
+        negative_slope: 0.1
+    use_weight_norm: true                 # Whether to apply weight normalization.
+
+###########################################################
+#       DISCRIMINATOR NETWORK ARCHITECTURE SETTING        #
+###########################################################
+discriminator_type: HiFiGANMultiScaleMultiPeriodDiscriminator
+discriminator_params:
+    scales: 3                              # Number of multi-scale discriminator.
+    scale_downsample_pooling: "AvgPool1d"  # Pooling operation for scale discriminator.
+    scale_downsample_pooling_params:
+        kernel_size: 4                     # Pooling kernel size.
+        stride: 2                          # Pooling stride.
+        padding: 2                         # Padding size.
+    scale_discriminator_params:
+        in_channels: 1                     # Number of input channels.
+        out_channels: 1                    # Number of output channels.
+        kernel_sizes: [15, 41, 5, 3]       # List of kernel sizes.
+        channels: 128                      # Initial number of channels.
+        max_downsample_channels: 1024      # Maximum number of channels in downsampling conv layers.
+        max_groups: 16                     # Maximum number of groups in downsampling conv layers.
+        bias: true
+        downsample_scales: [4, 4, 4, 4, 1] # Downsampling scales.
+        nonlinear_activation: "LeakyReLU"  # Nonlinear activation.
+        nonlinear_activation_params:
+            negative_slope: 0.1
+    follow_official_norm: true             # Whether to follow the official norm setting.
+    periods: [2, 3, 5, 7, 11]              # List of period for multi-period discriminator.
+    period_discriminator_params:
+        in_channels: 1                     # Number of input channels.
+        out_channels: 1                    # Number of output channels.
+        kernel_sizes: [5, 3]               # List of kernel sizes.
+        channels: 32                       # Initial number of channels.
+        downsample_scales: [3, 3, 3, 3, 1] # Downsampling scales.
+        max_downsample_channels: 1024      # Maximum number of channels in downsampling conv layers.
+        bias: true                         # Whether to use bias parameter in conv layer."
+        nonlinear_activation: "LeakyReLU"  # Nonlinear activation.
+        nonlinear_activation_params:       # Nonlinear activation paramters.
+            negative_slope: 0.1
+        use_weight_norm: true              # Whether to apply weight normalization.
+        use_spectral_norm: false           # Whether to apply spectral normalization.
+
+###########################################################
+#                   STFT LOSS SETTING                     #
+###########################################################
+use_stft_loss: false                 # Whether to use multi-resolution STFT loss.
+use_mel_loss: true                   # Whether to use Mel-spectrogram loss.
+mel_loss_params:
+    fs: 24000
+    fft_size: 2048
+    hop_size: 300
+    win_length: 1200
+    window: "hann"
+    num_mels: 80
+    fmin: 0
+    fmax: 12000
+    log_base: null
+generator_adv_loss_params:
+    average_by_discriminators: false # Whether to average loss by #discriminators.
+discriminator_adv_loss_params:
+    average_by_discriminators: false # Whether to average loss by #discriminators.
+use_feat_match_loss: true
+feat_match_loss_params:
+    average_by_discriminators: false # Whether to average loss by #discriminators.
+    average_by_layers: false         # Whether to average loss by #layers in each discriminator.
+    include_final_outputs: false     # Whether to include final outputs in feat match loss calculation.
+
+###########################################################
+#               ADVERSARIAL LOSS SETTING                  #
+###########################################################
+lambda_aux: 45.0       # Loss balancing coefficient for STFT loss.
+lambda_adv: 1.0        # Loss balancing coefficient for adversarial loss.
+lambda_feat_match: 2.0 # Loss balancing coefficient for feat match loss..
+
+###########################################################
+#                  DATA LOADER SETTING                    #
+###########################################################
+batch_size: 16              # Batch size.
+batch_max_steps: 8400       # Length of each audio in batch. Make sure dividable by hop_size.
+pin_memory: true            # Whether to pin memory in Pytorch DataLoader.
+num_workers: 2              # Number of workers in Pytorch DataLoader.
+remove_short_samples: false # Whether to remove samples the length of which are less than batch_max_steps.
+allow_cache: false          # Whether to allow cache in dataset. If true, it requires cpu memory.
+
+###########################################################
+#             OPTIMIZER & SCHEDULER SETTING               #
+###########################################################
+generator_optimizer_type: Adam
+generator_optimizer_params:
+    lr: 2.0e-4
+    betas: [0.5, 0.9]
+    weight_decay: 0.0
+generator_scheduler_type: MultiStepLR
+generator_scheduler_params:
+    gamma: 0.5
+    milestones:
+        - 200000
+        - 400000
+        - 600000
+        - 800000
+generator_grad_norm: -1
+discriminator_optimizer_type: Adam
+discriminator_optimizer_params:
+    lr: 2.0e-4
+    betas: [0.5, 0.9]
+    weight_decay: 0.0
+discriminator_scheduler_type: MultiStepLR
+discriminator_scheduler_params:
+    gamma: 0.5
+    milestones:
+        - 200000
+        - 400000
+        - 600000
+        - 800000
+discriminator_grad_norm: -1
+
+###########################################################
+#                    INTERVAL SETTING                     #
+###########################################################
+generator_train_start_steps: 1     # Number of steps to start to train discriminator.
+discriminator_train_start_steps: 0 # Number of steps to start to train discriminator.
+train_max_steps: 2500000           # Number of training steps.
+save_interval_steps: 10000         # Interval steps to save checkpoint.
+eval_interval_steps: 1000          # Interval steps to evaluate the network.
+log_interval_steps: 100            # Interval steps to record the training log.
+
+###########################################################
+#                     OTHER SETTING                       #
+###########################################################
+num_save_intermediate_results: 4  # Number of results to be saved as intermediate results.
diff --git a/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml b/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
deleted file mode 100644
index 29ff94d0..00000000
--- a/egs/jsut_song/voc1/conf/parallel_wavegan.v1.yaml
+++ /dev/null
@@ -1,121 +0,0 @@
-# This is the hyperparameter configuration file for Parallel WaveGAN.
-# This configuration requires 12 GB GPU memory and takes ~3 days on TITAN V.
-# You need to change the setting depneding on your dataset.
-
-###########################################################
-#                FEATURE EXTRACTION SETTING               #
-###########################################################
-sampling_rate: 22050     # Sampling rate.
-fft_size: 1024           # FFT size.
-hop_size: 256            # Hop size.
-win_length: null         # Window length.
-                         # If set to null, it will be the same as fft_size.
-window: "hann"           # Window function.
-num_mels: 80             # Number of mel basis.
-fmin: 80                 # Minimum freq in mel basis calculation.
-fmax: 7600               # Maximum frequency in mel basis calculation.
-global_gain_scale: 1.0   # Will be multiplied to all of waveform.
-trim_silence: false       # Whether to trim the start and end of silence.
-trim_threshold_in_db: 60 # Need to tune carefully if the recording is not good.
-trim_frame_size: 2048    # Frame size in trimming.
-trim_hop_size: 512       # Hop size in trimming.
-format: "hdf5"           # Feature file format. "npy" or "hdf5" is supported.
-
-###########################################################
-#         GENERATOR NETWORK ARCHITECTURE SETTING          #
-###########################################################
-generator_params:
-    in_channels: 1        # Number of input channels.
-    out_channels: 1       # Number of output channels.
-    kernel_size: 3        # Kernel size of dilated convolution.
-    layers: 30            # Number of residual block layers.
-    stacks: 3             # Number of stacks i.e., dilation cycles.
-    residual_channels: 64 # Number of channels in residual conv.
-    gate_channels: 128    # Number of channels in gated conv.
-    skip_channels: 64     # Number of channels in skip conv.
-    aux_channels: 80      # Number of channels for auxiliary feature conv.
-                          # Must be the same as num_mels.
-    aux_context_window: 2 # Context window size for auxiliary feature.
-                          # If set to 2, previous 2 and future 2 frames will be considered.
-    dropout: 0.0          # Dropout rate. 0.0 means no dropout applied.
-    use_weight_norm: true # Whether to use weight norm.
-                          # If set to true, it will be applied to all of the conv layers.
-    upsample_net: "ConvInUpsampleNetwork" # Upsampling network architecture.
-    upsample_params:                      # Upsampling network parameters.
-        upsample_scales: [4, 4, 4, 4]     # Upsampling scales. Prodcut of these must be the same as hop size.
-
-###########################################################
-#       DISCRIMINATOR NETWORK ARCHITECTURE SETTING        #
-###########################################################
-discriminator_params:
-    in_channels: 1        # Number of input channels.
-    out_channels: 1       # Number of output channels.
-    kernel_size: 3        # Number of output channels.
-    layers: 10            # Number of conv layers.
-    conv_channels: 64     # Number of chnn layers.
-    bias: true            # Whether to use bias parameter in conv.
-    use_weight_norm: true # Whether to use weight norm.
-                          # If set to true, it will be applied to all of the conv layers.
-    nonlinear_activation: "LeakyReLU" # Nonlinear function after each conv.
-    nonlinear_activation_params:      # Nonlinear function parameters
-        negative_slope: 0.2           # Alpha in LeakyReLU.
-
-###########################################################
-#                   STFT LOSS SETTING                     #
-###########################################################
-stft_loss_params:
-    fft_sizes: [1024, 2048, 512]  # List of FFT size for STFT-based loss.
-    hop_sizes: [120, 240, 50]     # List of hop size for STFT-based loss
-    win_lengths: [600, 1200, 240] # List of window length for STFT-based loss.
-    window: "hann_window"         # Window function for STFT-based loss
-
-###########################################################
-#               ADVERSARIAL LOSS SETTING                  #
-###########################################################
-lambda_adv: 4.0  # Loss balancing coefficient.
-
-###########################################################
-#                  DATA LOADER SETTING                    #
-###########################################################
-batch_size: 6              # Batch size.
-batch_max_steps: 25600     # Length of each audio in batch. Make sure dividable by hop_size.
-pin_memory: true           # Whether to pin memory in Pytorch DataLoader.
-num_workers: 2             # Number of workers in Pytorch DataLoader.
-remove_short_samples: true # Whether to remove samples the length of which are less than batch_max_steps.
-allow_cache: true          # Whether to allow cache in dataset. If true, it requires cpu memory.
-
-###########################################################
-#             OPTIMIZER & SCHEDULER SETTING               #
-###########################################################
-generator_optimizer_params:
-    lr: 0.0001             # Generator's learning rate.
-    eps: 1.0e-6            # Generator's epsilon.
-    weight_decay: 0.0      # Generator's weight decay coefficient.
-generator_scheduler_params:
-    step_size: 200000      # Generator's scheduler step size.
-    gamma: 0.5             # Generator's scheduler gamma.
-                           # At each step size, lr will be multiplied by this parameter.
-generator_grad_norm: 10    # Generator's gradient norm.
-discriminator_optimizer_params:
-    lr: 0.00005            # Discriminator's learning rate.
-    eps: 1.0e-6            # Discriminator's epsilon.
-    weight_decay: 0.0      # Discriminator's weight decay coefficient.
-discriminator_scheduler_params:
-    step_size: 200000      # Discriminator's scheduler step size.
-    gamma: 0.5             # Discriminator's scheduler gamma.
-                           # At each step size, lr will be multiplied by this parameter.
-discriminator_grad_norm: 1 # Discriminator's gradient norm.
-
-###########################################################
-#                    INTERVAL SETTING                     #
-###########################################################
-discriminator_train_start_steps: 100000 # Number of steps to start to train discriminator.
-train_max_steps: 400000                 # Number of training steps.
-save_interval_steps: 5000               # Interval steps to save checkpoint.
-eval_interval_steps: 1000               # Interval steps to evaluate the network.
-log_interval_steps: 100                 # Interval steps to record the training log.
-
-###########################################################
-#                     OTHER SETTING                       #
-###########################################################
-num_save_intermediate_results: 4  # Number of results to be saved as intermediate results.
diff --git a/egs/jsut_song/voc1/local/data_download.sh b/egs/jsut_song/voc1/local/data_download.sh
deleted file mode 100755
index f5f251f2..00000000
--- a/egs/jsut_song/voc1/local/data_download.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 Tomoki Hayashi
-#  MIT License (https://opensource.org/licenses/MIT)
-
-download_dir=$1
-
-# check arguments
-if [ $# != 1 ]; then
-    echo "Usage: $0 <download_dir>"
-    exit 1
-fi
-
-set -euo pipefail
-
-cwd=$(pwd)
-if [ ! -e "${download_dir}/jsut_ver1.1" ]; then
-    mkdir -p "${download_dir}"
-    cd "${download_dir}" || exit 1;
-    wget http://ss-takashi.sakura.ne.jp/corpus/jsut_ver1.1.zip
-    unzip -o ./*.zip
-    rm ./*.zip
-    cd "${cwd}" || exit 1;
-    echo "Successfully downloaded data."
-else
-    echo "Already exists. Skipped."
-fi
-
-if [ ! -e "${download_dir}/jsut_lab" ]; then
-    cd "${download_dir}" || exit 1;
-    git clone https://github.com/r9y9/jsut-lab
-    for name in loanword128 repeat500 voiceactress100 basic5000 onomatopee300 travel1000 countersuffix26 precedent130 utparaphrase512; do
-        cp -vr "jsut-lab/${name}" jsut_ver1.1/
-    done
-    cd - || exit 1;
-    echo "Successfully downloaded context label."
-else
-    echo "Already exists. Skipped."
-fi
diff --git a/egs/jsut_song/voc1/local/data_prep.py b/egs/jsut_song/voc1/local/data_prep.py
new file mode 100644
index 00000000..3b3022f9
--- /dev/null
+++ b/egs/jsut_song/voc1/local/data_prep.py
@@ -0,0 +1,294 @@
+"""
+Reference from ESPnet's egs2/nit_song070/svs1/local/data_prep.py
+https://github.com/espnet/espnet/blob/master/egs2/nit_song070/svs1/local/data_prep.py
+"""
+
+import argparse
+import os
+import re
+
+import librosa
+import numpy as np
+import soundfile as sf
+
+from espnet2.fileio.score_scp import SingingScoreWriter
+
+
+DEV_LIST = ["003"]
+TEST_LIST = ["004", "007", "010"]
+
+
+class DatasetType:
+    TRAIN = 0
+    DEV = 1
+    EVAL = 2
+
+
+
+def train_check(song):
+    return not test_check(song) and not dev_check(song)
+
+
+def dev_check(song):
+    for dev in DEV_LIST:
+        if dev in song:
+            return True
+    return False
+
+
+def test_check(song):
+    for test in TEST_LIST:
+        if test in song:
+            return True
+    return False
+
+dataset_check_functions = {
+    DatasetType.TRAIN: train_check,
+    DatasetType.DEV: dev_check,
+    DatasetType.EVAL: test_check
+}
+
+
+def pack_zero(string, size=4):
+    if len(string) < size:
+        string = "0" * (size - len(string)) + string
+    return string
+
+
+def create_score(phns, midis, syb_dur):
+    # Transfer into 'score' format
+    assert len(phns) == len(midis)
+    assert len(midis) == len(syb_dur)
+    st = 0
+    index_phn = 0
+    note_list = []
+    while index_phn < len(phns):
+        midi = midis[index_phn]
+        note_info = [st]
+        st += syb_dur[index_phn]
+        syb = [phns[index_phn]]
+        index_phn += 1
+        if (
+            index_phn < len(phns)
+            and syb_dur[index_phn] == syb_dur[index_phn - 1]
+            and midis[index_phn] == midis[index_phn - 1]
+        ):
+            syb.append(phns[index_phn])
+            index_phn += 1
+        syb = "_".join(syb)
+        note_info.extend([st, syb, midi, syb])
+        note_list.append(note_info)
+        # multi notes in one syllable
+        while index_phn < len(phns) and phns[index_phn] == phns[index_phn - 1]:
+            note_info = [st]
+            st += syb_dur[index_phn]
+            note_info.extend([st, "—", midis[index_phn], phns[index_phn]])
+            note_list.append(note_info)
+            index_phn += 1
+    return note_list
+
+
+def process_utterance(
+    writer,
+    wavscp,
+    text,
+    utt2spk,
+    label,
+    audio_dir,
+    wav_dumpdir,
+    label_id,
+    label_file,
+    tgt_sr=24000,
+):
+    labels = open(label_file, "r", encoding="utf-8")
+    label_info = labels.read().split("\n")
+    prev_end = 0
+    phns, notes, phn_dur = [], [], []
+    seg_start = 0
+    seg_id = 0
+    seg_len_phns = 0
+    segs = []  # store (seg_id, seg_start, seg_end, seg_len_phns)
+
+    sil = ["pau", "sil"]
+    sil = set(sil)
+
+    pattern = re.compile(
+        r"([a-z])@([a-zA-Z]+)\^([a-zA-Z]+)-([a-zA-Z]+)\+([a-zA-Z]+)=([a-zA-Z]+)_"
+    )  # extract phones from the start of the string
+
+    for i in range(len(label_info)):
+        note_label = label_info[i].split(" ")
+        start = int(note_label[0])
+        end = int(note_label[1])
+        if start < prev_end:
+            continue
+        detailed_info = note_label[2].split("E:")
+        # assert len(detailed_info) > 2, "{}".format(label_id)
+        if len(detailed_info) < 2:
+            print("skip {} for no note label".format(label_id))
+            continue
+        aligned_note = detailed_info[1].split("]")[0]
+        phones = re.findall(pattern, detailed_info[0])
+        assert (
+            len(phones) == 1
+        ), f"len(phones)={len(phones)} is not 1. Find more or no patterns from {detailed_info[0]}"
+        phn = phones[0][3]
+        if phn in sil:
+            if seg_len_phns > 0:
+                segs.append([pack_zero(str(seg_id)), seg_start, end, seg_len_phns])
+                seg_id += 1
+            seg_start = end
+            seg_len_phns = 0
+            continue
+        if aligned_note == "xx":
+            notes.append(0)  # rest
+        else:
+            assert (
+                aligned_note.isdigit()
+            ), f"aligned_note={aligned_note} is not an integer"
+            aligned_note = int(aligned_note)
+            notes.append(aligned_note)
+        phns.append(phn)
+        phn_dur.append(end - start)
+        seg_len_phns += 1
+
+    start_phn_index = 0
+    print(label_id)
+    for seg in segs:
+        print(seg)
+        y, sr = sf.read(
+            os.path.join(audio_dir, label_id) + ".wav",
+            start=int(seg[1] * 48000 * 16 // 1e7),
+            stop=int(seg[2] * 48000 * 16 // 1e7),
+        )
+        onset_env = librosa.onset.onset_strength(y=y, sr=sr)
+        # estimate a static tempo for midi format
+        tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=sr)[0]
+        tempo = int(tempo)
+
+        # note to midi index
+        seg_notes = notes[start_phn_index : start_phn_index + seg[3]]
+
+        # duration type convert
+        seg_phn_dur = [
+            float(dur / 1e7)
+            for dur in phn_dur[start_phn_index : start_phn_index + seg[3]]
+        ]
+
+        note_list = create_score(
+            phns[start_phn_index : start_phn_index + seg[3]],
+            seg_notes,
+            seg_phn_dur,
+        )
+
+        text.write(
+            "jsut_song_{}_{} {}\n".format(
+                label_id,
+                seg[0],
+                " ".join(phns[start_phn_index : start_phn_index + seg[3]]),
+            )
+        )
+        utt2spk.write("jsut_song_{}_{} {}\n".format(label_id, seg[0], "jsut_song"))
+
+        # apply bit convert, there is a known issue in direct convert in format wavscp
+        cmd = (
+            f"sox -t wav -r 48000 -b 16 -c 1 -L -e signed-integer "
+            f"{os.path.join(audio_dir, label_id)}.wav -c 1 -t wavpcm -b 16 -r {tgt_sr} "
+            f"{os.path.join(wav_dumpdir, 'jsut_song_'+label_id+'_'+seg[0])}.wav "
+            f"trim {float(seg[1] / 1e7)} {float((seg[2]-seg[1]) / 1e7)}"
+        )
+        os.system(cmd)
+
+        wavscp.write(
+            "jsut_song_{}_{} {}.wav\n".format(
+                label_id,
+                seg[0],
+                os.path.join(wav_dumpdir, "jsut_song_" + label_id + "_" + seg[0]),
+            )
+        )
+
+        running_dur = 0
+        assert (
+            len(seg_phn_dur)
+            == len(phns[start_phn_index : start_phn_index + seg[3]])
+            == seg[3]
+        )
+        label_entry = []
+        for i in range(seg[3]):
+            start = running_dur
+            end = running_dur + seg_phn_dur[i]
+            label_entry.append(
+                "{:.3f} {:.3f} {}".format(start, end, phns[start_phn_index + i])
+            )
+            running_dur += seg_phn_dur[i]
+
+        label.write(
+            "jsut_song_{}_{} {}\n".format(label_id, seg[0], " ".join(label_entry))
+        )
+        score = dict(
+            tempo=tempo, item_list=["st", "et", "lyric", "midi", "phns"], note=note_list
+        )
+        writer["jsut_song_{}_{}".format(label_id, seg[0])] = score
+        start_phn_index += seg[3]
+
+
+def process_subset(args, set_name, dataset_type):
+    writer = SingingScoreWriter(
+        args.score_dump, os.path.join(args.tgt_dir, set_name, "score.scp")
+    )
+    wavscp = open(
+        os.path.join(args.tgt_dir, set_name, "wav.scp"), "w", encoding="utf-8"
+    )
+    label = open(os.path.join(args.tgt_dir, set_name, "label"), "w", encoding="utf-8")
+    text = open(os.path.join(args.tgt_dir, set_name, "text"), "w", encoding="utf-8")
+    utt2spk = open(
+        os.path.join(args.tgt_dir, set_name, "utt2spk"), "w", encoding="utf-8"
+    )
+
+    assert dataset_type in dataset_check_functions
+    check_func = dataset_check_functions[dataset_type]
+    for label_file in os.listdir(args.lab_srcdir):
+        if not label_file.endswith(".lab"):
+            continue
+
+        if not check_func(label_file):
+            continue
+
+        label_id = label_file.split(".")[0]
+        process_utterance(
+            writer,
+            wavscp,
+            text,
+            utt2spk,
+            label,
+            args.wav_srcdir,
+            args.wav_dumpdir,
+            label_id,
+            os.path.join(args.lab_srcdir, label_file),
+            tgt_sr=args.sr,
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Prepare Data for JSUT-Song Database")
+    parser.add_argument("--lab_srcdir", type=str, help="label source directory")
+    parser.add_argument("--wav_srcdir", type=str, help="wav source directory")
+    parser.add_argument("--tgt_dir", type=str, default="data")
+    parser.add_argument(
+        "--score_dump", type=str, default="score_dump", help="score dump directory"
+    )
+    parser.add_argument(
+        "--wav_dumpdir", type=str, help="wav dump directoy (rebit)", default="wav_dump"
+    )
+    parser.add_argument("--sr", type=int, help="sampling rate (Hz)")
+
+    parser.add_argument("--train_set", type=str, help="name of training data directory")
+    parser.add_argument("--dev_set", type=str, help="name of development data direcotry")
+    parser.add_argument("--eval_set", type=str, help="name of evaluation data direcotry")
+    args = parser.parse_args()
+
+    for name, dataset_type in zip(
+        [args.train_set, args.dev_set, args.eval_set],
+        [DatasetType.TRAIN, DatasetType.DEV, DatasetType.EVAL],
+    ):
+        process_subset(args, name, dataset_type)
\ No newline at end of file
diff --git a/egs/jsut_song/voc1/local/data_prep.sh b/egs/jsut_song/voc1/local/data_prep.sh
deleted file mode 100755
index b2044d8f..00000000
--- a/egs/jsut_song/voc1/local/data_prep.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/bash
-
-# Copyright 2020 Tomoki Hayashi
-#  MIT License (https://opensource.org/licenses/MIT)
-
-# shellcheck disable=SC1091
-. ./path.sh || exit 1;
-
-fs=22050
-num_dev=100
-num_eval=100
-train_set="train_nodev"
-dev_set="dev"
-eval_set="eval"
-shuffle=false
-
-# shellcheck disable=SC1091
-. utils/parse_options.sh || exit 1;
-
-db_root=$1
-data_dir=$2
-
-# check arguments
-if [ $# != 2 ]; then
-    echo "Usage: $0 [Options] <db_root> <data_dir>"
-    echo ""
-    echo "Options:"
-    echo "    --num_dev: number of development uttreances (default=250)."
-    echo "    --num_eval: number of evaluation uttreances (default=250)."
-    echo "    --train_set: name of train set (default=train_nodev)."
-    echo "    --dev_set: name of dev set (default=dev)."
-    echo "    --eval_set: name of eval set (default=eval)."
-    echo "    --shuffle: whether to perform shuffle in making dev / eval set (default=false)."
-    exit 1
-fi
-
-set -euo pipefail
-
-[ ! -e "${data_dir}/all" ] && mkdir -p "${data_dir}/all"
-
-# set filenames
-scp="${data_dir}/all/wav.scp"
-
-# check file existence
-[ -e "${scp}" ] && rm "${scp}"
-
-# make all scp
-find "${db_root}" -follow -name "*.wav" | sort | while read -r filename; do
-    id=$(basename "${filename}" | sed -e "s/\.[^\.]*$//g")
-    echo "${id} cat ${filename} | sox -t wav - -c 1 -b 16 -t wav - rate ${fs} |" >> "${scp}"
-done
-
-# split
-num_all=$(wc -l < "${scp}")
-num_deveval=$((num_dev + num_eval))
-num_train=$((num_all - num_deveval))
-if [ ${num_eval} -ne 0 ]; then
-    utils/split_data.sh \
-        --num_first "${num_train}" \
-        --num_second "${num_deveval}" \
-        --shuffle "${shuffle}" \
-        "${data_dir}/all" \
-        "${data_dir}/${train_set}" \
-        "${data_dir}/deveval"
-    utils/split_data.sh \
-        --num_first "${num_dev}" \
-        --num_second "${num_eval}" \
-        --shuffle "${shuffle}" \
-        "${data_dir}/deveval" \
-        "${data_dir}/${dev_set}" \
-        "${data_dir}/${eval_set}"
-else
-    utils/split_data.sh \
-        --num_first "${num_train}" \
-        --num_second "${num_deveval}" \
-        --shuffle "${shuffle}" \
-        "${data_dir}/all" \
-        "${data_dir}/${train_set}" \
-        "${data_dir}/${dev_set}"
-    cp -r "${data_dir}/${dev_set}" "${data_dir}/${eval_set}"
-fi
-
-# remove tmp directories
-rm -rf "${data_dir}/all"
-rm -rf "${data_dir}/deveval"
-
-echo "Successfully prepared data."
diff --git a/egs/jsut_song/voc1/run.sh b/egs/jsut_song/voc1/run.sh
index c3872e47..88e770b5 100755
--- a/egs/jsut_song/voc1/run.sh
+++ b/egs/jsut_song/voc1/run.sh
@@ -7,31 +7,21 @@
 . ./path.sh || exit 1;
 
 # basic settings
-stage=0        # stage to start
+stage=-1        # stage to start
 stop_stage=100 # stage to stop
 verbose=1      # verbosity level (lower is less info)
 n_gpus=1       # number of gpus in training
 n_jobs=4       # number of parallel jobs in feature extraction
 
 # NOTE(kan-bayashi): renamed to conf to avoid conflict in parse_options.sh
-conf=conf/parallel_wavegan.v1.yaml
+conf=conf/hifigan.v1.yaml
 
 # directory path setting
-db_root=/path/to/database # direcotry including wavfiles (MODIFY BY YOURSELF)
-                          # each wav filename in the directory should be unique
-                          # e.g.
-                          # /path/to/database
-                          # ├── utt_1.wav
-                          # ├── utt_2.wav
-                          # │   ...
-                          # └── utt_N.wav
+db_root=downloads # direcotry to download data and labels from https://sites.google.com/site/shinnosuketakamichi/publication/jsut-song
 dumpdir=dump # directory to dump features
 
 # subset setting
 shuffle=false # whether to shuffle the data to create subset
-num_dev=100   # the number of development data
-num_eval=100  # the number of evaluation data
-              # (if set to 0, the same dev set is used as eval set)
 
 # training related setting
 tag=""     # tag for directory to save model
@@ -54,17 +44,45 @@ eval_set="eval"         # name of evaluation data direcotry
 
 set -euo pipefail
 
+if [ "${stage}" -le -1 ] && [ "${stop_stage}" -ge -1 ]; then
+    echo "Stage -1: Data download"
+    if [ -e "${db_root}/todai_child" ] && [ -e "${db_root}/jsut-song_ver1/child_song/wav" ]; then
+        echo "The JSUT-song corpus exists. Skip downloading."
+    
+    elif [ -e "${db_root}/jsut-song_ver1.zip" ] && [ -e "${db_root}/jsut-song_label.zip" ]; then
+        echo "Unzipping downloaded zip files for JSUT-song corpus."
+        unzip ${db_root}/jsut-song_ver1.zip -d ${db_root}
+        unzip ${db_root}/jsut-song_label.zip -d ${db_root}
+        rm ${db_root}/jsut-song_ver1.zip
+        rm ${db_root}/jsut-song_label.zip
+
+    else
+    	echo "ERROR: The JSUT-song corpus does not exist."
+    	echo "ERROR: Please download from https://sites.google.com/site/shinnosuketakamichi/publication/jsut-song"
+        echo "and locate it at ${db_root}"
+        echo "Please ensure that you've downloaded songs (jsut-song_ver1.zip) and labels (jsut-song_label.zip) to ${db_root} before proceeding"
+        # Terms from https://sites.google.com/site/shinnosuketakamichi/publication/jsut-song
+    	exit 1
+    fi
+fi
+
 if [ "${stage}" -le 0 ] && [ "${stop_stage}" -ge 0 ]; then
     echo "Stage 0: Data preparation"
-    local/data_prep.sh \
-        --fs "$(yq ".sampling_rate" "${conf}")" \
-        --num_dev "${num_dev}" \
-        --num_eval "${num_eval}" \
+    mkdir -p score_dump
+    mkdir -p wav_dump
+    python local/data_prep.py \
+        --lab_srcdir ${db_root}/todai_child \
+        --wav_srcdir ${db_root}/jsut-song_ver1/child_song/wav \
+        --score_dump score_dump \
+        --wav_dumpdir wav_dump \
+        --sr "$(yq ".sampling_rate" "${conf}")" \
         --train_set "${train_set}" \
         --dev_set "${dev_set}" \
-        --eval_set "${eval_set}" \
-        --shuffle "${shuffle}" \
-        "${db_root}" data
+        --eval_set "${eval_set}"
+    for src_data in ${train_set} ${dev_set} ${eval_set}; do
+        utils/utt2spk_to_spk2utt.pl < data/${src_data}/utt2spk > data/${src_data}/spk2utt
+        utils/fix_data_dir.sh --utt_extra_files "label score.scp" data/${src_data}
+    done
 fi
 
 stats_ext=$(grep -q "hdf5" <(yq ".format" "${conf}") && echo "h5" || echo "npy")
@@ -130,13 +148,13 @@ if [ "${stage}" -le 1 ] && [ "${stop_stage}" -ge 1 ]; then
 fi
 
 if [ -z "${tag}" ]; then
-    expdir="exp/${train_set}_$(basename "${conf}" .yaml)"
+    expdir="exp/${train_set}_jsut_song_$(basename "${conf}" .yaml)"
     if [ -n "${pretrain}" ]; then
         pretrain_tag=$(basename "$(dirname "${pretrain}")")
         expdir+="_${pretrain_tag}"
     fi
 else
-    expdir="exp/${train_set}_${tag}"
+    expdir="exp/${train_set}_jsut_song_${tag}"
 fi
 if [ "${stage}" -le 2 ] && [ "${stop_stage}" -ge 2 ]; then
     echo "Stage 2: Network training"

From 35dd311bb6484e8f12f9d81da3adb6942cef3063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CJionghao=E2=80=9D?= <jionghah@andrew.cmu.ed>
Date: Wed, 14 Feb 2024 21:56:48 -0500
Subject: [PATCH 4/4] commit data.sh

---
 egs/jsut_song/voc1/local/data.sh | 76 ++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100755 egs/jsut_song/voc1/local/data.sh

diff --git a/egs/jsut_song/voc1/local/data.sh b/egs/jsut_song/voc1/local/data.sh
new file mode 100755
index 00000000..0fbd09a0
--- /dev/null
+++ b/egs/jsut_song/voc1/local/data.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Reference from ESPnet's egs2/nit_song070/svs1/local/data.sh
+# https://github.com/espnet/espnet/blob/master/egs2/nit_song070/svs1/local/data.sh
+
+
+set -e
+set -u
+set -o pipefail
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+db_root=$1
+data_dir=$2
+
+SECONDS=0
+stage=-1
+stop_stage=100
+fs=24000
+g2p=None
+
+log "$0 $*"
+
+. utils/parse_options.sh || exit 1;
+
+if [ -z "${db_root}" ]; then
+    log "Fill the value of 'db_root' of db.sh"
+    exit 1
+fi
+
+mkdir -p ${db_root}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    log "stage -1: Data Download"
+    if [ -e "${db_root}/todai_child" ] && [ -e "${db_root}/jsut-song_ver1/child_song/wav" ]; then
+        echo "The JSUT-song corpus exists. Skip downloading."
+    
+    elif [ -e "${db_root}/jsut-song_ver1.zip" ] && [ -e "${db_root}/jsut-song_label.zip" ]; then
+        echo "Unzipping downloaded zip files for JSUT-song corpus."
+        unzip ${db_root}/jsut-song_ver1.zip -d ${db_root}
+        unzip ${db_root}/jsut-song_label.zip -d ${db_root}
+        rm ${db_root}/jsut-song_ver1.zip
+        rm ${db_root}/jsut-song_label.zip
+
+    if [ ! -e "${db_root}/jsut-song_ver1.zip" ] || [ ! -e "${db_root}/jsut-song_label.zip" ]; then
+    	echo "ERROR: The JSUT-song corpus does not exist."
+    	echo "ERROR: Please download from https://sites.google.com/site/shinnosuketakamichi/publication/jsut-song"
+        echo "and locate it at ${db_root}"
+        echo "Please ensure that you've downloaded songs (jsut-song_ver1.zip) and labels (jsut-song_label.zip) to ${db_root} before proceeding"
+        # Terms from https://sites.google.com/site/shinnosuketakamichi/publication/jsut-song
+    	exit 1
+    fi
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    log "stage 0: Data preparaion "
+
+    mkdir -p score_dump
+    mkdir -p wav_dump
+    python local/data_prep.py \
+        --lab_srcdir ${db_root}/todai_child \
+        --wav_srcdir ${db_root}/jsut-song_ver1/child_song/wav \
+        --score_dump score_dump \
+        --wav_dumpdir wav_dump \
+        --sr ${fs}
+    for src_data in ${train_set} ${train_dev} ${eval_set}; do
+        utils/utt2spk_to_spk2utt.pl < ${data_dir}/${src_data}/utt2spk > ${data_dir}/${src_data}/spk2utt
+        utils/fix_data_dir.sh --utt_extra_files "label score.scp" ${data_dir}/${src_data}
+    done
+fi
\ No newline at end of file