From 46b492a9fe80197e4867c38cf682334c34c754bb Mon Sep 17 00:00:00 2001 From: OmaymaMahjoub Date: Fri, 26 Jan 2024 10:06:56 +0100 Subject: [PATCH] feat: update the merge json files code --- README.md | 4 +--- marl_eval/json_tools/merge_json_files.py | 12 +++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ec068934..4131218e 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,7 @@ Here `run_1` to `run_n` correspond to the number of independent runs in a given #### Data Tooling [**Pull Neptune Data**](marl_eval/json_tools/pull_neptune_data.py): `pull_neptune_data` connects to a Neptune project, retrieves experiment data from specific systems and tags, and downloads it to a local directory. This function is particularly useful when there is a need to pull data from multiple experiments. -[**JSON Files Merging Script**](marl_eval/json_tools/merge_json_files.py): We offer a function called `concatenate_files` that reads json files from a specified directory, concatenates their contents into a single structured dictionary, -and ensures uniqueness of seed numbers within the data. It handles nested json structures and saves the concatenated -result into a new json file. Designed primarily for managing and aggregating json data from multiple files in experimental setups. +[**JSON Files Merging Script**](marl_eval/json_tools/merge_json_files.py): We offer a function called `concatenate_files` that can be found in `marl_eval/utils` that reads json files from a specified directory and concatenates their contents into a single structured dictionary, and ensures uniqueness of seed numbers within the data. It handles nested json structures and saves the concatenated result into a new json file. Designed primarily for managing and aggregating json data from multiple files in experimental setups. > 📌 Using `pull_neptune_data` alongside `concatenate_files` forms an effective workflow, first generating multiple JSON files from experiments and then merging them into a single file, ready for use in marl-eval. This approach streamlines data preparation. diff --git a/marl_eval/json_tools/merge_json_files.py b/marl_eval/json_tools/merge_json_files.py index 206bdc49..7e603df2 100644 --- a/marl_eval/json_tools/merge_json_files.py +++ b/marl_eval/json_tools/merge_json_files.py @@ -25,11 +25,13 @@ def _read_json_files(directory: str) -> list: """Reads all JSON files in a directory and returns a list of JSON objects.""" json_data = [] - for filename in os.listdir(directory): - if filename.endswith(".json"): - file_path = os.path.join(directory, filename) - with open(file_path) as file: - json_data.append(json.load(file)) + for root, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".json"): + file_path = os.path.join(root, filename) + with open(file_path) as file: + json_data.append(json.load(file)) + return json_data