From fe8bbfdf2fef3bed8ebd461dcd33dfdd28604d02 Mon Sep 17 00:00:00 2001 From: Jeffrey Rennie Date: Fri, 15 May 2020 10:53:36 -0700 Subject: [PATCH] fix: avoid running out of disk space (#558) by cleaning up after each inner synth run --- autosynth/synth.py | 147 ++++++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 68 deletions(-) diff --git a/autosynth/synth.py b/autosynth/synth.py index 0b4c8f6b5..103e70d57 100644 --- a/autosynth/synth.py +++ b/autosynth/synth.py @@ -519,82 +519,93 @@ def _inner_main(temp_dir: str) -> int: working_repo_path = synthtool_git.clone(f"https://github.com/{args.repository}.git") - os.chdir(working_repo_path) + try: + os.chdir(working_repo_path) - git.configure_git(args.github_user, args.github_email) + git.configure_git(args.github_user, args.github_email) - git.setup_branch(branch) + git.setup_branch(branch) - if args.synth_path: - os.chdir(args.synth_path) - - metadata_path = os.path.join(args.metadata_path or "", "synth.metadata") - - flags = autosynth.flags.parse_flags() - # Override flags specified in synth.py with flags specified in environment vars. - for key in flags.keys(): - env_value = os.environ.get(key, "") - if env_value: - flags[key] = False if env_value.lower() == "false" else env_value - - metadata = load_metadata(metadata_path) - multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS] - multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS] - if (not multiple_commits and not multiple_prs) or not metadata: - if change_pusher.check_if_pr_already_exists(branch): - return 0 - - synth_log = Synthesizer( - metadata_path, - args.extra_args, - deprecated_execution=args.deprecated_execution, - ).synthesize(base_synth_log_path) - - if not has_changes(): - logger.info("No changes. :)") - sys.exit(EXIT_CODE_SKIPPED) - - git.commit_all_changes(pr_title) - change_pusher.push_changes(1, branch, pr_title, synth_log) - return 1 + if args.synth_path: + os.chdir(args.synth_path) - else: - if not multiple_prs and change_pusher.check_if_pr_already_exists(branch): - return 0 # There's already an existing PR - - # Enumerate the versions to loop over. - sources = metadata.get("sources", []) - source_versions = [ - git_source.enumerate_versions_for_working_repo(metadata_path, sources) - ] - # Add supported source version types below: - source_versions.extend( - git_source.enumerate_versions(sources, pathlib.Path(temp_dir)) - ) + metadata_path = os.path.join(args.metadata_path or "", "synth.metadata") - # Prepare to call synthesize loop. - synthesizer = Synthesizer( - metadata_path, args.extra_args, args.deprecated_execution, "synth.py", - ) - x = SynthesizeLoopToolbox( - source_versions, - branch, - temp_dir, - metadata_path, - args.synth_path, - base_synth_log_path, - ) - if not multiple_commits: - change_pusher = SquashingChangePusher(change_pusher) + flags = autosynth.flags.parse_flags() + # Override flags specified in synth.py with flags specified in environment vars. + for key in flags.keys(): + env_value = os.environ.get(key, "") + if env_value: + flags[key] = False if env_value.lower() == "false" else env_value + + metadata = load_metadata(metadata_path) + multiple_commits = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_COMMITS] + multiple_prs = flags[autosynth.flags.AUTOSYNTH_MULTIPLE_PRS] + if (not multiple_commits and not multiple_prs) or not metadata: + if change_pusher.check_if_pr_already_exists(branch): + return 0 + + synth_log = Synthesizer( + metadata_path, + args.extra_args, + deprecated_execution=args.deprecated_execution, + ).synthesize(base_synth_log_path) + + if not has_changes(): + logger.info("No changes. :)") + sys.exit(EXIT_CODE_SKIPPED) - # Call the loop. - commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer) + git.commit_all_changes(pr_title) + change_pusher.push_changes(1, branch, pr_title, synth_log) + return 1 - if commit_count == 0: - logger.info("No changes. :)") - sys.exit(EXIT_CODE_SKIPPED) + else: + if not multiple_prs and change_pusher.check_if_pr_already_exists(branch): + return 0 # There's already an existing PR + + # Enumerate the versions to loop over. + sources = metadata.get("sources", []) + source_versions = [ + git_source.enumerate_versions_for_working_repo(metadata_path, sources) + ] + # Add supported source version types below: + source_versions.extend( + git_source.enumerate_versions(sources, pathlib.Path(temp_dir)) + ) + + # Prepare to call synthesize loop. + synthesizer = Synthesizer( + metadata_path, args.extra_args, args.deprecated_execution, "synth.py", + ) + x = SynthesizeLoopToolbox( + source_versions, + branch, + temp_dir, + metadata_path, + args.synth_path, + base_synth_log_path, + ) + if not multiple_commits: + change_pusher = SquashingChangePusher(change_pusher) - return commit_count + # Call the loop. + commit_count = synthesize_loop(x, multiple_prs, change_pusher, synthesizer) + + if commit_count == 0: + logger.info("No changes. :)") + sys.exit(EXIT_CODE_SKIPPED) + + return commit_count + finally: + if args.synth_path: + # We're generating code in a mono repo. The state left behind will + # probably be useful for generating the next API. + pass + else: + # We're generating a single API in a single repo, and using a different + # repo to generate the next API. So the next synth will not be able to + # use any of this state. Clean it up to avoid running out of disk space. + executor.check_call(["git", "clean", "-fdx"], cwd=working_repo_path) if __name__ == "__main__":