diff --git a/misc/e2elive.py b/misc/e2elive.py index 90e12f796..f4db36b47 100644 --- a/misc/e2elive.py +++ b/misc/e2elive.py @@ -74,7 +74,9 @@ def main(): if not (source_is_tar or (sourcenet and os.path.isdir(sourcenet))): tarname = args.s3_source_net if not tarname: - raise Exception("Must provide either local or s3 network to run test against") + raise Exception( + "Must provide either local or s3 network to run test against" + ) tarname = f"{tarname}.tar.bz2" # fetch test data from S3 @@ -82,10 +84,13 @@ def main(): import boto3 from botocore.config import Config from botocore import UNSIGNED + s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED)) tarpath = os.path.join(tempdir, tarname) prefix = "indexer/e2e4" - success = firstFromS3Prefix(s3, bucket, prefix, tarname, outpath=tarpath) + success = firstFromS3Prefix( + s3, bucket, prefix, tarname, outpath=tarpath, max_s3_keys=300 + ) if not success: raise Exception( f"failed to locate tarname={tarname} from AWS S3 path {bucket}/{prefix}" diff --git a/misc/util.py b/misc/util.py index 19b79f24a..c279a758e 100644 --- a/misc/util.py +++ b/misc/util.py @@ -158,7 +158,13 @@ def ensure_test_db(connection_string, keep_temps=False): # whoever calls this will need to import boto and get the s3 client def firstFromS3Prefix( - s3, bucket, prefix, desired_filename, outdir=None, outpath=None + s3, + bucket, + prefix, + desired_filename, + outdir=None, + outpath=None, + max_s3_keys=50, ) -> bool: haystack = [] found_needle = False @@ -175,7 +181,7 @@ def report(): atexit.register(report) - response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=50) + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=max_s3_keys) if (not response.get("KeyCount")) or ("Contents" not in response): raise Exception("nothing found in s3://{}/{}".format(bucket, prefix)) for x in response["Contents"]: