expose max_s3_keys haystack size in firstFromS3's needle search and t…

…emporarily set it to 300

misc/e2elive.py

@@ -74,18 +74,23 @@ def main():
     if not (source_is_tar or (sourcenet and os.path.isdir(sourcenet))):
         tarname = args.s3_source_net
         if not tarname:
-            raise Exception("Must provide either local or s3 network to run test against")
+            raise Exception(
+                "Must provide either local or s3 network to run test against"
+            )
         tarname = f"{tarname}.tar.bz2"
 
         # fetch test data from S3
         bucket = "algorand-testdata"
         import boto3
         from botocore.config import Config
         from botocore import UNSIGNED
+
         s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
         tarpath = os.path.join(tempdir, tarname)
         prefix = "indexer/e2e4"
-        success = firstFromS3Prefix(s3, bucket, prefix, tarname, outpath=tarpath)
+        success = firstFromS3Prefix(
+            s3, bucket, prefix, tarname, outpath=tarpath, max_s3_keys=300
+        )
         if not success:
             raise Exception(
                 f"failed to locate tarname={tarname} from AWS S3 path {bucket}/{prefix}"

misc/util.py

@@ -158,7 +158,13 @@ def ensure_test_db(connection_string, keep_temps=False):
 
 # whoever calls this will need to import boto and get the s3 client
 def firstFromS3Prefix(
-    s3, bucket, prefix, desired_filename, outdir=None, outpath=None
+    s3,
+    bucket,
+    prefix,
+    desired_filename,
+    outdir=None,
+    outpath=None,
+    max_s3_keys=50,
 ) -> bool:
     haystack = []
     found_needle = False
@@ -175,7 +181,7 @@ def report():
 
     atexit.register(report)
 
-    response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=50)
+    response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=max_s3_keys)
     if (not response.get("KeyCount")) or ("Contents" not in response):
         raise Exception("nothing found in s3://{}/{}".format(bucket, prefix))
     for x in response["Contents"]: