Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support PEP 658/714 #1457

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Support PEP 658/714 #1457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b032a72
Support PEP 658, without hashes
thejcannon Jul 5, 2023
4f622d4
Update s3_management/manage.py
thejcannon Jul 12, 2023
8b99f03
Merge branch 'main' into pep658
thejcannon Oct 27, 2023
5759ca8
.key
thejcannon Oct 27, 2023
bd690c4
its a set
thejcannon Oct 27, 2023
9b542c1
clearer
thejcannon Oct 27, 2023
142e600
ok get it working with hashes, and post-rebase
thejcannon Oct 27, 2023
9736eee
quotes
thejcannon Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions manywheel/upload.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,17 @@ for cuda_ver in "${CUDA_VERSIONS[@]}"; do

# Upload the wheels to s3
if [[ -d "$wheel_dir" ]]; then
pushd "$wheel_dir"
find . -type f -exec sh -c 'unzip -j {} -d . "*.dist-info/METADATA" && mv METADATA {}.metadata' \;
echo "Uploading all of: $(ls $wheel_dir) to $s3_wheel_dir"
ls "$wheel_dir" | xargs -I {} aws s3 cp "$wheel_dir"/{} "$s3_wheel_dir" --acl public-read
ls . | xargs -I {} aws s3 cp {} "$s3_wheel_dir" --acl public-read
popd
fi

if [[ -d "$libtorch_dir" ]]; then
pushd "$libtorch_dir"
echo "Uploading all of: $(ls $libtorch_dir) to $s3_libtorch_dir"
ls "$libtorch_dir" | xargs -I {} aws s3 cp "$libtorch_dir"/{} "$s3_libtorch_dir" --acl public-read
ls . | xargs -I {} aws s3 cp {} "$s3_libtorch_dir" --acl public-read
popd
fi
done
40 changes: 29 additions & 11 deletions s3_management/manage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
CLIENT = boto3.client('s3')
thejcannon marked this conversation as resolved.
Show resolved Hide resolved
BUCKET = S3.Bucket('pytorch')

ACCEPTED_FILE_EXTENSIONS = ("whl", "zip", "tar.gz")
ACCEPTED_FILE_EXTENSIONS = ("whl", "zip", "tar.gz", ".whl.metadata")
ACCEPTED_SUBDIR_PATTERNS = [
r"cu[0-9]+", # for cuda
r"rocm[0-9]+\.[0-9]+", # for rocm
Expand Down Expand Up @@ -148,8 +148,15 @@ def between_bad_dates(package_build_time: datetime):


class S3Index:
def __init__(self: S3IndexType, objects: List[S3Object], prefix: str) -> None:
def __init__(
self: S3IndexType,
objects: List[S3Object],
# Maps an object's key to the sha256 of the relevant .metadata (if it exists)
object_metadatas: Dict[str, str],
prefix: str,
) -> None:
self.objects = objects
self.object_metadatas = object_metadatas
self.prefix = prefix.rstrip("/")
self.html_name = PREFIXES_WITH_HTML[self.prefix]
# should dynamically grab subdirectories like whl/test/cu101
Expand Down Expand Up @@ -279,8 +286,14 @@ def to_simple_package_html(
out.append(' <body>')
out.append(' <h1>Links for {}</h1>'.format(package_name.lower().replace("_","-")))
for obj in sorted(self.gen_file_list(subdir, package_name)):
attributes = []
if metadata_sha256 := self.object_metadatas.get(obj.key):
# Serve the PEP 658 and PEP 714 metadata attributes
attributes += f'data-dist-info-metadata="{metadata_sha256}"'
attributes += f'data-core-metadata="{metadata_sha256}"'
attributes = " ".join(attributes)
maybe_fragment = f"#sha256={obj.checksum}" if obj.checksum else ""
out.append(f' <a href="/{obj.key}{maybe_fragment}">{path.basename(obj.key).replace("%2B","+")}</a><br/>')
out.append(f' <a href="/{obj.key}{maybe_fragment}">{path.basename(obj.key).replace("%2B","+")} {attributes}</a><br/>')
# Adding html footer
out.append(' </body>')
out.append('</html>')
Expand Down Expand Up @@ -407,6 +420,8 @@ def from_S3(cls: Type[S3IndexType], prefix: str) -> S3IndexType:
prefix = prefix.rstrip("/")
obj_names = cls.fetch_object_names(prefix)
objects = []
object_metadatas = {}

def fetch_metadata(key: str) :
return CLIENT.head_object(Bucket=BUCKET.name, Key=key, ChecksumMode="Enabled")

Expand All @@ -420,14 +435,17 @@ def fetch_metadata(key: str) :
sha256 = response.get("Metadata", {}).get("checksum-sha256")
sanitized_key = obj_key.replace("+", "%2B")
size = response.get("ContentLength")
s3_object = S3Object(
key=sanitized_key,
orig_key=obj_key,
checksum=sha256,
size=int(size) if size else size,
)
objects.append(s3_object)
return cls(objects, prefix)
if sanitized_key.endswith(".metadata"):
object_metadatas[sanitized_key[:-len(".metadata")]] = sha256
else:
s3_object = S3Object(
key=sanitized_key,
orig_key=obj_key,
checksum=sha256,
size=int(size) if size else size,
)
objects.append(s3_object)
return cls(objects, object_metadatas, prefix)

@classmethod
def undelete_prefix(cls: Type[S3IndexType], prefix: str) -> None:
Expand Down