Skip to content

Commit

Permalink
Review fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
yifanmai committed Aug 8, 2023
1 parent 9377622 commit 7a4b92b
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 15 deletions.
4 changes: 4 additions & 0 deletions src/helm/benchmark/model_deployment_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ class ClientSpec(ObjectSpec):

@dataclass(frozen=True)
class ModelDeployment:
"""A model deployment is an accessible instance of this model (e.g. a hosted endpoint).
A model can have model deployments."""

name: str
"""Name of the model deployment."""

Expand Down
28 changes: 13 additions & 15 deletions src/helm/benchmark/model_metadata_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,31 @@
@dataclass(frozen=True)
class ModelMetadata:
name: str
"""Name of the model e.g. "meta/llama-2"."""

# Organization that originally created the model (e.g. "EleutherAI")
# Note that this may be different from group or the prefix of the model `name`
# ("together" in "together/gpt-j-6b") as the hosting organization
# may be different from the creator organization. We also capitalize
# this field properly to later display in the UI.
# TODO: in the future, we want to cleanup the naming in the following ways:
# - make the creator_organization an identifier with a separate display name
# - have a convention like <hosting_organization><creator_organization>/<model_name>
creator_organization: Optional[str] = None
"""Organization that originally created the model (e.g. "meta")."""

# How this model is available (e.g., limited)
access: Optional[str] = None
"""How this model is available (e.g., limited).
If there are multiple deployments, this should be the most permissive access across
all deployments."""

# Whether we have yet to evaluate this model
todo: bool = False
"""Whether we have yet to evaluate this model."""

# When was the model released
release_date: Optional[date] = None
"""When the model was released."""

# The number of parameters
# This should be a string as the number of parameters is usually a round number (175B),
# but we set it as an int for plotting purposes.
num_parameters: Optional[int] = None
"""The number of model parameters.
This should be a string as the number of parameters is usually a round number (175B),
but we set it as an int for plotting purposes."""

# Tags corresponding to the properties of the model.
tags: List[str] = field(default_factory=list)
""""""


@dataclass(frozen=True)
Expand Down

0 comments on commit 7a4b92b

Please sign in to comment.