[DO NOT MERGE] Feat flex checksum

.changelog/9ebe24c4791541e0840da49eab6f9d97.json

@@ -0,0 +1,11 @@
+{
+    "id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97",
+    "type": "feature",
+    "description": "Enable HTTP checksums in supported services by default. New config fields, RequestChecksumCalculation and ResponseChecksumValidation, allow the caller to opt-out of this new default behavior. This feature also replaces the default MD5 checksum with CRC32.",
+    "modules": [
+        ".",
+        "config",
+        "service/internal/checksum",
+        "service/s3"
+    ]
+}

aws/checksum.go

@@ -0,0 +1,33 @@
+package aws
+
+// RequestChecksumCalculation controls request checksum calculation workflow
+type RequestChecksumCalculation int
+
+const (
+	// RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation
+	RequestChecksumCalculationUnset RequestChecksumCalculation = iota
+
+	// RequestChecksumCalculationWhenSupported indicates request checksum will be calculated
+	// if the operation supports input checksums
+	RequestChecksumCalculationWhenSupported
+
+	// RequestChecksumCalculationWhenRequired indicates request checksum will be calculated
+	// if required by the operation or if user elects to set a checksum algorithm in request
+	RequestChecksumCalculationWhenRequired
+)
+
+// ResponseChecksumValidation controls response checksum validation workflow
+type ResponseChecksumValidation int
+
+const (
+	// ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation
+	ResponseChecksumValidationUnset ResponseChecksumValidation = iota
+
+	// ResponseChecksumValidationWhenSupported indicates response checksum will be validated
+	// if the operation supports output checksums
+	ResponseChecksumValidationWhenSupported
+
+	// ResponseChecksumValidationWhenRequired indicates response checksum will only
+	// be validated if the operation requires output checksum validation
+	ResponseChecksumValidationWhenRequired
+)

aws/config.go

@@ -165,6 +165,33 @@ type Config struct {
 
 	// Controls how a resolved AWS account ID is handled for endpoint routing.
 	AccountIDEndpointMode AccountIDEndpointMode
+
+	// RequestChecksumCalculation determines when request checksum calculation is performed.
+	//
+	// There are two possible values for this setting:
+	//
+	// 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated
+	//    if the operation supports it, regardless of whether the user sets an algorithm in the request.
+	//
+	// 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user
+	//    explicitly sets a checksum algorithm in the request.
+	//
+	// This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION
+	// or the shared config profile attribute "request_checksum_calculation".
+	RequestChecksumCalculation RequestChecksumCalculation
+
+	// ResponseChecksumValidation determines when response checksum validation is performed
+	//
+	// There are two possible values for this setting:
+	//
+	// 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated
+	//    if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request.
+	//
+	// 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user
+	//    explicitly sets the validation mode to ENABLED in the request
+	// This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or
+	// the shared config profile attribute "response_checksum_validation".
+	ResponseChecksumValidation ResponseChecksumValidation
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java

@@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration {
 
     private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode";
 
+    private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation";
+
+    private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation";
+
     private static final List<AwsConfigField> AWS_CONFIG_FIELDS = ListUtils.of(
             AwsConfigField.builder()
                     .name(REGION_CONFIG_NAME)
@@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration {
                     .type(SdkGoTypes.Aws.AccountIDEndpointMode)
                     .documentation("Indicates how aws account ID is applied in endpoint2.0 routing")
                     .servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(REQUEST_CHECKSUM_CALCULATION)
+                    .type(SdkGoTypes.Aws.RequestChecksumCalculation)
+                    .documentation("Indicates how user opt-in/out request checksum calculation")
+                    .servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(RESPONSE_CHECKSUM_VALIDATION)
+                    .type(SdkGoTypes.Aws.ResponseChecksumValidation)
+                    .documentation("Indicates how user opt-in/out response checksum validation")
+                    .servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait)
                     .build()
     );
 

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java

@@ -22,6 +22,7 @@
 import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
 import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
 import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.knowledge.TopDownIndex;
 import software.amazon.smithy.model.shapes.MemberShape;
 import software.amazon.smithy.model.shapes.OperationShape;
 import software.amazon.smithy.model.shapes.ServiceShape;
@@ -73,9 +74,7 @@ public byte getOrder() {
     @Override
     public void processFinalizedModel(GoSettings settings, Model model) {
         ServiceShape service = settings.getService(model);
-        for (ShapeId operationId : service.getAllOperations()) {
-            final OperationShape operation = model.expectShape(operationId, OperationShape.class);
-
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
             // Create a symbol provider because one is not available in this call.
             SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings);
 
@@ -128,8 +127,7 @@ public void writeAdditionalFiles(
         boolean supportsComputeInputChecksumsWorkflow = false;
         boolean supportsChecksumValidationWorkflow = false;
 
-        for (ShapeId operationID : service.getAllOperations()) {
-            OperationShape operation = model.expectShape(operationID, OperationShape.class);
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
             if (!hasChecksumTrait(model, service, operation)) {
                 continue;
             }
@@ -178,26 +176,44 @@ public List<RuntimeClientPlugin> getClientPlugins() {
     }
 
     // return true if operation shape is decorated with `httpChecksum` trait.
-    private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         return operation.hasTrait(HttpChecksumTrait.class);
     }
 
-    private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         if (!hasChecksumTrait(model, service, operation)) {
             return false;
         }
         HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
         return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
     }
 
-    private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    public static boolean hasInputChecksumTrait(Model model, ServiceShape service) {
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
+                if (hasInputChecksumTrait(model, service, operation)) {
+                    return true;
+                }
+        }
+        return false;
+    }
+
+        private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         if (!hasChecksumTrait(model, service, operation)) {
             return false;
         }
         HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
         return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty();
     }
 
+    public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) {
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
+            if (hasOutputChecksumTrait(model, service, operation)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private boolean isS3ServiceShape(Model model, ServiceShape service) {
         String serviceId = service.expectTrait(ServiceTrait.class).getSdkId();
         return serviceId.equalsIgnoreCase("S3");
@@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper(
                                     return $T(stack, $T{
                                         GetAlgorithm: $L,
                                         RequireChecksum: $L,
+                                        RequestChecksumCalculation: options.RequestChecksumCalculation,
                                         EnableTrailingChecksum: $L,
                                         EnableComputeSHA256PayloadHash: true,
                                         EnableDecodedContentLengthHeader: $L,
@@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper(
                     writer.write("""
                                     return $T(stack, $T{
                                         GetValidationMode: $L,
+                                        ResponseChecksumValidation: options.ResponseChecksumValidation,
                                         ValidationAlgorithms: $L,
                                         IgnoreMultipartValidation: $L,
                                         LogValidationSkipped: true,
@@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper(
                                     AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
                             SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions",
                                     AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
-
                             getRequestValidationModeAccessorFuncName(operationName),
                             convertToGoStringList(responseAlgorithms),
                             ignoreMultipartChecksumValidationMap.getOrDefault(

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java

@@ -39,6 +39,8 @@ public static final class Aws {
         public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired");
         public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled");
 
+        public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation");
+        public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation");
 
         public static final class Middleware {
             public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints");

config/config.go

@@ -83,6 +83,12 @@ var defaultAWSConfigResolvers = []awsConfigResolver{
 
 	// Sets the AccountIDEndpointMode if present in env var or shared config profile
 	resolveAccountIDEndpointMode,
+
+	// Sets the RequestChecksumCalculation if present in env var or shared config profile
+	resolveRequestChecksumCalculation,
+
+	// Sets the ResponseChecksumValidation if present in env var or shared config profile
+	resolveResponseChecksumValidation,
 }
 
 // A Config represents a generic configuration value or set of values. This type

config/env_config.go

@@ -83,6 +83,9 @@ const (
 
 	awsAccountIDEnv             = "AWS_ACCOUNT_ID"
 	awsAccountIDEndpointModeEnv = "AWS_ACCOUNT_ID_ENDPOINT_MODE"
+
+	awsRequestChecksumCalculation = "AWS_REQUEST_CHECKSUM_CALCULATION"
+	awsResponseChecksumValidation = "AWS_RESPONSE_CHECKSUM_VALIDATION"
 )
 
 var (
@@ -296,6 +299,12 @@ type EnvConfig struct {
 
 	// Indicates whether account ID will be required/ignored in endpoint2.0 routing
 	AccountIDEndpointMode aws.AccountIDEndpointMode
+
+	// Indicates whether request checksum should be calculated
+	RequestChecksumCalculation aws.RequestChecksumCalculation
+
+	// Indicates whether response checksum should be validated
+	ResponseChecksumValidation aws.ResponseChecksumValidation
 }
 
 // loadEnvConfig reads configuration values from the OS's environment variables.
@@ -400,6 +409,13 @@ func NewEnvConfig() (EnvConfig, error) {
 		return cfg, err
 	}
 
+	if err := setRequestChecksumCalculationFromEnvVal(&cfg.RequestChecksumCalculation, []string{awsRequestChecksumCalculation}); err != nil {
+		return cfg, err
+	}
+	if err := setResponseChecksumValidationFromEnvVal(&cfg.ResponseChecksumValidation, []string{awsResponseChecksumValidation}); err != nil {
+		return cfg, err
+	}
+
 	return cfg, nil
 }
 
@@ -432,6 +448,14 @@ func (c EnvConfig) getAccountIDEndpointMode(context.Context) (aws.AccountIDEndpo
 	return c.AccountIDEndpointMode, len(c.AccountIDEndpointMode) > 0, nil
 }
 
+func (c EnvConfig) getRequestChecksumCalculation(context.Context) (aws.RequestChecksumCalculation, bool, error) {
+	return c.RequestChecksumCalculation, c.RequestChecksumCalculation > 0, nil
+}
+
+func (c EnvConfig) getResponseChecksumValidation(context.Context) (aws.ResponseChecksumValidation, bool, error) {
+	return c.ResponseChecksumValidation, c.ResponseChecksumValidation > 0, nil
+}
+
 // GetRetryMaxAttempts returns the value of AWS_MAX_ATTEMPTS if was specified,
 // and not 0.
 func (c EnvConfig) GetRetryMaxAttempts(ctx context.Context) (int, bool, error) {
@@ -528,6 +552,45 @@ func setAIDEndPointModeFromEnvVal(m *aws.AccountIDEndpointMode, keys []string) e
 	return nil
 }
 
+func setRequestChecksumCalculationFromEnvVal(m *aws.RequestChecksumCalculation, keys []string) error {
+	for _, k := range keys {
+		value := os.Getenv(k)
+		if len(value) == 0 {
+			continue
+		}
+
+		switch strings.ToLower(value) {
+		case checksumWhenSupported:
+			*m = aws.RequestChecksumCalculationWhenSupported
+		case checksumWhenRequired:
+			*m = aws.RequestChecksumCalculationWhenRequired
+		default:
+			return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value)
+		}
+	}
+	return nil
+}
+
+func setResponseChecksumValidationFromEnvVal(m *aws.ResponseChecksumValidation, keys []string) error {
+	for _, k := range keys {
+		value := os.Getenv(k)
+		if len(value) == 0 {
+			continue
+		}
+
+		switch strings.ToLower(value) {
+		case checksumWhenSupported:
+			*m = aws.ResponseChecksumValidationWhenSupported
+		case checksumWhenRequired:
+			*m = aws.ResponseChecksumValidationWhenRequired
+		default:
+			return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value)
+		}
+
+	}
+	return nil
+}
+
 // GetRegion returns the AWS Region if set in the environment. Returns an empty
 // string if not set.
 func (c EnvConfig) getRegion(ctx context.Context) (string, bool, error) {

config/env_config_test.go

@@ -514,7 +514,6 @@ func TestNewEnvConfig(t *testing.T) {
 			Config: EnvConfig{
 				AccountIDEndpointMode: aws.AccountIDEndpointModeRequired,
 			},
-			WantErr: false,
 		},
 		47: {
 			Env: map[string]string{
@@ -523,6 +522,52 @@ func TestNewEnvConfig(t *testing.T) {
 			Config:  EnvConfig{},
 			WantErr: true,
 		},
+		48: {
+			Env: map[string]string{
+				"AWS_REQUEST_CHECKSUM_CALCULATION": "WHEN_SUPPORTED",
+			},
+			Config: EnvConfig{
+				RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported,
+			},
+		},
+		49: {
+			Env: map[string]string{
+				"AWS_REQUEST_CHECKSUM_CALCULATION": "when_required",
+			},
+			Config: EnvConfig{
+				RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+			},
+		},
+		50: {
+			Env: map[string]string{
+				"AWS_REQUEST_CHECKSUM_CALCULATION": "blabla",
+			},
+			Config:  EnvConfig{},
+			WantErr: true,
+		},
+		51: {
+			Env: map[string]string{
+				"AWS_RESPONSE_CHECKSUM_VALIDATION": "WHEN_SUPPORTED",
+			},
+			Config: EnvConfig{
+				ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported,
+			},
+		},
+		52: {
+			Env: map[string]string{
+				"AWS_RESPONSE_CHECKSUM_VALIDATION": "when_Required",
+			},
+			Config: EnvConfig{
+				ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+			},
+		},
+		53: {
+			Env: map[string]string{
+				"AWS_RESPONSE_CHECKSUM_VALIDATION": "blabla",
+			},
+			Config:  EnvConfig{},
+			WantErr: true,
+		},
 	}
 
 	for i, c := range cases {