From 47368f99b7c2fecf35ed5bd1f8d342072954e26e Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Fri, 13 Oct 2023 14:01:55 +0400 Subject: [PATCH] *: Limit DeleteObjects request object amount Closes #844. Signed-off-by: Evgenii Baidakov --- api/handler/api.go | 9 +++++---- api/handler/delete.go | 5 +++++ api/handler/util.go | 4 ++++ api/layer/neofs.go | 3 +++ cmd/s3-gw/app.go | 5 +++++ cmd/s3-gw/app_settings.go | 5 +++++ config/config.yaml | 6 ++++++ docs/aws_s3_compat.md | 2 ++ docs/configuration.md | 13 +++++++++++++ 9 files changed, 48 insertions(+), 4 deletions(-) diff --git a/api/handler/api.go b/api/handler/api.go index 572c7628..3b739cc4 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -25,10 +25,11 @@ type ( // Config contains data which handler needs to keep. Config struct { - Policy PlacementPolicy - DefaultMaxAge int - NotificatorEnabled bool - CopiesNumber uint32 + Policy PlacementPolicy + DefaultMaxAge int + NotificatorEnabled bool + CopiesNumber uint32 + MaxDeletePerRequest int } PlacementPolicy interface { diff --git a/api/handler/delete.go b/api/handler/delete.go index 9437c3f3..a6d1aefc 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -176,6 +176,11 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re return } + if len(requested.Objects) > h.cfg.MaxDeletePerRequest { + h.logAndSendError(w, "too many objects to delete", reqInfo, layer.ErrTooManyObjectForDeletion) + return + } + removed := make(map[string]*layer.VersionedObject) toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects)) for _, obj := range requested.Objects { diff --git a/api/handler/util.go b/api/handler/util.go index 4c75beff..cd0e2207 100644 --- a/api/handler/util.go +++ b/api/handler/util.go @@ -43,6 +43,10 @@ func transformToS3Error(err error) error { return s3errors.GetAPIError(s3errors.ErrUnsupportedMetadata) } + if errors.Is(err, layer.ErrTooManyObjectForDeletion) { + return s3errors.GetAPIError(s3errors.ErrBadRequest) + } + return s3errors.GetAPIError(s3errors.ErrInternalError) } diff --git a/api/layer/neofs.go b/api/layer/neofs.go index c06481d2..690ffa8b 100644 --- a/api/layer/neofs.go +++ b/api/layer/neofs.go @@ -130,6 +130,9 @@ var ErrAccessDenied = errors.New("access denied") // ErrMetaEmptyParameterValue describes situation when meta parameter was passed but with empty value. var ErrMetaEmptyParameterValue = errors.New("meta empty parameter value") +// ErrTooManyObjectForDeletion is returned if user is trying to delete to many objects per request. +var ErrTooManyObjectForDeletion = errors.New("to many objects for deletion") + // NeoFS represents virtual connection to NeoFS network. type NeoFS interface { // CreateContainer creates and saves parameterized container in NeoFS. diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index c86a24b3..c6be35db 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -686,6 +686,11 @@ func (a *App) initHandler() { cfg.CopiesNumber = val } + cfg.MaxDeletePerRequest = a.cfg.GetInt(cfgMaxObjectToDeletePerRequest) + if cfg.MaxDeletePerRequest == 0 { + cfg.MaxDeletePerRequest = defaultMaxObjectDeletePerRequest + } + var err error a.api, err = handler.New(a.log, a.obj, a.nc, cfg) if err != nil { diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 53a2dd9f..74c98555 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -28,6 +28,8 @@ const ( defaultMaxClientsCount = 100 defaultMaxClientsDeadline = time.Second * 30 + + defaultMaxObjectDeletePerRequest = 1000 ) const ( // Settings. @@ -121,6 +123,9 @@ const ( // Settings. // Number of the object copies to consider PUT to NeoFS successful. cfgSetCopiesNumber = "neofs.set_copies_number" + // Maximum number of objects to be deleted per request limit by this value. + cfgMaxObjectToDeletePerRequest = "s3.max_object_to_delete_per_request" + // Timeout between retrieving actual epoch from NeoFS. Actual only if slicer.enabled = true. cfgEpochUpdateInterval = "neofs.epoch_update_interval" diff --git a/config/config.yaml b/config/config.yaml index e2972b9d..cb6fddbf 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -147,3 +147,9 @@ allowed_access_key_id_prefixes: # Allows to use slicer for Object uploading. internal_slicer: false + +# S3 + +s3: + # Maximum number of objects to be deleted per request limit by this value. + max_object_to_delete_per_request: 1000 diff --git a/docs/aws_s3_compat.md b/docs/aws_s3_compat.md index a67c2f86..98f6a4e6 100644 --- a/docs/aws_s3_compat.md +++ b/docs/aws_s3_compat.md @@ -28,6 +28,8 @@ Reference: | 🔵 | WriteGetObjectResponse | Waiting for Lambda to be developed | | 🟢 | GetObjectAttributes | | +* DeleteObjects limited by max amount of objects which can be deleted per request. See `max_object_to_delete_per_request` parameter. + ## ACL For now there are some limitations: diff --git a/docs/configuration.md b/docs/configuration.md index 3cf92b47..d563f94a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -455,3 +455,16 @@ neofs: | Parameter | Type | Default value | Description | |---------------------|----------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `set_copies_number` | `uint32` | `0` | Number of the object copies to consider PUT to NeoFS successful.
Default value `0` means that object will be processed according to the container's placement policy | + +# `s3` section + +Contains parameters to configure requests runtime. + +```yaml +s3: + max_object_to_delete_per_request: 1000 +``` + +| Parameter | Type | Default value | Description | +|------------------------------------|-------|---------------|-------------------------------------------------------------------------------------------------------------------------------| +| `max_object_to_delete_per_request` | `int` | `1000` | Allows to set maximum object amount which can be deleted per request. If amount is higher, the `Bad request` will be returned |