From a70392673bdbaa96be766bbd0fde2d56d66f1279 Mon Sep 17 00:00:00 2001 From: Kevin Yeh Date: Thu, 11 Jul 2024 12:28:53 -0400 Subject: [PATCH] BCDA-7900: Set CCLF/Opt Out imports in dev/test to only import specific subfolders (#974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🎫 Ticket https://jira.cms.gov/browse/BCDA-7900 ## 🛠 Changes - Update dev/test CCLF and Opt Out S3 Importers to only import specific subfolders ## ℹ️ Context As part of improving integration tests, it was observed that dev and test were competing for the same files in BFD Test S3. This changes the app code for those environments to avoid competition and only look for files in environment-relevant subfolders (e.g. `bcda/in/test/` instead of `bcda/in/`.) ## 🧪 Validation See #963 for successful tests, yay! --- bcda/cclf/s3_fileprocessor.go | 6 +++++ bcda/cclf/s3_fileprocessor_test.go | 14 +++++++++++ optout/utils.go | 22 +++++++++++++++++ optout/utils_test.go | 38 ++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/bcda/cclf/s3_fileprocessor.go b/bcda/cclf/s3_fileprocessor.go index 4d1f44d6d..ee3737bbd 100644 --- a/bcda/cclf/s3_fileprocessor.go +++ b/bcda/cclf/s3_fileprocessor.go @@ -33,6 +33,12 @@ func (processor *S3FileProcessor) LoadCclfFiles(path string) (cclfMap map[string continue } + // ignore files for different environments, and don't add it to the skipped count + if !optout.IsForCurrentEnv(*obj.Key) { + processor.Handler.Infof("Skipping file for different environment: %s/%s", bucket, *obj.Key) + continue + } + // validate the top level zipped folder cmsID, err := getCMSID(*obj.Key) if err != nil { diff --git a/bcda/cclf/s3_fileprocessor_test.go b/bcda/cclf/s3_fileprocessor_test.go index c69e1987a..6da721cb6 100644 --- a/bcda/cclf/s3_fileprocessor_test.go +++ b/bcda/cclf/s3_fileprocessor_test.go @@ -89,6 +89,20 @@ func (s *S3ProcessorTestSuite) TestLoadCclfFiles() { } } +func (s *S3ProcessorTestSuite) TestLoadCclfFiles_SkipOtherEnvs() { + cleanupEnvVars := testUtils.SetEnvVars(s.T(), []testUtils.EnvVar{{Name: "ENV", Value: "someenv"}}) + defer cleanupEnvVars() + + s3Bucket, cleanupS3 := testUtils.CreateZipsInS3(s.T(), testUtils.ZipInput{ZipName: "blah/not-someenv/T.BCD.A0001.ZCY18.D181120.T1000000", CclfNames: []string{"test", "zip"}}) + defer cleanupS3() + + cclfMap, skipped, failure, err := s.processor.LoadCclfFiles(s3Bucket) + assert.Nil(s.T(), err) + assert.Equal(s.T(), 0, skipped) + assert.Equal(s.T(), 0, failure) + assert.Empty(s.T(), cclfMap) +} + func (s *S3ProcessorTestSuite) TestLoadCclfFiles_SingleFile() { cmsID, key := "A0001", metadataKey{perfYear: 18, fileType: models.FileTypeDefault} tests := []struct { diff --git a/optout/utils.go b/optout/utils.go index 4421812f7..cfa0b2b8c 100644 --- a/optout/utils.go +++ b/optout/utils.go @@ -5,8 +5,10 @@ import ( "fmt" "regexp" "strconv" + "strings" "time" + "github.com/CMSgov/bcda-app/conf" "github.com/pkg/errors" ) @@ -31,6 +33,13 @@ func ParseMetadata(filename string) (OptOutFilenameMetadata, error) { return metadata, err } + // ignore files for different environments + if !IsForCurrentEnv(filename) { + fmt.Printf("Skipping file for different environment: %s.\n", filename) + err := fmt.Errorf("Skipping file for different environment: %s", filename) + return metadata, err + } + filenameDate := matches[3] t, err := time.Parse("D060102.T150405", filenameDate) if err != nil || t.IsZero() { @@ -104,3 +113,16 @@ func ConvertDt(s string) (time.Time, error) { } return t, nil } + +// Checks if the given S3 filePath is for the current environment; this is necessary for lower environments +// since they share a single BFD S3 bucket and will upload files under a subdirectory for the given env. +func IsForCurrentEnv(filePath string) bool { + env := conf.GetEnv("ENV") + + // We do not expect or require subdirectories for local dev or production; always return true. + if env != "dev" && env != "test" { + return true + } + + return strings.Contains(filePath, fmt.Sprintf("/%s/", env)) +} diff --git a/optout/utils_test.go b/optout/utils_test.go index 175203c7c..a31b1b32f 100644 --- a/optout/utils_test.go +++ b/optout/utils_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/CMSgov/bcda-app/conf" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -13,6 +14,10 @@ type OptOutTestSuite struct { suite.Suite } +func TestOptOutTestSuite(t *testing.T) { + suite.Run(t, new(OptOutTestSuite)) +} + func (s *OptOutTestSuite) TestParseMetadata() { assert := assert.New(s.T()) @@ -31,6 +36,39 @@ func (s *OptOutTestSuite) TestParseMetadata() { assert.Nil(err) } +func (s *OptOutTestSuite) TestParseMetadata_CorrectEnv() { + assert := assert.New(s.T()) + + originalValue := conf.GetEnv("ENV") + if err := conf.SetEnv(s.T(), "ENV", "someenv"); err != nil { + assert.Failf("Error %s env value %s to %s\n", err.Error(), "ENV", "someenv") + } + defer func() { + conf.SetEnv(s.T(), "ENV", originalValue) + }() + + expTime, _ := time.Parse(time.RFC3339, "2019-12-20T21:09:42Z") + metadata, err := ParseMetadata("blah/someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420") + assert.Equal("T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420", metadata.Name) + assert.Equal(expTime.Format("010203040506"), metadata.Timestamp.Format("010203040506")) + assert.Nil(err) +} + +func (s *OptOutTestSuite) TestParseMetadata_DifferentEnv() { + assert := assert.New(s.T()) + + originalValue := conf.GetEnv("ENV") + if err := conf.SetEnv(s.T(), "ENV", "someenv"); err != nil { + assert.Failf("Error %s env value %s to %s\n", err.Error(), "ENV", "someenv") + } + defer func() { + conf.SetEnv(s.T(), "ENV", originalValue) + }() + + _, err := ParseMetadata("blah/not-someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420") + assert.EqualError(err, "Skipping file for different environment: blah/not-someenv/T#EFT.ON.ACO.NGD1800.DPRF.D191220.T2109420") +} + func (s *OptOutTestSuite) TestParseMetadata_InvalidData() { assert := assert.New(s.T())