diff --git a/bcda/bcdacli/cli_test.go b/bcda/bcdacli/cli_test.go index bb4ab5b02..3a5c77eb4 100644 --- a/bcda/bcdacli/cli_test.go +++ b/bcda/bcdacli/cli_test.go @@ -29,6 +29,7 @@ import ( "github.com/CMSgov/bcda-app/bcda/utils" "github.com/CMSgov/bcda-app/conf" logger "github.com/CMSgov/bcda-app/log" + "github.com/pkg/errors" "github.com/go-chi/chi/v5" "github.com/pborman/uuid" @@ -703,31 +704,47 @@ func (s *CLITestSuite) TestImportCCLFDirectory() { assert := assert.New(s.T()) hook := test.NewLocal(testUtils.GetLogger(logger.API)) - postgrestest.DeleteCCLFFilesByCMSID(s.T(), s.db, targetACO) - defer postgrestest.DeleteCCLFFilesByCMSID(s.T(), s.db, targetACO) + type test struct { + path string + err error + expectedLogs []string + } - path, cleanup := testUtils.CopyToTemporaryDirectory(s.T(), "../../shared_files/cclf/archives/valid2/") - defer cleanup() + tests := []test{ + {path: "../../shared_files/cclf/archives/valid2/", err: errors.New("Files skipped or failed import. See logs for more details."), expectedLogs: []string{"Successfully imported 2 files.", "Failed to import 0 files.", "Skipped 1 files."}}, + {path: "../../shared_files/cclf/archives/invalid_bcd/", err: errors.New("one or more files failed to import correctly"), expectedLogs: []string{"error returned from ImportCCLFDirectory", "", ""}}, + {path: "../../shared_files/cclf/archives/skip/", err: errors.New("Files failed to import or no files were imported. See logs for more details."), expectedLogs: []string{"Successfully imported 0 files.", "Failed to import 0 files.", "Skipped 1 files."}}, + } - args := []string{"bcda", "import-cclf-directory", constants.DirectoryArg, path} - err := s.testApp.Run(args) - assert.NotNil(err) - var success, failed, skipped bool - for _, entry := range hook.AllEntries() { - if strings.Contains(entry.Message, "Successfully imported 2 files.") { - success = true - } - if strings.Contains(entry.Message, "Failed to import 0 files.") { - failed = true + for _, tc := range tests { + postgrestest.DeleteCCLFFilesByCMSID(s.T(), s.db, targetACO) + defer postgrestest.DeleteCCLFFilesByCMSID(s.T(), s.db, targetACO) + path, cleanup := testUtils.CopyToTemporaryDirectory(s.T(), tc.path) + defer cleanup() + args := []string{"bcda", "import-cclf-directory", constants.DirectoryArg, path} + err := s.testApp.Run(args) + if tc.err == nil { + assert.Nil(err) + } else { + assert.NotNil(err) } - if strings.Contains(entry.Message, "Skipped 1 files.") { - skipped = true + + var success, failed, skipped bool + for _, entry := range hook.AllEntries() { + if strings.Contains(entry.Message, tc.expectedLogs[0]) { + success = true + } + if strings.Contains(entry.Message, tc.expectedLogs[1]) { + failed = true + } + if strings.Contains(entry.Message, tc.expectedLogs[2]) { + skipped = true + } } + assert.True(success) + assert.True(failed) + assert.True(skipped) } - assert.True(success) - assert.True(failed) - assert.True(skipped) - } func (s *CLITestSuite) TestDeleteDirectoryContents() { diff --git a/bcda/cclf/fileprocessor.go b/bcda/cclf/fileprocessor.go index ec91c0454..8ad8a1827 100644 --- a/bcda/cclf/fileprocessor.go +++ b/bcda/cclf/fileprocessor.go @@ -14,6 +14,7 @@ import ( "github.com/CMSgov/bcda-app/bcda/utils" "github.com/CMSgov/bcda-app/conf" "github.com/CMSgov/bcda-app/log" + "github.com/CMSgov/bcda-app/optout" "github.com/pkg/errors" ) @@ -56,8 +57,17 @@ func (p *processor) walk(path string, info os.FileInfo, err error) error { return nil } + // ignore the opt out file, and don't add it to the skipped count + optOut, _ := optout.IsOptOut(info.Name()) + if optOut { + fmt.Print("Skipping opt-out file: ", info.Name()) + log.API.Info("Skipping opt-out file: ", info.Name()) + return nil + } + zipReader, err := zip.OpenReader(filepath.Clean(path)) if err != nil { + p.skipped = p.skipped + 1 msg := fmt.Sprintf("Skipping %s: file could not be opened as a CCLF archive. %s", path, err.Error()) fmt.Println(msg) diff --git a/bcda/cclf/fileprocessor_test.go b/bcda/cclf/fileprocessor_test.go index e3411a266..6de2eefe3 100644 --- a/bcda/cclf/fileprocessor_test.go +++ b/bcda/cclf/fileprocessor_test.go @@ -66,7 +66,7 @@ func (s *FileProcessorTestSuite) TestProcessCCLFArchives() { }{ {filepath.Join(s.basePath, "cclf/archives/valid/"), 2, 1, 1, 1}, {filepath.Join(s.basePath, "cclf/archives/bcd/"), 2, 1, 1, 1}, - {filepath.Join(s.basePath, "cclf/mixed/with_invalid_filenames/"), 2, 5, 1, 1}, + {filepath.Join(s.basePath, "cclf/mixed/with_invalid_filenames/"), 2, 4, 1, 1}, {filepath.Join(s.basePath, "cclf/mixed/0/valid_names/"), 3, 3, 3, 0}, {filepath.Join(s.basePath, "cclf/archives/8/valid/"), 5, 0, 0, 5}, {filepath.Join(s.basePath, "cclf/files/9/valid_names/"), 0, 4, 0, 0}, @@ -112,7 +112,7 @@ func (s *FileProcessorTestSuite) TestProcessCCLFArchives_ExpireFiles() { assert.Nil(err) cclfList := cclfMap["A0001"][key] assert.Equal(2, len(cclfList)) - assert.Equal(5, skipped) + assert.Equal(4, skipped) // assert that this file is still here. _, err = os.Open(filePath) assert.Nil(err) @@ -127,7 +127,7 @@ func (s *FileProcessorTestSuite) TestProcessCCLFArchives_ExpireFiles() { assert.Nil(err) cclfList = cclfMap["A0001"][key] assert.Equal(2, len(cclfList)) - assert.Equal(5, skipped) + assert.Equal(4, skipped) // assert that this file is not still here. _, err = os.Open(filePath) diff --git a/optout/utils.go b/optout/utils.go index 0ab8ced67..4421812f7 100644 --- a/optout/utils.go +++ b/optout/utils.go @@ -24,17 +24,14 @@ const ( func ParseMetadata(filename string) (OptOutFilenameMetadata, error) { var metadata OptOutFilenameMetadata - // Beneficiary Data Sharing Preferences File sent by 1-800-Medicare: P#EFT.ON.ACO.NGD1800.DPRF.Dyymmdd.Thhmmsst - // Prefix: T = test, P = prod; - filenameRegexp := regexp.MustCompile(`((P|T)\#EFT)\.ON\.ACO\.NGD1800\.DPRF\.(D\d{6}\.T\d{6})\d`) - filenameMatches := filenameRegexp.FindStringSubmatch(filename) - if len(filenameMatches) < 4 { + isOptOut, matches := IsOptOut(filename) + if !isOptOut { fmt.Printf("Invalid filename for file: %s.\n", filename) err := fmt.Errorf("invalid filename for file: %s", filename) return metadata, err } - filenameDate := filenameMatches[3] + filenameDate := matches[3] t, err := time.Parse("D060102.T150405", filenameDate) if err != nil || t.IsZero() { fmt.Printf("Failed to parse date '%s' from file: %s.\n", filenameDate, filename) @@ -43,11 +40,20 @@ func ParseMetadata(filename string) (OptOutFilenameMetadata, error) { } metadata.Timestamp = t - metadata.Name = filenameMatches[0] + metadata.Name = matches[0] return metadata, nil } +func IsOptOut(filename string) (isOptOut bool, matches []string) { + filenameRegexp := regexp.MustCompile(`((P|T)\#EFT)\.ON\.ACO\.NGD1800\.DPRF\.(D\d{6}\.T\d{6})\d`) + matches = filenameRegexp.FindStringSubmatch(filename) + if len(matches) > 3 { + isOptOut = true + } + return isOptOut, matches +} + func ParseRecord(metadata *OptOutFilenameMetadata, b []byte) (*OptOutRecord, error) { ds := string(bytes.TrimSpace(b[effectiveDtStart:effectiveDtEnd])) dt, err := ConvertDt(ds) diff --git a/shared_files/cclf/archives/skip/T.BCD.ACOB.ZC0Y18.D181120.T0001000 b/shared_files/cclf/archives/skip/T.BCD.ACOB.ZC0Y18.D181120.T0001000 new file mode 100644 index 000000000..ad62601b0 Binary files /dev/null and b/shared_files/cclf/archives/skip/T.BCD.ACOB.ZC0Y18.D181120.T0001000 differ