Skip to content

Commit

Permalink
Remove additional console logs from opt-out and CCLF imports (#976)
Browse files Browse the repository at this point in the history 
## 🎫 Ticket

N/A

## 🛠 Changes

- Remove extraneous console logs

## ℹ️ Context

These logs were duplicated so that they would show up in Jenkins. Now
that we are running these imports in lambdas, we can remove them.

These logs are currently showing up in the "bad logs" area in the splunk
dashboard since they don't have a "level" associated with them.

## 🧪 Validation

Unit Testing.
  • Loading branch information
@kyeah
kyeah authored Jul 18, 2024
1 parent 87bd47a commit ff3e30e
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 37 deletions.
22 changes: 3 additions & 19 deletions bcda/suppression/suppression.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,10 @@ func (importer OptOutImporter) ImportSuppressionDirectory(path string) (success,
for _, metadata := range *suppresslist {
err = importer.validate(metadata)
if err != nil {
fmt.Printf("Failed to validate suppression file: %s.\n", metadata)
importer.Logger.Errorf("Failed to validate suppression file: %s", metadata)
failure++
} else {
if err = importer.ImportSuppressionData(metadata); err != nil {
fmt.Printf("Failed to import suppression file: %s.\n", metadata)
importer.Logger.Errorf("Failed to import suppression file: %s ", metadata)
failure++
} else {
Expand All @@ -71,13 +69,11 @@ func (importer OptOutImporter) ImportSuppressionDirectory(path string) (success,
}

func (importer OptOutImporter) validate(metadata *optout.OptOutFilenameMetadata) error {
fmt.Printf("Validating suppression file %s...\n", metadata)
importer.Logger.Infof("Validating suppression file %s...", metadata)

count := 0
sc, close, err := importer.FileHandler.OpenFile(metadata)
if err != nil {
fmt.Printf("Could not read file %s.\n", metadata)
err = errors.Wrapf(err, "could not read file %s", metadata)
importer.Logger.Error(err)
return err
Expand All @@ -91,7 +87,6 @@ func (importer OptOutImporter) validate(metadata *optout.OptOutFilenameMetadata)
if count == 0 {
if metaInfo != headerCode {
// invalid file header found
fmt.Printf("Invalid file header for file: %s.\n", metadata.FilePath)
err := fmt.Errorf("invalid file header for file: %s", metadata.FilePath)
importer.Logger.Error(err)
return err
Expand All @@ -106,23 +101,20 @@ func (importer OptOutImporter) validate(metadata *optout.OptOutFilenameMetadata)
// trailer info
expectedCount, err := strconv.Atoi(string(bytes.TrimSpace(b[recCountStart:recCountEnd])))
if err != nil {
fmt.Printf("Failed to parse record count from file: %s.\n", metadata.FilePath)
err = fmt.Errorf("failed to parse record count from file: %s", metadata.FilePath)
importer.Logger.Error(err)
return err
}
// subtract the single count from the header
count--
if count != expectedCount {
fmt.Printf("Incorrect number of records found from file: '%s'. Expected record count: %d, Actual record count: %d.\n", metadata.FilePath, expectedCount, count)
err = fmt.Errorf("incorrect number of records found from file: '%s'. Expected record count: %d, Actual record count: %d", metadata.FilePath, expectedCount, count)
importer.Logger.Error(err)
return err
}
}
}

fmt.Printf("Successfully validated suppression file %s.\n", metadata)
importer.Logger.Infof("Successfully validated suppression file %s.", metadata)
return nil
}
Expand All @@ -137,7 +129,6 @@ func (importer OptOutImporter) ImportSuppressionData(metadata *optout.OptOutFile
}

if err = importer.Saver.SaveOptOutRecord(*suppression); err != nil {
fmt.Println("Could not create suppression record.")
err = errors.Wrap(err, "could not create suppression record")
importer.Logger.Error(err)
return err
Expand All @@ -154,7 +145,6 @@ func (importer OptOutImporter) ImportSuppressionData(metadata *optout.OptOutFile
}

func (importer OptOutImporter) importSuppressionMetadata(metadata *optout.OptOutFilenameMetadata, importFunc func(uint, []byte) error) error {
fmt.Printf("Importing suppression file %s...\n", metadata)
importer.Logger.Infof("Importing suppression file %s...", metadata)

var (
Expand All @@ -168,7 +158,6 @@ func (importer OptOutImporter) importSuppressionMetadata(metadata *optout.OptOut
}

if suppressionMetaFile.ID, err = importer.Saver.SaveFile(suppressionMetaFile); err != nil {
fmt.Printf("Could not create suppression file record for file: %s. \n", metadata)
err = errors.Wrapf(err, "could not create suppression file record for file: %s.", metadata)
importer.Logger.Error(err)
return err
Expand All @@ -180,7 +169,6 @@ func (importer OptOutImporter) importSuppressionMetadata(metadata *optout.OptOut

sc, close, err := importer.FileHandler.OpenFile(metadata)
if err != nil {
fmt.Printf("Could not read file %s.\n", metadata)
err = errors.Wrapf(err, "could not read file %s", metadata)
importer.Logger.Error(err)
return err
Expand All @@ -202,21 +190,17 @@ func (importer OptOutImporter) importSuppressionMetadata(metadata *optout.OptOut
}
importedCount++
if importedCount%importer.ImportStatusInterval == 0 {
fmt.Printf("Suppression records imported: %d\n", importedCount)
importer.Logger.Infof("Suppression records imported: %d\n", importedCount)
}
}
}

successMsg := fmt.Sprintf("Successfully imported %d records from suppression file %s.", importedCount, metadata)
fmt.Println(successMsg)
importer.Logger.Infof(successMsg)
importer.Logger.Infof("Successfully imported %d records from suppression file %s.", importedCount, metadata)
return nil
}

func (importer OptOutImporter) updateImportStatus(metadata *optout.OptOutFilenameMetadata, status string) {
if err := importer.Saver.UpdateImportStatus(*metadata, status); err != nil {
fmt.Printf("Could not update suppression file record for file_id: %s. \n", metadata.String())
err = errors.Wrapf(err, "could not update suppression file record for file_id: %s.", metadata.String())
importer.Logger.Error(err)
importer.Logger.Error(errors.Wrapf(err, "could not update suppression file record for file_id: %s.", metadata.String()))
}
}
24 changes: 6 additions & 18 deletions optout/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,18 @@ func ParseMetadata(filename string) (OptOutFilenameMetadata, error) {
var metadata OptOutFilenameMetadata
isOptOut, matches := IsOptOut(filename)
if !isOptOut {
fmt.Printf("Invalid filename for file: %s.\n", filename)
err := fmt.Errorf("invalid filename for file: %s", filename)
return metadata, err
return metadata, fmt.Errorf("invalid filename for file: %s", filename)
}

// ignore files for different environments
if !IsForCurrentEnv(filename) {
fmt.Printf("Skipping file for different environment: %s.\n", filename)
err := fmt.Errorf("Skipping file for different environment: %s", filename)
return metadata, err
return metadata, fmt.Errorf("Skipping file for different environment: %s", filename)
}

filenameDate := matches[3]
t, err := time.Parse("D060102.T150405", filenameDate)
if err != nil || t.IsZero() {
fmt.Printf("Failed to parse date '%s' from file: %s.\n", filenameDate, filename)
err = errors.Wrapf(err, "failed to parse date '%s' from file: %s", filenameDate, filename)
return metadata, err
return metadata, errors.Wrapf(err, "failed to parse date '%s' from file: %s", filenameDate, filename)
}

metadata.Timestamp = t
Expand All @@ -67,26 +61,20 @@ func ParseRecord(metadata *OptOutFilenameMetadata, b []byte) (*OptOutRecord, err
ds := string(bytes.TrimSpace(b[effectiveDtStart:effectiveDtEnd]))
dt, err := ConvertDt(ds)
if err != nil {
fmt.Printf("Failed to parse the effective date '%s' from file: %s.\n", ds, metadata.FilePath)
err = errors.Wrapf(err, "failed to parse the effective date '%s' from file: %s", ds, metadata.FilePath)
return nil, err
return nil, errors.Wrapf(err, "failed to parse the effective date '%s' from file: %s", ds, metadata.FilePath)
}
ds = string(bytes.TrimSpace(b[samhsaEffectiveDtStart:samhsaEffectiveDtEnd]))
samhsaDt, err := ConvertDt(ds)
if err != nil {
fmt.Printf("Failed to parse the samhsa effective date '%s' from file: %s.\n", ds, metadata.FilePath)
err = errors.Wrapf(err, "failed to parse the samhsa effective date '%s' from file: %s", ds, metadata.FilePath)
return nil, err
return nil, errors.Wrapf(err, "failed to parse the samhsa effective date '%s' from file: %s", ds, metadata.FilePath)
}
keyval := string(bytes.TrimSpace(b[lKeyStart:lKeyEnd]))
if keyval == "" {
keyval = "0"
}
lk, err := strconv.Atoi(keyval)
if err != nil {
fmt.Printf("Failed to parse beneficiary link key from file: %s.\n", metadata.FilePath)
err = errors.Wrapf(err, "failed to parse beneficiary link key from file: %s", metadata.FilePath)
return nil, err
return nil, errors.Wrapf(err, "failed to parse beneficiary link key from file: %s", metadata.FilePath)
}

return &OptOutRecord{
Expand Down

0 comments on commit ff3e30e

Please sign in to comment.