From 9040d9fe5142a83772c9fe19931c17ea37ef5b45 Mon Sep 17 00:00:00 2001 From: tma5 Date: Tue, 15 Oct 2024 17:12:04 -0400 Subject: [PATCH 1/2] deliminates data stream and route with an underscore --- ...=> covid-all-monthly-vaccination_csv.json} | 0 ...event1.json => dextesting_testevent1.json} | 0 .../v2/{ehdi-csv.json => ehdi_csv.json} | 0 .../v2/{eicr-fhir.json => eicr_fhir.json} | 0 ...csv.json => generic-immunization_csv.json} | 0 ...sv.json => influenza-vaccination_csv.json} | 0 ...e.json => pulsenet_localsequencefile.json} | 0 ...r.json => routine-immunization_other.json} | 0 ...ntion-csv.json => rsv-prevention_csv.json} | 0 .../configs/testing/file_destinations.yml | 96 +++++++++++++++++++ .../internal/metadata/v2/manifest.go | 8 +- upload-server/internal/ui/ui.go | 8 +- upload-server/testing/cli_test.go | 8 +- 13 files changed, 111 insertions(+), 9 deletions(-) rename upload-configs/v2/{covid-all-monthly-vaccination-csv.json => covid-all-monthly-vaccination_csv.json} (100%) rename upload-configs/v2/{dextesting-testevent1.json => dextesting_testevent1.json} (100%) rename upload-configs/v2/{ehdi-csv.json => ehdi_csv.json} (100%) rename upload-configs/v2/{eicr-fhir.json => eicr_fhir.json} (100%) rename upload-configs/v2/{generic-immunization-csv.json => generic-immunization_csv.json} (100%) rename upload-configs/v2/{influenza-vaccination-csv.json => influenza-vaccination_csv.json} (100%) rename upload-configs/v2/{pulsenet-localsequencefile.json => pulsenet_localsequencefile.json} (100%) rename upload-configs/v2/{routine-immunization-other.json => routine-immunization_other.json} (100%) rename upload-configs/v2/{rsv-prevention-csv.json => rsv-prevention_csv.json} (100%) create mode 100644 upload-server/configs/testing/file_destinations.yml diff --git a/upload-configs/v2/covid-all-monthly-vaccination-csv.json b/upload-configs/v2/covid-all-monthly-vaccination_csv.json similarity index 100% rename from upload-configs/v2/covid-all-monthly-vaccination-csv.json rename to upload-configs/v2/covid-all-monthly-vaccination_csv.json diff --git a/upload-configs/v2/dextesting-testevent1.json b/upload-configs/v2/dextesting_testevent1.json similarity index 100% rename from upload-configs/v2/dextesting-testevent1.json rename to upload-configs/v2/dextesting_testevent1.json diff --git a/upload-configs/v2/ehdi-csv.json b/upload-configs/v2/ehdi_csv.json similarity index 100% rename from upload-configs/v2/ehdi-csv.json rename to upload-configs/v2/ehdi_csv.json diff --git a/upload-configs/v2/eicr-fhir.json b/upload-configs/v2/eicr_fhir.json similarity index 100% rename from upload-configs/v2/eicr-fhir.json rename to upload-configs/v2/eicr_fhir.json diff --git a/upload-configs/v2/generic-immunization-csv.json b/upload-configs/v2/generic-immunization_csv.json similarity index 100% rename from upload-configs/v2/generic-immunization-csv.json rename to upload-configs/v2/generic-immunization_csv.json diff --git a/upload-configs/v2/influenza-vaccination-csv.json b/upload-configs/v2/influenza-vaccination_csv.json similarity index 100% rename from upload-configs/v2/influenza-vaccination-csv.json rename to upload-configs/v2/influenza-vaccination_csv.json diff --git a/upload-configs/v2/pulsenet-localsequencefile.json b/upload-configs/v2/pulsenet_localsequencefile.json similarity index 100% rename from upload-configs/v2/pulsenet-localsequencefile.json rename to upload-configs/v2/pulsenet_localsequencefile.json diff --git a/upload-configs/v2/routine-immunization-other.json b/upload-configs/v2/routine-immunization_other.json similarity index 100% rename from upload-configs/v2/routine-immunization-other.json rename to upload-configs/v2/routine-immunization_other.json diff --git a/upload-configs/v2/rsv-prevention-csv.json b/upload-configs/v2/rsv-prevention_csv.json similarity index 100% rename from upload-configs/v2/rsv-prevention-csv.json rename to upload-configs/v2/rsv-prevention_csv.json diff --git a/upload-server/configs/testing/file_destinations.yml b/upload-server/configs/testing/file_destinations.yml new file mode 100644 index 000000000..e07d9eb54 --- /dev/null +++ b/upload-server/configs/testing/file_destinations.yml @@ -0,0 +1,96 @@ +version: 1.0 + +az-edav: &az-edav + endpoint: "http://azurite:10000/devstoreaccount1" + storage_account: "devstoreaccount1" + storage_key: $AZURITE_KEY + container_name: edav + +az-ehdi: &az-ehdi + endpoint: "http://azurite:10000/devstoreaccount1" + storage_account: "devstoreaccount1" + storage_key: $AZURITE_KEY + container_name: ehdi + +az-eicr: &az-eicr + endpoint: "http://azurite:10000/devstoreaccount1" + storage_account: "devstoreaccount1" + storage_key: $AZURITE_KEY + container_name: eicr + +az-ncird: &az-ncird + endpoint: "http://azurite:10000/devstoreaccount1" + storage_account: "devstoreaccount1" + storage_key: $AZURITE_KEY + container_name: ncird + +# if needed, properties of an anchor can be overridden +# in a delivery target +# +# path_template can be defined at data_stream level +# where it is valid for all targets unless +# a target defines its own path_template property +programs: + - data_stream_id: pulsenet + data_stream_route: localsequencefile + delivery_targets: + - name: edav + type: file + path: ./uploads/edav + - data_stream_id: ehdi + data_stream_route: csv + delivery_targets: + - name: ehdi + type: file + path: ./uploads/ehdi + - data_stream_id: eicr + data_stream_route: fhir + delivery_targets: + - name: eicr + type: file + path: ./uploads/eicr + - data_stream_id: rsp-prevention + data_stream_route: csv + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - data_stream_id: generic-immunization + data_stream_route: csv + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - data_stream_id: influenza-vaccination + data_stream_route: csv + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - data_stream_id: routine-immunization + data_stream_route: other + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - data_stream_id: rsp-prevention + data_stream_route: csv + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - data_stream_id: dextesting + data_stream_route: testevent1 + delivery_targets: + - name: ncird + type: file + path: ./uploads/ncird + - name: eicr + type: file + path: ./uploads/eicr + - name: edav + type: file + path: ./uploads/edav + - name: ehdi + type: file + path: ./uploads/ehdi diff --git a/upload-server/internal/metadata/v2/manifest.go b/upload-server/internal/metadata/v2/manifest.go index b73663193..2337f49a3 100644 --- a/upload-server/internal/metadata/v2/manifest.go +++ b/upload-server/internal/metadata/v2/manifest.go @@ -9,13 +9,13 @@ import ( "fmt" ) -type Config struct { +type ConfigIdentification struct { DataStreamID string DataStreamRoute string } -func (c *Config) Path() string { - path := fmt.Sprintf("%s/%s-%s.json", "v2", c.DataStreamID, c.DataStreamRoute) +func (c *ConfigIdentification) Path() string { + path := fmt.Sprintf("%s/%s_%s.json", "v2", c.DataStreamID, c.DataStreamRoute) return path } @@ -29,7 +29,7 @@ func NewFromManifest(manifest handler.MetaData) (validation.ConfigLocation, erro return nil, errors.Join(validation.ErrFailure, &validation.ErrorMissing{Field: "data_stream_route"}) } - return &Config{ + return &ConfigIdentification{ DataStreamID: dataStreamID, DataStreamRoute: dataStreamRoute, }, nil diff --git a/upload-server/internal/ui/ui.go b/upload-server/internal/ui/ui.go index 21de361ff..b3f1145a3 100644 --- a/upload-server/internal/ui/ui.go +++ b/upload-server/internal/ui/ui.go @@ -5,6 +5,7 @@ import ( "embed" "encoding/json" "fmt" + v2 "github.com/cdcgov/data-exchange-upload/upload-server/internal/metadata/v2" "io" "net/http" "net/url" @@ -126,7 +127,12 @@ func GetRouter(uploadUrl string, infoUrl string) *mux.Router { dataStream := r.FormValue("data_stream_id") dataStreamRoute := r.FormValue("data_stream_route") - config, err := metadata.Cache.GetConfig(r.Context(), fmt.Sprintf("v2/%s-%s.json", dataStream, dataStreamRoute)) + configId := v2.ConfigIdentification{ + DataStreamID: dataStream, + DataStreamRoute: dataStreamRoute, + } + + config, err := metadata.Cache.GetConfig(r.Context(), configId.Path()) if err != nil { http.Error(rw, err.Error(), http.StatusNotFound) return diff --git a/upload-server/testing/cli_test.go b/upload-server/testing/cli_test.go index 481860868..d1633f717 100644 --- a/upload-server/testing/cli_test.go +++ b/upload-server/testing/cli_test.go @@ -289,7 +289,7 @@ func TestGetFileDeliveryPrefixDate(t *testing.T) { "data_stream_id": "test_stream", "data_stream_route": "test_route", } - metadata.Cache.SetConfig("v2/test_stream-test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FolderStructure: metadata.FolderStructureDate, }, @@ -316,7 +316,7 @@ func TestGetFileDeliveryPrefixRoot(t *testing.T) { "data_stream_id": "test_stream", "data_stream_route": "test_route", } - metadata.Cache.SetConfig("v2/test_stream-test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FolderStructure: metadata.FolderStructureRoot, }, @@ -341,7 +341,7 @@ func TestDeliveryFilenameSuffixUploadId(t *testing.T) { "data_stream_route": "test_route", } tuid := "1234" - metadata.Cache.SetConfig("v2/test_stream-test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FilenameSuffix: metadata.FilenameSuffixUploadId, }, @@ -364,7 +364,7 @@ func TestDeliveryFilenameSuffixNone(t *testing.T) { "data_stream_route": "test_route", } tuid := "1234" - metadata.Cache.SetConfig("v2/test_stream-test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FilenameSuffix: "", }, From 6abbea043f528d6d8756914e7e544c3ce8c223ac Mon Sep 17 00:00:00 2001 From: tma5 Date: Thu, 17 Oct 2024 10:00:04 -0400 Subject: [PATCH 2/2] tweak test data stream and route names --- upload-server/testing/cli_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/upload-server/testing/cli_test.go b/upload-server/testing/cli_test.go index d1633f717..258a71a55 100644 --- a/upload-server/testing/cli_test.go +++ b/upload-server/testing/cli_test.go @@ -286,10 +286,10 @@ func TestGetFileDeliveryPrefixDate(t *testing.T) { ctx := context.TODO() m := map[string]string{ "version": "2.0", - "data_stream_id": "test_stream", - "data_stream_route": "test_route", + "data_stream_id": "test-stream", + "data_stream_route": "test-route", } - metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test-stream_test-route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FolderStructure: metadata.FolderStructureDate, }, @@ -313,10 +313,10 @@ func TestGetFileDeliveryPrefixRoot(t *testing.T) { ctx := context.TODO() m := map[string]string{ "version": "2.0", - "data_stream_id": "test_stream", - "data_stream_route": "test_route", + "data_stream_id": "test-stream", + "data_stream_route": "test-route", } - metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test-stream_test-route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FolderStructure: metadata.FolderStructureRoot, }, @@ -337,11 +337,11 @@ func TestDeliveryFilenameSuffixUploadId(t *testing.T) { ctx := context.TODO() m := map[string]string{ "version": "2.0", - "data_stream_id": "test_stream", - "data_stream_route": "test_route", + "data_stream_id": "test-stream", + "data_stream_route": "test-route", } tuid := "1234" - metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test-stream_test-route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FilenameSuffix: metadata.FilenameSuffixUploadId, }, @@ -360,11 +360,11 @@ func TestDeliveryFilenameSuffixNone(t *testing.T) { ctx := context.TODO() m := map[string]string{ "version": "2.0", - "data_stream_id": "test_stream", - "data_stream_route": "test_route", + "data_stream_id": "test-stream", + "data_stream_route": "test-route", } tuid := "1234" - metadata.Cache.SetConfig("v2/test_stream_test_route.json", &validation.ManifestConfig{ + metadata.Cache.SetConfig("v2/test-stream_test-route.json", &validation.ManifestConfig{ Copy: validation.CopyConfig{ FilenameSuffix: "", },