diff --git a/examples/healthcare/README.md b/examples/healthcare/README.md index a3f9a00414..5ed7b41936 100644 --- a/examples/healthcare/README.md +++ b/examples/healthcare/README.md @@ -31,9 +31,9 @@ The sample data is located in the input/ folder. 1. At this point you have loaded the sample data. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Staging Http Server [http://localhost:8010/v1/search](http://localhost:8010/v1/search). *Your port may be different if you changed it during setup* -1. To run the conformance flows simply press the **Run** button next to the final flow. +1. To run the harmonize flows simply press the **Run** button next to the final flow. -1. Now you have conformed the data into your final database. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Final Http Server [http://localhost:8011/v1/search](http://localhost:8011/v1/search). *Your port may be different if you changed it during setup* +1. Now you have harmonized the data into your final database. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Final Http Server [http://localhost:8011/v1/search](http://localhost:8011/v1/search). *Your port may be different if you changed it during setup* # Entities @@ -43,7 +43,7 @@ Entities represent the data you are modeling. For this example we provide the ** Flows are sets of plugins that work together to create an envelope document. - [Input Flows](#input-flows) work on incoming data and store it in the Staging database. -- [Conformance Flows](#conformance-flows) work on staged data and transform and store it into the Final database. +- [Harmonize Flows](#harmonize-flows) work on staged data and transform and store it into the Final database. ## Input Flows @@ -53,9 +53,9 @@ The hl7 Flow is intended to ingest C-CDA C32 Hl7 XML files. When running the hl7 ### nppes The nppes Flow is intended to ingest NPPES csv files. This flow will split each row of the NPPES file into a separate XML document in the staging database. When running the hl7 flow simply point it at input/nppes. Set the collection to **nppes** and set the document type to **Delimited Text**. -## Conformance Flows +## Harmonize Flows -There is only one conformance flow provided. This final flow will create a conformed XML document that contains the original C32 xml as the content of an envelope. It will also extract various data from th3 C32 into the header section for easier queryability. +There is only one harmonize flow provided. This final flow will create a harmonized XML document that contains the original C32 xml as the content of an envelope. It will also extract various data from th3 C32 into the header section for easier queryability. ## Final REST services diff --git a/examples/healthcare/plugins/entities/Patients/conformance/REST/services/metadata/providers.xml b/examples/healthcare/plugins/entities/Patients/harmonize/REST/services/metadata/providers.xml similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/REST/services/metadata/providers.xml rename to examples/healthcare/plugins/entities/Patients/harmonize/REST/services/metadata/providers.xml diff --git a/examples/healthcare/plugins/entities/Patients/conformance/REST/services/providers.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/REST/services/providers.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/REST/services/providers.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/REST/services/providers.xqy diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/collector/collector.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/final/collector/collector.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/collector/collector.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/final/collector/collector.xqy diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/content/content.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/final/content/content.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/content/content.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/final/content/content.xqy diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/final.xml b/examples/healthcare/plugins/entities/Patients/harmonize/final/final.xml similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/final.xml rename to examples/healthcare/plugins/entities/Patients/harmonize/final/final.xml diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/headers/headers.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/final/headers/headers.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/headers/headers.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/final/headers/headers.xqy diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/triples/triples.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/final/triples/triples.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/triples/triples.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/final/triples/triples.xqy diff --git a/examples/healthcare/plugins/entities/Patients/conformance/final/writer/writer.xqy b/examples/healthcare/plugins/entities/Patients/harmonize/final/writer/writer.xqy similarity index 100% rename from examples/healthcare/plugins/entities/Patients/conformance/final/writer/writer.xqy rename to examples/healthcare/plugins/entities/Patients/harmonize/final/writer/writer.xqy diff --git a/examples/hr-hub/README.md b/examples/hr-hub/README.md index b199032a76..37dcbbea4c 100644 --- a/examples/hr-hub/README.md +++ b/examples/hr-hub/README.md @@ -23,7 +23,7 @@ Global Corp has exported the Employee data from a relational database. They are Acme Tech uses a SAAS for managing employees. The data is provided as JSON documents that came straight out of the SAAS REST API. # What is this example? -In this example we are loading the CSV table dumps from Global Corp and the JSON documents from Acme Tech into the Hub staging area. We then conform the two data sources into the final area by extracting common header fields. The header fields we extract are: +In this example we are loading the CSV table dumps from Global Corp and the JSON documents from Acme Tech into the Hub staging area. We then harmonize the two data sources into the final area by extracting common header fields. The header fields we extract are: - Employee ID - Employee Hire Date diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/collector/collector.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/collector/collector.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/collector/collector.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/collector/collector.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/content/content.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/content/content.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/content/content.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/content/content.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/conform-acme-tech.xml b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/harmonize-acme-tech.xml similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/conform-acme-tech.xml rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/harmonize-acme-tech.xml diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/headers/headers.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/headers/headers.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/headers/headers.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/headers/headers.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/triples/triples.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/triples/triples.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/triples/triples.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/triples/triples.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/writer/writer.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/writer/writer.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-acme-tech/writer/writer.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-acme-tech/writer/writer.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/collector/collector.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/collector/collector.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/collector/collector.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/collector/collector.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/content/content.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/content/content.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/content/content.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/content/content.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/conform-global-corp.xml b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/harmonize-global-corp.xml similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/conform-global-corp.xml rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/harmonize-global-corp.xml diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/headers/headers.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/headers/headers.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/headers/headers.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/headers/headers.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/triples/triples.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/triples/triples.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/triples/triples.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/triples/triples.sjs diff --git a/examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/writer/writer.sjs b/examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/writer/writer.sjs similarity index 100% rename from examples/hr-hub/plugins/entities/Employee/conformance/conform-global-corp/writer/writer.sjs rename to examples/hr-hub/plugins/entities/Employee/harmonize/harmonize-global-corp/writer/writer.sjs diff --git a/examples/tweets/README.md b/examples/tweets/README.md index eedb7d868b..7259575d41 100644 --- a/examples/tweets/README.md +++ b/examples/tweets/README.md @@ -22,7 +22,7 @@ The tweets are zipped into a .zip file in the input/ folder. **Entity Name:** Tweet **Ingest Flow Name:** ingest-tweets - **Conformance Flow Name:** conform-tweets + **Harmonize Flow Name:** harmonize-tweets **Plugin Type:** Javascript **Data Format:** JSON @@ -32,9 +32,9 @@ The tweets are zipped into a .zip file in the input/ folder. 1. At this point you have loaded the sample tweets. You can browse the data via [QConsole](http://localhost:8000/qconsole) in the data-hub-STAGING database or by searching the REST endpoint on the Staging Http Server [http://localhost:8010/v1/search](http://localhost:8010/v1/search). *Your port may be different if you changed it during setup* -1. To run the conformance flow simply press the **Run** button next to the "conform-tweets" flow. +1. To run the harmonize flow simply press the **Run** button next to the "harmonize-tweets" flow. -1. Now you have conformed the data into your final database. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Final Http Server [http://localhost:8011/v1/search](http://localhost:8011/v1/search). *Your port may be different if you changed it during setup* +1. Now you have harmonizeed the data into your final database. You can browse the data via [QConsole](http://localhost:8000/qconsole) or by searching the REST endpoint on the Final Http Server [http://localhost:8011/v1/search](http://localhost:8011/v1/search). *Your port may be different if you changed it during setup* # Entities @@ -44,13 +44,13 @@ Entities represent the data you are modeling. For this example you created the * Flows are sets of plugins that work together to create an envelope document. - [Input Flows](#input-flows) work on incoming data and store it in the Staging database. -- [Conformance Flows](#conformance-flows) work on staged data and transform and store it into the Final database. +- [Harmonize Flows](#harmonize-flows) work on staged data and transform and store it into the Final database. ## Input Flows ### input-tweets The auto-generated input Flow will ingest the compressed tweets as JSON files. -## Conformance Flows +## Harmonize Flows -This "conform-tweets" flow will create a conformed JSON document that contains the original tweet JSON as the content of an envelope. +This "harmonize-tweets" flow will create a harmonizeed JSON document that contains the original tweet JSON as the content of an envelope. diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java index d6b4171af0..d8efbe1585 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java @@ -38,8 +38,6 @@ import com.marklogic.appdeployer.AppConfig; import com.marklogic.appdeployer.ConfigDir; import com.marklogic.appdeployer.command.Command; -import com.marklogic.appdeployer.command.databases.DeploySchemasDatabaseCommand; -import com.marklogic.appdeployer.command.databases.DeployTriggersDatabaseCommand; import com.marklogic.appdeployer.command.modules.AllButAssetsModulesFinder; import com.marklogic.appdeployer.command.modules.AssetModulesFinder; import com.marklogic.appdeployer.command.security.DeployRolesCommand; @@ -267,12 +265,12 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) String dirStr = dir.toString(); boolean isInputDir = dirStr.matches(".*[/\\\\]input[/\\\\].*"); - boolean isConformanceDir = dirStr.matches(".*[/\\\\]conformance[/\\\\].*"); + boolean isHarmonizeDir = dirStr.matches(".*[/\\\\]harmonize[/\\\\].*"); if (isRest) { if (isInputDir) { loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), stagingClient)); } - else if (isConformanceDir) { + else if (isHarmonizeDir) { loadedFiles.addAll(hubModulesLoader.loadModules(dir.normalize().toAbsolutePath().toFile(), new AllButAssetsModulesFinder(), finalClient)); } return FileVisitResult.SKIP_SUBTREE; diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java index 8d698a8f9b..7d03cfeaf8 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java @@ -237,6 +237,9 @@ public void runInputFlow(Flow flow, HubConfig config) { flow.getEntityName(), flow.getName(), FlowType.INPUT.toString(), flow.getDataFormat()); + sourceOptions.setInputFileType("documents"); + sourceOptions.setTransformModule("/com.marklogic.hub/mlcp-flow-transform.xqy"); + sourceOptions.setTransformNamespace("http://marklogic.com/data-hub/mlcp-flow-transform"); mlcp.addSourceDirectory(config.modulesPath, sourceOptions); mlcp.loadContent(); } diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java index f3371ec84f..64bccbc095 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/Mlcp.java @@ -30,7 +30,7 @@ import com.marklogic.client.io.Format; public class Mlcp { - + public static final String DOCUMENT_TYPE_KEY = "-document_type"; public static final String INPUT_FILE_PATH_KEY = "-input_file_path"; public static final String INPUT_FILE_TYPE_KEY = "-input_file_type"; @@ -40,7 +40,7 @@ public class Mlcp { public static final String PORT_KEY = "-port"; public static final String USERNAME_KEY = "-username"; public static final String PASSWORD_KEY = "-password"; - + private static final Logger LOGGER = LoggerFactory.getLogger(Mlcp.class); private static final String DEFAULT_HADOOP_HOME_DIR = "./hadoop/"; @@ -109,24 +109,38 @@ public List getMlcpArguments() throws IOException, JSONException { String canonicalPath = file.getCanonicalPath(); List arguments = new ArrayList<>(); - + arguments.add(INPUT_FILE_PATH_KEY); arguments.add(canonicalPath); - + arguments.add(OUTPUT_URI_REPLACE_KEY); arguments.add("\""+canonicalPath+",''\""); - + arguments.add(INPUT_FILE_TYPE_KEY); arguments.add(sourceOptions.getInputFileType()); - addOtherArguments(arguments, sourceOptions.getOtherOptions()); - + + String other = sourceOptions.getOtherOptions(); + if (other != null) { + addOtherArguments(arguments, other); + } + //add document type only if it does not exist in the list if(!arguments.contains(DOCUMENT_TYPE_KEY)) { arguments.add(DOCUMENT_TYPE_KEY); arguments.add(sourceOptions.getDataFormat()); } - + + String transformModule = sourceOptions.getTransformModule(); + if (transformModule != null) { + arguments.add("-transform_module"); + arguments.add("\"" + transformModule +"\""); + arguments.add("-transform_namespace"); + arguments.add("\"" + sourceOptions.getTransformNamespace() +"\""); + arguments.add("-transform_param"); + arguments.add("\"" + sourceOptions.getTransformParams() + "\""); + } + return arguments; } @@ -142,9 +156,9 @@ private void addOtherArguments(List arguments, arguments.add(key); arguments.add(jsonObject.getString(key)); } - + } - + } } @@ -155,6 +169,8 @@ public static class SourceOptions { private String dataFormat = "json"; private String inputFileType; private String otherOptions; + private String transformModule; + private String transformNamespace; public SourceOptions(String entityName, String flowName, String flowType, Format dataFormat) { this.entityName = entityName; @@ -191,7 +207,7 @@ public String getInputFileType() { public void setInputFileType(String inputFileType) { this.inputFileType = inputFileType; } - + public String getOtherOptions() { return otherOptions; } @@ -199,6 +215,28 @@ public String getOtherOptions() { public void setOtherOptions(String otherOptions) { this.otherOptions = otherOptions; } + + protected String getTransformParams() { + return String.format( + "%s%s%s", + entityName, flowName, flowType); + } + + public String getTransformModule() { + return transformModule; + } + + public void setTransformModule(String transformModule) { + this.transformModule = transformModule; + } + + public String getTransformNamespace() { + return transformNamespace; + } + + public void setTransformNamespace(String transformNamespace) { + this.transformNamespace = transformNamespace; + } } public List getMlcpOptions(MlcpSource source) throws IOException, JSONException { @@ -218,7 +256,7 @@ public List getMlcpOptions(MlcpSource source) throws IOException, JSONEx List sourceArguments = source.getMlcpArguments(); mlcpOptions.addAll(sourceArguments); - + return mlcpOptions; } } diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/Scaffolding.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/Scaffolding.java index e46a82b62f..f4289d8c68 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/Scaffolding.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/Scaffolding.java @@ -63,7 +63,7 @@ public static void createFlow(String entityName, String flowName, throws IOException { File flowDir = getFlowDir(userlandDir, entityName, flowName, flowType); - if (flowType.equals(FlowType.CONFORMANCE)) { + if (flowType.equals(FlowType.HARMONIZE)) { File collectorDir = new File(flowDir, "collector"); collectorDir.mkdirs(); writeFile("scaffolding/" + flowType + "/" + pluginFormat + "/collector." + pluginFormat, diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/ScaffoldingValidator.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/ScaffoldingValidator.java index e7c7dc0d1b..89e8db7c67 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/ScaffoldingValidator.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/ScaffoldingValidator.java @@ -9,12 +9,12 @@ public class ScaffoldingValidator { public static boolean isUniqueRestServiceExtension(File pluginsDir, String name) { String entityNameFilter = "[a-zA-Z0-9_.-]+"; - String flowTypeFilter = "(" + FlowType.INPUT + "|" + FlowType.CONFORMANCE + ")"; + String flowTypeFilter = "(" + FlowType.INPUT + "|" + FlowType.HARMONIZE + ")"; String pluginFormatFilter = "(" + PluginFormat.XQUERY + "|" + PluginFormat.JAVASCRIPT + ")"; String absoluteFilePathFilter = Scaffolding.getAbsolutePath(pluginsDir.getAbsolutePath(), "entities", entityNameFilter, flowTypeFilter, "REST", "services", name + "." + pluginFormatFilter); return !checkIfFileExists(pluginsDir, absoluteFilePathFilter); } - + private static boolean checkIfFileExists(File rootDirectory, String absoluteFilePathFilter) { File[] list = rootDirectory.listFiles(); if (list != null) { @@ -32,7 +32,7 @@ private static boolean checkIfFileExists(File rootDirectory, String absoluteFile } return false; } - + public boolean checkIfFolderExists(File rootDirectory, String absoluteFilePathFilter) { File[] list = rootDirectory.listFiles(); if (list != null) { @@ -47,10 +47,10 @@ public boolean checkIfFolderExists(File rootDirectory, String absoluteFilePathFi } return false; } - + public static boolean isUniqueRestTransform(File pluginsDir, String name) { String entityNameFilter = "[a-zA-Z0-9_.-]+"; - String flowTypeFilter = "(" + FlowType.INPUT + "|" + FlowType.CONFORMANCE + ")"; + String flowTypeFilter = "(" + FlowType.INPUT + "|" + FlowType.HARMONIZE + ")"; String pluginFormatFilter = "(" + PluginFormat.XQUERY + "|" + PluginFormat.JAVASCRIPT + ")"; String absoluteFilePathFilter = Scaffolding.getAbsolutePath(pluginsDir.getAbsolutePath(), "entities", entityNameFilter, flowTypeFilter, "REST", "transforms", name + "." + pluginFormatFilter); return !checkIfFileExists(pluginsDir, absoluteFilePathFilter); diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/FlowType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/FlowType.java index b6b9fc9379..241bc8f273 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/FlowType.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/FlowType.java @@ -1,7 +1,7 @@ package com.marklogic.hub.flow; public enum FlowType { - INPUT("input"), CONFORMANCE("conformance"); + INPUT("input"), HARMONIZE("harmonize"); private String type; FlowType(String type) { diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy index 762080c9e7..285cd09955 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy @@ -314,10 +314,10 @@ declare function flow:get-flows( cts:uri-match($ENTITIES-DIR || "*") }) let $flows := - for $flow in $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/[^/]+/$")] - let $name := fn:replace($flow, $ENTITIES-DIR || $entity-name || "/(input|conformance)/([^/]+)/$", "$2") + for $flow in $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|harmonize)/[^/]+/$")] + let $name := fn:replace($flow, $ENTITIES-DIR || $entity-name || "/(input|harmonize)/([^/]+)/$", "$2") return - flow:get-flow($entity-name, $name, (), $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|conformance)/" || $name || "/.+")]) + flow:get-flow($entity-name, $name, (), $uris[fn:matches(., $ENTITIES-DIR || $entity-name || "/(input|harmonize)/" || $name || "/.+")]) return { @@ -415,7 +415,7 @@ declare function flow:run-collector( (), $module-uri, "collector", - "conformance", + "harmonize", $ex, (), xdmp:elapsed-time() - $before, @@ -430,7 +430,7 @@ declare function flow:run-collector( null-node {}, $module-uri, "collector", - "conformance", + "harmonize", null-node {}, json:to-array($resp), xdmp:elapsed-time() - $before, diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/collector.sjs b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/collector.sjs similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/collector.sjs rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/collector.sjs diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/content.sjs b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/content.sjs similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/content.sjs rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/content.sjs diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/headers.sjs b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/headers.sjs similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/headers.sjs rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/headers.sjs diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/triples.sjs b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/triples.sjs similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/triples.sjs rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/triples.sjs diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/writer.sjs b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/writer.sjs similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/sjs/writer.sjs rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/sjs/writer.sjs diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/collector.xqy b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/collector.xqy similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/collector.xqy rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/collector.xqy diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/content.xqy b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/content.xqy similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/content.xqy rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/content.xqy diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/headers.xqy b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/headers.xqy similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/headers.xqy rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/headers.xqy diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/triples.xqy b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/triples.xqy similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/triples.xqy rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/triples.xqy diff --git a/marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/writer.xqy b/marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/writer.xqy similarity index 100% rename from marklogic-data-hub/src/main/resources/scaffolding/conformance/xqy/writer.xqy rename to marklogic-data-hub/src/main/resources/scaffolding/harmonize/xqy/writer.xqy diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubInstallTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubInstallTest.java index 3e094db8f1..4d4b82cb92 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubInstallTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubInstallTest.java @@ -50,21 +50,21 @@ public void testInstallUserModules() throws IOException, ParserConfigurationExce dataHub.installUserModules(path); assertEquals( - getResource("data-hub-test/entities/test-entity/conformance/final/collector/collector.xqy"), - getModulesFile("/entities/test-entity/conformance/final/collector/collector.xqy")); + getResource("data-hub-test/entities/test-entity/harmonize/final/collector/collector.xqy"), + getModulesFile("/entities/test-entity/harmonize/final/collector/collector.xqy")); assertEquals( - getResource("data-hub-test/entities/test-entity/conformance/final/content/content.xqy"), - getModulesFile("/entities/test-entity/conformance/final/content/content.xqy")); + getResource("data-hub-test/entities/test-entity/harmonize/final/content/content.xqy"), + getModulesFile("/entities/test-entity/harmonize/final/content/content.xqy")); assertEquals( - getResource("data-hub-test/entities/test-entity/conformance/final/headers/headers.xqy"), - getModulesFile("/entities/test-entity/conformance/final/headers/headers.xqy")); + getResource("data-hub-test/entities/test-entity/harmonize/final/headers/headers.xqy"), + getModulesFile("/entities/test-entity/harmonize/final/headers/headers.xqy")); assertEquals( - getResource("data-hub-test/entities/test-entity/conformance/final/triples/triples.xqy"), - getModulesFile("/entities/test-entity/conformance/final/triples/triples.xqy")); + getResource("data-hub-test/entities/test-entity/harmonize/final/triples/triples.xqy"), + getModulesFile("/entities/test-entity/harmonize/final/triples/triples.xqy")); assertXMLEqual( - getXmlFromResource("data-hub-test/entities/test-entity/conformance/final/final.xml"), - getModulesDocument("/entities/test-entity/conformance/final/final.xml")); + getXmlFromResource("data-hub-test/entities/test-entity/harmonize/final/final.xml"), + getModulesDocument("/entities/test-entity/harmonize/final/final.xml")); assertEquals( @@ -85,14 +85,14 @@ public void testInstallUserModules() throws IOException, ParserConfigurationExce getModulesDocument("/Default/" + HubConfig.DEFAULT_STAGING_NAME + "/rest-api/options/doctors.xml")); assertXMLEqual( - getXmlFromResource("data-hub-test/entities/test-entity/conformance/REST/options/patients.xml"), + getXmlFromResource("data-hub-test/entities/test-entity/harmonize/REST/options/patients.xml"), getModulesDocument("/Default/" + HubConfig.DEFAULT_FINAL_NAME + "/rest-api/options/patients.xml")); assertXMLEqual( getXmlFromResource("data-hub-helpers/test-conf-metadata.xml"), getModulesDocument("/marklogic.rest.transform/test-conf-transform/assets/metadata.xml")); assertEquals( - getResource("data-hub-test/entities/test-entity/conformance/REST/transforms/test-conf-transform.xqy"), + getResource("data-hub-test/entities/test-entity/harmonize/REST/transforms/test-conf-transform.xqy"), getModulesFile("/marklogic.rest.transform/test-conf-transform/assets/transform.xqy")); assertXMLEqual( diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java index 444050c6c8..c428590304 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java @@ -40,7 +40,7 @@ public static void setup() throws IOException { Scaffolding.createEntity(ENTITY, pluginsDir); Scaffolding.createFlow(ENTITY, "testinput", FlowType.INPUT, PluginFormat.JAVASCRIPT, Format.XML, pluginsDir); - Scaffolding.createFlow(ENTITY, "testconformance", FlowType.CONFORMANCE, + Scaffolding.createFlow(ENTITY, "testharmonize", FlowType.HARMONIZE, PluginFormat.JAVASCRIPT, Format.XML, pluginsDir); new DataHub(getHubConfig()).installUserModules(pluginsDir.toString()); @@ -56,8 +56,8 @@ public static void teardown() throws IOException { public void runFlows() throws IOException, ParserConfigurationException, SAXException { FlowManager fm = new FlowManager(stagingClient); Flow inputFlow = fm.getFlow(ENTITY, "testinput", FlowType.INPUT); - Flow conformanceFlow = fm.getFlow(ENTITY, "testconformance", - FlowType.CONFORMANCE); + Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", + FlowType.HARMONIZE); URL url = HubTestBase.class.getClassLoader().getResource("e2e-test/input"); HubConfig config = getHubConfig(url.getPath()); @@ -65,9 +65,9 @@ public void runFlows() throws IOException, ParserConfigurationException, SAXExce assertXMLEqual(getXmlFromResource("e2e-test/staged.xml"), stagingDocMgr.read("/input.xml").next().getContent(new DOMHandle()).get()); - JobFinishedListener conformanceFlowListener = new JobFinishedListener(); - fm.runFlow(conformanceFlow, 10, conformanceFlowListener); - conformanceFlowListener.waitForFinish(); + JobFinishedListener harmonizeFlowListener = new JobFinishedListener(); + fm.runFlow(harmonizeFlow, 10, harmonizeFlowListener); + harmonizeFlowListener.waitForFinish(); assertXMLEqual(getXmlFromResource("e2e-test/final.xml"), finalDocMgr.read("/input.xml").next().getContent(new DOMHandle()).get()); } } diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java index 7cd82d6f43..b328c7e8e0 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java @@ -39,7 +39,7 @@ public static void setup() throws IOException { Scaffolding.createEntity(ENTITY, pluginsDir); Scaffolding.createFlow(ENTITY, "testinput", FlowType.INPUT, PluginFormat.XQUERY, Format.XML, pluginsDir); - Scaffolding.createFlow(ENTITY, "testconformance", FlowType.CONFORMANCE, + Scaffolding.createFlow(ENTITY, "testharmonize", FlowType.HARMONIZE, PluginFormat.XQUERY, Format.XML, pluginsDir); new DataHub(getHubConfig()).installUserModules(pluginsDir.toString()); @@ -55,8 +55,8 @@ public static void teardown() throws IOException { public void runFlows() throws IOException, ParserConfigurationException, SAXException { FlowManager fm = new FlowManager(stagingClient); Flow inputFlow = fm.getFlow(ENTITY, "testinput", FlowType.INPUT); - Flow conformanceFlow = fm.getFlow(ENTITY, "testconformance", - FlowType.CONFORMANCE); + Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", + FlowType.HARMONIZE); URL url = HubTestBase.class.getClassLoader().getResource("e2e-test/input"); HubConfig config = getHubConfig(url.getPath()); @@ -64,9 +64,9 @@ public void runFlows() throws IOException, ParserConfigurationException, SAXExce assertXMLEqual(getXmlFromResource("e2e-test/staged.xml"), stagingDocMgr.read("/input.xml").next().getContent(new DOMHandle()).get()); - JobFinishedListener conformanceFlowListener = new JobFinishedListener(); - fm.runFlow(conformanceFlow, 10, conformanceFlowListener); - conformanceFlowListener.waitForFinish(); + JobFinishedListener harmonizeFlowListener = new JobFinishedListener(); + fm.runFlow(harmonizeFlow, 10, harmonizeFlowListener); + harmonizeFlowListener.waitForFinish(); assertXMLEqual(getXmlFromResource("e2e-test/final.xml"), finalDocMgr.read("/input.xml").next().getContent(new DOMHandle()).get()); } } diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java index 45db55236d..161b875ff3 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/EntityManagerTest.java @@ -49,16 +49,16 @@ public static void setup() throws IOException { installStagingDoc("/incoming/employee1.xml", meta, getResource("flow-manager-test/input/employee1.xml")); installStagingDoc("/incoming/employee2.xml", meta, getResource("flow-manager-test/input/employee2.xml")); runInModules( - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/collector/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/headers/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/triples/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/content/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/collector/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/headers/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/triples/\")," + - "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/content/\")"); - installModule("/entities/test/conformance/my-test-flow1/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); - installModule("/entities/test/conformance/my-test-flow2/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/collector/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/headers/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/triples/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/content/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/collector/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/headers/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/triples/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/content/\")"); + installModule("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); + installModule("/entities/test/harmonize/my-test-flow2/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); } @AfterClass @@ -85,7 +85,7 @@ public void testGetEntities() { ServerCollector c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); ServerPlugin t = (ServerPlugin)flow1.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -103,7 +103,7 @@ public void testGetEntities() { c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); t = (ServerPlugin)flow2.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -131,7 +131,7 @@ public void testGetEntity() { ServerCollector c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); ServerPlugin t = (ServerPlugin)flow1.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -148,7 +148,7 @@ public void testGetEntity() { c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); t = (ServerPlugin)flow2.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java index 5192059a9c..8015365706 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java @@ -63,19 +63,19 @@ public static void setup() throws IOException { meta.getCollections().add("tester"); installStagingDoc("/employee1.xml", meta, getResource("flow-manager-test/input/employee1.xml")); installStagingDoc("/employee2.xml", meta, getResource("flow-manager-test/input/employee2.xml")); - runInModules("xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/collector/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/headers/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/triples/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow1/content/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/collector/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/headers/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/triples/\")," - + "xdmp:directory-create(\"/entities/test/conformance/my-test-flow2/content/\")"); + runInModules("xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/collector/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/headers/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/triples/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow1/content/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/collector/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/headers/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/triples/\")," + + "xdmp:directory-create(\"/entities/test/harmonize/my-test-flow2/content/\")"); installModule( - "/entities/test/conformance/my-test-flow1/collector/collector.xqy", + "/entities/test/harmonize/my-test-flow1/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); installModule( - "/entities/test/conformance/my-test-flow2/collector/collector.xqy", + "/entities/test/harmonize/my-test-flow2/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); } @@ -106,9 +106,9 @@ public void testSimpleFlowFromXml() throws IOException, ParserConfigurationExcep @Test public void testEmptyFlow() throws IOException { - runInModules("xdmp:directory-create(\"/entities/test/conformance/empty-flow/\")"); - installModule("/entities/test/conformance/empty-flow/empty-flow.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); - installModule("/entities/test/conformance/empty-flow/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); + runInModules("xdmp:directory-create(\"/entities/test/harmonize/empty-flow/\")"); + installModule("/entities/test/harmonize/empty-flow/empty-flow.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); + installModule("/entities/test/harmonize/empty-flow/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy"); FlowManager fm = new FlowManager(stagingClient); SimpleFlow flow1 = (SimpleFlow)fm.getFlow("test", "empty-flow"); @@ -116,7 +116,7 @@ public void testEmptyFlow() throws IOException { ServerCollector c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/empty-flow/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/empty-flow/collector/collector.xqy", c.getModule()); ServerPlugin t = (ServerPlugin)flow1.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -128,12 +128,12 @@ public void testEmptyFlow() throws IOException { DefaultWriter w = (DefaultWriter)flow1.getWriter(); assertNotNull(w); - runInModules("xdmp:directory-delete(\"/entities/test/conformance/empty-flow/\")"); + runInModules("xdmp:directory-delete(\"/entities/test/harmonize/empty-flow/\")"); } @Test public void testSimpleFlowToXml() throws IOException, ParserConfigurationException, SAXException { - SimpleFlow flow = new SimpleFlow("test", "my-test-flow", FlowType.CONFORMANCE, Format.XML); + SimpleFlow flow = new SimpleFlow("test", "my-test-flow", FlowType.HARMONIZE, Format.XML); flow.setCollector(new QueryCollector()); flow.setContentPlugin(new ContentPlugin()); flow.setHeaderPlugin(new HeadersPlugin()); @@ -146,7 +146,7 @@ public void testSimpleFlowToXml() throws IOException, ParserConfigurationExcepti @Test public void testGetFlows() { - installModule("/entities/test/conformance/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); + installModule("/entities/test/harmonize/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); FlowManager fm = new FlowManager(stagingClient); List flows = fm.getFlows("test"); @@ -157,11 +157,11 @@ public void testGetFlows() { assertEquals("my-test-flow1", flow1.getName()); assertEquals(Format.XML, flow1.getDataFormat()); assertEquals("test", flow1.getEntityName()); - assertEquals(FlowType.CONFORMANCE, flow1.getType()); + assertEquals(FlowType.HARMONIZE, flow1.getType()); ServerCollector c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); ServerPlugin t = (ServerPlugin)flow1.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -178,7 +178,7 @@ public void testGetFlows() { c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); t = (ServerPlugin)flow2.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -192,7 +192,7 @@ public void testGetFlows() { @Test public void getTestFlow() { - installModule("/entities/test/conformance/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1-json.xml"); + installModule("/entities/test/harmonize/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1-json.xml"); FlowManager fm = new FlowManager(stagingClient); SimpleFlow flow1 = (SimpleFlow)fm.getFlow("test", "my-test-flow1"); @@ -202,7 +202,7 @@ public void getTestFlow() { ServerCollector c = (ServerCollector)flow1.getCollector(); assertEquals(PluginType.XQUERY, c.getType()); - assertEquals("/entities/test/conformance/my-test-flow1/collector/collector.xqy", c.getModule()); + assertEquals("/entities/test/harmonize/my-test-flow1/collector/collector.xqy", c.getModule()); ServerPlugin t = (ServerPlugin)flow1.getContentPlugin(); assertEquals(PluginType.XQUERY, t.getType()); @@ -216,7 +216,7 @@ public void getTestFlow() { @Test public void testRunFlow() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { - installModule("/entities/test/conformance/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); + installModule("/entities/test/harmonize/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml"); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); JobFinishedListener listener = new JobFinishedListener(); @@ -226,14 +226,14 @@ public void testRunFlow() throws SAXException, IOException, ParserConfigurationE listener.waitForFinish(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed/conformed1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed/conformed2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); } @Test public void testRunFlowWithHeader() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { - installModule("/entities/test/conformance/my-test-flow-with-header/collector/collector.xqy", "flow-manager-test/my-test-flow-with-header/collector/collector.xqy"); - installModule("/entities/test/conformance/my-test-flow-with-header/headers/headers.xqy", "flow-manager-test/my-test-flow-with-header/headers/headers.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-header/collector/collector.xqy", "flow-manager-test/my-test-flow-with-header/collector/collector.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-header/headers/headers.xqy", "flow-manager-test/my-test-flow-with-header/headers/headers.xqy"); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); @@ -244,19 +244,19 @@ public void testRunFlowWithHeader() throws SAXException, IOException, ParserConf listener.waitForFinish(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed-with-header/conformed1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed-with-header/conformed2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); - runInModules("xdmp:directory-delete(\"/entities/test/conformance/my-test-flow-with-header/\")"); + runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-header/\")"); } @Test public void testRunFlowWithAll() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { - installModule("/entities/test/conformance/my-test-flow-with-all/my-test-flow-with-all.xml", "flow-manager-test/my-test-flow-with-all/my-test-flow-with-all.xml"); - installModule("/entities/test/conformance/my-test-flow-with-all/collector/collector.xqy", "flow-manager-test/my-test-flow-with-all/collector/collector.xqy"); - installModule("/entities/test/conformance/my-test-flow-with-all/headers/headers.xqy", "flow-manager-test/my-test-flow-with-all/headers/headers.xqy"); - installModule("/entities/test/conformance/my-test-flow-with-all/content/content.xqy", "flow-manager-test/my-test-flow-with-all/content/content.xqy"); - installModule("/entities/test/conformance/my-test-flow-with-all/triples/triples.xqy", "flow-manager-test/my-test-flow-with-all/triples/triples.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-all/my-test-flow-with-all.xml", "flow-manager-test/my-test-flow-with-all/my-test-flow-with-all.xml"); + installModule("/entities/test/harmonize/my-test-flow-with-all/collector/collector.xqy", "flow-manager-test/my-test-flow-with-all/collector/collector.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-all/headers/headers.xqy", "flow-manager-test/my-test-flow-with-all/headers/headers.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-all/content/content.xqy", "flow-manager-test/my-test-flow-with-all/content/content.xqy"); + installModule("/entities/test/harmonize/my-test-flow-with-all/triples/triples.xqy", "flow-manager-test/my-test-flow-with-all/triples/triples.xqy"); JobFinishedListener listener = new JobFinishedListener(); assertEquals(2, getStagingDocCount()); @@ -267,9 +267,9 @@ public void testRunFlowWithAll() throws SAXException, IOException, ParserConfigu listener.waitForFinish(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed-with-all/conformed1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); - assertXMLEqual(getXmlFromResource("flow-manager-test/conformed-with-all/conformed2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); + assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); - runInModules("xdmp:directory-delete(\"/entities/test/conformance/my-test-flow-with-all/\")"); + runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-all/\")"); } } diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java index a9c21039a7..a91d657098 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingTest.java @@ -62,8 +62,8 @@ public void createXqyInputFlow() throws IOException, SAXException { } @Test - public void createXqyConformanceFlow() throws IOException, SAXException { - createConformanceFlow(PluginFormat.XQUERY, Format.XML); + public void createXqyHarmonizeFlow() throws IOException, SAXException { + createHarmonizeFlow(PluginFormat.XQUERY, Format.XML); } @Test @@ -72,8 +72,8 @@ public void createSjsInputFlow() throws IOException, SAXException { } @Test - public void createSjsConformanceFlow() throws IOException, SAXException { - createConformanceFlow(PluginFormat.JAVASCRIPT, Format.JSON); + public void createSjsHarmonizeFlow() throws IOException, SAXException { + createHarmonizeFlow(PluginFormat.JAVASCRIPT, Format.JSON); } private void createInputFlow(PluginFormat pluginFormat, Format dataFormat) throws IOException, SAXException { @@ -115,7 +115,7 @@ private void createInputFlow(PluginFormat pluginFormat, Format dataFormat) throw assertTrue(triplesContent.exists()); } - private void createConformanceFlow(PluginFormat pluginFormat, Format dataFormat) throws IOException, SAXException { + private void createHarmonizeFlow(PluginFormat pluginFormat, Format dataFormat) throws IOException, SAXException { assertFalse(pluginsDir.exists()); Scaffolding.createEntity("my-fun-test", pluginsDir); @@ -125,12 +125,12 @@ private void createConformanceFlow(PluginFormat pluginFormat, Format dataFormat) assertTrue(entityDir.exists()); assertEquals(new File(pluginPath + "/entities/my-fun-test").toPath(), entityDir.toPath()); - Scaffolding.createFlow("my-fun-test", "test-conformance", FlowType.CONFORMANCE, pluginFormat, dataFormat, pluginsDir); - File flowDir = Scaffolding.getFlowDir(pluginsDir, "my-fun-test", "test-conformance", FlowType.CONFORMANCE); - assertEquals(new File(pluginPath + "/entities/my-fun-test/conformance/test-conformance").toPath(), flowDir.toPath()); + Scaffolding.createFlow("my-fun-test", "test-harmonize", FlowType.HARMONIZE, pluginFormat, dataFormat, pluginsDir); + File flowDir = Scaffolding.getFlowDir(pluginsDir, "my-fun-test", "test-harmonize", FlowType.HARMONIZE); + assertEquals(new File(pluginPath + "/entities/my-fun-test/harmonize/test-harmonize").toPath(), flowDir.toPath()); assertTrue(flowDir.exists()); - File flowDescriptor = new File(flowDir, "test-conformance.xml"); + File flowDescriptor = new File(flowDir, "test-harmonize.xml"); assertTrue(flowDescriptor.exists()); String flowXML ="simple" + dataFormat.getDefaultMimetype() + ""; assertXMLEqual(flowXML, IOUtils.toString(new FileInputStream(flowDescriptor))); @@ -160,7 +160,7 @@ private void createConformanceFlow(PluginFormat pluginFormat, Format dataFormat) public void createXqyRestExtension() throws IOException { String entityName = "my-fun-test"; String extensionName = "myExtension"; - FlowType flowType = FlowType.CONFORMANCE; + FlowType flowType = FlowType.HARMONIZE; PluginFormat pluginFormat = PluginFormat.XQUERY; try { Scaffolding.createRestExtension(entityName, extensionName, flowType, pluginFormat, pluginsDir); @@ -206,7 +206,7 @@ public void createSjsRestExtension() throws IOException { public void createXqyRestTransform() throws IOException { String entityName = "my-fun-test"; String transformName = "myTransform"; - FlowType flowType = FlowType.CONFORMANCE; + FlowType flowType = FlowType.HARMONIZE; PluginFormat pluginFormat = PluginFormat.XQUERY; try { Scaffolding.createRestTransform(entityName, transformName, flowType, pluginFormat, pluginsDir); @@ -225,7 +225,7 @@ public void createXqyRestTransform() throws IOException { public void createSjsRestTransform() throws IOException { String entityName = "my-fun-test"; String transformName = "myTransform"; - FlowType flowType = FlowType.CONFORMANCE; + FlowType flowType = FlowType.HARMONIZE; PluginFormat pluginFormat = PluginFormat.JAVASCRIPT; try { Scaffolding.createRestTransform(entityName, transformName, flowType, pluginFormat, pluginsDir); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingValidatorTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingValidatorTest.java index 972adbe1e8..a772539573 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingValidatorTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/ScaffoldingValidatorTest.java @@ -31,51 +31,51 @@ public class ScaffoldingValidatorTest extends HubTestBase { public static void setupClass() throws IOException { XMLUnit.setIgnoreWhitespace(true); } - + @Before public void setup() throws IOException { PluginFormat pluginFormat = PluginFormat.XQUERY; createPlugins(TEST_ENTITY_NAME, FlowType.INPUT, pluginFormat); - createPlugins(TEST_ENTITY_NAME, FlowType.CONFORMANCE, pluginFormat); + createPlugins(TEST_ENTITY_NAME, FlowType.HARMONIZE, pluginFormat); } - + @After public void teardown() throws IOException { FileUtils.deleteDirectory(pluginsDir); } - + private void createPlugins(String entityName, FlowType flowType, PluginFormat pluginFormat) throws IOException { - + String flowName = entityName + flowType + "-flow"; String flowTypePath = Scaffolding.getAbsolutePath(pluginPath, "entities", entityName, flowType.toString()); String flowPath = Scaffolding.getAbsolutePath(flowTypePath, flowName); - + List plugins = new ArrayList<>(); - if (flowType.equals(FlowType.CONFORMANCE)) { + if (flowType.equals(FlowType.HARMONIZE)) { plugins.add(createPluginObj(flowPath, "collector", flowType, pluginFormat)); } plugins.add(createPluginObj(flowPath, "content", flowType, pluginFormat)); plugins.add(createPluginObj(flowPath, "headers", flowType, pluginFormat)); plugins.add(createPluginObj(flowPath, "triples", flowType, pluginFormat)); - + for (Plugin plugin : plugins) { createFile(plugin.getParentDirectory(), plugin.getFilename(), plugin.getTemplateFilePath()); } } - + private Plugin createPluginObj(String flowPath, String pluginType, FlowType flowType, PluginFormat pluginFormat) { String parentDirectory = Scaffolding.getAbsolutePath(flowPath, pluginType); String filename = pluginType + "." + pluginFormat; String templateFilePath = "scaffolding/" + flowType + "/" + pluginFormat + "/" + pluginType + "." + pluginFormat; return new Plugin(parentDirectory, filename, templateFilePath); } - + private class Plugin { - + private String parentDirectory; private String filename; private String templateFilePath; - + public Plugin(String parentDirectory, String filename, String templateFilePath) { this.parentDirectory = parentDirectory; this.filename = filename; @@ -94,13 +94,13 @@ public String getTemplateFilePath() { return templateFilePath; } } - + private static void createFile(String parentDirectory, String filename, String templateFilePath) throws IOException { File parentDirectoryFile = new File(parentDirectory); parentDirectoryFile.mkdirs(); writeFile(templateFilePath, Paths.get(parentDirectoryFile.getPath(), filename)); } - + private static void writeFile(String srcFile, Path dstFile) throws IOException { InputStream inputStream = Scaffolding.class.getClassLoader() @@ -112,27 +112,27 @@ private static void writeFile(String srcFile, Path dstFile) public void testIsUniqueRestServiceExtension() throws IOException { String restServiceExtensionName = "test-rest-service"; boolean isUnique = ScaffoldingValidator.isUniqueRestServiceExtension(pluginsDir, restServiceExtensionName); - assertTrue("The rest service extension "+ restServiceExtensionName + " is not yet existing so it should be unique.", isUnique); + assertTrue("The rest service extension "+ restServiceExtensionName + " is not yet existing so it should be unique.", isUnique); try { - Scaffolding.createRestExtension(TEST_ENTITY_NAME, restServiceExtensionName, FlowType.CONFORMANCE, PluginFormat.XQUERY, pluginsDir); + Scaffolding.createRestExtension(TEST_ENTITY_NAME, restServiceExtensionName, FlowType.HARMONIZE, PluginFormat.XQUERY, pluginsDir); } catch (ScaffoldingValidationException e) { Assert.fail(e.getMessage()); } isUnique = ScaffoldingValidator.isUniqueRestServiceExtension(pluginsDir, restServiceExtensionName); - assertFalse("At this point, the rest service extension "+ restServiceExtensionName + " is already existing so it should not be unique.", isUnique); + assertFalse("At this point, the rest service extension "+ restServiceExtensionName + " is already existing so it should not be unique.", isUnique); } - + @Test public void testIsUniqueRestTransform() throws IOException { String restTransformName = "test-rest-transform"; boolean isUnique = ScaffoldingValidator.isUniqueRestTransform(pluginsDir, restTransformName); - assertTrue("The rest transform "+ restTransformName + " is not yet existing so it should be unique.", isUnique); + assertTrue("The rest transform "+ restTransformName + " is not yet existing so it should be unique.", isUnique); try { - Scaffolding.createRestTransform(TEST_ENTITY_NAME, restTransformName, FlowType.CONFORMANCE, PluginFormat.XQUERY, pluginsDir); + Scaffolding.createRestTransform(TEST_ENTITY_NAME, restTransformName, FlowType.HARMONIZE, PluginFormat.XQUERY, pluginsDir); } catch (ScaffoldingValidationException e) { Assert.fail(e.getMessage()); } isUnique = ScaffoldingValidator.isUniqueRestTransform(pluginsDir, restTransformName); - assertFalse("At this point, the rest service extension "+ restTransformName + " is already existing so it should not be unique.", isUnique); + assertFalse("At this point, the rest service extension "+ restTransformName + " is already existing so it should not be unique.", isUnique); } } diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/REST/options/patients.xml b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/REST/options/patients.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/REST/options/patients.xml rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/REST/options/patients.xml diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/REST/transforms/test-conf-transform.xqy b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/REST/transforms/test-conf-transform.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/REST/transforms/test-conf-transform.xqy rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/REST/transforms/test-conf-transform.xqy diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/collector/collector.xqy b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/collector/collector.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/collector/collector.xqy rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/collector/collector.xqy diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/content/content.xqy b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/content/content.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/content/content.xqy rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/content/content.xqy diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/final.xml b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/final.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/final.xml rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/final.xml diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/headers/headers.xqy b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/headers/headers.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/headers/headers.xqy rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/headers/headers.xqy diff --git a/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/triples/triples.xqy b/marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/triples/triples.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/conformance/final/triples/triples.xqy rename to marklogic-data-hub/src/test/resources/data-hub-test/entities/test-entity/harmonize/final/triples/triples.xqy diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-all/conformed1.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-all/harmonized1.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-all/conformed1.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-all/harmonized1.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-all/conformed2.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-all/harmonized2.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-all/conformed2.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-all/harmonized2.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-header/conformed1.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-header/harmonized1.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-header/conformed1.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-header/harmonized1.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-header/conformed2.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-header/harmonized2.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed-with-header/conformed2.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized-with-header/harmonized2.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed/conformed1.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized/harmonized1.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed/conformed1.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized/harmonized1.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/conformed/conformed2.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/harmonized/harmonized2.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/flow-manager-test/conformed/conformed2.xml rename to marklogic-data-hub/src/test/resources/flow-manager-test/harmonized/harmonized2.xml diff --git a/marklogic-data-hub/src/test/resources/flow-manager-test/simple-flow.xml b/marklogic-data-hub/src/test/resources/flow-manager-test/simple-flow.xml index 197bebb8f8..856478bd86 100644 --- a/marklogic-data-hub/src/test/resources/flow-manager-test/simple-flow.xml +++ b/marklogic-data-hub/src/test/resources/flow-manager-test/simple-flow.xml @@ -1,7 +1,7 @@ my-test-flow test - conformance + harmonize simple application/xml diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/collector/collector.xqy b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/collector/collector.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/collector/collector.xqy rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/collector/collector.xqy diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/content/content.xqy b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/content/content.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/content/content.xqy rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/content/content.xqy diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/headers/headers.xqy b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/headers/headers.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/headers/headers.xqy rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/headers/headers.xqy diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/traceme.xml b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/traceme.xml similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/traceme.xml rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/traceme.xml diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/triples/triples.xqy b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/triples/triples.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/triples/triples.xqy rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/triples/triples.xqy diff --git a/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/writer/writer.xqy b/marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/writer/writer.xqy similarity index 100% rename from marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/conformance/traceme/writer/writer.xqy rename to marklogic-data-hub/src/test/resources/tracing-test/entities/trace-entity/harmonize/traceme/writer/writer.xqy diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy index cbb0b9b3e5..58e8666757 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy @@ -9,7 +9,7 @@ import com.marklogic.hub.DataHub; import com.marklogic.gradle.task.InstallHubTask; import com.marklogic.gradle.task.UninstallHubTask; import com.marklogic.gradle.task.CreateEntityTask; -import com.marklogic.gradle.task.CreateConformanceFlowTask; +import com.marklogic.gradle.task.CreateHarmonizeFlowTask; import com.marklogic.gradle.task.CreateInputFlowTask; import com.marklogic.gradle.task.RunFlowTask; import com.marklogic.hub.HubConfig; @@ -35,7 +35,7 @@ class DataHubPlugin implements Plugin { String scaffoldGroup = "MarkLogic Data Hub Scaffolding" project.task("createEntity", group: scaffoldGroup, type: CreateEntityTask) - project.task("createConformanceFlow", group: scaffoldGroup, type: CreateConformanceFlowTask) + project.task("createHarmonizeFlow", group: scaffoldGroup, type: CreateHarmonizeFlowTask) project.task("createInputFlow", group: scaffoldGroup, type: CreateInputFlowTask) String flowGroup = "MarkLogic Data Hub Flow Management" diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/CreateConformanceFlowTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/CreateConformanceFlowTask.groovy index 2658b4d562..b5b462d40c 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/CreateConformanceFlowTask.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/CreateConformanceFlowTask.groovy @@ -3,10 +3,10 @@ package com.marklogic.gradle.task import com.marklogic.hub.flow.FlowType; import org.gradle.api.tasks.TaskAction -class CreateConformanceFlowTask extends CreateFlowTask { +class CreateHarmonizeFlowTask extends CreateFlowTask { @TaskAction - void createConformanceFlow() { - createFlow(FlowType.CONFORMANCE) + void createHarmonizeFlow() { + createFlow(FlowType.HARMONIZE) } } diff --git a/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java b/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java index 5fd76228de..b32218e0e6 100644 --- a/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java +++ b/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java @@ -45,7 +45,6 @@ public class EnvironmentConfiguration { private static final String ML_AUTH = "mlAuth"; private static final String USER_PLUGIN_DIR = "userPluginDir"; private static final String ASSET_INSTALL_TIME_FILE = "assetInstallTimeFile"; - private static final String MLCP_OPTIONS_DIR = "mlcp-options"; @Autowired private Environment environment; @@ -255,7 +254,7 @@ public void saveOrUpdateFlowMlcpOptionsToFile(String entityName, String flowName } private String getMlcpOptionsFilePath(String entityName, String flowName) { - return "." + File.separator + MLCP_OPTIONS_DIR + File.separator + entityName + "-" + flowName + ".txt"; + return "." + File.separator + entityName + "-" + flowName + ".txt"; } public String getFlowMlcpOptionsFromFile(String entityName, String flowName) throws IOException { diff --git a/quick-start/src/main/java/com/marklogic/hub/factory/EntityModelFactory.java b/quick-start/src/main/java/com/marklogic/hub/factory/EntityModelFactory.java index 235cf8605b..561c3d7d69 100644 --- a/quick-start/src/main/java/com/marklogic/hub/factory/EntityModelFactory.java +++ b/quick-start/src/main/java/com/marklogic/hub/factory/EntityModelFactory.java @@ -37,12 +37,12 @@ public EntityModelFactory(List entities) { } public EntityModel createNewEntity(File userPluginDir, String entityName, - String inputFlowName, String conformFlowName, PluginFormat pluginFormat, + String inputFlowName, String harmonizeFlowName, PluginFormat pluginFormat, Format dataFormat) throws IOException { EntityModel entityModel = new EntityModel(); entityModel.setEntityName(entityName); entityModel.setInputFlows(new ArrayList<>()); - entityModel.setConformFlows(new ArrayList<>()); + entityModel.setHarmonizeFlows(new ArrayList<>()); Scaffolding.createEntity(entityName, userPluginDir); @@ -53,12 +53,12 @@ public EntityModel createNewEntity(File userPluginDir, String entityName, entityModel.getInputFlows().add(inputFlow); } - if (conformFlowName != null) { - FlowModel conformFlow = flowModelFactory.createNewFlow( - userPluginDir, conformFlowName, - FlowType.CONFORMANCE, pluginFormat, + if (harmonizeFlowName != null) { + FlowModel harmonizeFlow = flowModelFactory.createNewFlow( + userPluginDir, harmonizeFlowName, + FlowType.HARMONIZE, pluginFormat, dataFormat); - entityModel.getConformFlows().add(conformFlow); + entityModel.getHarmonizeFlows().add(harmonizeFlow); } return entityModel; @@ -69,7 +69,7 @@ public EntityModel createEntity(String entityName, String entityFilePath, SyncSt entityModel.setEntityName(entityName); //this will be updated after traversing its modules entityModel.setSynched(this.entitiesInServer.containsKey(entityName)); - + FlowModelFactory flowModelFactory = new FlowModelFactory( this.entitiesInServer.get(entityName), entityName); RestModelFactory restModelFactory = new RestModelFactory(entityName); @@ -79,19 +79,19 @@ public EntityModel createEntity(String entityName, String entityFilePath, SyncSt } //set the values of modules of the entity such as flows, rest, etc. - private void setEntityModules(EntityModel entityModel, String entityFilePath, + private void setEntityModules(EntityModel entityModel, String entityFilePath, FlowModelFactory flowModelFactory, RestModelFactory restModelFactory, SyncStatusService syncStatusService) { this.setEntityModules(entityModel, entityFilePath, FlowType.INPUT, flowModelFactory, restModelFactory, syncStatusService); - this.setEntityModules(entityModel, entityFilePath, FlowType.CONFORMANCE, flowModelFactory, restModelFactory, syncStatusService); + this.setEntityModules(entityModel, entityFilePath, FlowType.HARMONIZE, flowModelFactory, restModelFactory, syncStatusService); } - private void setEntityModules(EntityModel entityModel, String entityFilePath, - FlowType flowType, FlowModelFactory flowModelFactory, + private void setEntityModules(EntityModel entityModel, String entityFilePath, + FlowType flowType, FlowModelFactory flowModelFactory, RestModelFactory restModelFactory, SyncStatusService syncStatusService) { String modulesParentDirectory = entityFilePath + File.separator + flowType.toString(); List folderNames = FileUtil.listDirectFolders(modulesParentDirectory); - + List flows = new ArrayList<>(); RestModel restModel = null; for (String folderName : folderNames) { @@ -109,8 +109,8 @@ private void setEntityModules(EntityModel entityModel, String entityFilePath, entityModel.setInputFlows(flows); entityModel.setInputRest(restModel); } else { - entityModel.setConformFlows(flows); - entityModel.setConformRest(restModel); + entityModel.setHarmonizeFlows(flows); + entityModel.setHarmonizeRest(restModel); } } diff --git a/quick-start/src/main/java/com/marklogic/hub/model/EntityModel.java b/quick-start/src/main/java/com/marklogic/hub/model/EntityModel.java index 7cfc359134..6a79dc90e3 100644 --- a/quick-start/src/main/java/com/marklogic/hub/model/EntityModel.java +++ b/quick-start/src/main/java/com/marklogic/hub/model/EntityModel.java @@ -7,79 +7,79 @@ public class EntityModel { private String entityName; - private List inputFlows; - private List conformFlows; - private boolean isSynched; - private RestModel inputRest; - private RestModel conformRest; + private List inputFlows; + private List harmonizeFlows; + private boolean isSynched; + private RestModel inputRest; + private RestModel harmonizeRest; public String getEntityName() { return entityName; - } + } public void setEntityName(String entityName) { this.entityName = entityName; - } + } - public List getInputFlows() { - return inputFlows; - } - - public Map getInputFlowsAsMap() { + public List getInputFlows() { + return inputFlows; + } + + public Map getInputFlowsAsMap() { Map flowModels = new HashMap<>(); - + if (inputFlows != null) { for (FlowModel model : inputFlows) { flowModels.put(model.getFlowName(), model); } } - + return flowModels; } - public void setInputFlows(List inputFlows) { - this.inputFlows = inputFlows; - } - - public List getConformFlows() { - return conformFlows; - } - - public Map getConformFlowsAsMap() { - Map flowModels = new HashMap<>(); - - if (conformFlows != null) { - for (FlowModel model : conformFlows) { - flowModels.put(model.getFlowName(), model); - } - } - - return flowModels; - } - - public void setConformFlows(List conformFlows) { - this.conformFlows = conformFlows; - } - - public boolean isSynched() { - return isSynched; - } - - public void setSynched(boolean isSynched) { - this.isSynched = isSynched; - } - - public void setInputFlowsSynched(boolean synched) { - if (inputFlows != null) { - for (FlowModel model : inputFlows) { - model.setSynched(synched); - } - } - } - - public void setConformFlowsSynched(boolean synched) { - if (conformFlows != null) { - for (FlowModel model : conformFlows) { + public void setInputFlows(List inputFlows) { + this.inputFlows = inputFlows; + } + + public List getHarmonizeFlows() { + return harmonizeFlows; + } + + public Map getHarmonizeFlowsAsMap() { + Map flowModels = new HashMap<>(); + + if (harmonizeFlows != null) { + for (FlowModel model : harmonizeFlows) { + flowModels.put(model.getFlowName(), model); + } + } + + return flowModels; + } + + public void setHarmonizeFlows(List harmonizeFlows) { + this.harmonizeFlows = harmonizeFlows; + } + + public boolean isSynched() { + return isSynched; + } + + public void setSynched(boolean isSynched) { + this.isSynched = isSynched; + } + + public void setInputFlowsSynched(boolean synched) { + if (inputFlows != null) { + for (FlowModel model : inputFlows) { + model.setSynched(synched); + } + } + } + + public void setHarmonizeFlowsSynched(boolean synched) { + if (harmonizeFlows != null) { + for (FlowModel model : harmonizeFlows) { model.setSynched(synched); } } @@ -93,14 +93,14 @@ public void setInputRest(RestModel inputRest) { this.inputRest = inputRest; } - public RestModel getConformRest() { - return conformRest; + public RestModel getHarmonizeRest() { + return harmonizeRest; } - public void setConformRest(RestModel conformRest) { - this.conformRest = conformRest; + public void setHarmonizeRest(RestModel harmonizeRest) { + this.harmonizeRest = harmonizeRest; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -110,7 +110,7 @@ public String toString() { sb.append("isSynched="); sb.append(isSynched); sb.append("}"); - + return sb.toString(); } } diff --git a/quick-start/src/main/java/com/marklogic/hub/model/FlowType.java b/quick-start/src/main/java/com/marklogic/hub/model/FlowType.java index 61ccbb5a04..2663bd855d 100644 --- a/quick-start/src/main/java/com/marklogic/hub/model/FlowType.java +++ b/quick-start/src/main/java/com/marklogic/hub/model/FlowType.java @@ -2,34 +2,34 @@ public enum FlowType { - INPUT("input", "Input Flow"), CONFORM("conformance", "Conformance Flow"); - - public String name; - public String type; - - FlowType(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public String getType() { - return type; - } - - public static FlowType getFlowType(String type) { - for (FlowType flowType : FlowType.values()) { - if (flowType.getName().equals(type)) { - return flowType; - } - } - return null; - } - - public String toString() { - return name; - } + INPUT("input", "Input Flow"), HARMONIZE("harmonize", "Harmonize Flow"); + + public String name; + public String type; + + FlowType(String name, String type) { + this.name = name; + this.type = type; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public static FlowType getFlowType(String type) { + for (FlowType flowType : FlowType.values()) { + if (flowType.getName().equals(type)) { + return flowType; + } + } + return null; + } + + public String toString() { + return name; + } } diff --git a/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java b/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java index db1456fa81..21310c18fb 100644 --- a/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java +++ b/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java @@ -85,14 +85,14 @@ public Entity getEntity(String entityName) { } public EntityModel createEntity(String EntityName, String inputFlowName, - String conformFlowName, PluginFormat pluginFormat, Format dataFormat) { + String harmonizeFlowName, PluginFormat pluginFormat, Format dataFormat) { EntityModelFactory EntityModelFactory = new EntityModelFactory(); EntityModel EntityModel; try { File pluginDir = new File(environmentConfiguration.getUserPluginDir()); EntityModel = EntityModelFactory.createNewEntity(pluginDir, EntityName, - inputFlowName, conformFlowName, pluginFormat, dataFormat); + inputFlowName, harmonizeFlowName, pluginFormat, dataFormat); } catch (IOException e) { throw new EntityManagerException(e.getMessage(), e); } diff --git a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java index b9816e6c52..35ef076184 100644 --- a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java +++ b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java @@ -3,7 +3,6 @@ import java.io.File; import java.util.ArrayList; import java.util.Enumeration; -import java.util.Iterator; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpSession; diff --git a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/EntityApiController.java b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/EntityApiController.java index b82993eaf5..2cef47900b 100644 --- a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/EntityApiController.java +++ b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/EntityApiController.java @@ -34,115 +34,116 @@ @Scope("session") public class EntityApiController implements InitializingBean, DisposableBean, FileSystemEventListener { - @Autowired - private EnvironmentConfiguration environmentConfiguration; - - @Autowired - private DataHubService dataHubService; - - @Autowired - private EntityManagerService entityManagerService; - - @Autowired - private FileSystemWatcherService watcherService; - - @Autowired - private SyncStatusService syncStatusService; - - @RequestMapping(method = RequestMethod.GET) - @ResponseBody - public List getEntities(HttpSession session) { - LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); - List entities = entityManagerService.getEntities(); - loginForm.setEntities(entities); - return entities; - } - - @RequestMapping(value = "display", method = RequestMethod.POST) - @ResponseBody - public EntityModel displayEntity(@RequestBody String entityName, HttpSession session) { - LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); - loginForm.selectEntity(entityName); - - return loginForm.getSelectedEntity(); - } - - @RequestMapping(method = RequestMethod.POST, consumes = { MediaType.APPLICATION_JSON_UTF8_VALUE }, produces = { - MediaType.APPLICATION_JSON_UTF8_VALUE }) - @ResponseBody - public LoginForm saveEntity(@RequestBody EntityForm entityForm, BindingResult bindingResult, HttpSession session) { - LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); - List entities = loginForm.getEntities(); - - entityForm.validate(entities); - - EntityModel entityModel = entityManagerService.createEntity(entityForm.getEntityName(), - entityForm.getInputFlowName(), entityForm.getConformFlowName(), entityForm.getPluginFormat(), - entityForm.getDataFormat()); - - entities.add(entityModel); - loginForm.setSelectedEntity(entityModel); - return loginForm; - } - - /** - * Get a list of entities that has changed. This API does not return until a - * change has occurred. - * - * @param session - * @return - */ - @RequestMapping(value = "status-change", method = RequestMethod.GET) - public LoginForm getStatusChange(HttpServletRequest request) { - HttpSession session = request.getSession(); - - synchronized (syncStatusService) { - try { - syncStatusService.wait(); - } catch (InterruptedException e) { - } - - // refresh the list of entities saved in the session - LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); - if (null == loginForm) { - loginForm = new LoginForm(); - } - - // add checking if data hub is installed and the server is - // acceptable. Something may have changed the server or removed the - // data hub outside the app - loginForm.setInstalled(dataHubService.isInstalled()); - loginForm.setServerVersionAccepted(dataHubService.isServerAcceptable()); - if (loginForm.isInstalled()) { - List entities = entityManagerService.getEntities(); - loginForm.setEntities(entities); - loginForm.refreshSelectedEntity(); - } - - // refresh the session loginForm - session.setAttribute("loginForm", loginForm); - - return loginForm; - } - } - - @Override - public void afterPropertiesSet() throws Exception { - String pluginDir = environmentConfiguration.getUserPluginDir(); - watcherService.watch(pluginDir, this); - } - - @Override - public void destroy() throws Exception { - synchronized (syncStatusService) { - syncStatusService.notifyAll(); - } - } - - @Override - public void onWatchEvent(Path path, WatchEvent event) { - synchronized (syncStatusService) { - syncStatusService.notifyAll(); - } - } + @Autowired + private EnvironmentConfiguration environmentConfiguration; + + @Autowired + private DataHubService dataHubService; + + @Autowired + private EntityManagerService entityManagerService; + + @Autowired + private FileSystemWatcherService watcherService; + + @Autowired + private SyncStatusService syncStatusService; + + @RequestMapping(method = RequestMethod.GET) + @ResponseBody + public List getEntities(HttpSession session) { + LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); + List entities = entityManagerService.getEntities(); + loginForm.setEntities(entities); + return entities; + } + + @RequestMapping(value = "display", method = RequestMethod.POST) + @ResponseBody + public EntityModel displayEntity(@RequestBody String entityName, HttpSession session) { + LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); + loginForm.selectEntity(entityName); + + return loginForm.getSelectedEntity(); + } + + @RequestMapping(method = RequestMethod.POST, consumes = { MediaType.APPLICATION_JSON_UTF8_VALUE }, produces = { + MediaType.APPLICATION_JSON_UTF8_VALUE }) + @ResponseBody + public LoginForm saveEntity(@RequestBody EntityForm entityForm, BindingResult bindingResult, HttpSession session) { + LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); + List entities = loginForm.getEntities(); + + entityForm.validate(entities); + + EntityModel entityModel = entityManagerService.createEntity(entityForm.getEntityName(), + entityForm.getInputFlowName(), entityForm.getHarmonizeFlowName(), entityForm.getPluginFormat(), + entityForm.getDataFormat()); + + entities.add(entityModel); + loginForm.setSelectedEntity(entityModel); + return loginForm; + } + + /** + * Get a list of entities that has changed. This API does not return until a + * change has occurred. + * + * @param session + * @return + */ + @RequestMapping(value = "status-change", method = RequestMethod.GET) + public LoginForm getStatusChange(HttpServletRequest request) { + HttpSession session = request.getSession(); + + synchronized (syncStatusService) { + try { + syncStatusService.wait(); + } + catch (InterruptedException e) { + } + + // refresh the list of entities saved in the session + LoginForm loginForm = (LoginForm) session.getAttribute("loginForm"); + if (null == loginForm) { + loginForm = new LoginForm(); + } + + // add checking if data hub is installed and the server is + // acceptable. Something may have changed the server or removed the + // data hub outside the app + loginForm.setInstalled(dataHubService.isInstalled()); + loginForm.setServerVersionAccepted(dataHubService.isServerAcceptable()); + if (loginForm.isInstalled()) { + List entities = entityManagerService.getEntities(); + loginForm.setEntities(entities); + loginForm.refreshSelectedEntity(); + } + + // refresh the session loginForm + session.setAttribute("loginForm", loginForm); + + return loginForm; + } + } + + @Override + public void afterPropertiesSet() throws Exception { + String pluginDir = environmentConfiguration.getUserPluginDir(); + watcherService.watch(pluginDir, this); + } + + @Override + public void destroy() throws Exception { + synchronized (syncStatusService) { + syncStatusService.notifyAll(); + } + } + + @Override + public void onWatchEvent(Path path, WatchEvent event) { + synchronized (syncStatusService) { + syncStatusService.notifyAll(); + } + } } diff --git a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java index fbf4d5799f..a40cbef904 100644 --- a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java +++ b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java @@ -88,8 +88,8 @@ public EntityModel saveFlow(@RequestBody FlowForm flowForm, flowForm.getPluginFormat(), flowForm.getDataFormat()); - if (flowForm.getFlowType().equals(FlowType.CONFORMANCE)) { - selectedEntity.getConformFlows().add(flowModel); + if (flowForm.getFlowType().equals(FlowType.HARMONIZE)) { + selectedEntity.getHarmonizeFlows().add(flowModel); } else { selectedEntity.getInputFlows().add(flowModel); } @@ -100,7 +100,7 @@ private List getAllFlowsOfSelectedEntity( EntityModel selectedEntity) { List flowList = new ArrayList<>(); flowList.addAll(selectedEntity.getInputFlows()); - flowList.addAll(selectedEntity.getConformFlows()); + flowList.addAll(selectedEntity.getHarmonizeFlows()); return flowList; } diff --git a/quick-start/src/main/java/com/marklogic/hub/web/form/EntityForm.java b/quick-start/src/main/java/com/marklogic/hub/web/form/EntityForm.java index 7eac726346..0b9f4d642e 100644 --- a/quick-start/src/main/java/com/marklogic/hub/web/form/EntityForm.java +++ b/quick-start/src/main/java/com/marklogic/hub/web/form/EntityForm.java @@ -11,7 +11,7 @@ public class EntityForm extends BaseForm { private String entityName; private String inputFlowName; - private String conformFlowName; + private String harmonizeFlowName; private PluginFormat pluginFormat; private Format dataFormat; @@ -31,12 +31,12 @@ public void setInputFlowName(String inputFlowName) { this.inputFlowName = inputFlowName; } - public String getConformFlowName() { - return conformFlowName; + public String getHarmonizeFlowName() { + return harmonizeFlowName; } - public void setConformFlowName(String conformFlowName) { - this.conformFlowName = conformFlowName; + public void setHarmonizeFlowName(String harmonizeFlowName) { + this.harmonizeFlowName = harmonizeFlowName; } public PluginFormat getPluginFormat() { @@ -69,10 +69,10 @@ public void validate(List entityList) { } } if ((this.inputFlowName == null || "".equals(this.inputFlowName.trim())) - && (this.conformFlowName == null || "" - .equals(this.conformFlowName.trim()))) { + && (this.harmonizeFlowName == null || "" + .equals(this.harmonizeFlowName.trim()))) { throw new FormValidationException( - "Either the Ingest Flow Name or the Conformance Flow Name must be supplied."); + "Either the Ingest Flow Name or the Harmonize Flow Name must be supplied."); } } } diff --git a/quick-start/src/main/resources/static/app/services/modalService.js b/quick-start/src/main/resources/static/app/services/modalService.js index 8b1ceb308b..d624c83bdb 100644 --- a/quick-start/src/main/resources/static/app/services/modalService.js +++ b/quick-start/src/main/resources/static/app/services/modalService.js @@ -8,7 +8,7 @@ .controller('loadDataModalController', LoadDataModalController) .controller('entityModalController', EntityModalController) .controller('flowModalController', FlowModalController); - + function GetByFieldAndValue() { return function(field, value, collection) { var i=0, len=collection.length; @@ -91,7 +91,7 @@ $scope.mlcpInitialCommand = ''; $scope.mlcpCommand = ''; $scope.groups = []; - + $scope.ok = function() { $uibModalInstance.close($scope.loadDataForm); }; @@ -99,7 +99,7 @@ $scope.cancel = function() { $uibModalInstance.dismiss(); }; - + $scope.download = function() { $scope.loading = true; DataHub.downloadMlcpOptionsFile($scope.loadDataForm) @@ -153,7 +153,7 @@ $scope.loadTree(data, node); }); }; - + $scope.loadTree = function(data, node) { if (node == null) { // jshint ignore:line //initialize root @@ -163,7 +163,7 @@ } $scope.showInputPathTreeBrowser = true; }; - + $scope.searchPathThenHideTree = function(basePath, node) { DataHub.searchPath(basePath).success(function(data) { $scope.loadTree(data, node); @@ -172,16 +172,16 @@ $scope.showInputPathTreeBrowser = false; }); }; - + $scope.dataForTheTree = []; - + $scope.searchPathThenHideTree($scope.loadDataForm.inputPath); $scope.mlcpInitialCommand = constructInitialMlcpCommand(DataHub); - + $scope.updateMlcpCommand = function() { $scope.mlcpCommand = updateMlcpCommand($scope.mlcpInitialCommand, $scope.loadDataForm, $scope.groups); }; - + $scope.loadSettings = function() { DataHub.getJsonFile('/json/inputOptions.json') .success(function(data) { @@ -197,9 +197,9 @@ $scope.updateMlcpCommand(); }); }; - + $scope.loadSettings(); - + $scope.isText = function(type) { if(type === 'string' || type === 'comma-list' || type === 'number' || type === 'character') { return true; @@ -207,40 +207,41 @@ return false; } }; - + $scope.hideInputPathTreeBrowser = function() { $scope.showInputPathTreeBrowser = false; }; - + $scope.showBasedOnCategoryAndInputFileType = function(category, inputFileType) { return showBasedOnCategoryAndInputFileType(category, inputFileType); }; - + $scope.showIfHasNoFilterFieldOrWithSpecifiedValue = function(field,value,collection) { if(angular.isUndefined(field) || $filter('GetByFieldAndValue')(field,value,collection)) { return true; } return false; }; - + $scope.makeDefaultValueReadOnlyIfApplicable = function($event) { var elem = $event.currentTarget; - var readOnlyLengthData = elem.getAttribute("data-read-only-length"); + var readOnlyLengthData = elem.getAttribute('data-read-only-length'); if(readOnlyLengthData) { - var readOnlyLength = parseInt(readOnlyLengthData); - if (($event.which != 37 && ($event.which != 39)) - && ((elem.selectionStart < readOnlyLength) - || ((elem.selectionStart === readOnlyLength) && ($event.which === 8)))) { + var readOnlyLength = parseInt(readOnlyLengthData); + if (($event.which !== 37 && ($event.which !== 39)) && + ((elem.selectionStart < readOnlyLength) || + ((elem.selectionStart === readOnlyLength) && + ($event.which === 8)))) { $event.preventDefault(); - return false; - } + return false; + } } }; } - + /* - * update $scope.groups and add a ReadOnlyLength - * for options with default value to disable removal of default value + * update $scope.groups and add a ReadOnlyLength + * for options with default value to disable removal of default value * for options with type 'comma-list', set it to the length of the default value * for options with type 'string', set it to -1 which means it should be readonly */ @@ -250,14 +251,15 @@ if(setting.Value) { if(setting.Type === 'comma-list') { setting.ReadOnlyLength = setting.Value.length; - } else if(setting.Type === 'string') { + } + else if(setting.Type === 'string') { setting.ReadOnlyLength = -1; - } + } } }); }); } - + function updateGroupsBasedOnPreviousSettings(groups, otherOptions) { if(otherOptions !== null) { var optionsMap = $.parseJSON(otherOptions); @@ -271,7 +273,7 @@ }); } } - + function showBasedOnCategoryAndInputFileType(category, inputFileType) { if(category === 'Delimited text options' && inputFileType !== 'delimited_text') { return false; @@ -280,7 +282,7 @@ } return true; } - + function constructInitialMlcpCommand(DataHub) { var mlcpCommand = 'mlcp'; var mlcpExtension = '.sh'; @@ -294,13 +296,13 @@ mlcpCommand += ' -password ' + DataHub.status.mlPassword; return mlcpCommand; } - + function updateMlcpCommand(initialMlcpCommand, loadDataForm, groups) { var mlcpCommand = initialMlcpCommand; mlcpCommand += ' -input_file_path ' + loadDataForm.inputPath; mlcpCommand += ' -input_file_type ' + loadDataForm.inputFileType; mlcpCommand += ' -output_uri_replace "' + loadDataForm.inputPath + ',\'\'"'; - + var otherOptions = []; $.each(groups, function(i, group) { if(showBasedOnCategoryAndInputFileType(group.category, loadDataForm.inputFileType)) { @@ -316,7 +318,7 @@ }); } }); - + loadDataForm.otherOptions = otherOptions.length > 0 ? JSON.stringify(otherOptions) : null; return mlcpCommand; } diff --git a/quick-start/src/main/resources/static/top/flows.html b/quick-start/src/main/resources/static/top/flows.html index 98e342cdc2..2c5896face 100644 --- a/quick-start/src/main/resources/static/top/flows.html +++ b/quick-start/src/main/resources/static/top/flows.html @@ -11,9 +11,9 @@

-

Conformance Flows

+

Harmonize Flows

    -
  • +

  • {{flow.flowName}}

    diff --git a/quick-start/src/main/resources/static/top/modal/entityModal.html b/quick-start/src/main/resources/static/top/modal/entityModal.html index 5f4a5c1426..b40d7dfd86 100644 --- a/quick-start/src/main/resources/static/top/modal/entityModal.html +++ b/quick-start/src/main/resources/static/top/modal/entityModal.html @@ -22,8 +22,8 @@
    - - + +
    @@ -49,4 +49,4 @@
    - \ No newline at end of file +