diff --git a/examples/online-store/entity-config/databases/final-database.json b/examples/online-store/entity-config/databases/final-database.json new file mode 100644 index 0000000000..10849361e5 --- /dev/null +++ b/examples/online-store/entity-config/databases/final-database.json @@ -0,0 +1 @@ +{"path-namespace":[{"prefix":"es", "namespace-uri":"http://marklogic.com/entity-services"}], "range-element-index":[{"collation":"http://marklogic.com/collation/codepoint", "invalid-values":"reject", "localname":"sku", "namespace-uri":null, "range-value-positions":false, "scalar-type":"string"}]} \ No newline at end of file diff --git a/examples/online-store/entity-config/databases/staging-database.json b/examples/online-store/entity-config/databases/staging-database.json new file mode 100644 index 0000000000..10849361e5 --- /dev/null +++ b/examples/online-store/entity-config/databases/staging-database.json @@ -0,0 +1 @@ +{"path-namespace":[{"prefix":"es", "namespace-uri":"http://marklogic.com/entity-services"}], "range-element-index":[{"collation":"http://marklogic.com/collation/codepoint", "invalid-values":"reject", "localname":"sku", "namespace-uri":null, "range-value-positions":false, "scalar-type":"string"}]} \ No newline at end of file diff --git a/examples/online-store/entity-config/final-entity-options.xml b/examples/online-store/entity-config/final-entity-options.xml new file mode 100644 index 0000000000..cb861f22c5 --- /dev/null +++ b/examples/online-store/entity-config/final-entity-options.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + //*:instance/Order/id + + + + + + + + + + + unfiltered + + + //*:instance/(Product|Order) + + + + true + + + \ No newline at end of file diff --git a/examples/online-store/entity-config/staging-entity-options.xml b/examples/online-store/entity-config/staging-entity-options.xml new file mode 100644 index 0000000000..cb861f22c5 --- /dev/null +++ b/examples/online-store/entity-config/staging-entity-options.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + //*:instance/Order/id + + + + + + + + + + + unfiltered + + + //*:instance/(Product|Order) + + + + true + + + \ No newline at end of file diff --git a/examples/online-store/gradle/wrapper/gradle-wrapper.jar b/examples/online-store/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..29953ea141 Binary files /dev/null and b/examples/online-store/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/online-store/gradlew b/examples/online-store/gradlew new file mode 100644 index 0000000000..4453ccea33 --- /dev/null +++ b/examples/online-store/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save ( ) { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/examples/online-store/gradlew.bat b/examples/online-store/gradlew.bat new file mode 100644 index 0000000000..f9553162f1 --- /dev/null +++ b/examples/online-store/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/123.properties b/examples/online-store/plugins/entities/Product/harmonize/123/123.properties new file mode 100644 index 0000000000..a48af9149e --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/123.properties @@ -0,0 +1,8 @@ +# +#Thu Jan 03 14:14:30 MST 2019 +collectorModule=collector.xqy +dataFormat=xml +codeFormat=xqy +mainCodeFormat=xqy +mainModule=main.xqy +collectorCodeFormat=xqy diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/collector.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/collector.xqy new file mode 100644 index 0000000000..cacecd88fa --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/collector.xqy @@ -0,0 +1,20 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare option xdmp:mapping "false"; + +(:~ + : Collect IDs plugin + : + : @param $options - a map containing options. Options are sent from Java + : + : @return - a sequence of ids or uris + :) +declare function plugin:collect( + $options as map:map) as xs:string* +{ + (: by default we return the URIs in the same collection as the Entity name :) + cts:uris((), (), cts:collection-query('entity')) +}; + diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/content.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/content.xqy new file mode 100644 index 0000000000..cf6a81cb7e --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/content.xqy @@ -0,0 +1,29 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Content Plugin + : + : @param $id - the identifier returned by the collector + : @param $options - a map containing options. Options are sent from Java + : + : @return - your transformed content + :) +declare function plugin:create-content( + $id as xs:string, + $options as map:map) as item()? +{ + let $doc := fn:doc($id) + return + if ($doc/es:envelope) then + $doc/es:envelope/es:instance/node() + else if ($doc/envelope/instance) then + $doc/envelope/instance + else + $doc +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/headers.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/headers.xqy new file mode 100644 index 0000000000..d8ee319d1c --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/headers.xqy @@ -0,0 +1,24 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Headers Plugin + : + : @param $id - the identifier returned by the collector + : @param $content - the output of your content plugin + : @param $options - a map containing options. Options are sent from Java + : + : @return - zero or more header nodes + :) +declare function plugin:create-headers( + $id as xs:string, + $content as item()?, + $options as map:map) as node()* +{ + () +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/main.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/main.xqy new file mode 100644 index 0000000000..981eeeb456 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/main.xqy @@ -0,0 +1,55 @@ +xquery version "1.0-ml"; + +(: Your plugin must be in this namespace for the DHF to recognize it:) +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +(: + : This module exposes helper functions to make your life easier + : See documentation at: + : https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib + :) +import module namespace dhf = "http://marklogic.com/dhf" + at "/com.marklogic.hub/dhf.xqy"; + +(: include modules to construct various parts of the envelope :) +import module namespace content = "http://marklogic.com/data-hub/plugins" at "content.xqy"; +import module namespace headers = "http://marklogic.com/data-hub/plugins" at "headers.xqy"; +import module namespace triples = "http://marklogic.com/data-hub/plugins" at "triples.xqy"; + +(: include the writer module which persists your envelope into MarkLogic :) +import module namespace writer = "http://marklogic.com/data-hub/plugins" at "writer.xqy"; + +declare option xdmp:mapping "false"; + +(:~ + : Plugin Entry point + : + : @param $id - the identifier returned by the collector + : @param $options - a map containing options. Options are sent from Java + : + :) +declare function plugin:main( + $id as xs:string, + $options as map:map) +{ + let $content-context := dhf:content-context() + let $content := dhf:run($content-context, function() { + content:create-content($id, $options) + }) + + let $header-context := dhf:headers-context($content) + let $headers := dhf:run($header-context, function() { + headers:create-headers($id, $content, $options) + }) + + let $triple-context := dhf:triples-context($content, $headers) + let $triples := dhf:run($triple-context, function() { + triples:create-triples($id, $content, $headers, $options) + }) + + let $envelope := dhf:make-envelope($content, $headers, $triples, map:get($options, "dataFormat")) + return + (: writers must be invoked this way. + see: https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib#run-writer :) + dhf:run-writer(xdmp:function(xs:QName("writer:write")), $id, $envelope, $options) +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/triples.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/triples.xqy new file mode 100644 index 0000000000..e2d7e1bdd0 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/triples.xqy @@ -0,0 +1,26 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Triples Plugin + : + : @param $id - the identifier returned by the collector + : @param $content - the output of your content plugin + : @param $headers - the output of your headers plugin + : @param $options - a map containing options. Options are sent from Java + : + : @return - zero or more triples + :) +declare function plugin:create-triples( + $id as xs:string, + $content as item()?, + $headers as item()*, + $options as map:map) as sem:triple* +{ + () +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/123/writer.xqy b/examples/online-store/plugins/entities/Product/harmonize/123/writer.xqy new file mode 100644 index 0000000000..c817114a1a --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/123/writer.xqy @@ -0,0 +1,22 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare option xdmp:mapping "false"; + +(:~ + : Writer Plugin + : + : @param $id - the identifier returned by the collector + : @param $envelope - the final envelope + : @param $options - a map containing options. Options are sent from Java + : + : @return - nothing + :) +declare function plugin:write( + $id as xs:string, + $envelope as item(), + $options as map:map) as empty-sequence() +{ + xdmp:document-insert($id, $envelope, xdmp:default-permissions(), 'entity') +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/Test.properties b/examples/online-store/plugins/entities/Product/harmonize/Test/Test.properties new file mode 100644 index 0000000000..259defc3bf --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/Test.properties @@ -0,0 +1,9 @@ +# +#Fri Jan 18 09:03:33 MST 2019 +mainModule=main.sjs +collectorCodeFormat=sjs +mapping=Test-2 +mainCodeFormat=sjs +codeFormat=sjs +collectorModule=collector.sjs +dataFormat=json diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/collector.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/collector.sjs new file mode 100644 index 0000000000..c48fdc8102 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/collector.sjs @@ -0,0 +1,15 @@ +/* + * Collect IDs plugin + * + * @param options - a map containing options. Options are sent from Java + * + * @return - an array of ids or uris + */ +function collect(options) { + // by default we return the URIs in the same collection as the Entity name + return cts.uris(null, null, cts.collectionQuery(options.entity)); +} + +module.exports = { + collect: collect +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/content.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/content.sjs new file mode 100644 index 0000000000..965b57a9e3 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/content.sjs @@ -0,0 +1,81 @@ +'use strict' + +/* +* Create Content Plugin +* +* @param id - the identifier returned by the collector +* @param options - an object containing options. Options are sent from Java +* +* @return - your content +*/ +function createContent(id, options) { + let doc = cts.doc(id); + + let source; + + // for xml we need to use xpath + if(doc && xdmp.nodeKind(doc) === 'element' && doc instanceof XMLDocument) { + source = doc + } + // for json we need to return the instance + else if(doc && doc instanceof Document) { + source = fn.head(doc.root); + } + // for everything else + else { + source = doc; + } + + return extractInstanceProduct(source); +} + +/** +* Creates an object instance from some source document. +* @param source A document or node that contains +* data for populating a Product +* @return An object with extracted data and +* metadata about the instance. +*/ +function extractInstanceProduct(source) { + // the original source documents + let attachments = source; + // now check to see if we have XML or json, then create a node clone from the root of the instance + if (source instanceof Element || source instanceof ObjectNode) { + let instancePath = '/*:envelope/*:instance'; + if(source instanceof Element) { + //make sure we grab content root only + instancePath += '/node()[not(. instance of processing-instruction() or . instance of comment())]'; + } + source = new NodeBuilder().addNode(fn.head(source.xpath(instancePath))).toNode(); + } + else{ + source = new NodeBuilder().addNode(fn.head(source)).toNode(); + } + /* These mappings were generated using mapping: Test, version: 2 on 2019-01-18T16:03:33.128249Z.*/ + let sku = !fn.empty(fn.head(source.xpath('//SKU'))) ? xs.string(fn.head(fn.head(source.xpath('//SKU')))) : null; + let title = !fn.empty(fn.head(source.xpath('//title'))) ? xs.string(fn.head(fn.head(source.xpath('//title')))) : null; + let price = !fn.empty(fn.head(source.xpath('//price'))) ? xs.decimal(fn.head(fn.head(source.xpath('//price')))) : null; + + // return the instance object + return { + '$attachments': attachments, + '$type': 'Product', + '$version': '0.0.1', + 'sku': sku, + 'title': title, + 'price': price + } +}; + + +function makeReferenceObject(type, ref) { + return { + '$type': type, + '$ref': ref + }; +} + +module.exports = { + createContent: createContent +}; + diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/headers.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/headers.sjs new file mode 100644 index 0000000000..efb96d569d --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/headers.sjs @@ -0,0 +1,16 @@ +/* + * Create Headers Plugin + * + * @param id - the identifier returned by the collector + * @param content - the output of your content plugin + * @param options - an object containing options. Options are sent from Java + * + * @return - an object of headers + */ +function createHeaders(id, content, options) { + return {}; +} + +module.exports = { + createHeaders: createHeaders +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/main.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/main.sjs new file mode 100644 index 0000000000..adf3ab8fb3 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/main.sjs @@ -0,0 +1,43 @@ +// dhf.sjs exposes helper functions to make your life easier +// See documentation at: +// https://marklogic.github.io/marklogic-data-hub/docs/server-side/ +const dhf = require('/data-hub/4/dhf.sjs'); + +const contentPlugin = require('./content.sjs'); +const headersPlugin = require('./headers.sjs'); +const triplesPlugin = require('./triples.sjs'); +const writerPlugin = require('./writer.sjs'); + +/* + * Plugin Entry point + * + * @param id - the identifier returned by the collector + * @param options - a map containing options. Options are sent from Java + * + */ +function main(id, options) { + var contentContext = dhf.contentContext(); + var content = dhf.run(contentContext, function() { + return contentPlugin.createContent(id, options); + }); + + var headerContext = dhf.headersContext(content); + var headers = dhf.run(headerContext, function() { + return headersPlugin.createHeaders(id, content, options); + }); + + var tripleContext = dhf.triplesContext(content, headers); + var triples = dhf.run(tripleContext, function() { + return triplesPlugin.createTriples(id, content, headers, options); + }); + + var envelope = dhf.makeEnvelope(content, headers, triples, options.dataFormat); + + // writers must be invoked this way. + // see: https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib#run-writer + dhf.runWriter(writerPlugin, id, envelope, options); +} + +module.exports = { + main: main +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/triples.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/triples.sjs new file mode 100644 index 0000000000..2bc6de7af0 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/triples.sjs @@ -0,0 +1,18 @@ +/* + * Create Triples Plugin + * + * @param id - the identifier returned by the collector + * @param content - the output of your content plugin + * @param headers - the output of your heaaders plugin + * @param options - an object containing options. Options are sent from Java + * + * @return - an array of triples + */ +function createTriples(id, content, headers, options) { + return []; +} + +module.exports = { + createTriples: createTriples +}; + diff --git a/examples/online-store/plugins/entities/Product/harmonize/Test/writer.sjs b/examples/online-store/plugins/entities/Product/harmonize/Test/writer.sjs new file mode 100644 index 0000000000..a957b56137 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/Test/writer.sjs @@ -0,0 +1,14 @@ +/*~ + * Writer Plugin + * + * @param id - the identifier returned by the collector + * @param envelope - the final envelope + * @param options - an object options. Options are sent from Java + * + * @return - nothing + */ +function write(id, envelope, options) { + xdmp.documentInsert(id, envelope, xdmp.defaultPermissions(), options.entity); +} + +module.exports = write; diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/abc.properties b/examples/online-store/plugins/entities/Product/harmonize/abc/abc.properties new file mode 100644 index 0000000000..101a55863d --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/abc.properties @@ -0,0 +1,8 @@ +# +#Thu Jan 03 14:20:06 MST 2019 +collectorModule=collector.xqy +dataFormat=xml +codeFormat=xqy +mainCodeFormat=xqy +mainModule=main.xqy +collectorCodeFormat=xqy diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/collector.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/collector.xqy new file mode 100644 index 0000000000..999fd43c74 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/collector.xqy @@ -0,0 +1,20 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare option xdmp:mapping "false"; + +(:~ + : Collect IDs plugin + : + : @param $options - a map containing options. Options are sent from Java + : + : @return - a sequence of ids or uris + :) +declare function plugin:collect( + $options as map:map) as xs:string* +{ + (: by default we return the URIs in the same collection as the Entity name :) + cts:uris((), (), cts:collection-query(map:get($options, "entity"))) +}; + diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/content.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/content.xqy new file mode 100644 index 0000000000..cf6a81cb7e --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/content.xqy @@ -0,0 +1,29 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Content Plugin + : + : @param $id - the identifier returned by the collector + : @param $options - a map containing options. Options are sent from Java + : + : @return - your transformed content + :) +declare function plugin:create-content( + $id as xs:string, + $options as map:map) as item()? +{ + let $doc := fn:doc($id) + return + if ($doc/es:envelope) then + $doc/es:envelope/es:instance/node() + else if ($doc/envelope/instance) then + $doc/envelope/instance + else + $doc +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/headers.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/headers.xqy new file mode 100644 index 0000000000..d8ee319d1c --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/headers.xqy @@ -0,0 +1,24 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Headers Plugin + : + : @param $id - the identifier returned by the collector + : @param $content - the output of your content plugin + : @param $options - a map containing options. Options are sent from Java + : + : @return - zero or more header nodes + :) +declare function plugin:create-headers( + $id as xs:string, + $content as item()?, + $options as map:map) as node()* +{ + () +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/main.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/main.xqy new file mode 100644 index 0000000000..981eeeb456 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/main.xqy @@ -0,0 +1,55 @@ +xquery version "1.0-ml"; + +(: Your plugin must be in this namespace for the DHF to recognize it:) +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +(: + : This module exposes helper functions to make your life easier + : See documentation at: + : https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib + :) +import module namespace dhf = "http://marklogic.com/dhf" + at "/com.marklogic.hub/dhf.xqy"; + +(: include modules to construct various parts of the envelope :) +import module namespace content = "http://marklogic.com/data-hub/plugins" at "content.xqy"; +import module namespace headers = "http://marklogic.com/data-hub/plugins" at "headers.xqy"; +import module namespace triples = "http://marklogic.com/data-hub/plugins" at "triples.xqy"; + +(: include the writer module which persists your envelope into MarkLogic :) +import module namespace writer = "http://marklogic.com/data-hub/plugins" at "writer.xqy"; + +declare option xdmp:mapping "false"; + +(:~ + : Plugin Entry point + : + : @param $id - the identifier returned by the collector + : @param $options - a map containing options. Options are sent from Java + : + :) +declare function plugin:main( + $id as xs:string, + $options as map:map) +{ + let $content-context := dhf:content-context() + let $content := dhf:run($content-context, function() { + content:create-content($id, $options) + }) + + let $header-context := dhf:headers-context($content) + let $headers := dhf:run($header-context, function() { + headers:create-headers($id, $content, $options) + }) + + let $triple-context := dhf:triples-context($content, $headers) + let $triples := dhf:run($triple-context, function() { + triples:create-triples($id, $content, $headers, $options) + }) + + let $envelope := dhf:make-envelope($content, $headers, $triples, map:get($options, "dataFormat")) + return + (: writers must be invoked this way. + see: https://github.com/marklogic/marklogic-data-hub/wiki/dhf-lib#run-writer :) + dhf:run-writer(xdmp:function(xs:QName("writer:write")), $id, $envelope, $options) +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/triples.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/triples.xqy new file mode 100644 index 0000000000..e2d7e1bdd0 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/triples.xqy @@ -0,0 +1,26 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare namespace es = "http://marklogic.com/entity-services"; + +declare option xdmp:mapping "false"; + +(:~ + : Create Triples Plugin + : + : @param $id - the identifier returned by the collector + : @param $content - the output of your content plugin + : @param $headers - the output of your headers plugin + : @param $options - a map containing options. Options are sent from Java + : + : @return - zero or more triples + :) +declare function plugin:create-triples( + $id as xs:string, + $content as item()?, + $headers as item()*, + $options as map:map) as sem:triple* +{ + () +}; diff --git a/examples/online-store/plugins/entities/Product/harmonize/abc/writer.xqy b/examples/online-store/plugins/entities/Product/harmonize/abc/writer.xqy new file mode 100644 index 0000000000..9c7dfeec99 --- /dev/null +++ b/examples/online-store/plugins/entities/Product/harmonize/abc/writer.xqy @@ -0,0 +1,22 @@ +xquery version "1.0-ml"; + +module namespace plugin = "http://marklogic.com/data-hub/plugins"; + +declare option xdmp:mapping "false"; + +(:~ + : Writer Plugin + : + : @param $id - the identifier returned by the collector + : @param $envelope - the final envelope + : @param $options - a map containing options. Options are sent from Java + : + : @return - nothing + :) +declare function plugin:write( + $id as xs:string, + $envelope as item(), + $options as map:map) as empty-sequence() +{ + xdmp:document-insert($id, $envelope, xdmp:default-permissions(), map:get($options, "entity")) +}; diff --git a/examples/online-store/plugins/mappings/Test/Test-0.mapping.json b/examples/online-store/plugins/mappings/Test/Test-0.mapping.json new file mode 100644 index 0000000000..bbcc9ffc61 --- /dev/null +++ b/examples/online-store/plugins/mappings/Test/Test-0.mapping.json @@ -0,0 +1,10 @@ +{ + "language" : "zxx", + "name" : "Test", + "description" : "This is a test mapping", + "version" : 0, + "targetEntityType" : "http://example.org/Product-0.0.1/Product", + "sourceContext" : "", + "sourceURI" : "", + "properties" : { } +} \ No newline at end of file diff --git a/examples/online-store/plugins/mappings/Test/Test-1.mapping.json b/examples/online-store/plugins/mappings/Test/Test-1.mapping.json new file mode 100644 index 0000000000..856754a572 --- /dev/null +++ b/examples/online-store/plugins/mappings/Test/Test-1.mapping.json @@ -0,0 +1,10 @@ +{ + "language" : "zxx", + "name" : "Test", + "description" : "This is a test mapping", + "version" : 1, + "targetEntityType" : "http://example.org/Product-0.0.1/Product", + "sourceContext" : "//", + "sourceURI" : "1000200", + "properties" : { } +} \ No newline at end of file diff --git a/examples/online-store/plugins/mappings/Test/Test-2.mapping.json b/examples/online-store/plugins/mappings/Test/Test-2.mapping.json new file mode 100644 index 0000000000..461766993e --- /dev/null +++ b/examples/online-store/plugins/mappings/Test/Test-2.mapping.json @@ -0,0 +1,20 @@ +{ + "language" : "zxx", + "name" : "Test", + "description" : "This is a test mapping", + "version" : 2, + "targetEntityType" : "http://example.org/Product-0.0.1/Product", + "sourceContext" : "//", + "sourceURI" : "1000200", + "properties" : { + "price" : { + "sourcedFrom" : "price" + }, + "sku" : { + "sourcedFrom" : "SKU" + }, + "title" : { + "sourcedFrom" : "title" + } + } +} \ No newline at end of file diff --git a/examples/online-store/src/main/entity-config/final-entity-options.xml b/examples/online-store/src/main/entity-config/final-entity-options.xml new file mode 100644 index 0000000000..398aacabfd --- /dev/null +++ b/examples/online-store/src/main/entity-config/final-entity-options.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + //*:instance/Order/id + + + + + + + + + + + unfiltered + + + //*:instance/(Product|Order) + + + + true + + + \ No newline at end of file diff --git a/examples/online-store/src/main/entity-config/staging-entity-options.xml b/examples/online-store/src/main/entity-config/staging-entity-options.xml new file mode 100644 index 0000000000..398aacabfd --- /dev/null +++ b/examples/online-store/src/main/entity-config/staging-entity-options.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + //*:instance/Order/id + + + + + + + + + + + unfiltered + + + //*:instance/(Product|Order) + + + + true + + + \ No newline at end of file diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/EntityManager.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/EntityManager.java index b7ee9fa69f..fab4dfa7cf 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/EntityManager.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/EntityManager.java @@ -16,9 +16,11 @@ package com.marklogic.hub; -import com.marklogic.hub.impl.EntityManagerImpl; +import com.marklogic.hub.entity.HubEntity; +import java.io.IOException; import java.util.HashMap; +import java.util.List; /** * Manages existing entities' MarkLogic Server database index settings and query options. @@ -57,4 +59,10 @@ public interface EntityManager { boolean deployFinalQueryOptions(); boolean deployStagingQueryOptions(); + + List getEntities(); + + HubEntity saveEntity(HubEntity entity, Boolean rename) throws IOException; + + void deleteEntity(String entity) throws IOException; } diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java index a08b4cac7d..6e6585d819 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java @@ -97,6 +97,13 @@ public interface HubProject { */ Path getHubSecurityDir(); + /** + * Gets the path for the hub's triggers directory + * + * @return the path for the hub's triggers directory + */ + Path getHubTriggersDir(); + /** * Gets the path for the user config directory * diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommand.java new file mode 100644 index 0000000000..6b3e65b85e --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommand.java @@ -0,0 +1,195 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.hub.deploy.commands; + +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.appdeployer.command.CommandContext; +import com.marklogic.appdeployer.command.SortOrderConstants; +import com.marklogic.appdeployer.command.modules.LoadModulesCommand; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.document.JSONDocumentManager; +import com.marklogic.client.ext.modulesloader.Modules; +import com.marklogic.client.ext.modulesloader.impl.EntityDefModulesFinder; +import com.marklogic.client.ext.modulesloader.impl.MappingDefModulesFinder; +import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; +import com.marklogic.client.ext.util.DocumentPermissionsParser; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.hub.HubConfig; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Component; +import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Date; +import java.util.regex.Pattern; + +/** + * Loads user artifacts like mappings and entities. This will be deployed after triggers + */ +@Component +public class LoadUserArtifactsCommand extends LoadModulesCommand { + + @Autowired + private HubConfig hubConfig; + + private DocumentPermissionsParser documentPermissionsParser = new DefaultDocumentPermissionsParser(); + + public void setForceLoad(boolean forceLoad) { + this.forceLoad = forceLoad; + } + + private boolean forceLoad = false; + + public LoadUserArtifactsCommand() { + super(); + setExecuteSortOrder(SortOrderConstants.DEPLOY_TRIGGERS + 1); + } + + boolean isArtifactDir(Path dir, Path startPath) { + String dirStr = dir.toString(); + String startPathStr = Pattern.quote(startPath.toString()); + String regex = startPathStr + "[/\\\\][^/\\\\]+$"; + return dirStr.matches(regex); + } + + private PropertiesModuleManager getModulesManager() { + String timestampFile = hubConfig.getHubProject().getUserModulesDeployTimestampFile(); + PropertiesModuleManager pmm = new PropertiesModuleManager(timestampFile); + + // Need to delete ml-javaclient-utils timestamp file as well as modules present in the standard gradle locations are now + // loaded by the modules loader in the parent class which adds these entries to the ml-javaclient-utils timestamp file + String filePath = hubConfig.getAppConfig().getModuleTimestampsPath(); + File defaultTimestampFile = new File(filePath); + + if (forceLoad) { + pmm.deletePropertiesFile(); + if (defaultTimestampFile.exists()){ + defaultTimestampFile.delete(); + } + } + return pmm; + } + + @Override + public void execute(CommandContext context) { + AppConfig config = context.getAppConfig(); + + DatabaseClient stagingClient = hubConfig.newStagingClient(); + DatabaseClient finalClient = hubConfig.newFinalClient(); + + Path userModulesPath = hubConfig.getHubPluginsDir(); + String baseDir = userModulesPath.normalize().toAbsolutePath().toString(); + Path startPath = userModulesPath.resolve("entities"); + Path mappingPath = userModulesPath.resolve("mappings"); + + JSONDocumentManager finalDocMgr = finalClient.newJSONDocumentManager(); + JSONDocumentManager stagingDocMgr = stagingClient.newJSONDocumentManager(); + + DocumentWriteSet finalEntityDocumentWriteSet = finalDocMgr.newWriteSet(); + DocumentWriteSet stagingEntityDocumentWriteSet = stagingDocMgr.newWriteSet(); + DocumentWriteSet finalMappingDocumentWriteSet = finalDocMgr.newWriteSet(); + DocumentWriteSet stagingMappingDocumentWriteSet = stagingDocMgr.newWriteSet(); + PropertiesModuleManager propertiesModuleManager = getModulesManager(); + + try { + if (startPath.toFile().exists()) { + //first let's do the entities + Files.walkFileTree(startPath, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String currentDir = dir.normalize().toAbsolutePath().toString(); + if (isArtifactDir(dir, startPath.toAbsolutePath())) { + Modules modules = new EntityDefModulesFinder().findModules(dir.toString()); + DocumentMetadataHandle meta = new DocumentMetadataHandle(); + meta.getCollections().add("http://marklogic.com/entity-services/models"); + documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); + for (Resource r : modules.getAssets()) { + if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { + InputStream inputStream = r.getInputStream(); + StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); + inputStream.close(); + finalEntityDocumentWriteSet.add("/entities/" + r.getFilename(), meta, handle); + stagingEntityDocumentWriteSet.add("/entities/" + r.getFilename(), meta, handle); + propertiesModuleManager.saveLastLoadedTimestamp(r.getFile(), new Date()); + } + } + return FileVisitResult.CONTINUE; + } else { + return FileVisitResult.CONTINUE; + } + } + }); + + //now let's do the mappings path + if (mappingPath.toFile().exists()) { + Files.walkFileTree(mappingPath, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String currentDir = dir.normalize().toAbsolutePath().toString(); + + if (isArtifactDir(dir, mappingPath.toAbsolutePath())) { + Modules modules = new MappingDefModulesFinder().findModules(dir.toString()); + DocumentMetadataHandle meta = new DocumentMetadataHandle(); + meta.getCollections().add("http://marklogic.com/data-hub/mappings"); + documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); + for (Resource r : modules.getAssets()) { + if (forceLoad || propertiesModuleManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { + InputStream inputStream = r.getInputStream(); + StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); + inputStream.close(); + finalMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); + stagingMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); + propertiesModuleManager.saveLastLoadedTimestamp(r.getFile(), new Date()); + } + } + return FileVisitResult.CONTINUE; + } else { + return FileVisitResult.CONTINUE; + } + } + }); + } + if (stagingEntityDocumentWriteSet.size() > 0) { + finalDocMgr.write(finalEntityDocumentWriteSet); + stagingDocMgr.write(stagingEntityDocumentWriteSet); + } + if (stagingMappingDocumentWriteSet.size() > 0) { + finalDocMgr.write(finalMappingDocumentWriteSet); + stagingDocMgr.write(stagingMappingDocumentWriteSet); + } + } + + } catch (IOException e) { + e.printStackTrace(); + //throw new RuntimeException(e); + } + } + + public void setHubConfig(HubConfig hubConfig) { + this.hubConfig = hubConfig; + } + +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java index 6c204f6baf..9c7c4d4148 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java @@ -21,43 +21,31 @@ import com.marklogic.appdeployer.command.modules.LoadModulesCommand; import com.marklogic.client.DatabaseClient; import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.document.JSONDocumentManager; import com.marklogic.client.document.XMLDocumentManager; -import com.marklogic.client.ext.modulesloader.Modules; +import com.marklogic.client.ext.file.CacheBusterDocumentFileProcessor; import com.marklogic.client.ext.modulesloader.ModulesManager; -import com.marklogic.client.ext.modulesloader.impl.AssetFileLoader; -import com.marklogic.client.ext.modulesloader.impl.DefaultModulesLoader; -import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager; +import com.marklogic.client.ext.modulesloader.impl.*; import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; import com.marklogic.client.ext.util.DocumentPermissionsParser; -import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; import com.marklogic.client.io.StringHandle; -import com.marklogic.client.ext.file.CacheBusterDocumentFileProcessor; -import com.marklogic.client.ext.modulesloader.impl.EntityDefModulesFinder; -import com.marklogic.client.ext.modulesloader.impl.MappingDefModulesFinder; -import com.marklogic.client.ext.modulesloader.impl.SearchOptionsFinder; -import com.marklogic.client.ext.modulesloader.impl.UserModulesFinder; import com.marklogic.hub.EntityManager; import com.marklogic.hub.FlowManager; import com.marklogic.hub.HubConfig; import com.marklogic.hub.deploy.util.HubFileFilter; import com.marklogic.hub.error.LegacyFlowsException; import com.marklogic.hub.flow.Flow; -import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Date; import java.util.List; -import java.util.regex.Pattern; + /** * Extends ml-app-deployer's LoadModulesCommand, which expects to load from every path defined by "mlModulePaths", so @@ -77,7 +65,6 @@ public class LoadUserModulesCommand extends LoadModulesCommand { @Autowired private FlowManager flowManager; - private DocumentPermissionsParser documentPermissionsParser = new DefaultDocumentPermissionsParser(); private ThreadPoolTaskExecutor threadPoolTaskExecutor; @@ -116,6 +103,7 @@ private PropertiesModuleManager getModulesManager() { private AssetFileLoader getAssetFileLoader(AppConfig config, PropertiesModuleManager moduleManager) { AssetFileLoader assetFileLoader = new AssetFileLoader(hubConfig.newModulesDbClient(), moduleManager); assetFileLoader.addDocumentFileProcessor(new CacheBusterDocumentFileProcessor()); + //Add file extensions to HubFileFilter.accept() to prevent mappings, entities files being loaded to Modules db assetFileLoader.addFileFilter(new HubFileFilter()); assetFileLoader.setPermissions(config.getModulePermissions()); return assetFileLoader; @@ -148,20 +136,6 @@ boolean isHarmonizeRestDir(Path dir) { return dir.endsWith("REST") && dir.toString().matches(".*[/\\\\]harmonize[/\\\\].*"); } - boolean isEntityDir(Path dir, Path startPath) { - String dirStr = dir.toString(); - String startPathStr = Pattern.quote(startPath.toString()); - String regex = startPathStr + "[/\\\\][^/\\\\]+$"; - return dirStr.matches(regex); - } - - boolean isMappingDir(Path dir, Path startPath) { - String dirStr = dir.toString(); - String startPathStr = Pattern.quote(startPath.toString()); - String regex = startPathStr + "[/\\\\][^/\\\\]+$"; - return dirStr.matches(regex); - } - boolean isFlowPropertiesFile(Path dir) { Path parent = dir.getParent(); return dir.toFile().isFile() && @@ -217,14 +191,6 @@ public void execute(CommandContext context) { modulesLoader.loadModules("classpath*:/ml-modules-final", new SearchOptionsFinder(), finalClient); } - //for now we'll use two different document managers - JSONDocumentManager finalEntityDocMgr = finalClient.newJSONDocumentManager(); - JSONDocumentManager stagingEntityDocMgr = stagingClient.newJSONDocumentManager(); - JSONDocumentManager finalMappingDocMgr = finalClient.newJSONDocumentManager(); - JSONDocumentManager stagingMappingDocMgr = stagingClient.newJSONDocumentManager(); - DocumentWriteSet finalMappingDocumentWriteSet = finalMappingDocMgr.newWriteSet(); - DocumentWriteSet stagingMappingDocumentWriteSet = stagingMappingDocMgr.newWriteSet(); - AllButAssetsModulesFinder allButAssetsModulesFinder = new AllButAssetsModulesFinder(); Path dir = Paths.get(hubConfig.getHubProject().getProjectDirString(), HubConfig.ENTITY_CONFIG_DIR); @@ -258,25 +224,6 @@ else if (isHarmonizeRestDir(dir)) { modulesLoader.loadModules(currentDir, allButAssetsModulesFinder, finalClient); return FileVisitResult.SKIP_SUBTREE; } - else if (isEntityDir(dir, startPath.toAbsolutePath())) { - Modules modules = new EntityDefModulesFinder().findModules(dir.toString()); - DocumentMetadataHandle meta = new DocumentMetadataHandle(); - meta.getCollections().add("http://marklogic.com/entity-services/models"); - documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); - for (Resource r : modules.getAssets()) { - if (forceLoad || modulesManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { - InputStream inputStream = r.getInputStream(); - StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); - inputStream.close(); - finalEntityDocMgr.write("/entities/" + r.getFilename(), meta, handle); - - // Uncomment to send entity model to staging db as well - stagingEntityDocMgr.write("/entities/" + r.getFilename(), meta, handle); - modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date()); - } - } - return FileVisitResult.CONTINUE; - } else { return FileVisitResult.CONTINUE; } @@ -296,58 +243,9 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) return FileVisitResult.CONTINUE; } }); - - //now let's do the mappings path - if (mappingPath.toFile().exists()) { - Files.walkFileTree(mappingPath, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - String currentDir = dir.normalize().toAbsolutePath().toString(); - - if (isMappingDir(dir, mappingPath.toAbsolutePath())) { - Modules modules = new MappingDefModulesFinder().findModules(dir.toString()); - DocumentMetadataHandle meta = new DocumentMetadataHandle(); - meta.getCollections().add("http://marklogic.com/data-hub/mappings"); - documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); - for (Resource r : modules.getAssets()) { - if (forceLoad || modulesManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { - InputStream inputStream = r.getInputStream(); - StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); - inputStream.close(); - finalMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); - stagingMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); - modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date()); - } - } - return FileVisitResult.CONTINUE; - } else { - return FileVisitResult.CONTINUE; - } - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) { - Flow flow = flowManager.getFlowFromProperties(file); - StringHandle handle = new StringHandle(flow.serialize()); - handle.setFormat(Format.XML); - documentWriteSet.add(flow.getFlowDbPath(), handle); - modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date()); - } - return FileVisitResult.CONTINUE; - } - }); - } - if (documentWriteSet.size() > 0) { documentManager.write(documentWriteSet); } - - if (stagingMappingDocumentWriteSet.size() > 0) { - finalMappingDocMgr.write(finalMappingDocumentWriteSet); - stagingMappingDocMgr.write(stagingMappingDocumentWriteSet); - } } threadPoolTaskExecutor.shutdown(); } catch (IOException e) { diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/EntityDeploymentUtil.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/EntityDeploymentUtil.java new file mode 100644 index 0000000000..fab982559a --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/EntityDeploymentUtil.java @@ -0,0 +1,101 @@ +/* + * Copyright 2012-2019 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.hub.deploy.util; + +import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.marker.AbstractWriteHandle; +import com.marklogic.client.io.marker.DocumentMetadataWriteHandle; +import com.marklogic.client.io.marker.JSONWriteHandle; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/* + * Singleton to aid with Entity deployment. Needed as bridge between walking the entities file structure + * on modules deploy and inserting entity JSON models in the database after triggers are created. + */ +public class EntityDeploymentUtil { + private ConcurrentHashMap metaMap = new ConcurrentHashMap(); + private ConcurrentHashMap contentMap = new ConcurrentHashMap(); + + private static EntityDeploymentUtil instance; + + public static synchronized EntityDeploymentUtil getInstance() { + if(instance == null){ + instance = new EntityDeploymentUtil(); + } + return instance; + } + + public Set getEntityURIs() { + return contentMap.keySet(); + } + + public void enqueueEntity(String uri, DocumentMetadataHandle meta, JSONWriteHandle content) { + metaMap.put(uri, meta); + contentMap.put(uri, content); + } + + public WriteEvent dequeueEntity(String uri) { + DocumentMetadataHandle meta = metaMap.get(uri); + JSONWriteHandle content = contentMap.get(uri); + return new WriteEventImpl(uri, meta, content); + } + + public void reset() { + metaMap.clear(); + contentMap.clear(); + } + + private class WriteEventImpl implements WriteEvent { + private String uri; + private DocumentMetadataHandle meta; + private JSONWriteHandle content; + + public WriteEventImpl(String uri, DocumentMetadataHandle meta, JSONWriteHandle content) { + this.uri = uri; + this.meta = meta; + this.content = content; + } + + @Override + public String getTargetUri() { + return uri; + } + + @Override + public AbstractWriteHandle getContent() { + return content; + } + + @Override + public DocumentMetadataWriteHandle getMetadata() { + return meta; + } + + @Override + public long getJobRecordNumber() { + return 0; + } + + @Override + public long getBatchRecordNumber() { + return 0; + } + } + +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/HubFileFilter.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/HubFileFilter.java index 45aed5aec7..2b2482e5f7 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/HubFileFilter.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/util/HubFileFilter.java @@ -28,6 +28,7 @@ public boolean accept(File f) { boolean result = f != null && !f.getName().startsWith(".") && !f.getName().endsWith("entity.json") && + !f.getName().endsWith("mapping.json") && !f.getName().equals(f.getParentFile().getName() + ".properties") && !f.toString().matches(".*[/\\\\]REST[/\\\\].*") && diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/AbstractEntity.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/AbstractEntity.java deleted file mode 100644 index ebee34f7ba..0000000000 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/AbstractEntity.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2012-2019 MarkLogic Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.marklogic.hub.entity; - -import com.marklogic.hub.FlowManager; -import com.marklogic.hub.flow.Flow; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import java.util.ArrayList; -import java.util.List; - -/** - * Abstract Base class for entities - */ -public abstract class AbstractEntity implements Entity { - - private String name; - private ArrayList flows = new ArrayList(); - - public AbstractEntity(Element xml) { - deserialize(xml); - } - - public AbstractEntity(String name) { - this.name = name; - } - - private void deserialize(Node xml) { - NodeList children = xml.getChildNodes(); - for (int i = 0; i < children.getLength(); i++) { - Node node = children.item(i); - if (node.getNodeType() != Node.ELEMENT_NODE) { - continue; - } - - String nodeName = node.getLocalName(); - switch(nodeName) { - case "name": - this.name = node.getTextContent(); - break; - case "flows": - deserialize(node); - break; - case "flow": - flows.add(FlowManager.flowFromXml((Element)node)); - break; - } - } - } - - @Override - public String getName() { - return name; - } - - @Override - public String serialize() { - return null; - } - - @Override - public List getFlows() { - return flows; - } - -} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionType.java new file mode 100644 index 0000000000..67767f9de9 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionType.java @@ -0,0 +1,214 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class DefinitionType extends JsonPojo { + protected String name; + protected String description; + protected String primaryKey; + protected List required; + protected List pii; + protected List elementRangeIndex; + protected List rangeIndex; + protected List wordLexicon; + + protected List properties; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(String primaryKey) { + this.primaryKey = primaryKey; + } + + public List getRequired() { + return required; + } + + public void setRequired(List required) { + this.required = required; + } + + public List getPii() { + return pii; + } + + public void setPii(List pii) { + this.pii = pii; + } + + public List getRangeIndex() { + return rangeIndex; + } + + public void setRangeIndex(List rangeIndex) { + this.rangeIndex = rangeIndex; + } + + public List getElementRangeIndex() { + return elementRangeIndex; + } + + public void setElementRangeIndex(List elementRangeIndex) { + this.elementRangeIndex = elementRangeIndex; + } + + public List getWordLexicon() { + return wordLexicon; + } + + public void setWordLexicon(List wordLexicon) { + this.wordLexicon = wordLexicon; + } + + public List getProperties() { + return properties; + } + + public void setProperties(List properties) { + this.properties = properties; + } + + public static DefinitionType fromJson(String name, JsonNode node) { + DefinitionType definitionType = new DefinitionType(); + definitionType.setName(name); + + definitionType.setDescription(getValue(node, "description")); + definitionType.setPrimaryKey(getValue(node, "primaryKey")); + + ArrayList required = new ArrayList<>(); + JsonNode requiredNodes = node.get("required"); + if (requiredNodes != null) { + for (final JsonNode n : requiredNodes) { + required.add(n.asText()); + } + } + definitionType.setRequired(required); + + ArrayList pii = new ArrayList<>(); + JsonNode piiNodes = node.get("pii"); + if (piiNodes != null) { + for (final JsonNode n : piiNodes) { + pii.add(n.asText()); + } + } + definitionType.setPii(pii); + + ArrayList elementRangeIndexes = new ArrayList<>(); + JsonNode elementRangeIndexNodes = node.get("elementRangeIndex"); + if (elementRangeIndexNodes != null) { + for (final JsonNode n : elementRangeIndexNodes) { + elementRangeIndexes.add(n.asText()); + } + } + definitionType.setElementRangeIndex(elementRangeIndexes); + + ArrayList rangeIndexes = new ArrayList<>(); + JsonNode rangeIndexNodes = node.get("rangeIndex"); + if (rangeIndexNodes != null) { + for (final JsonNode n : rangeIndexNodes) { + rangeIndexes.add(n.asText()); + } + } + definitionType.setRangeIndex(rangeIndexes); + + ArrayList wordLexicons = new ArrayList<>(); + JsonNode wordLexiconNodes = node.get("wordLexicon"); + if (wordLexiconNodes != null) { + for (final JsonNode n : wordLexiconNodes) { + wordLexicons.add(n.asText()); + } + } + definitionType.setWordLexicon(wordLexicons); + + ArrayList properties = new ArrayList<>(); + JsonNode propertiesNode = node.get("properties"); + if (propertiesNode != null) { + Iterator fieldItr = propertiesNode.fieldNames(); + while(fieldItr.hasNext()) { + String key = fieldItr.next(); + JsonNode propertyNode = propertiesNode.get(key); + if (propertyNode != null) { + properties.add(PropertyType.fromJson(key, propertyNode)); + } + } + } + definitionType.setProperties(properties); + + return definitionType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + writeStringIf(node, "description", description); + writeStringIf(node, "primaryKey", primaryKey); + + ArrayNode requiredArray = JsonNodeFactory.instance.arrayNode(); + required.forEach(requiredArray::add); + node.set("required", requiredArray); + + ArrayNode piiArray = JsonNodeFactory.instance.arrayNode(); + pii.forEach(piiArray::add); + node.set("pii", piiArray); + + ArrayNode elementRangeIndexArray = JsonNodeFactory.instance.arrayNode(); + elementRangeIndex.forEach(elementRangeIndexArray ::add); + node.set("elementRangeIndex", elementRangeIndexArray); + + ArrayNode rangeIndexArray = JsonNodeFactory.instance.arrayNode(); + rangeIndex.forEach(rangeIndexArray ::add); + node.set("rangeIndex", rangeIndexArray); + + ArrayNode wordLexiconArray = JsonNodeFactory.instance.arrayNode(); + wordLexicon.forEach(wordLexiconArray::add); + node.set("wordLexicon", wordLexiconArray); + + ObjectNode propertiesObj = JsonNodeFactory.instance.objectNode(); + + for (PropertyType prop : properties) { + propertiesObj.set(prop.getName(), prop.toJson()); + } + node.set("properties", propertiesObj); + return node; + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionsType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionsType.java new file mode 100644 index 0000000000..2451132077 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/DefinitionsType.java @@ -0,0 +1,43 @@ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.HashMap; +import java.util.Map; + +public class DefinitionsType extends JsonPojo { + protected Map definitions; + + public Map getDefinitions() { + if (definitions == null) { + definitions = new HashMap<>(); + } + return this.definitions; + } + + public void addDefinition(String name, DefinitionType definitionType) { + getDefinitions().put(name, definitionType); + } + + public void removeDefinition(String name) { + getDefinitions().remove(name); + } + + public static DefinitionsType fromJson(JsonNode json) { + DefinitionsType definitionsType = new DefinitionsType(); + json.fields().forEachRemaining((Map.Entry field) -> { + definitionsType.addDefinition(field.getKey(), DefinitionType.fromJson(field.getKey(), field.getValue())); + }); + return definitionsType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + this.getDefinitions().forEach((definitionName, definitionType) -> { + node.set(definitionName, definitionType.toJson()); + }); + return node; + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/Entity.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/Entity.java deleted file mode 100644 index c27fde3bc1..0000000000 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/Entity.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2012-2019 MarkLogic Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.marklogic.hub.entity; - -import com.marklogic.hub.flow.Flow; - -import java.util.List; - -/** - * Entity interface, holds basics about a defined entity object - */ -public interface Entity { - /** - * Gets the Entity name - * - * @return the entity name - */ - String getName(); - - /** - * Serializes the Entity as an XML string - * - * @return the serialized XML string - */ - String serialize(); - - /** - * Returns all flows registered to the entity - * - * @return a list of flows - */ - List getFlows(); -} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/EntityImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/EntityImpl.java deleted file mode 100644 index 3c69395ba7..0000000000 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/EntityImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2012-2019 MarkLogic Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.marklogic.hub.entity; - -import org.w3c.dom.Element; - -/** - * An implementation of the Entity base class - */ -public class EntityImpl extends AbstractEntity { - - public EntityImpl(Element xml) { - super(xml); - } - - public EntityImpl(String name) { - super(name); - } - -} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/HubEntity.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/HubEntity.java new file mode 100644 index 0000000000..541325b458 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/HubEntity.java @@ -0,0 +1,73 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + + +public class HubEntity extends JsonPojo { + + protected String filename; + protected InfoType info; + protected DefinitionsType definitions; + + public String getFilename() { + return filename; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public InfoType getInfo() { + return info; + } + + public void setInfo(InfoType info) { + this.info = info; + } + + public DefinitionsType getDefinitions() { + return definitions; + } + + public void setDefinitions(DefinitionsType definition) { + this.definitions = definition; + } + + + @Override + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + writeObjectIf(node, "info", info); + + node.set("definitions",definitions.toJson()); + + return node; + } + + public static HubEntity fromJson(String filename, JsonNode node) { + HubEntity hubEntity = new HubEntity(); + hubEntity.setFilename(filename); + hubEntity.setInfo(InfoType.fromJson(node.get("info"))); + + String title = hubEntity.getInfo().getTitle(); + hubEntity.setDefinitions(DefinitionsType.fromJson(node.get("definitions"))); + return hubEntity; + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/InfoType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/InfoType.java new file mode 100644 index 0000000000..e68ad78e63 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/InfoType.java @@ -0,0 +1,144 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class InfoType extends JsonPojo { + + protected String title; + protected String version; + protected String baseUri; + protected String description; + + /** + * Gets the value of the title property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getTitle() { + return title; + } + + /** + * Sets the value of the title property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setTitle(String value) { + this.title = value; + } + + /** + * Gets the value of the version property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getVersion() { + return version; + } + + /** + * Sets the value of the version property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setVersion(String value) { + this.version = value; + } + + /** + * Gets the value of the baseUri property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getBaseUri() { + return baseUri; + } + + /** + * Sets the value of the baseUri property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setBaseUri(String value) { + this.baseUri = value; + } + + /** + * Gets the value of the description property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getDescription() { + return description; + } + + /** + * Sets the value of the description property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setDescription(String value) { + this.description = value; + } + + public static InfoType fromJson(JsonNode node) { + InfoType infoType = new InfoType(); + infoType.title = getValue(node, "title"); + infoType.version = getValue(node, "version"); + infoType.baseUri = getValue(node, "baseUri"); + infoType.description = getValue(node, "description"); + return infoType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + writeStringIf(node, "title", title); + writeStringIf(node, "version", version); + writeStringIf(node, "baseUri", baseUri); + writeStringIf(node, "description", description); + return node; + } + +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/ItemType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/ItemType.java new file mode 100644 index 0000000000..69bc1a053f --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/ItemType.java @@ -0,0 +1,77 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ItemType extends JsonPojo { + @JsonProperty(value = "$ref") + protected String ref; + protected String datatype; + protected String collation; + + public String getRef() { + return ref; + } + + public void setRef(String ref) { + this.ref = ref; + } + + public String getDatatype() { + return datatype; + } + + public void setDatatype(String datatype) { + this.datatype = datatype; + } + + public String getCollation() { + return collation; + } + + public void setCollation(String collation) { + this.collation = collation; + } + + public boolean hasValues() { + return ( + (ref != null && !ref.isEmpty()) || + (datatype != null && !datatype.isEmpty()) || + (collation != null && !collation.isEmpty()) + ); + } + + public static ItemType fromJson(JsonNode node) { + ItemType itemType = new ItemType(); + itemType.setRef(getValue(node, "$ref")); + itemType.setDatatype(getValue(node, "datatype")); + itemType.setCollation(getValue(node, "collation")); + return itemType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + writeStringIf(node, "$ref", ref); + writeStringIf(node, "datatype", datatype); + writeStringIf(node, "collation", collation); + return node; + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/JsonPojo.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/JsonPojo.java new file mode 100644 index 0000000000..8439c02ed8 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/JsonPojo.java @@ -0,0 +1,66 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public abstract class JsonPojo { + + protected static String getValue(JsonNode node, String key) { + String value = null; + JsonNode n = node.get(key); + if (n != null && !(n instanceof NullNode)) { + value = n.asText(); + } + return value; + } + + protected static Integer getIntValue(JsonNode node, String key) { + return getIntValue(node, key, null); + } + + protected static Integer getIntValue(JsonNode node, String key, Integer defaultValue) { + Integer value = defaultValue; + JsonNode n = node.get(key); + if (n != null && !(n instanceof NullNode)) { + value = n.asInt(); + } + return value; + } + + public abstract JsonNode toJson(); + + protected static void writeObjectIf(ObjectNode node, String key, JsonPojo o) { + if (o != null) { + node.set(key, o.toJson()); + } + } + + protected static void writeStringIf(ObjectNode node, String key, String value) { + if (value != null) { + node.put(key, value); + } + } + + protected static void writeNumberIf(ObjectNode node, String key, Integer value) { + if (value != null) { + node.put(key, value); + } + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/PropertyType.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/PropertyType.java new file mode 100644 index 0000000000..b40046d8a5 --- /dev/null +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/entity/PropertyType.java @@ -0,0 +1,114 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.marklogic.hub.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class PropertyType extends JsonPojo { + + protected String name; + protected String datatype; + protected String description; + + @JsonProperty(value="$ref") + protected String ref; + + protected String collation; + + ItemType items; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDatatype() { + return datatype; + } + + public void setDatatype(String datatype) { + this.datatype = datatype; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getRef() { + return ref; + } + + public void setRef(String ref) { + this.ref = ref; + } + + public String getCollation() { + return collation; + } + + public void setCollation(String collation) { + this.collation = collation; + } + + public ItemType getItems() { + return items; + } + + public void setItems(ItemType items) { + this.items = items; + } + + public static PropertyType fromJson(String name, JsonNode defs) { + PropertyType propertyType = new PropertyType(); + propertyType.name = name; + propertyType.datatype = getValue(defs, "datatype"); + propertyType.description = getValue(defs, "description"); + propertyType.ref = getValue(defs, "$ref"); + propertyType.collation = getValue(defs, "collation"); + + JsonNode itemsNode = defs.get("items"); + if (itemsNode != null) { + propertyType.setItems(ItemType.fromJson(itemsNode)); + } + return propertyType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + + writeStringIf(node, "datatype", datatype); + writeStringIf(node, "description", description); + writeStringIf(node, "$ref", ref); + writeStringIf(node, "collation", collation); + + if (items != null && items.hasValues()) { + node.set("items", items.toJson()); + } + + return node; + } +} diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java index 4c0c993b3e..3a880be9fe 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java @@ -90,6 +90,9 @@ public class DataHubImpl implements DataHub { @Autowired private LoadUserModulesCommand loadUserModulesCommand; + @Autowired + private LoadUserArtifactsCommand loadUserArtifactsCommand; + @Autowired private DeployHubAmpsCommand deployHubAmpsCommand; @@ -665,6 +668,7 @@ private void updateModuleCommandList(Map> commandsMap) { List commands = new ArrayList(); commands.add(loadHubModulesCommand); commands.add(loadUserModulesCommand); + commands.add(loadUserArtifactsCommand); for (Command c : commandsMap.get("mlModuleCommands")) { if (c instanceof LoadModulesCommand) { diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/EntityManagerImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/EntityManagerImpl.java index 7379791728..938ab6ff3c 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/EntityManagerImpl.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/EntityManagerImpl.java @@ -33,7 +33,9 @@ import com.marklogic.hub.EntityManager; import com.marklogic.hub.HubConfig; import com.marklogic.hub.HubProject; +import com.marklogic.hub.entity.HubEntity; import com.marklogic.hub.error.EntityServicesGenerationException; +import com.marklogic.hub.util.FileUtil; import com.marklogic.hub.util.HubModuleManager; import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -243,6 +245,85 @@ private List getModifiedRawEntities(long minimumFileTimestampToLoad) { return entities; } + public List getEntities() { + List entities = new ArrayList<>(); + Path entitiesPath = hubConfig.getHubEntitiesDir(); + List entityNames = FileUtil.listDirectFolders(entitiesPath.toFile()); + ObjectMapper objectMapper = new ObjectMapper(); + for (String entityName : entityNames) { + File[] entityDefs = entitiesPath.resolve(entityName).toFile().listFiles((dir, name) -> name.endsWith(ENTITY_FILE_EXTENSION)); + for (File entityDef : entityDefs) { + try { + FileInputStream fileInputStream = new FileInputStream(entityDef); + JsonNode node = objectMapper.readTree(fileInputStream); + entities.add(HubEntity.fromJson(entityDef.getAbsolutePath(), node)); + fileInputStream.close(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + return entities; + } + + public HubEntity saveEntity(HubEntity entity, Boolean rename) throws IOException { + JsonNode node = entity.toJson(); + ObjectMapper objectMapper = new ObjectMapper(); + String fullpath = entity.getFilename(); + String title = entity.getInfo().getTitle(); + + if (rename) { + String filename = new File(fullpath).getName(); + String entityFromFilename = filename.substring(0, filename.indexOf(ENTITY_FILE_EXTENSION)); + if (!entityFromFilename.equals(entity.getInfo().getTitle())) { + // The entity name was changed since the files were created. Update + // the path. + + // Update the name of the entity definition file + File origFile = new File(fullpath); + File newFile = new File(origFile.getParent() + File.separator + title + ENTITY_FILE_EXTENSION); + if (!origFile.renameTo(newFile)) { + throw new IOException("Unable to rename " + origFile.getAbsolutePath() + " to " + + newFile.getAbsolutePath()); + } + ; + + // Update the directory name + File origDirectory = new File(origFile.getParent()); + File newDirectory = new File(origDirectory.getParent() + File.separator + title); + if (!origDirectory.renameTo(newDirectory)) { + throw new IOException("Unable to rename " + origDirectory.getAbsolutePath() + " to " + + newDirectory.getAbsolutePath()); + } + + fullpath = newDirectory.getAbsolutePath() + File.separator + title + ENTITY_FILE_EXTENSION; + entity.setFilename(fullpath); + } + } + else { + Path dir = hubConfig.getHubEntitiesDir().resolve(title); + if (!dir.toFile().exists()) { + dir.toFile().mkdirs(); + } + fullpath = Paths.get(dir.toString(), title + ENTITY_FILE_EXTENSION).toString(); + } + + + String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); + FileUtils.writeStringToFile(new File(fullpath), json); + + return entity; + } + + public void deleteEntity(String entity) throws IOException { + Path dir = hubConfig.getHubEntitiesDir().resolve(entity); + if (dir.toFile().exists()) { + FileUtils.deleteDirectory(dir.toFile()); + } + } + private class PiiGenerator extends ResourceManager { private static final String NAME = "ml:piiGenerator"; private RequestParameters params = new RequestParameters(); diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java index 3681ec8714..bf7871b595 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java @@ -107,6 +107,10 @@ public void createProject(String projectDirString) { @Override public Path getHubSchemasDir() { return getHubConfigDir().resolve("schemas"); } + @Override public Path getHubTriggersDir() { + return getHubConfigDir().resolve("triggers"); + } + @Override public Path getUserConfigDir() { return this.projectDir.resolve(USER_CONFIG_DIR); } @@ -158,6 +162,7 @@ public void createProject(String projectDirString) { File databasesDir = getHubDatabaseDir().toFile(); File serversDir = getHubServersDir().toFile(); File securityDir = getHubSecurityDir().toFile(); + File triggersDir = getHubTriggersDir().toFile(); boolean newConfigInitialized = hubConfigDir.exists() && @@ -169,7 +174,9 @@ public void createProject(String projectDirString) { serversDir.exists() && serversDir.isDirectory() && securityDir.exists() && - securityDir.isDirectory(); + securityDir.isDirectory() && + triggersDir.exists() && + triggersDir.isDirectory(); return buildGradle.exists() && gradleProperties.exists() && @@ -243,6 +250,13 @@ public void createProject(String projectDirString) { getHubSchemasDir().toFile().mkdirs(); getUserSchemasDir().toFile().mkdirs(); + //create hub triggers + Path hubTriggersDir = getHubTriggersDir(); + hubTriggersDir.toFile().mkdirs(); + writeResourceFile("hub-internal-config/triggers/ml-dh-entity-create.json", hubTriggersDir.resolve("ml-dh-entity-create.json"), true); + writeResourceFile("hub-internal-config/triggers/ml-dh-entity-modify.json", hubTriggersDir.resolve("ml-dh-entity-modify.json"), true); + writeResourceFile("hub-internal-config/triggers/ml-dh-entity-delete.json", hubTriggersDir.resolve("ml-dh-entity-delete.json"), true); + Path gradlew = projectDir.resolve("gradlew"); writeResourceFile("scaffolding/gradlew", gradlew); makeExecutable(gradlew); diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-create.json b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-create.json new file mode 100644 index 0000000000..430363be68 --- /dev/null +++ b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-create.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-create", + "description": "MarkLogic Data Hub entity model creation trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "create" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-delete.json b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-delete.json new file mode 100644 index 0000000000..9e1275abf1 --- /dev/null +++ b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-delete.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-delete", + "description": "MarkLogic Data Hub entity model delete trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "delete" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-delete-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-modify.json b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-modify.json new file mode 100644 index 0000000000..5b35490faf --- /dev/null +++ b/marklogic-data-hub/src/main/resources/hub-internal-config/triggers/ml-dh-entity-modify.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-modify", + "description": "MarkLogic Data Hub entity model update trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "modify" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/extensions/scaffold-content.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/extensions/scaffold-content.xqy index ca5d8a4091..8a7e7834d1 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/extensions/scaffold-content.xqy +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/extensions/scaffold-content.xqy @@ -338,6 +338,9 @@ service:generate-lets($model, $entity-type-name, $mapping, $entity) let $properties := map:get($entity-type, "properties") let $required-properties := ( map:get($entity-type, "primaryKey"), + if (fn:empty(map:get($entity-type, "required"))) then + () + else json:array-values(map:get($entity-type, "required")) ) for $property-name in map:keys($properties) diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/impl/hub-entities.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/impl/hub-entities.xqy index 13fd7122b0..70186003eb 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/impl/hub-entities.xqy +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/impl/hub-entities.xqy @@ -38,16 +38,19 @@ declare function hent:get-model($entity-name as xs:string, $used-models as xs:st return let $model-map as map:map? := $model let $refs := $model//*[fn:local-name(.) = '$ref'][fn:starts-with(., "#/definitions")] ! fn:replace(., "#/definitions/", "") - let $_ := - let $definitions := map:get($model-map, "definitions") - for $ref in $refs[fn:not(. = $used-models)] - let $other-model as map:map? := hent:get-model($ref, ($used-models, $entity-name)) - let $other-defs := map:get($other-model, "definitions") - for $key in map:keys($other-defs) - return - map:put($definitions, $key, map:get($other-defs, $key)) - return - $model-map + let $definitions := map:get($model-map, "definitions") + let $_ := + for $ref in $refs[fn:not(. = $used-models)] + let $m := + if (fn:empty(map:get($definitions, $ref))) then + let $other-model as map:map? := hent:get-model($ref, ($used-models, $entity-name)) + let $other-defs := map:get($other-model, "definitions") + for $key in map:keys($other-defs) + return + map:put($definitions, $key, map:get($other-defs, $key)) + else () + return () + return $model-map }; declare function hent:uber-model() as map:map @@ -100,6 +103,7 @@ declare %private function hent:fix-options($nodes as node()*) typeswitch($n) case element(search:options) return element { fn:node-name($n) } { + $n/namespace::node(), , @@ -108,7 +112,10 @@ declare %private function hent:fix-options($nodes as node()*) case element(search:additional-query) return () case element(search:return-facets) return true case element() return - element { fn:node-name($n) } { hent:fix-options(($n/@*, $n/node())) } + element { fn:node-name($n) } { + $n/namespace::node(), + hent:fix-options(($n/@*, $n/node())) + } case text() return fn:replace($n, "es:", "*:") default return $n diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/rest-api/lib/endpoint-util.sjs b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/rest-api/lib/endpoint-util.sjs index 11f7dc7d21..61a0ebb8ff 100644 --- a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/rest-api/lib/endpoint-util.sjs +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/rest-api/lib/endpoint-util.sjs @@ -1,5 +1,5 @@ /* - * Copyright 2018 MarkLogic Corporation + * Copyright 2012-2019 MarkLogic Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-delete-trigger.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-delete-trigger.xqy new file mode 100644 index 0000000000..db2634fcd5 --- /dev/null +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-delete-trigger.xqy @@ -0,0 +1,27 @@ +xquery version '1.0-ml'; + +import module namespace es = "http://marklogic.com/entity-services" + at "/MarkLogic/entity-services/entity-services.xqy"; +import module namespace tde = "http://marklogic.com/xdmp/tde" + at "/MarkLogic/tde.xqy"; +import module namespace trgr = 'http://marklogic.com/xdmp/triggers' at '/MarkLogic/triggers.xqy'; + +declare variable $ENTITY-MODEL-COLLECTION as xs:string := "http://marklogic.com/entity-services/models"; +declare variable $TDE-COLLECTION as xs:string := "http://marklogic.com/entity-services/models"; + +declare variable $trgr:uri as xs:string external; + +let $entity-def := fn:doc($trgr:uri) +let $tde-uri := $trgr:uri || ".tde.xml" +return ( + xdmp:invoke-function( + function() { + if (fn:doc-available($trgr:uri)) then + xdmp:document-delete($trgr:uri) + else (), + if (fn:doc-available($tde-uri)) then + xdmp:document-delete($tde-uri) + else () + }, map:entry("database", xdmp:schema-database()) + ) +); diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-trigger.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-trigger.xqy new file mode 100644 index 0000000000..cdbc2203aa --- /dev/null +++ b/marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/4/triggers/entity-model-trigger.xqy @@ -0,0 +1,54 @@ +xquery version '1.0-ml'; + +import module namespace es = "http://marklogic.com/entity-services" + at "/MarkLogic/entity-services/entity-services.xqy"; +import module namespace tde = "http://marklogic.com/xdmp/tde" + at "/MarkLogic/tde.xqy"; +import module namespace trgr = 'http://marklogic.com/xdmp/triggers' at '/MarkLogic/triggers.xqy'; + +declare variable $ENTITY-MODEL-COLLECTION as xs:string := "http://marklogic.com/entity-services/models"; + +declare variable $trgr:uri as xs:string external; + +declare function local:make-TDE-flexible($node as node()) { + typeswitch($node) + case document-node() return document {fn:map(local:make-TDE-flexible#1, $node/node())} + case element(tde:template)|element(tde:templates)|element(tde:rows)|element(tde:row)|element(tde:columns)|element(tde:triples)|element(tde:triple) + return element {fn:node-name($node)} { $node/@*, fn:map(local:make-TDE-flexible#1, $node/node()) } + case element(tde:column) + return element {fn:node-name($node)} { + $node/@*, + ($node/node() except $node/(tde:nullable|tde:invalid-values)), + element tde:nullable {fn:true()}, + element tde:invalid-values {"ignore"} + } + case element(tde:subject)|element(tde:predicate)|element(tde:object) + return element {fn:node-name($node)} { + $node/@*, + ($node/node() except $node/tde:invalid-values), + element tde:invalid-values {"ignore"} + } + default return $node +}; + +let $entity-def := fn:doc($trgr:uri) +let $_validate := es:model-validate($entity-def) +let $default-permissions := xdmp:default-permissions() +return ( + xdmp:invoke-function( + function() { + xdmp:document-insert( + $trgr:uri, + $entity-def, + $default-permissions, + $ENTITY-MODEL-COLLECTION + ) + }, map:entry("database", xdmp:schema-database()) + ), + tde:template-insert( + $trgr:uri || ".tde.xml", + local:make-TDE-flexible(es:extraction-template-generate($entity-def)), + $default-permissions, + ("ml-data-hub-tde") + ) +); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java index 9dbbbecfe5..310a574175 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java @@ -37,6 +37,7 @@ import com.marklogic.client.ext.modulesloader.ssl.SimpleX509TrustManager; import com.marklogic.client.io.*; import com.marklogic.hub.deploy.commands.LoadHubModulesCommand; +import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand; import com.marklogic.hub.deploy.commands.LoadUserModulesCommand; import com.marklogic.hub.error.DataHubConfigurationException; import com.marklogic.hub.flow.CodeFormat; @@ -121,6 +122,9 @@ public class HubTestBase { @Autowired protected LoadUserModulesCommand loadUserModulesCommand; + @Autowired + protected LoadUserArtifactsCommand loadUserArtifactsCommand; + @Autowired protected Scaffolding scaffolding; @@ -884,6 +888,8 @@ protected void installUserModules(HubConfig hubConfig, boolean force) { LoadModulesCommand loadModulesCommand = new LoadModulesCommand(); commands.add(loadModulesCommand); + loadUserArtifactsCommand.setForceLoad(force); + commands.add(loadUserArtifactsCommand); SimpleAppDeployer deployer = new SimpleAppDeployer(((HubConfigImpl)hubConfig).getManageClient(), ((HubConfigImpl)hubConfig).getAdminManager()); deployer.setCommands(commands); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java index 820d88470f..7d4314ee80 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java @@ -119,7 +119,7 @@ public void testInstallUserModules() throws IOException, ParserConfigurationExce HubConfig hubConfig = getHubAdminConfig(); int totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null); - installUserModules(hubConfig, true); + installUserModules(hubConfig, false); assertEquals( getResource("data-hub-test/plugins/entities/test-entity/harmonize/final/collector.xqy"), @@ -217,6 +217,12 @@ public void testInstallUserModules() throws IOException, ParserConfigurationExce getResource("data-hub-test/plugins/entities/test-entity/input/REST/transforms/test-input-transform.xqy"), getModulesFile("/marklogic.rest.transform/test-input-transform/assets/transform.xqy")); + /* + ** The following tests would fail as installUserModules() is run with "forceLoad" option set to true as the + * LoadUserModulesCommand runs first and the timestamp file it creates will be deleted by LoadUserArtifactsCommand + * as currently these 2 commands share the timestamp file + */ + String timestampFile = hubConfig.getHubProject().getUserModulesDeployTimestampFile(); PropertiesModuleManager propsManager = new PropertiesModuleManager(timestampFile); propsManager.initialize(); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java index db8e120ed4..35774fe45b 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java @@ -166,7 +166,7 @@ public void upgrade300To403ToCurrentVersion() throws Exception { adminHubConfig.refreshProject(); dataHub.upgradeHub(); - + // Confirm that the directories have been backed up Assertions.assertTrue(adminHubConfig.getHubProject().getProjectDir() .resolve("src/main/hub-internal-config-4.0.3").toFile().exists()); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommandTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommandTest.java similarity index 60% rename from marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommandTest.java rename to marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommandTest.java index 74fcdf5f7f..eee2513e6e 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommandTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/LoadUserArtifactsCommandTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -32,50 +33,63 @@ @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = ApplicationConfig.class) -public class LoadUserModulesCommandTest extends HubTestBase { - - public LoadUserModulesCommand loadUserModulesCommand; +public class LoadUserArtifactsCommandTest extends HubTestBase { @BeforeEach public void setup() { - loadUserModulesCommand = new LoadUserModulesCommand(); - loadUserModulesCommand.setHubConfig(getHubAdminConfig()); + loadUserArtifactsCommand.setHubConfig(getHubAdminConfig()); } @Test public void testIsEntityDir() { Path startPath = Paths.get("/tmp/my-project/plugins/entities"); Path dir = Paths.get("/tmp/my-project/plugins/entities/my-entity"); - assertTrue(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertTrue(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("/tmp/my-project/plugins/entities"); dir = Paths.get("/tmp/my-project/plugins/entities"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("/tmp/my-project/plugins/entities"); dir = Paths.get("/tmp/my-project/plugins/entities/my-entity/input"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("/tmp/my-project/plugins/entities"); dir = Paths.get("/tmp/my-project/plugins/entities/my-entity/input/my-input-flow"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); + + startPath = Paths.get("/tmp/my-project/plugins/mappings"); + dir = Paths.get("/tmp/my-project/plugins/mappings/my-mappings/input"); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); + + startPath = Paths.get("/tmp/my-project/plugins/mappings"); + dir = Paths.get("/tmp/my-project/plugins/mappings/my-mappings"); + assertTrue(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); // test windows paths startPath = Paths.get("c:\\temp\\my-project\\plugins\\entities"); dir = Paths.get("c:\\temp\\my-project\\plugins\\entities\\my-entity"); - assertTrue(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertTrue(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("c:\\temp\\my-project\\plugins\\entities"); dir = Paths.get("c:\\temp\\my-project\\plugins\\entities"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("c:\\temp\\my-project\\plugins\\entities"); dir = Paths.get("c:\\temp\\my-project\\plugins\\entities\\my-entity\\input"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); startPath = Paths.get("c:\\temp\\my-project\\plugins\\entities"); dir = Paths.get("c:\\temp\\my-project\\plugins\\entities\\my-entity\\input\\my-input-flow"); - assertFalse(loadUserModulesCommand.isEntityDir(dir, startPath)); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); + + startPath = Paths.get("c:\\temp\\my-project\\plugins\\mappings"); + dir = Paths.get("c:\\temp\\my-project\\plugins\\mappings\\my-mappings\\path1\\path2"); + assertFalse(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); + + startPath = Paths.get("c:\\temp\\my-project\\plugins\\mappings"); + dir = Paths.get("c:\\temp\\my-project\\plugins\\mappings\\my-mappings"); + assertTrue(loadUserArtifactsCommand.isArtifactDir(dir, startPath)); } } diff --git a/marklogic-data-hub/src/test/resources/es-alignment-test/Order.entity.json b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.entity.json new file mode 100644 index 0000000000..2fb2fc424c --- /dev/null +++ b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.entity.json @@ -0,0 +1,50 @@ +{ "info": { + "title": "DHExample", + "description": "Data Hub Example", + "version": "1.0.0", + "baseUri": "http://marklogic.com/data-hub/" + }, + "definitions": { + "Order": { + "properties": { + "id": { "datatype": "int" }, + "purchasedItems": { + "datatype": "array", + "items": { + "$ref": "#/definitions/Item" + } + }, + "customer": { + "$ref": "#/definitions/Customer" + }, + "transactionDateTime": { "datatype": "dateTime" }, + "totalCost": { "datatype": "double" } + }, + "required": ["id", "transactionDateTime", "totalCost"], + "primaryKey": "id", + "pathRangeIndex": ["id", "totalCost"] + }, + "Customer": { + "properties": { + "id": { "datatype": "int" }, + "name": { "datatype": "string" } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pii": ["name"], + "pathRangeIndex": ["id"] + }, + "Item": { + "properties": { + "id": { "datatype": "int" }, + "name": { "datatype": "string" }, + "description": { "datatype": "string" }, + "rating": { "datatype": "float" } + }, + "required": ["id", "name"], + "primaryKey": "id", + "pathRangeIndex": ["id", "rating"], + "wordLexicon": ["description"] + } + } +} \ No newline at end of file diff --git a/marklogic-data-hub/src/test/resources/es-alignment-test/Order.instance.xml b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.instance.xml new file mode 100644 index 0000000000..09a7f79870 --- /dev/null +++ b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.instance.xml @@ -0,0 +1,19 @@ + + 123 + + + 123 + some string + some string + 123 + + + + + 123 + some string + + + 2000-01-23T17:00:26.789186-08:00 + 123 + \ No newline at end of file diff --git a/marklogic-data-hub/src/test/resources/es-alignment-test/Order.search.xml b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.search.xml new file mode 100644 index 0000000000..3900b215cd --- /dev/null +++ b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.search.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + //es:instance/Order/totalCost + + + + + + + + + //es:instance/Item/rating + + + + + + + + + + //es:instance/Order/id + + + //es:instance/Order/totalCost + + + + + //es:instance/Customer/id + + + + + //es:instance/Item/id + + + //es:instance/Item/rating + + + + + + + + unfiltered + + + //es:instance/(Order|Customer|Item) + + + + + + instance + + + + es:instance + + + + + + false + + + \ No newline at end of file diff --git a/marklogic-data-hub/src/test/resources/es-alignment-test/Order.tde.xml b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.tde.xml new file mode 100644 index 0000000000..46afc87d7f --- /dev/null +++ b/marklogic-data-hub/src/test/resources/es-alignment-test/Order.tde.xml @@ -0,0 +1,246 @@ + + + +Extraction Template Generated from Entity Type Document +graph uri: http://marklogic.com/data-hub/DHExample-1.0.0 + + //*:instance[*:info/*:version = "1.0.0"] + + + + + RDF + "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + + + RDF_TYPE + sem:iri(concat($RDF, "type")) + + + + + es + http://marklogic.com/entity-services + + + + + ./Customer + + + subject-iri + sem:iri(concat("http://marklogic.com/data-hub/DHExample-1.0.0/Customer/", fn:encode-for-uri(xs:string(./id)))) + + + + + + $subject-iri + + + $RDF_TYPE + + + sem:iri("http://marklogic.com/data-hub/DHExample-1.0.0/Customer") + + + + + $subject-iri + + + sem:iri("http://www.w3.org/2000/01/rdf-schema#isDefinedBy") + + + fn:base-uri(.) + + + + + + ./Customer + + + DHExample + Customer + sparse + + + id + int + id + + + name + string + name + + + + + + + ./Order + + + subject-iri + sem:iri(concat("http://marklogic.com/data-hub/DHExample-1.0.0/Order/", fn:encode-for-uri(xs:string(./id)))) + + + + + + $subject-iri + + + $RDF_TYPE + + + sem:iri("http://marklogic.com/data-hub/DHExample-1.0.0/Order") + + + + + $subject-iri + + + sem:iri("http://www.w3.org/2000/01/rdf-schema#isDefinedBy") + + + fn:base-uri(.) + + + + + + ./Order + + + DHExample + Order + sparse + + + id + int + id + + + customer + int + customer/Customer + true + + + transactionDateTime + dateTime + transactionDateTime + + + totalCost + double + totalCost + + + + + + + ./purchasedItems + + + DHExample + Order_purchasedItems + sparse + + + + id + int + ../id + + + + purchasedItems_id + int + Item + + + + + + + + + ./Item + + + subject-iri + sem:iri(concat("http://marklogic.com/data-hub/DHExample-1.0.0/Item/", fn:encode-for-uri(xs:string(./id)))) + + + + + + $subject-iri + + + $RDF_TYPE + + + sem:iri("http://marklogic.com/data-hub/DHExample-1.0.0/Item") + + + + + $subject-iri + + + sem:iri("http://www.w3.org/2000/01/rdf-schema#isDefinedBy") + + + fn:base-uri(.) + + + + + + ./Item + + + DHExample + Item + sparse + + + id + int + id + + + name + string + name + + + description + string + description + true + + + rating + float + rating + true + + + + + + + \ No newline at end of file diff --git a/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-create.json b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-create.json new file mode 100644 index 0000000000..430363be68 --- /dev/null +++ b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-create.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-create", + "description": "MarkLogic Data Hub entity model creation trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "create" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-delete.json b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-delete.json new file mode 100644 index 0000000000..9e1275abf1 --- /dev/null +++ b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-delete.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-delete", + "description": "MarkLogic Data Hub entity model delete trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "delete" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-delete-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-modify.json b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-modify.json new file mode 100644 index 0000000000..5b35490faf --- /dev/null +++ b/marklogic-data-hub/src/test/resources/upgrade-projects/dhf403from300/src/main/hub-internal-config/triggers/ml-dh-entity-modify.json @@ -0,0 +1,31 @@ +{ + "name": "ml-dh-entity-modify", + "description": "MarkLogic Data Hub entity model update trigger", + "event": { + "data-event": { + "collection-scope": { + "uri": "http://marklogic.com/entity-services/models" + }, + "document-content": { + "update-kind": "modify" + }, + "when": "post-commit" + } + }, + "module": "data-hub/4/triggers/entity-model-trigger.xqy", + "module-db": "%%mlModulesDbName%%", + "module-root": "/", + "enabled": true, + "recursive": true, + "task-priority": "normal", + "permission": [ + { + "role-name": "%%mlHubAdminRole%%", + "capability": "update" + }, + { + "role-name": "%%mlHubUserRole%%", + "capability": "read" + } + ] +} diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy index 279a7309ec..1f8546eb8c 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/DataHubPlugin.groovy @@ -25,6 +25,7 @@ import com.marklogic.hub.ApplicationConfig import com.marklogic.hub.deploy.commands.GeneratePiiCommand import com.marklogic.hub.deploy.commands.LoadHubModulesCommand import com.marklogic.hub.deploy.commands.LoadUserModulesCommand +import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand import com.marklogic.hub.impl.* import org.gradle.api.GradleException import org.gradle.api.Plugin @@ -43,6 +44,7 @@ class DataHubPlugin implements Plugin { private HubConfigImpl hubConfig private LoadHubModulesCommand loadHubModulesCommand private LoadUserModulesCommand loadUserModulesCommand + private LoadUserArtifactsCommand loadUserArtifactsCommand private MappingManagerImpl mappingManager private FlowManagerImpl flowManager private EntityManagerImpl entityManager @@ -118,6 +120,10 @@ class DataHubPlugin implements Plugin { // This isn't likely to be used, but it's being kept for regression purposes for now project.task("hubDeployUserModules", group: deployGroup, type: DeployUserModulesTask, description: "Installs user modules from the plugins and src/main/entity-config directories.") + project.task("hubDeployUserArtifacts", group: deployGroup, type: DeployUserArtifactsTask, + description: "Installs user artifacts such as entities and mappings.") + .mustRunAfter(["hubDeployUserModules"]) + // HubWatchTask extends ml-gradle's WatchTask to ensure that modules are loaded from the hub-specific locations. project.tasks.replace("mlWatch", HubWatchTask) @@ -154,6 +160,7 @@ class DataHubPlugin implements Plugin { scaffolding = ctx.getBean(ScaffoldingImpl.class) loadHubModulesCommand = ctx.getBean(LoadHubModulesCommand.class) loadUserModulesCommand = ctx.getBean(LoadUserModulesCommand.class) + loadUserArtifactsCommand = ctx.getBean(LoadUserArtifactsCommand.class) mappingManager = ctx.getBean(MappingManagerImpl.class) flowManager = ctx.getBean(FlowManagerImpl.class) entityManager = ctx.getBean(EntityManagerImpl.class) @@ -201,6 +208,7 @@ class DataHubPlugin implements Plugin { project.extensions.add("scaffolding", scaffolding) project.extensions.add("loadHubModulesCommand", loadHubModulesCommand) project.extensions.add("loadUserModulesCommand", loadUserModulesCommand) + project.extensions.add("loadUserArtifactsCommand", loadUserArtifactsCommand) project.extensions.add("mappingManager", mappingManager) project.extensions.add("flowManager", flowManager) project.extensions.add("entityManager", entityManager) diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployUserArtifactsTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployUserArtifactsTask.groovy new file mode 100644 index 0000000000..84d7f2f537 --- /dev/null +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/DeployUserArtifactsTask.groovy @@ -0,0 +1,37 @@ +/* + * Copyright 2012-2018 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.marklogic.gradle.task + +import org.gradle.api.tasks.TaskAction + +class DeployUserArtifactsTask extends HubTask { + + @TaskAction + void deployUserModules() { + if (!isHubInstalled()) { + println("Data Hub is not installed.") + return + } + + def cmd = getLoadUserArtifactsCommand() + cmd.setForceLoad(true); + + cmd.execute(getCommandContext()) + + } +} diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy index e9962aeeeb..c1ddffcc1c 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/HubTask.groovy @@ -24,6 +24,7 @@ import com.marklogic.client.DatabaseClient import com.marklogic.hub.* import com.marklogic.hub.deploy.commands.GeneratePiiCommand import com.marklogic.hub.deploy.commands.LoadHubModulesCommand +import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand import com.marklogic.hub.deploy.commands.LoadUserModulesCommand import com.marklogic.hub.job.JobManager import com.marklogic.hub.scaffold.Scaffolding @@ -62,6 +63,11 @@ abstract class HubTask extends DefaultTask { getProject().property("loadUserModulesCommand") } + @Internal + LoadUserArtifactsCommand getLoadUserArtifactsCommand() { + getProject().property("loadUserArtifactsCommand") + } + @Internal MappingManager getMappingManager() { getProject().property("mappingManager") diff --git a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateEntityTaskTest.groovy b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateEntityTaskTest.groovy index aa2c0bc9e7..9758358909 100644 --- a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateEntityTaskTest.groovy +++ b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateEntityTaskTest.groovy @@ -17,6 +17,7 @@ package com.marklogic.gradle.task +import com.marklogic.hub.HubConfig import org.gradle.testkit.runner.UnexpectedBuildFailure import org.gradle.testkit.runner.UnexpectedBuildSuccess @@ -29,6 +30,7 @@ class CreateEntityTaskTest extends BaseTest { def setupSpec() { createGradleFiles() runTask('hubInit') + clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); } def "create entity with no name"() { @@ -48,19 +50,26 @@ class CreateEntityTaskTest extends BaseTest { entityName=my-new-entity } """ - + getStagingDocCount("http://marklogic.com/entity-services/models") == 0 + def modCount = getModulesDocCount(); when: - def result = runTask('hubCreateEntity') + def result = runTask('hubCreateEntity', 'hubDeployUserArtifacts') then: notThrown(UnexpectedBuildFailure) result.task(":hubCreateEntity").outcome == SUCCESS + result.task(":hubDeployUserArtifacts").outcome == SUCCESS File entityFile = Paths.get(testProjectDir.root.toString(), "plugins", "entities", "my-new-entity", "my-new-entity.entity.json").toFile() entityFile.isFile() == true String entityActual = entityFile.getText('UTF-8') String entityExpected = new File("src/test/resources/my-new-entity.entity.json").getText('UTF-8') assert(entityActual == entityExpected) + + File entityDir = Paths.get(testProjectDir.root.toString(), "plugins", "entities", "my-new-entity").toFile() + entityDir.isDirectory() == true + getStagingDocCount("http://marklogic.com/entity-services/models") == 1 + getModulesDocCount() == modCount } } diff --git a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateMappingTaskTest.groovy b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateMappingTaskTest.groovy index 6b89963dd0..5966fbe846 100644 --- a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateMappingTaskTest.groovy +++ b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/CreateMappingTaskTest.groovy @@ -17,6 +17,7 @@ package com.marklogic.gradle.task +import com.marklogic.hub.HubConfig import org.gradle.testkit.runner.UnexpectedBuildFailure import org.gradle.testkit.runner.UnexpectedBuildSuccess @@ -29,6 +30,7 @@ class CreateMappingTaskTest extends BaseTest { def setupSpec() { createGradleFiles() runTask('hubInit') + clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); } def "create mapping with no name"() { @@ -46,18 +48,21 @@ class CreateMappingTaskTest extends BaseTest { propertiesFile << """ ext { mappingName=my-new-mapping + entityName=my-new-entity } """ when: - def result = runTask('hubCreateMapping') + def result = runTask('hubCreateEntity', 'hubCreateMapping', 'hubDeployUserArtifacts' ) then: notThrown(UnexpectedBuildFailure) result.task(":hubCreateMapping").outcome == SUCCESS + result.task(":hubDeployUserArtifacts").outcome == SUCCESS File mappingDir = Paths.get(testProjectDir.root.toString(), "plugins", "mappings", "my-new-mapping").toFile() mappingDir.isDirectory() == true + getStagingDocCount("http://marklogic.com/data-hub/mappings") == 1 } } diff --git a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionType.java b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionType.java index ad091fb76d..00ac7439fe 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionType.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionType.java @@ -109,10 +109,9 @@ public void setProperties(List properties) { this.properties = properties; } - public static DefinitionType fromJson(String name, JsonNode defs) { + public static DefinitionType fromJson(String name, JsonNode node) { DefinitionType definitionType = new DefinitionType(); definitionType.setName(name); - JsonNode node = defs.get(name); definitionType.setDescription(getValue(node, "description")); definitionType.setPrimaryKey(getValue(node, "primaryKey")); diff --git a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionsType.java b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionsType.java index 0df3f84314..c49e73fcaf 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionsType.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/DefinitionsType.java @@ -16,6 +16,10 @@ */ package com.marklogic.quickstart.model.entity_services; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + import java.util.HashMap; import java.util.Map; @@ -30,4 +34,28 @@ public Map getDefinitions() { return this.definitions; } + public void addDefinition(String name, DefinitionType definitionType) { + getDefinitions().put(name, definitionType); + } + + public void removeDefinition(String name) { + getDefinitions().remove(name); + } + + public static DefinitionsType fromJson(JsonNode json) { + DefinitionsType definitionsType = new DefinitionsType(); + json.fields().forEachRemaining((Map.Entry field) -> { + definitionsType.addDefinition(field.getKey(), DefinitionType.fromJson(field.getKey(), field.getValue())); + }); + return definitionsType; + } + + public JsonNode toJson() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + this.getDefinitions().forEach((definitionName, definitionType) -> { + node.set(definitionName, definitionType.toJson()); + }); + + return node; + } } diff --git a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/EntityModel.java b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/EntityModel.java index ed01c99cfd..8a3fa9e11f 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/EntityModel.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/model/entity_services/EntityModel.java @@ -17,6 +17,7 @@ package com.marklogic.quickstart.model.entity_services; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonUnwrapped; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -29,7 +30,7 @@ public class EntityModel extends JsonPojo { protected String filename; protected HubUIData hubUi; protected InfoType info; - protected DefinitionType definition; + protected DefinitionsType definitions; public List inputFlows; public List harmonizeFlows; @@ -87,8 +88,9 @@ public void setInfo(InfoType value) { * {@link DefinitionsType } * */ - public DefinitionType getDefinition() { - return definition; + @JsonUnwrapped + public DefinitionsType getDefinitions() { + return definitions; } /** @@ -99,8 +101,8 @@ public DefinitionType getDefinition() { * {@link DefinitionsType } * */ - public void setDefinition(DefinitionType value) { - this.definition = value; + public void setDefinitions(DefinitionsType value) { + this.definitions = value; } public List getInputFlows() { @@ -125,7 +127,8 @@ public static EntityModel fromJson(String filename, JsonNode node) { entityModel.setInfo(InfoType.fromJson(node.get("info"))); String title = entityModel.getInfo().getTitle(); - entityModel.setDefinition(DefinitionType.fromJson(title, node.get("definitions"))); + + entityModel.setDefinitions(DefinitionsType.fromJson(node.get("definitions"))); return entityModel; } @@ -133,9 +136,7 @@ public JsonNode toJson() { ObjectNode node = JsonNodeFactory.instance.objectNode(); writeObjectIf(node, "info", info); - ObjectNode definitions = JsonNodeFactory.instance.objectNode(); - definitions.set(info.getTitle(), definition.toJson()); - node.set("definitions",definitions); + node.set("definitions",definitions.toJson()); return node; } diff --git a/quick-start/src/main/java/com/marklogic/quickstart/service/DataHubService.java b/quick-start/src/main/java/com/marklogic/quickstart/service/DataHubService.java index bbc557c258..15f6119a42 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/service/DataHubService.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/service/DataHubService.java @@ -20,6 +20,7 @@ import com.marklogic.appdeployer.impl.SimpleAppDeployer; import com.marklogic.hub.DataHub; import com.marklogic.hub.HubConfig; +import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand; import com.marklogic.hub.deploy.commands.LoadUserModulesCommand; import com.marklogic.hub.deploy.util.HubDeployStatusListener; import com.marklogic.hub.error.CantUpgradeException; @@ -56,6 +57,9 @@ public class DataHubService { @Autowired private LoadUserModulesCommand loadUserModulesCommand; + @Autowired + private LoadUserArtifactsCommand loadUserArtifactsCommand; + public boolean install(HubConfig config, HubDeployStatusListener listener) throws DataHubException { logger.info("Installing Data Hub"); try { @@ -177,7 +181,12 @@ private void installUserModules(HubConfig hubConfig, boolean forceLoad, DeployUs List commands = new ArrayList<>(); loadUserModulesCommand.setHubConfig(hubConfig); loadUserModulesCommand.setForceLoad(forceLoad); + + loadUserArtifactsCommand.setHubConfig(hubConfig); + loadUserArtifactsCommand.setForceLoad(forceLoad); + commands.add(loadUserModulesCommand); + commands.add(loadUserArtifactsCommand); SimpleAppDeployer deployer = new SimpleAppDeployer(((HubConfigImpl)hubConfig).getManageClient(), ((HubConfigImpl)hubConfig).getAdminManager()); deployer.setCommands(commands); diff --git a/quick-start/src/main/java/com/marklogic/quickstart/service/EntityManagerService.java b/quick-start/src/main/java/com/marklogic/quickstart/service/EntityManagerService.java index 7fe56dc17f..2e64c675a4 100644 --- a/quick-start/src/main/java/com/marklogic/quickstart/service/EntityManagerService.java +++ b/quick-start/src/main/java/com/marklogic/quickstart/service/EntityManagerService.java @@ -22,23 +22,23 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.hub.EntityManager; import com.marklogic.hub.HubConfig; +import com.marklogic.hub.entity.HubEntity; import com.marklogic.hub.error.DataHubProjectException; import com.marklogic.hub.flow.FlowType; import com.marklogic.hub.impl.HubConfigImpl; import com.marklogic.hub.scaffold.Scaffolding; +import com.marklogic.hub.util.FileUtil; import com.marklogic.hub.validate.EntitiesValidator; import com.marklogic.quickstart.model.FlowModel; import com.marklogic.quickstart.model.PluginModel; import com.marklogic.quickstart.model.entity_services.EntityModel; import com.marklogic.quickstart.model.entity_services.HubUIData; import com.marklogic.quickstart.model.entity_services.InfoType; -import com.marklogic.hub.util.FileUtil; import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -96,27 +96,21 @@ public List getLegacyEntities() throws IOException { public List getEntities() throws IOException { Map hubUiData = getUiData(); List entities = new ArrayList<>(); - Path entitiesPath = hubConfig.getHubEntitiesDir(); - List entityNames = FileUtil.listDirectFolders(entitiesPath.toFile()); - ObjectMapper objectMapper = new ObjectMapper(); - for (String entityName : entityNames) { - File[] entityDefs = entitiesPath.resolve(entityName).toFile().listFiles((dir, name) -> name.endsWith(ENTITY_FILE_EXTENSION)); - for (File entityDef : entityDefs) { - FileInputStream fileInputStream = new FileInputStream(entityDef); - JsonNode node = objectMapper.readTree(fileInputStream); - fileInputStream.close(); - EntityModel entityModel = EntityModel.fromJson(entityDef.getAbsolutePath(), node); - if (entityModel != null) { - HubUIData data = hubUiData.get(entityModel.getInfo().getTitle()); - if (data == null) { - data = new HubUIData(); - } - entityModel.setHubUi(data); - entityModel.inputFlows = flowManagerService.getFlows(entityName, FlowType.INPUT); - entityModel.harmonizeFlows = flowManagerService.getFlows(entityName, FlowType.HARMONIZE); - - entities.add(entityModel); + + List entityList = em.getEntities(); + + for (HubEntity entity : entityList) { + EntityModel entityModel = EntityModel.fromJson(entity.getFilename(), entity.toJson()); + if (entityModel != null) { + HubUIData data = hubUiData.get(entityModel.getInfo().getTitle()); + if (data == null) { + data = new HubUIData(); } + entityModel.setHubUi(data); + entityModel.inputFlows = flowManagerService.getFlows(entity.getInfo().getTitle(), FlowType.INPUT); + entityModel.harmonizeFlows = flowManagerService.getFlows(entity.getInfo().getTitle(), FlowType.HARMONIZE); + + entities.add(entityModel); } } @@ -143,52 +137,23 @@ public EntityModel createEntity(EntityModel newEntity) throws IOException { public EntityModel saveEntity(EntityModel entity) throws IOException { JsonNode node = entity.toJson(); - ObjectMapper objectMapper = new ObjectMapper(); String fullpath = entity.getFilename(); - String title = entity.getInfo().getTitle(); + + HubEntity hubEntity = HubEntity.fromJson(fullpath, node); + if (fullpath == null) { - Path dir = hubConfig.getHubEntitiesDir().resolve(title); - if (!dir.toFile().exists()) { - dir.toFile().mkdirs(); - } - fullpath = Paths.get(dir.toString(), title + ENTITY_FILE_EXTENSION).toString(); + em.saveEntity(hubEntity, false); } else { - String filename = new File(fullpath).getName(); - String entityFromFilename = filename.substring(0, filename.indexOf(ENTITY_FILE_EXTENSION)); - if (!entityFromFilename.equals(entity.getName())) { - // The entity name was changed since the files were created. Update - // the path. - - // Update the name of the entity definition file - File origFile = new File(fullpath); - File newFile = new File(origFile.getParent() + File.separator + title + ENTITY_FILE_EXTENSION); - if (!origFile.renameTo(newFile)) { - throw new IOException("Unable to rename " + origFile.getAbsolutePath() + " to " + - newFile.getAbsolutePath()); - }; - - // Update the directory name - File origDirectory = new File(origFile.getParent()); - File newDirectory = new File(origDirectory.getParent() + File.separator + title); - if (!origDirectory.renameTo(newDirectory)) { - throw new IOException("Unable to rename " + origDirectory.getAbsolutePath() + " to " + - newDirectory.getAbsolutePath()); - } + HubEntity renamedEntity = em.saveEntity(hubEntity, true); + entity.setFilename(renamedEntity.getFilename()); - fullpath = newDirectory.getAbsolutePath() + File.separator + title + ENTITY_FILE_EXTENSION; - entity.setFilename(fullpath); - - // Redeploy the flows - dataHubService.reinstallUserModules(hubConfig, null, null); - } + // Redeploy the flows + dataHubService.reinstallUserModules(hubConfig, null, null); } - String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(node); - FileUtils.writeStringToFile(new File(fullpath), json); - return entity; } @@ -196,7 +161,7 @@ public void deleteEntity(String entity) throws IOException { Path dir = hubConfig.getHubEntitiesDir().resolve(entity); if (dir.toFile().exists()) { watcherService.unwatch(dir.getParent().toString()); - FileUtils.deleteDirectory(dir.toFile()); + em.deleteEntity(entity); } } diff --git a/quick-start/src/main/ui/app/entities/definitions.model.ts b/quick-start/src/main/ui/app/entities/definitions.model.ts new file mode 100644 index 0000000000..22d94e61ef --- /dev/null +++ b/quick-start/src/main/ui/app/entities/definitions.model.ts @@ -0,0 +1,21 @@ +import { DefinitionType } from './definition.model'; + +export class DefinitionsType { + + set(definitionName:string, definitionType: DefinitionType) { + (this as any)[definitionName] = definitionType; + } + + get(definitionName: string) { + return (this as any)[definitionName]; + } + + fromJSON(json: any) { + for (const definitionKey of Object.keys(json)) { + const definitionType = new DefinitionType(); + definitionType.fromJSON(json[definitionKey]); + (this as any)[definitionKey] = definitionType; + } + return this; + } +} diff --git a/quick-start/src/main/ui/app/entities/entities.service.ts b/quick-start/src/main/ui/app/entities/entities.service.ts index ad7e16af7e..41777997a2 100644 --- a/quick-start/src/main/ui/app/entities/entities.service.ts +++ b/quick-start/src/main/ui/app/entities/entities.service.ts @@ -4,6 +4,7 @@ import { ProjectService } from '../projects'; import { Subject } from 'rxjs/Subject'; import { SettingsService } from '../settings'; +import { DefinitionsType } from './definitions.model'; import { Entity } from './entity.model'; import { Flow } from './flow.model'; import { Plugin } from './plugin.model'; @@ -19,6 +20,7 @@ import * as _ from 'lodash'; @Injectable() export class EntitiesService { + static readonly entityRefPrefix = '#/definitions/'; coreDataTypes: Array = EntityConsts.coreDataTypes; entityRefDataTypes: Array = []; @@ -36,7 +38,7 @@ export class EntitiesService { getEntities() { this.http.get(this.url('/entities/')).map((res: Response) => { - let entities: Array = res.json(); + const entities: Array = res.json(); return entities.map((entity) => { return new Entity().fromJSON(entity); }); @@ -52,17 +54,18 @@ export class EntitiesService { // } createEntity(entity: Entity) { - return this.http.post(this.url('/entities/create'), entity).map((res:Response) => { + return this.http.post(this.url('/entities/create'), entity).map((res: Response) => { return new Entity().fromJSON(res.json()); }); } saveEntity(entity: Entity) { - let resp = this.http.put(this.url(`/entities/${entity.name}`), entity).map((res: Response) => { + const expandedEntity = this.expandEntity(entity); + const resp = this.http.put(this.url(`/entities/${expandedEntity.name}`), expandedEntity).map((res: Response) => { return new Entity().fromJSON(res.json()); }).share(); resp.subscribe((newEntity: Entity) => { - let index = _.findIndex(this.entities, { 'name': newEntity.name }); + const index = _.findIndex(this.entities, { 'name': newEntity.name }); if (index >= 0) { this.entities[index] = newEntity; } else { @@ -74,9 +77,49 @@ export class EntitiesService { return resp; } + expandEntity(entity: Entity) { + const supportingEntites: Array = this.findSupportingEntities(entity); + const definitions = new DefinitionsType(); + definitions.set(entity.name, entity.definition); + supportingEntites.forEach((supportingEntity) => { + definitions.set(supportingEntity.name, supportingEntity.definition); + }); + entity.definitions = definitions; + return entity; + } + + /* + * find entities that are referenced in entities recursively. Avoid infinite loop by tracking + * entities visited. + */ + findSupportingEntities(entity: Entity, visitedEntities: Array = []): Array { + if (visitedEntities.length === 0) { + visitedEntities.push(entity.name); + } + let supportingEntities: Array = []; + this.entityReferencesInEntity(entity).forEach((prop: PropertyType) => { + const ref = prop.$ref || prop.items.$ref; + const refEntityName = ref.substr(EntitiesService.entityRefPrefix.length); + if (visitedEntities.findIndex((entityName: string) => refEntityName === entityName) < 0) { + const refEntity = this.findEntityByName(refEntityName); + visitedEntities.push(refEntityName); + const refSupportingEntities = this.findSupportingEntities(refEntity, visitedEntities); + supportingEntities.push(refEntity); + if (refSupportingEntities.length > 0) { + supportingEntities = supportingEntities.concat(refSupportingEntities); + } + } + }); + return supportingEntities; + } + + findEntityByName(entityName: string): Entity { + return _.find(this.entities, { 'name': entityName }); + } + editEntity(entity: Entity) { - let result = new Subject(); - let actions = { + const result = new Subject(); + const actions = { save: () => { result.next(null); result.complete(); @@ -86,7 +129,7 @@ export class EntitiesService { } }; - let editDialog = this.dialogService.showCustomDialog({ + const editDialog = this.dialogService.showCustomDialog({ component: EntityEditorComponent, providers: [ { provide: 'entity', useValue: entity }, @@ -100,18 +143,26 @@ export class EntitiesService { return result.asObservable(); } + entityReferencesInEntity(entity: Entity) { + if (entity.definition && entity.definition.properties) { + return entity.definition.properties.filter((prop: PropertyType) => { + return prop.$ref || (prop.items && prop.items.$ref); + }); + } else { + return []; + } + } + deleteEntity(entityToDelete: Entity) { // remove references to this entity this.entities.forEach((entity: Entity) => { - if (entity.definition && entity.definition.properties) { - entity.definition.properties.forEach((prop: PropertyType) => { - if (prop.$ref && prop.$ref.endsWith(entityToDelete.name)) { - prop.$ref = null; - } else if (prop.items && prop.items.$ref && prop.items.$ref.endsWith(entityToDelete.name)) { - prop.items.$ref = null; - } - }); - } + this.entityReferencesInEntity(entity).forEach((prop: PropertyType) => { + if (prop.$ref && prop.$ref.endsWith(entityToDelete.name)) { + prop.$ref = null; + } else if (prop.items && prop.items.$ref && prop.items.$ref.endsWith(entityToDelete.name)) { + prop.items.$ref = null; + } + }); const connectionName = `${entity.name}-${entityToDelete.name}`; if (entity.hubUi && entity.hubUi.vertices && entity.hubUi.vertices[connectionName]) { @@ -126,7 +177,7 @@ export class EntitiesService { } deleteFlow(flow: Flow, flowType: string) { - let resp = this.http.delete(this.url(`/entities/${flow.entityName}/flows/${flow.flowName}/${flowType}`)).share(); + const resp = this.http.delete(this.url(`/entities/${flow.entityName}/flows/${flow.flowName}/${flowType}`)).share(); resp.subscribe(() => { this.entities.forEach((entity: Entity) => { if (entity.name === flow.entityName) { @@ -187,7 +238,7 @@ export class EntitiesService { runInputFlow(flow: Flow, mlcpOptions: any) { const url = this.url(`/entities/${flow.entityName}/flows/input/${flow.flowName}/run`); - let options = { + const options = { mlcpPath: this.settingsService.mlcpPath, mlcpOptions: mlcpOptions }; @@ -217,10 +268,10 @@ export class EntitiesService { if (entity.definition && entity.definition.properties) { entity.definition.properties.forEach((property: PropertyType) => { - if (property.$ref && !property.$ref.startsWith('#/definitions/')) { + if (property.$ref && !property.$ref.startsWith(EntitiesService.entityRefPrefix)) { this.externalRefDataTypes.push(property.$ref); } else if (property.datatype === 'array') { - if (property.items && property.items.$ref && !property.items.$ref.startsWith('#/definitions/')) { + if (property.items && property.items.$ref && !property.items.$ref.startsWith(EntitiesService.entityRefPrefix)) { this.externalRefDataTypes.push(property.items.$ref); } } diff --git a/quick-start/src/main/ui/app/entities/entity.model.ts b/quick-start/src/main/ui/app/entities/entity.model.ts index f39052523d..773fb6f2b1 100644 --- a/quick-start/src/main/ui/app/entities/entity.model.ts +++ b/quick-start/src/main/ui/app/entities/entity.model.ts @@ -1,5 +1,6 @@ import { InfoType } from './info.model'; import { DefinitionType } from './definition.model'; +import { DefinitionsType } from './definitions.model'; import { HubUIData } from './hubuidata.model'; import { Point } from '../entity-modeler/math-helper'; import { Flow } from './flow.model'; @@ -11,6 +12,7 @@ export class Entity { info: InfoType; definition: DefinitionType; + definitions: DefinitionsType; inputFlows: Array; harmonizeFlows: Array; @@ -91,6 +93,10 @@ export class Entity { this.transform = `translate(${this.x}, ${this.y}) scale(${this.scale})`; } + set definitionsType(_definitions: DefinitionsType) { + this.definitions = _definitions; + } + constructor() {} fromJSON(json) { @@ -108,6 +114,13 @@ export class Entity { this.definition = new DefinitionType().fromJSON(json.definition); } + if (json.definitions) { + this.definitions = new DefinitionsType().fromJSON(json.definitions); + if (!json.definition) { + this.definition = this.definitions.get(this.name); + } + } + this.inputFlows = []; if (json.inputFlows && _.isArray(json.inputFlows)) { for (let flow of json.inputFlows) { @@ -122,6 +135,7 @@ export class Entity { } } + return this; } diff --git a/quick-start/src/test/java/com/marklogic/quickstart/service/EntityManagerServiceTest.java b/quick-start/src/test/java/com/marklogic/quickstart/service/EntityManagerServiceTest.java index f49bd4f88e..91bff761ec 100644 --- a/quick-start/src/test/java/com/marklogic/quickstart/service/EntityManagerServiceTest.java +++ b/quick-start/src/test/java/com/marklogic/quickstart/service/EntityManagerServiceTest.java @@ -42,10 +42,12 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @ExtendWith(SpringExtension.class) @@ -141,9 +143,9 @@ public void saveEntity() throws IOException { List entities = entityMgrService.getEntities(); assertEquals(2, entities.size()); - String[] expected = {ENTITY, ENTITY2}; - String[] actual = { entities.get(0).getName(), entities.get(1).getName() }; - assertArrayEquals(expected, actual); + List expected = Arrays.asList(ENTITY, ENTITY2); + List actual = Arrays.asList(entities.get(0).getName(), entities.get(1).getName()); + assertTrue(expected.containsAll(actual)); } @Test @@ -224,9 +226,10 @@ public void changeEntityName() throws IOException { // Load the entity, then check the flows to make sure they know the right entity name final String FLOW_NAME = "sjs-json-input-flow"; List inputFlows = entities.get(0).getInputFlows(); + List flowNameList = inputFlows.stream().map(flow -> flow.flowName).collect(Collectors.toList()); assertEquals(RENAMED_ENTITY, inputFlows.get(0).entityName); - assertEquals(FLOW_NAME, inputFlows.get(0).flowName); + assertTrue(flowNameList.contains(FLOW_NAME)); assertEquals(FlowType.INPUT, inputFlows.get(0).flowType); //cleanup.