diff --git a/README.md b/README.md index 1c017b8..d164cfe 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,9 @@ assertEquals(onePlusOne, anotherOnePlusOne) // Top level type_universe ::= ... definition ::= '(' 'define' symbol ')' -stmt ::= | +stmt ::= | | + +include_file ::= `(include_file )` // Domain domain_definition ::= | @@ -400,6 +402,43 @@ Unlike record elements, product element defintions must include identifiers. (product int_pair first::int second::int) ``` +#### Type Domain Includes + +It is possible to split type universes among multiple files, which allows type domains defined in another project to be +permuted. For example: + +``` +// root.ion: +(include_file "sibling.ion") +(include_file "sub-dir/thing.ion") +(include_file "/other-project/toy-ast.ion") +``` + +While discussing further details, it is helpful to introduce two terms: an "includer" is a file which includes another +using `include_file`, and the "includee" is a file which is included. + +The `root.ion` universe will contain all type domains from all includees and may still define additional type domains +of its own. + +`include_file` statements are allowed in includees. Any attempt to include a file that has already been seen will be +ignored. + +When resolving the file to include, the following locations are searched: + +- The directory containing the includer (if the path to the includee does not start with `/`) +- The directory containing the "main" type universe that was passed to `pig` on the command-line. +- Any directories specified with the `--include` or `-I` arguments, in the order they were specified. + +The first matching file found wins and any other matching files ignored. + +Note that paths starting with `/` do not actually refer to the root of any file system, but instead are treated as +relative to the include directories. + +Paths specified with `include_file` may only contain alphanumeric or one of the following characters: +`-`, `_`, `.` or `/`. Additionally, two consecutive periods ".." (i.e. a parent directory) are not allowed. + +Lastly, `include_file` can only be used at the top-level within a `.ion` file. It is not allowed anywhere within a +`(domain ...)` clause. #### Using PIG In Your Project @@ -408,16 +447,14 @@ Unlike record elements, product element defintions must include identifiers. At build time and before compilation of your application or library, the following should be executed: ``` -pig \ - -u \ - -t kotlin \ - -n \ - -o path/to/package/ +pig -u -t kotlin -n -o [ -I ] ``` - ``: path to the Ion text file containing the type universe -- ``: path to the file for the generated code +- ``: path to the file for the generated code - ``: the name used in the `package` statement at the top of the output file +- ``: search path to external include directory (optional). Can be specified more than once, +i.e. `pig ... -I -I -I ` Execute: `pig --help` for all command-line options. diff --git a/pig/build.gradle b/pig/build.gradle index 02b0347..3e6d91b 100644 --- a/pig/build.gradle +++ b/pig/build.gradle @@ -43,6 +43,7 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.2' testImplementation 'org.junit.jupiter:junit-jupiter-params:5.6.2' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.6.2' + testImplementation 'com.google.jimfs:jimfs:1.2' } application { diff --git a/pig/src/org/partiql/pig/cmdline/Command.kt b/pig/src/org/partiql/pig/cmdline/Command.kt index fa5ede4..1042b97 100644 --- a/pig/src/org/partiql/pig/cmdline/Command.kt +++ b/pig/src/org/partiql/pig/cmdline/Command.kt @@ -15,10 +15,36 @@ package org.partiql.pig.cmdline -import java.io.File +import java.nio.file.Path +/** Represents command line options specified by the user. */ sealed class Command { + + /** The `--help` command. */ object ShowHelp : Command() + + /** + * Returned by [CommandLineParser] when the user has specified invalid command-line arguments + * + * - [message]: an error message to be displayed to the user. + */ data class InvalidCommandLineArguments(val message: String) : Command() - data class Generate(val typeUniverseFile: File, val outputFile: File, val target: TargetLanguage) : Command() -} \ No newline at end of file + + /** + * Contains the details of a *valid* command-line specified by the user. + * + * - [typeUniverseFilePath]: the path to the type universe file. + * - [outputFilePath]: the path to the output file. (This makes the assumption that there is only one output file.) + * - [includePaths]: directories to be searched when looking for files included with `include_file`. + * - [target]: specifies the target language and any other parameters unique to the target language. + */ + data class Generate( + val typeUniverseFilePath: Path, + val outputFilePath: Path, + val includePaths: List, + val target: TargetLanguage + ) : Command() +} + + + diff --git a/pig/src/org/partiql/pig/cmdline/CommandLineParser.kt b/pig/src/org/partiql/pig/cmdline/CommandLineParser.kt index e8f11c7..d801b70 100644 --- a/pig/src/org/partiql/pig/cmdline/CommandLineParser.kt +++ b/pig/src/org/partiql/pig/cmdline/CommandLineParser.kt @@ -16,8 +16,9 @@ package org.partiql.pig.cmdline import joptsimple.* -import java.io.File import java.io.PrintStream +import java.nio.file.Path +import java.nio.file.Paths class CommandLineParser { @@ -30,7 +31,7 @@ class CommandLineParser { HTML } - private object languageTargetTypeValueConverter : ValueConverter { + private object LanguageTargetTypeValueConverter : ValueConverter { private val lookup = LanguageTargetType.values().associateBy { it.name.toLowerCase() } override fun convert(value: String?): LanguageTargetType { @@ -47,6 +48,12 @@ class CommandLineParser { } } + private object PathValueConverter : ValueConverter { + override fun convert(value: String?): Path = Paths.get(value).toAbsolutePath().normalize() + override fun valueType(): Class = Path::class.java + override fun valuePattern(): String? = null + } + private val formatter = object : BuiltinHelpFormatter(120, 2) { override fun format(options: MutableMap?): String { return """PartiQL I.R. Generator @@ -56,6 +63,8 @@ class CommandLineParser { | | --target=kotlin requires --namespace= | --target=custom requires --template= + | All paths specified in these command-line options are relative to the current working + | directory by default. | |Examples: | @@ -65,7 +74,9 @@ class CommandLineParser { """.trimMargin() } } - private val optParser = OptionParser().also { it.formatHelpWith(formatter) } + private val optParser = OptionParser().apply { + formatHelpWith(formatter) + } private val helpOpt = optParser.acceptsAll(listOf("help", "h", "?"), "prints this help") @@ -73,18 +84,22 @@ class CommandLineParser { private val universeOpt = optParser.acceptsAll(listOf("universe", "u"), "Type universe input file") .withRequiredArg() - .ofType(File::class.java) + .withValuesConvertedBy(PathValueConverter) .required() + private val includeSearchRootOpt = optParser.acceptsAll(listOf("include", "I"), "Include search path") + .withRequiredArg() + .withValuesConvertedBy(PathValueConverter) + .describedAs("Search path for files included with include_file. May be specified multiple times.") + private val outputOpt = optParser.acceptsAll(listOf("output", "o"), "Generated output file") .withRequiredArg() - .ofType(File::class.java) + .withValuesConvertedBy(PathValueConverter) .required() private val targetTypeOpt = optParser.acceptsAll(listOf("target", "t"), "Target language") .withRequiredArg() - //.ofType(LanguageTargetType::class.java) - .withValuesConvertedBy(languageTargetTypeValueConverter) + .withValuesConvertedBy(LanguageTargetTypeValueConverter) .required() private val namespaceOpt = optParser.acceptsAll(listOf("namespace", "n"), "Namespace for generated code") @@ -93,7 +108,7 @@ class CommandLineParser { private val templateOpt = optParser.acceptsAll(listOf("template", "e"), "Path to an Apache FreeMarker template") .withOptionalArg() - .ofType(File::class.java) + .withValuesConvertedBy(PathValueConverter) /** @@ -116,9 +131,15 @@ class CommandLineParser { optSet.has(helpOpt) -> Command.ShowHelp else -> { // !! is fine in this case since we define these options as .required() above. - val typeUniverseFile: File = optSet.valueOf(universeOpt)!! + val typeUniverseFile: Path = optSet.valueOf(universeOpt)!! val targetType = optSet.valueOf(targetTypeOpt)!! - val outputFile: File = optSet.valueOf(outputOpt)!! + val outputFile: Path = optSet.valueOf(outputOpt)!! + + // Always add the parent of the file containing the main type universe as an include root. + val includeSearchRoots = listOf( + typeUniverseFile.parent, + *optSet.valuesOf(includeSearchRootOpt)!!.toTypedArray() + ) if (targetType.requireNamespace) { if (!optSet.has(namespaceOpt)) { @@ -145,13 +166,11 @@ class CommandLineParser { LanguageTargetType.CUSTOM -> TargetLanguage.Custom(optSet.valueOf(templateOpt)) } - Command.Generate(typeUniverseFile, outputFile, target) + Command.Generate(typeUniverseFile, outputFile, includeSearchRoots, target) } } } catch(ex: OptionException) { Command.InvalidCommandLineArguments(ex.message!!) } - } - -} \ No newline at end of file +} diff --git a/pig/src/org/partiql/pig/cmdline/TargetLanguage.kt b/pig/src/org/partiql/pig/cmdline/TargetLanguage.kt index 3e7dc5c..1be15e4 100644 --- a/pig/src/org/partiql/pig/cmdline/TargetLanguage.kt +++ b/pig/src/org/partiql/pig/cmdline/TargetLanguage.kt @@ -16,9 +16,10 @@ package org.partiql.pig.cmdline import java.io.File +import java.nio.file.Path sealed class TargetLanguage { data class Kotlin(val namespace: String) : TargetLanguage() - data class Custom(val templateFile: File) : TargetLanguage() + data class Custom(val templateFile: Path) : TargetLanguage() object Html : TargetLanguage() } \ No newline at end of file diff --git a/pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt b/pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt index 5d7b20e..556651e 100644 --- a/pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt +++ b/pig/src/org/partiql/pig/domain/model/SemanticErrorContext.kt @@ -15,12 +15,12 @@ package org.partiql.pig.domain.model -import com.amazon.ionelement.api.IonLocation import com.amazon.ionelement.api.MetaContainer -import com.amazon.ionelement.api.location -import org.partiql.pig.errors.PigException +import org.partiql.pig.domain.parser.SourceLocation +import org.partiql.pig.domain.parser.sourceLocation import org.partiql.pig.errors.ErrorContext import org.partiql.pig.errors.PigError +import org.partiql.pig.errors.PigException /** * Encapsulates all error context information in an easily testable way. @@ -66,16 +66,16 @@ sealed class SemanticErrorContext(val msgFormatter: () -> String): ErrorContext : SemanticErrorContext({ "Cannot remove built-in type '$typeName'" }) data class DuplicateTypeDomainName(val domainName: String) - : SemanticErrorContext({ "Duplicate type domain tag: '${domainName} "}) + : SemanticErrorContext({ "Duplicate type domain tag: '${domainName}' "}) data class DuplicateRecordElementTag(val elementName: String) - : SemanticErrorContext({ "Duplicate record element tag: '${elementName} "}) + : SemanticErrorContext({ "Duplicate record element tag: '${elementName}' "}) data class DuplicateElementIdentifier(val elementName: String) - : SemanticErrorContext({ "Duplicate element identifier: '${elementName} "}) + : SemanticErrorContext({ "Duplicate element identifier: '${elementName}' "}) data class NameAlreadyUsed(val name: String, val domainName: String) - : SemanticErrorContext({ "Name '$name' was previously used in the `$domainName` type domain" }) + : SemanticErrorContext({ "Name '$name' was previously used in the '$domainName' type domain" }) data class CannotRemoveNonExistentSumVariant(val sumTypeName: String, val variantName: String) : SemanticErrorContext({ "Permuted sum type '${sumTypeName}' tries to remove variant '${variantName}' which " + @@ -109,9 +109,9 @@ sealed class SemanticErrorContext(val msgFormatter: () -> String): ErrorContext * Shortcut for throwing [PigException] with the specified metas and [PigError]. */ fun semanticError(blame: MetaContainer, context: ErrorContext): Nothing = - semanticError(blame.location, context) + semanticError(blame.sourceLocation, context) /** * Shortcut for throwing [PigException] with the specified metas and [PigError]. */ -fun semanticError(blame: IonLocation?, context: ErrorContext): Nothing = +fun semanticError(blame: SourceLocation?, context: ErrorContext): Nothing = throw PigException(PigError(blame, context)) diff --git a/pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt b/pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt index 7c4ddba..8dcbe10 100644 --- a/pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt +++ b/pig/src/org/partiql/pig/domain/parser/ParserErrorContext.kt @@ -15,14 +15,12 @@ package org.partiql.pig.domain.parser -import com.amazon.ionelement.api.IonElement -import com.amazon.ionelement.api.IonLocation -import com.amazon.ionelement.api.location import com.amazon.ionelement.api.ElementType import com.amazon.ionelement.api.IonElementException -import org.partiql.pig.errors.PigException import org.partiql.pig.errors.ErrorContext import org.partiql.pig.errors.PigError +import org.partiql.pig.errors.PigException +import javax.swing.text.html.parser.Parser /** * Variants of [ParserErrorContext] contain details about various parse errors that can be encountered @@ -33,7 +31,7 @@ import org.partiql.pig.errors.PigError sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext { override val message: String get() = msgFormatter() - /** Indicates that an []IonElectrolyteException] was thrown during parsing of a type universe. */ + /** Indicates that an [IonElementException] was thrown during parsing of a type universe. */ data class IonElementError(val ex: IonElementException) : ParserErrorContext({ ex.message!! }) { // This is for unit tests... we don't include IonElectrolyteException here since it doesn't implement @@ -51,15 +49,29 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext { data class InvalidTopLevelTag(val tag: String) : ParserErrorContext({ "Invalid top-level tag: '$tag'"}) - data class InvalidSumLevelTag(val tag: String) - : ParserErrorContext({ "Invalid tag for sum variant: '$tag'"}) - data class InvalidPermutedDomainTag(val tag: String) : ParserErrorContext({ "Invalid tag for permute_domain body: '$tag'"}) data class InvalidWithSumTag(val tag: String) : ParserErrorContext({ "Invalid tag for with body: '$tag'"}) + data class IncludeFileNotFound(val includeFilePath: String, val searchedPaths: List ) + : ParserErrorContext( + { + "Could not locate include file '$includeFilePath' at any of the following locations:\n" + + searchedPaths.joinToString("\n") + } + ) + + data class IncludeFilePathContainsIllegalCharacter(val c: Char) + : ParserErrorContext({ "Illegal character '$c' in include_file path" }) + + object IncludeFilePathContainsParentDirectory + : ParserErrorContext({ "include_file path contained parent directory, i.e. \"..\"" }) + + object IncludeFilePathMustNotStartWithRoot + : ParserErrorContext({ "include_file path must not start with '/'" }) + data class ExpectedTypeReferenceArityTag(val tag: String) : ParserErrorContext({ "Expected '*' or '?' but found '$tag'"}) @@ -79,8 +91,7 @@ sealed class ParserErrorContext(val msgFormatter: () -> String): ErrorContext { : ParserErrorContext({ "Element has multiple name annotations"}) } - -fun parseError(blame: IonLocation?, context: ErrorContext): Nothing = +fun parseError(blame: SourceLocation?, context: ErrorContext): Nothing = PigError(blame, context).let { throw when (context) { is ParserErrorContext.IonElementError -> { @@ -91,8 +102,3 @@ fun parseError(blame: IonLocation?, context: ErrorContext): Nothing = } } -fun parseError(blame: IonElement, context: ErrorContext): Nothing { - val loc = blame.metas.location - parseError(loc, context) -} - diff --git a/pig/src/org/partiql/pig/domain/parser/SourceLocation.kt b/pig/src/org/partiql/pig/domain/parser/SourceLocation.kt new file mode 100644 index 0000000..787d205 --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/SourceLocation.kt @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser + +import com.amazon.ionelement.api.IonLocation +import com.amazon.ionelement.api.MetaContainer + +/** + * Used to construct helpful error messages for the end-user, who will be able to the location of a given + * error which includes the file name, line & column. + */ +data class SourceLocation(val path: String, val location: IonLocation) { + override fun toString(): String { + return "$path:$location" + } +} + +internal const val SOURCE_LOCATION_META_TAG = "\$pig_source_location" + +internal val MetaContainer.sourceLocation + get() = this[SOURCE_LOCATION_META_TAG] as? SourceLocation diff --git a/pig/src/org/partiql/pig/domain/parser/TypeDomainParser.kt b/pig/src/org/partiql/pig/domain/parser/TypeDomainParser.kt index fb23328..6598f3c 100644 --- a/pig/src/org/partiql/pig/domain/parser/TypeDomainParser.kt +++ b/pig/src/org/partiql/pig/domain/parser/TypeDomainParser.kt @@ -15,275 +15,398 @@ package org.partiql.pig.domain.parser -import com.amazon.ionelement.api.* import com.amazon.ion.IonReader import com.amazon.ion.system.IonReaderBuilder -import org.partiql.pig.domain.model.* - -/** Parses a type universe contained in [universeText]. */ -fun parseTypeUniverse(universeText: String) = - IonReaderBuilder.standard().build(universeText).use { - parseTypeUniverse(it) +import com.amazon.ionelement.api.AnyElement +import com.amazon.ionelement.api.IonElement +import com.amazon.ionelement.api.IonElementException +import com.amazon.ionelement.api.IonElementLoaderOptions +import com.amazon.ionelement.api.IonLocation +import com.amazon.ionelement.api.MetaContainer +import com.amazon.ionelement.api.SexpElement +import com.amazon.ionelement.api.SymbolElement +import com.amazon.ionelement.api.createIonElementLoader +import com.amazon.ionelement.api.emptyMetaContainer +import com.amazon.ionelement.api.head +import com.amazon.ionelement.api.location +import com.amazon.ionelement.api.metaContainerOf +import com.amazon.ionelement.api.tag +import com.amazon.ionelement.api.tail +import org.partiql.pig.domain.parser.include.IncludeCycleHandler +import org.partiql.pig.domain.parser.include.IncludeResolutionException +import org.partiql.pig.domain.parser.include.IncludeResolver +import org.partiql.pig.domain.parser.include.InputSource +import org.partiql.pig.domain.model.Arity +import org.partiql.pig.domain.model.DataType +import org.partiql.pig.domain.model.NamedElement +import org.partiql.pig.domain.model.PermutedDomain +import org.partiql.pig.domain.model.PermutedSum +import org.partiql.pig.domain.model.Statement +import org.partiql.pig.domain.model.Transform +import org.partiql.pig.domain.model.TupleType +import org.partiql.pig.domain.model.TypeDomain +import org.partiql.pig.domain.model.TypeRef +import org.partiql.pig.domain.model.TypeUniverse +import org.partiql.pig.errors.ErrorContext +import org.partiql.pig.errors.PigError +import org.partiql.pig.errors.PigException +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path + +/** + * Parses the type universe specified in [mainTypeUniversePath]. + * + * @param mainTypeUniversePath Specifies the "main" type universe file where parsing will begin. The [FileSystem] + * of this path (as returned by [Path.getFileSystem]) will be utilized when searching for and reading files referenced + * by any `include_file` statement. Thus, it is possible to read type domains from a real file system for + * production or from an in-memory file system for tests. In the future this may also form the basis for reading type + * universes from `.jar` files. + * + * To read the type universe from a real file system, obtain the [Path] from the [java.nio.file.FileSystem.getPath] + * method on the default file system ([java.nio.file.FileSystems.getDefault]) or [java.nio.file.Paths.get]. + * + * @param includePaths specifies a list of directories to be searched (in order) when looking for files specified + * with `include_file`. The parent directory of [mainTypeUniversePath] is always added as a source root. + */ +internal fun parseMainTypeUniverse( + mainTypeUniversePath: Path, + includePaths: List +): TypeUniverse { + require(includePaths.all { it.fileSystem === mainTypeUniversePath.fileSystem}) { + "The mainTypeUniversePath and all includePaths must be from the same FileSystem" } -/** Parses a type universe in the specified [IonReader]. */ -fun parseTypeUniverse(reader: IonReader): TypeUniverse { - val elementLoader = createIonElementLoader(IonElementLoaderOptions(includeLocationMeta = true)) - - val domains = try { - val topLevelElements = elementLoader.loadAllElements(reader) - topLevelElements.map { topLevelValue -> - val topLevelSexp = topLevelValue.asSexp() - when (topLevelSexp.tag) { - "define" -> parseDefine(topLevelSexp) - "transform" -> parseTransform(topLevelSexp) - else -> parseError( - topLevelSexp.head, - ParserErrorContext.InvalidTopLevelTag(topLevelSexp.tag)) + val resolver = IncludeResolver( + searchDirs = listOf(mainTypeUniversePath.parent, *includePaths.toTypedArray()), + fileSystem = mainTypeUniversePath.fileSystem + ) + val includeFileManager = IncludeCycleHandler(mainTypeUniversePath, resolver) + val source = InputSource(mainTypeUniversePath.toAbsolutePath().normalize(), Files.newInputStream(mainTypeUniversePath)) + return TypeUniverseParser(source, includeFileManager).parse() +} + +/** + * Parses a type universe file. + * + * There must exist only one instance of this class per type universe file. (i.e. every `include_file` statement + * results in new instance of this class. + * + * When an `include_file` statement is encountered, [includeCycleHandler] will be utilized to deal with include + * search paths and include cycles. + */ +internal class TypeUniverseParser( + private val source: InputSource, + private val includeCycleHandler: IncludeCycleHandler +) { + /** Parses a type universe in the specified [IonReader]. */ + fun parse(): TypeUniverse { + val elementLoader = createIonElementLoader(IonElementLoaderOptions(includeLocationMeta = true)) + return IonReaderBuilder.standard().build(source.inputStream).use { reader: IonReader -> + val domains = try { + val topLevelElements = elementLoader.loadAllElements(reader) + topLevelElements.flatMap { topLevelValue -> + val topLevelSexp = topLevelValue.asSexp() + when (topLevelSexp.tag) { + "define" -> listOf(parseDefine(topLevelSexp)) + "transform" -> listOf(parseTransform(topLevelSexp)) + "include_file" -> parseIncludeFile(topLevelSexp) + else -> parseError( + topLevelSexp.head, + ParserErrorContext.InvalidTopLevelTag(topLevelSexp.tag)) + } + } + } catch (iee: IonElementException) { + parseError(iee.location?.toSourceLocation(), ParserErrorContext.IonElementError(iee)) + } + TypeUniverse(domains) } - } } - catch(iee: IonElementException) { - parseError(iee.location, ParserErrorContext.IonElementError(iee)) + + private fun parseError(blame: IonElement, context: ErrorContext): Nothing { + val loc = blame.metas.location?.toSourceLocation() + parseError(loc, context) } - return TypeUniverse(domains) -} + private fun IonLocation.toSourceLocation() = SourceLocation(source.absolutePath.toString(), this) -private fun parseDefine(sexp: SexpElement): Statement { - requireArityForTag(sexp, 2) - val args = sexp.tail // Skip tag - val name = args.head.symbolValue - val valueSexp = args.tail.head.asSexp() - - return when (valueSexp.tag) { - "domain" -> parseTypeDomain(name, valueSexp) - "permute_domain" -> parsePermuteDomain(name, valueSexp) - else -> parseError( - valueSexp.head, - ParserErrorContext.UnknownConstructor(valueSexp.tag)) - } -} + private fun MetaContainer.toSourceLocationMetas(): MetaContainer = this.location?.let { + metaContainerOf(SOURCE_LOCATION_META_TAG to it.toSourceLocation()) + } ?: emptyMetaContainer() -fun parseTransform(sexp: SexpElement): Statement { - requireArityForTag(sexp, 2) - return Transform( - sourceDomainTag = sexp.values[1].symbolValue, - destinationDomainTag = sexp.values[2].symbolValue, - metas = sexp.metas - ) -} + private fun parseIncludeFile(sexp: SexpElement): List { + requireArityForTag(sexp, 1) + val includeePath = sexp.tail.single().asString() + val includeePathText = includeePath.textValue.trim() -private fun parseTypeDomain(domainName: String, sexp: SexpElement): TypeDomain { - val args = sexp.tail // Skip tag - //val typesSexps = args.tail + includeePathText.forEach { + if (!isValidPathChar(it)) { + parseError(includeePath, ParserErrorContext.IncludeFilePathContainsIllegalCharacter(it)) + } + } - val userTypes = args.map { tlv -> - val tlvs = tlv.asSexp() - parseDomainLevelStatement(tlvs) - }.toList() + if(includeePathText.contains("..")) { + parseError(includeePath, ParserErrorContext.IncludeFilePathContainsParentDirectory) + } - return TypeDomain( - tag = domainName, - userTypes = userTypes, - metas = sexp.metas) -} + if(includeePathText.startsWith('/')) { + parseError(includeePath, ParserErrorContext.IncludeFilePathMustNotStartWithRoot) + } -private fun parseDomainLevelStatement(tlvs: SexpElement): DataType.UserType { - return when (tlvs.tag) { - "product" -> parseProductBody(tlvs.tail, tlvs.metas) - "record" -> parseRecordBody(tlvs.tail, tlvs.metas) - "sum" -> parseSum(tlvs) - else -> parseError(tlvs.head, ParserErrorContext.InvalidDomainLevelTag(tlvs.tag)) + return try { + includeCycleHandler.parseIncludedTypeUniverse(includeePath.textValue, source.absolutePath) + } catch(ex: IncludeResolutionException) { + throw PigException( + PigError( + sexp.metas.location?.toSourceLocation(), + ParserErrorContext.IncludeFileNotFound(ex.inputFilePathString, ex.consideredFilePaths) + ), + ex + ) + } } -} -private fun parseTypeRefs(values: List): List = - values.map { parseSingleTypeRef(it) } - -// Parses a sum-variant product or record (depending on the syntax used) -private fun parseVariant( - bodyArguments: List, - metas: MetaContainer -): DataType.UserType.Tuple { - val elements = bodyArguments.tail - - // If there are no elements, definitely not a record. - val isRecord = if(elements.none()) { - false - } else { - // if the head element is an s-exp that does not start with `?` or `*` then we're parsing a record - when (val headElem = elements.head) { - is SexpElement -> { - when (headElem.values.head.symbolValue) { - "?", "*" -> false - else -> true - } - } - is SymbolElement -> false - else -> parseError(elements.head, ParserErrorContext.ExpectedSymbolOrSexp(elements.head.type)) + private fun parseDefine(sexp: SexpElement): Statement { + requireArityForTag(sexp, 2) + val args = sexp.tail // Skip tag + val name = args.head.symbolValue + val valueSexp = args.tail.head.asSexp() + + return when (valueSexp.tag) { + "domain" -> parseTypeDomain(name, valueSexp) + "permute_domain" -> parsePermuteDomain(name, valueSexp) + else -> parseError( + valueSexp.head, + ParserErrorContext.UnknownConstructor(valueSexp.tag)) } } - return when { - isRecord -> { - parseRecordBody(bodyArguments, metas) - } else -> { - parseProductBody(bodyArguments, metas) - } + private fun parseTransform(sexp: SexpElement): Statement { + requireArityForTag(sexp, 2) + return Transform( + sourceDomainTag = sexp.values[1].symbolValue, + destinationDomainTag = sexp.values[2].symbolValue, + metas = sexp.metas.toSourceLocationMetas() + ) } -} -private fun parseProductBody(bodyArguments: List, metas: MetaContainer): DataType.UserType.Tuple { - val typeName = bodyArguments.head.symbolValue + private fun parseTypeDomain(domainName: String, sexp: SexpElement): TypeDomain { + val args = sexp.tail // Skip tag + //val typesSexps = args.tail - val namedElements = parseProductElements(bodyArguments.tail) + val userTypes = args.map { tlv -> + val tlvs = tlv.asSexp() + parseDomainLevelStatement(tlvs) + }.toList() - return DataType.UserType.Tuple(typeName, TupleType.PRODUCT, namedElements, metas) -} + return TypeDomain( + tag = domainName, + userTypes = userTypes, + metas = sexp.metas.toSourceLocationMetas()) + } -private fun parseProductElements(values: List): List = - values.map { - val identifier = when(it.annotations.size) { - // TODO: add tests for these errrors - 0 -> parseError(it, ParserErrorContext.MissingElementIdentifierAnnotation) - 1 -> it.annotations.single() - else -> parseError(it, ParserErrorContext.MultipleElementIdentifierAnnotations) + private fun parseDomainLevelStatement(tlvs: SexpElement): DataType.UserType { + return when (tlvs.tag) { + "product" -> parseProductBody(tlvs.tail, tlvs.metas.toSourceLocationMetas()) + "record" -> parseRecordBody(tlvs.tail, tlvs.metas.toSourceLocationMetas()) + "sum" -> parseSum(tlvs) + else -> parseError(tlvs.head, ParserErrorContext.InvalidDomainLevelTag(tlvs.tag)) } - - NamedElement( - tag = "", // NOTE: tag is not used in the s-expression representation of products! - identifier = identifier, - typeReference = parseSingleTypeRef(it), - metas = it.metas) } -private fun parseRecordBody(bodyArguments: List, metas: MetaContainer): DataType.UserType.Tuple { - val typeName = bodyArguments.head.symbolValue - val namedElements = parseRecordElements(bodyArguments.tail) - return DataType.UserType.Tuple(typeName, TupleType.RECORD, namedElements, metas) -} - -fun parseRecordElements(elementSexps: List): List = - elementSexps.asSequence() - .map { it.asSexp() } - .map { elementSexp -> - if(elementSexp.values.size != 2) { - parseError(elementSexp, ParserErrorContext.InvalidArity(2, elementSexp.size)) + // Parses a sum-variant product or record (depending on the syntax used) + private fun parseVariant( + bodyArguments: List, + metas: MetaContainer + ): DataType.UserType.Tuple { + val elements = bodyArguments.tail + + // If there are no elements, definitely not a record. + val isRecord = if(elements.none()) { + false + } else { + // if the head element is an s-exp that does not start with `?` or `*` then we're parsing a record + when (val headElem = elements.head) { + is SexpElement -> { + when (headElem.values.head.symbolValue) { + "?", "*" -> false + else -> true + } + } + is SymbolElement -> false + else -> parseError(elements.head, ParserErrorContext.ExpectedSymbolOrSexp(elements.head.type)) } - val tag = elementSexp.values[0].symbolValue - val identifier = when(elementSexp.annotations.size) { - 0 -> tag - 1 -> elementSexp.annotations.single() - else -> parseError(elementSexp, ParserErrorContext.MultipleElementIdentifierAnnotations) + } + + return when { + isRecord -> { + parseRecordBody(bodyArguments, metas) + } else -> { + parseProductBody(bodyArguments, metas) } - val typeRef = parseSingleTypeRef(elementSexp.values[1]) - NamedElement( - identifier = identifier, - tag = tag, - typeReference = typeRef, - metas = elementSexp.metas) } - .toList() + } + + private fun parseProductBody(bodyArguments: List, metas: MetaContainer): DataType.UserType.Tuple { + val typeName = bodyArguments.head.symbolValue -private fun parseSum(sexp: SexpElement): DataType.UserType.Sum { - val args = sexp.tail // Skip tag - val typeName = args.head.symbolValue + val namedElements = parseProductElements(bodyArguments.tail) - val variants = args.tail.map { - parseSumVariant(it.asSexp()) + return DataType.UserType.Tuple(typeName, TupleType.PRODUCT, namedElements, metas) } - return DataType.UserType.Sum(typeName, variants.toList(), sexp.metas) -} + private fun parseProductElements(values: List): List = + values.map { + val identifier = when(it.annotations.size) { + // TODO: add tests for these errrors + 0 -> parseError(it, ParserErrorContext.MissingElementIdentifierAnnotation) + 1 -> it.annotations.single() + else -> parseError(it, ParserErrorContext.MultipleElementIdentifierAnnotations) + } -private fun parseSumVariant(sexp: SexpElement): DataType.UserType.Tuple { - return parseVariant(sexp.values, sexp.metas) -} + NamedElement( + tag = "", // NOTE: tag is not used in the s-expression representation of products! + identifier = identifier, + typeReference = parseSingleTypeRef(it), + metas = it.metas.toSourceLocationMetas()) + } + + private fun parseRecordBody(bodyArguments: List, metas: MetaContainer): DataType.UserType.Tuple { + val typeName = bodyArguments.head.symbolValue + val namedElements = parseRecordElements(bodyArguments.tail) + return DataType.UserType.Tuple(typeName, TupleType.RECORD, namedElements, metas) + } -private fun parseSingleTypeRef(typeRefExp: IonElement): TypeRef { - val metas = typeRefExp.metas - return when (typeRefExp) { - is SymbolElement -> TypeRef(typeRefExp.textValue, Arity.Required, metas) - is SexpElement -> { - when (typeRefExp.tag) { - "?" -> { - requireArityForTag(typeRefExp, 1) - val typeName = typeRefExp.tail.head.symbolValue - TypeRef(typeName, Arity.Optional, metas) + private fun parseRecordElements(elementSexps: List): List = + elementSexps.asSequence() + .map { it.asSexp() } + .map { elementSexp -> + if(elementSexp.values.size != 2) { + parseError(elementSexp, ParserErrorContext.InvalidArity(2, elementSexp.size)) } - "*" -> { - requireArityForTag(typeRefExp, IntRange(2, 3)) - val typeName = typeRefExp.tail.head.symbolValue - val arityRange = typeRefExp.tail.tail - val minArity = arityRange.head.longValue - TypeRef(typeName, Arity.Variadic(minArity.toInt()), metas) + val tag = elementSexp.values[0].symbolValue + val identifier = when(elementSexp.annotations.size) { + 0 -> tag + 1 -> elementSexp.annotations.single() + else -> parseError(elementSexp, ParserErrorContext.MultipleElementIdentifierAnnotations) } - else -> parseError(typeRefExp.head, ParserErrorContext.ExpectedTypeReferenceArityTag(typeRefExp.tag)) + val typeRef = parseSingleTypeRef(elementSexp.values[1]) + NamedElement( + identifier = identifier, + tag = tag, + typeReference = typeRef, + metas = elementSexp.metas.toSourceLocationMetas()) } + .toList() + + private fun parseSum(sexp: SexpElement): DataType.UserType.Sum { + val args = sexp.tail // Skip tag + val typeName = args.head.symbolValue + + val variants = args.tail.map { + parseSumVariant(it.asSexp()) } - else -> parseError(typeRefExp, ParserErrorContext.ExpectedSymbolOrSexp(typeRefExp.type)) + + return DataType.UserType.Sum(typeName, variants.toList(), sexp.metas.toSourceLocationMetas()) + } + + private fun parseSumVariant(sexp: SexpElement): DataType.UserType.Tuple { + return parseVariant(sexp.values, sexp.metas.toSourceLocationMetas()) } -} -private fun parsePermuteDomain(domainName: String, sexp: SexpElement): PermutedDomain { - val args = sexp.tail // Skip tag - - val permutingDomain = args.head.symbolValue - val removedTypes = mutableListOf() - val newTypes = mutableListOf() - val permutedSums = mutableListOf() - - val alterSexps = args.tail - alterSexps.map { it.asSexp() }.forEach { alterSexp -> - when(alterSexp.head.symbolValue) { - "with" -> permutedSums.add(parseWithSum(alterSexp)) - "exclude" -> alterSexp.tail.mapTo(removedTypes) { it.symbolValue } - "include" -> alterSexp.tail.mapTo(newTypes) { parseDomainLevelStatement(it.asSexp()) } - else -> parseError(alterSexp, ParserErrorContext.InvalidPermutedDomainTag(alterSexp.head.symbolValue)) + private fun parseSingleTypeRef(typeRefExp: IonElement): TypeRef { + val metas = typeRefExp.metas.toSourceLocationMetas() + return when (typeRefExp) { + is SymbolElement -> TypeRef(typeRefExp.textValue, Arity.Required, metas) + is SexpElement -> { + when (typeRefExp.tag) { + "?" -> { + requireArityForTag(typeRefExp, 1) + val typeName = typeRefExp.tail.head.symbolValue + TypeRef(typeName, Arity.Optional, metas) + } + "*" -> { + requireArityForTag(typeRefExp, IntRange(2, 3)) + val typeName = typeRefExp.tail.head.symbolValue + val arityRange = typeRefExp.tail.tail + val minArity = arityRange.head.longValue + TypeRef(typeName, Arity.Variadic(minArity.toInt()), metas) + } + else -> parseError(typeRefExp.head, ParserErrorContext.ExpectedTypeReferenceArityTag(typeRefExp.tag)) + } + } + else -> parseError(typeRefExp, ParserErrorContext.ExpectedSymbolOrSexp(typeRefExp.type)) } } - return PermutedDomain( - tag = domainName, - permutesDomain = permutingDomain, - excludedTypes = removedTypes, - includedTypes = newTypes, - permutedSums = permutedSums, - metas = sexp.metas) -} + private fun parsePermuteDomain(domainName: String, sexp: SexpElement): PermutedDomain { + val args = sexp.tail // Skip tag + + val permutingDomain = args.head.symbolValue + val removedTypes = mutableListOf() + val newTypes = mutableListOf() + val permutedSums = mutableListOf() + + val alterSexps = args.tail + alterSexps.map { it.asSexp() }.forEach { alterSexp -> + when(alterSexp.head.symbolValue) { + "with" -> permutedSums.add(parseWithSum(alterSexp)) + "exclude" -> alterSexp.tail.mapTo(removedTypes) { it.symbolValue } + "include" -> alterSexp.tail.mapTo(newTypes) { parseDomainLevelStatement(it.asSexp()) } + else -> parseError(alterSexp, ParserErrorContext.InvalidPermutedDomainTag(alterSexp.head.symbolValue)) + } + } + + return PermutedDomain( + tag = domainName, + permutesDomain = permutingDomain, + excludedTypes = removedTypes, + includedTypes = newTypes, + permutedSums = permutedSums, + metas = sexp.metas.toSourceLocationMetas()) + } -private fun parseWithSum(sexp: SexpElement): PermutedSum { - val args = sexp.tail // Skip tag + private fun parseWithSum(sexp: SexpElement): PermutedSum { + val args = sexp.tail // Skip tag - val nameOfAlteredSum = args.head.symbolValue - val removedVariants = mutableListOf() - val addedVariants = mutableListOf() + val nameOfAlteredSum = args.head.symbolValue + val removedVariants = mutableListOf() + val addedVariants = mutableListOf() - args.tail.forEach { alterationValue -> - val alterationSexp = alterationValue.asSexp() - when (val alterationTag = alterationSexp.tag) { - "exclude" -> alterationSexp.tail.mapTo(removedVariants) { it.symbolValue } - "include" -> alterationSexp.tail.mapTo(addedVariants) { parseSumVariant(it.asSexp()) } - else -> parseError(alterationSexp, ParserErrorContext.InvalidWithSumTag(alterationTag)) + args.tail.forEach { alterationValue -> + val alterationSexp = alterationValue.asSexp() + when (val alterationTag = alterationSexp.tag) { + "exclude" -> alterationSexp.tail.mapTo(removedVariants) { it.symbolValue } + "include" -> alterationSexp.tail.mapTo(addedVariants) { parseSumVariant(it.asSexp()) } + else -> parseError(alterationSexp, ParserErrorContext.InvalidWithSumTag(alterationTag)) + } } - } - return PermutedSum(nameOfAlteredSum, removedVariants, addedVariants, sexp.metas) -} + return PermutedSum(nameOfAlteredSum, removedVariants, addedVariants, sexp.metas.toSourceLocationMetas()) + } -private fun requireArityForTag(sexp: SexpElement, arity: Int) { - // Note: arity does not include the tag! - val argCount = sexp.values.size - 1 - if(argCount != arity) { - parseError(sexp, ParserErrorContext.InvalidArityForTag(IntRange(arity, arity), sexp.head.symbolValue, argCount)) + private fun requireArityForTag(sexp: SexpElement, arity: Int) { + // Note: arity does not include the tag! + val argCount = sexp.values.size - 1 + if(argCount != arity) { + parseError(sexp, ParserErrorContext.InvalidArityForTag(IntRange(arity, arity), sexp.head.symbolValue, argCount)) + } } -} -private fun requireArityForTag(sexp: SexpElement, arityRange: IntRange) { - // Note: arity does not include the tag! - val argCount = sexp.values.size - 1 - if(argCount !in arityRange) { - parseError(sexp, ParserErrorContext.InvalidArityForTag(arityRange, sexp.head.symbolValue, argCount)) + private fun requireArityForTag(sexp: SexpElement, arityRange: IntRange) { + // Note: arity does not include the tag! + val argCount = sexp.values.size - 1 + if(argCount !in arityRange) { + parseError(sexp, ParserErrorContext.InvalidArityForTag(arityRange, sexp.head.symbolValue, argCount)) + } } } + +private val OTHER_VALID_PATH_CHARS = setOf('_', '-', '.', '/') + +/** + * Legal characters are: any letter or digit, plus any character in [OTHER_VALID_PATH_CHARS]. + * + * All other characters are illegal. + */ +private fun isValidPathChar(c: Char) = c.isLetterOrDigit() || OTHER_VALID_PATH_CHARS.contains(c) diff --git a/pig/src/org/partiql/pig/domain/parser/include/IncludeCycleHandler.kt b/pig/src/org/partiql/pig/domain/parser/include/IncludeCycleHandler.kt new file mode 100644 index 0000000..a0fb3c6 --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/include/IncludeCycleHandler.kt @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +import org.partiql.pig.domain.model.Statement +import org.partiql.pig.domain.parser.TypeUniverseParser +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path + +/** + * Prevents cycles in files included with the `include_file` statement from becoming a problem. + * + * This is accomplished by keeping track of all files "seen" by the parser and then making any attempt to + * include a file that was previously seen a no-op. + * + * @param mainTypeUniversePath The path to the main type universe that was passed on the command-line. This is the + * first "seen" file and does not require resolution because the user gave an explicit path to its location. + * + * @param resolver For identifying the full path to the file to be included. + * + * @see IncludeResolver + * @see FileSystem + */ +internal class IncludeCycleHandler( + mainTypeUniversePath: Path, + private val resolver: IncludeResolver +) { + private val seenFiles = HashSet().apply { add(mainTypeUniversePath.toAbsolutePath().normalize()) } + + /** + * Parses a universe file included with `include_file`. + * + * The return value is a [List] of [Statement]s that make up the type universe file. + * + * This function becomes a no-op in the event that the [includee] has been seen previously: an + * empty [List] is returned instead of the file being parsed again. + * + * @param includeePath The file requested to be included. + * @param includerPath The file in which the includee is to be included. + */ + fun parseIncludedTypeUniverse(includeePath: String, includerPath: Path): List { + + val resolvedIncludeFile = resolver.resolve(resolver.fileSystem.getPath(includeePath), includerPath) + + return if(!seenFiles.contains(resolvedIncludeFile)) { + seenFiles.add(resolvedIncludeFile) + Files.newInputStream(resolvedIncludeFile).use { + val source = InputSource(resolvedIncludeFile, it) + val parser = TypeUniverseParser(source, this) + parser.parse() + }.statements + } else { + listOf() + } + } +} diff --git a/pig/src/org/partiql/pig/domain/parser/include/IncludeResolutionException.kt b/pig/src/org/partiql/pig/domain/parser/include/IncludeResolutionException.kt new file mode 100644 index 0000000..b25239b --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/include/IncludeResolutionException.kt @@ -0,0 +1,25 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +import java.lang.Exception + +/** Thrown by [IncludeResolver] to indicate that it cannot locate an include file. */ +class IncludeResolutionException( + val inputFilePathString: String, + val consideredFilePaths: List +) : Exception("Could not locate include file '$inputFilePathString' in any considered path") + diff --git a/pig/src/org/partiql/pig/domain/parser/include/IncludeResolver.kt b/pig/src/org/partiql/pig/domain/parser/include/IncludeResolver.kt new file mode 100644 index 0000000..856d9a0 --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/include/IncludeResolver.kt @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +import java.nio.file.FileSystem +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path + +/** + * Manages the task of locating a file that was specified with the `include_file` statement. + * + * Terminology: + * + * - "includer": is a file containing an `include_file` statement. i.e. it is the file doing the including. + * - "includee": is a file being included by the includer. + * + * @param searchDirs A list of directories to search after searching the directory containing the includer, normally + * specified on the command-line. + * @param fileSystem The instance of [FileSystem] to be used. Allowing this to be specified instead of always using + * the default [FileSystem] allows an in-memory file system to be used during tests. For production uses, use the + * [FileSystem] returned from [FileSystems.getDefault] here. + * + * @throws InvalidIncludePathException if one of [searchDirs] does not exist or is not a directory. + */ +internal class IncludeResolver( + searchDirs: List, + internal val fileSystem: FileSystem +) { + + private val searchDirectories = searchDirs + .map { it.toAbsolutePath().normalize() } + .onEach { + validatePath(it) + if (!Files.isDirectory(it)) { + throw InvalidIncludePathException(it.toString()) + } + }.toTypedArray() + + private fun validatePath(it: Path) { + require(it.fileSystem === fileSystem) { "Path $it must not belong to a different FileSystem instance." } + } + + /** + * Searches for the absolute path of an included file, returning the first file found. + * + * The parent directory of the [includerFile] is searched first, followed by each of the source roots in turn. If + * [includeeFile] starts with `/`, the parent directory of [includerFile] is skipped. + * + * @return The absolute [Path] of the first file located. + */ + fun resolve(includeeFile: Path, includerFile: Path): Path { + val normalizedIncluder = includerFile.normalize().toAbsolutePath() + + // Determine list of all possible search roots + val includerParentDir = normalizedIncluder.parent + + val searchPaths = listOf(includerParentDir, *searchDirectories).distinct() + // distinct is needed because duplicate entries can arise in this list, for instance if the + // includer's parent directory is the same as an include directory. Primarily this is needed to + // prevent the directory from appearing twice in the error messages we display when we can't + // find an include file. + + val possibleIncludeeFiles = searchPaths + .map { + val appendPath = when { + includeeFile.isAbsolute -> includeeFile.toString().substring(1) + else -> includeeFile.toString() + } + it.resolve(appendPath) + } + + + // The resolved file is the first one that exists. + return possibleIncludeeFiles.firstOrNull { Files.exists(it) } + ?: throw IncludeResolutionException( + inputFilePathString = includeeFile.toString(), + consideredFilePaths = possibleIncludeeFiles.map { "$it" }) + } +} diff --git a/pig/src/org/partiql/pig/domain/parser/include/InputSource.kt b/pig/src/org/partiql/pig/domain/parser/include/InputSource.kt new file mode 100644 index 0000000..b29da7f --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/include/InputSource.kt @@ -0,0 +1,22 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +import java.io.InputStream +import java.nio.file.Path + +/** Provides a convenient way to associate an inputStream with its path. */ +class InputSource(val absolutePath: Path, val inputStream: InputStream) diff --git a/pig/src/org/partiql/pig/domain/parser/include/InvalidIncludePathException.kt b/pig/src/org/partiql/pig/domain/parser/include/InvalidIncludePathException.kt new file mode 100644 index 0000000..d191050 --- /dev/null +++ b/pig/src/org/partiql/pig/domain/parser/include/InvalidIncludePathException.kt @@ -0,0 +1,25 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +/** + * Thrown by [IncludeResolver] to indicate that one of the search roots passed to it is invalid. + * + * Search roots are normally specified on the command-line. + */ +class InvalidIncludePathException(invalidIncludePath: String) + : Exception("Specified include path '$invalidIncludePath' does not exist or is not a directory.") + diff --git a/pig/src/org/partiql/pig/errors/PigError.kt b/pig/src/org/partiql/pig/errors/PigError.kt index 8a96c1d..fdc775c 100644 --- a/pig/src/org/partiql/pig/errors/PigError.kt +++ b/pig/src/org/partiql/pig/errors/PigError.kt @@ -15,8 +15,7 @@ package org.partiql.pig.errors -import com.amazon.ionelement.api.IonLocation -import com.amazon.ionelement.api.locationToString +import org.partiql.pig.domain.parser.SourceLocation /** * [ErrorContext] instances provide information about an error message which can later be used @@ -29,7 +28,7 @@ interface ErrorContext { val message: String } -data class PigError(val location: IonLocation?, val context: ErrorContext) { - override fun toString(): String = "${locationToString(location)}: ${context.message}" +data class PigError(val location: SourceLocation?, val context: ErrorContext) { + override fun toString(): String = "${location?.toString() ?: ""}: ${context.message}" } diff --git a/pig/src/org/partiql/pig/generator/custom/generator.kt b/pig/src/org/partiql/pig/generator/custom/generator.kt index 205f1de..0f84b7c 100644 --- a/pig/src/org/partiql/pig/generator/custom/generator.kt +++ b/pig/src/org/partiql/pig/generator/custom/generator.kt @@ -17,12 +17,12 @@ package org.partiql.pig.generator.custom import org.partiql.pig.domain.model.TypeDomain import org.partiql.pig.generator.createDefaultFreeMarkerConfiguration -import java.io.File import java.io.PrintWriter +import java.nio.file.Path import java.time.OffsetDateTime fun applyCustomTemplate( - templateFile: File, + templatePath: Path, domains: List, output: PrintWriter ) { @@ -30,8 +30,8 @@ fun applyCustomTemplate( val cfg = createDefaultFreeMarkerConfiguration() - cfg.setDirectoryForTemplateLoading(templateFile.parentFile) - val template = cfg.getTemplate(templateFile.name)!! + cfg.setDirectoryForTemplateLoading(templatePath.parent.toFile()) + val template = cfg.getTemplate(templatePath.fileName.toString())!! template.process(renderModel, output) } diff --git a/pig/src/org/partiql/pig/main.kt b/pig/src/org/partiql/pig/main.kt index b0f6c9a..577561d 100644 --- a/pig/src/org/partiql/pig/main.kt +++ b/pig/src/org/partiql/pig/main.kt @@ -15,23 +15,27 @@ package org.partiql.pig -import com.amazon.ion.system.IonReaderBuilder import org.partiql.pig.cmdline.Command import org.partiql.pig.cmdline.CommandLineParser import org.partiql.pig.cmdline.TargetLanguage +import org.partiql.pig.domain.parser.include.InvalidIncludePathException import org.partiql.pig.errors.PigException import org.partiql.pig.domain.model.TypeUniverse -import org.partiql.pig.domain.parser.parseTypeUniverse +import org.partiql.pig.domain.parser.parseMainTypeUniverse import org.partiql.pig.generator.custom.applyCustomTemplate import org.partiql.pig.generator.html.applyHtmlTemplate import org.partiql.pig.generator.kotlin.applyKotlinTemplate import org.partiql.pig.generator.kotlin.convertToKTypeUniverse -import java.io.FileInputStream import java.io.PrintWriter import kotlin.system.exitProcess - fun progress(msg: String) = - println("pig: ${msg}") +fun progress(msg: String) = + println("pig: $msg") + +fun fatal(msg: String) { + System.err.print("pig: $msg") + exitProcess(-1) +} /** * Entry point for when pig is being invoked from the command-line. @@ -47,9 +51,12 @@ fun main(args: Array) { is Command.Generate -> { try { generateCode(command) - } catch (e: PigException) { - System.err.println("pig: ${e.error.location}: ${e.error.context.message}\n${e.stackTrace}") - exitProcess(-1) + } + catch(e: InvalidIncludePathException) { + fatal(e.message!!) + } + catch (e: PigException) { + fatal("${e.error.location}: ${e.error.context.message}\n${e.stackTrace}") } } } @@ -61,17 +68,20 @@ fun main(args: Array) { * having to `exec` pig as a separate process. */ fun generateCode(command: Command.Generate) { - progress("universe file: ${command.typeUniverseFile}") - progress("output file : ${command.outputFile}") - - progress("parsing the universe...") - val typeUniverse: TypeUniverse = FileInputStream(command.typeUniverseFile).use { inputStream -> - IonReaderBuilder.standard().build(inputStream).use { ionReader -> parseTypeUniverse(ionReader) } + progress("universe file: ${command.typeUniverseFilePath}") + progress("output file : ${command.outputFilePath}") + command.includePaths.forEach { + progress("include dir : $it") } + progress("parsing the universe...") + val typeUniverse: TypeUniverse = parseMainTypeUniverse( + mainTypeUniversePath = command.typeUniverseFilePath, + includePaths = command.includePaths + ) progress("permuting domains...") - PrintWriter(command.outputFile).use { printWriter -> + PrintWriter(command.outputFilePath.toFile()).use { printWriter -> when (command.target) { is TargetLanguage.Kotlin -> { progress("applying Kotlin pre-processing") diff --git a/pig/test-domains/README.md b/pig/test-domains/README.md new file mode 100644 index 0000000..69fcc0d --- /dev/null +++ b/pig/test-domains/README.md @@ -0,0 +1,37 @@ +The `.ion` files in this directory are used to test various aspects of the `include_file` statement. Parsing `main.ion` +and the files it includes is challenging if we can do it correctly then we should be able to handle any +`include_file` statement the user throws at us. + +The following concerns of `IncludeCycleHandler`, `IncludeResolver`, and `TypeDomainParser` are being tested here: + +- Cyclic includes are tolerated without exception. +- Aboslute and relative paths are respected during include resolution. +- Searching across multiple search roots works. + +Also, a concern of the `TypeDomainSemanticChecker` is tested as well: + +- Duplicate domains names are detected even if they are defined in different files. + +The file structure here is: + +``` +├── duplicate_domains.ion used by `TypeDomainSemanticCheckerTests` +├── include-missing-absolute.ion used by parser error handling tests and not included by main.ion +├── include-missing-relative.ion used by parser error handling tests and not included by main.ion +├── main.ion includes directly or indirectly all other files (unless noted otherwise) +├── root_a +│ ├── dir_x +│ │ ├── circular_universe_c.ion includes circular_universe_d.ion +│ │ ├── circular_universe_d.ion includes circular_universe_c.ion +│ │ ├── first_duplicated_domain_name.ion defines domain_dup +│ │ ├── include_b.ion includes . ./dir_a/universe_b +│ │ └── universe_a.ion includes include_b.ion +│ └── same-file-name.ion has same file name as root_b/same-file-name.ion (search stops here) +├── root_b +│ └── dir_z +│ │ ├── second_duplicated_domain_name.ion defines domain_dup +│ │ └── universe_b.ion includes nothing│ +│ └── same-file-name.ion has same file name as root_a/same-file-name.ion (but this one is ignored) +└── sibling-of-main.ion +``` + \ No newline at end of file diff --git a/pig/test-domains/duplicate_domains.ion b/pig/test-domains/duplicate_domains.ion new file mode 100644 index 0000000..db45680 --- /dev/null +++ b/pig/test-domains/duplicate_domains.ion @@ -0,0 +1,4 @@ + +(include_file "dir_x/first_duplicated_domain_name.ion") +(include_file "dir_z/second_duplicated_domain_name.ion") + diff --git a/pig/test-domains/include-missing.ion b/pig/test-domains/include-missing.ion new file mode 100644 index 0000000..e980333 --- /dev/null +++ b/pig/test-domains/include-missing.ion @@ -0,0 +1,4 @@ + +// This file attempts to include a missing file using an absolute path. + +(include_file "dir_x/does-not-exist.ion") diff --git a/pig/test-domains/main.ion b/pig/test-domains/main.ion new file mode 100644 index 0000000..7f7438f --- /dev/null +++ b/pig/test-domains/main.ion @@ -0,0 +1,6 @@ + + +(include_file "sibling-of-main.ion") +(include_file "dir_x/universe_a.ion") +(include_file "dir_x/circular_universe_c.ion") +(include_file "same_file_name.ion") diff --git a/pig/test-domains/root_a/dir_x/circular_universe_c.ion b/pig/test-domains/root_a/dir_x/circular_universe_c.ion new file mode 100644 index 0000000..9e797ba --- /dev/null +++ b/pig/test-domains/root_a/dir_x/circular_universe_c.ion @@ -0,0 +1,4 @@ + +(define domain_c (domain (product foo))) +// make a circular reference +(include_file "circular_universe_d.ion") diff --git a/pig/test-domains/root_a/dir_x/circular_universe_d.ion b/pig/test-domains/root_a/dir_x/circular_universe_d.ion new file mode 100644 index 0000000..f0da743 --- /dev/null +++ b/pig/test-domains/root_a/dir_x/circular_universe_d.ion @@ -0,0 +1,5 @@ + +(define domain_d (domain (product foo))) + +// make a circular reference +(include_file "circular_universe_c.ion") diff --git a/pig/test-domains/root_a/dir_x/first_duplicated_domain_name.ion b/pig/test-domains/root_a/dir_x/first_duplicated_domain_name.ion new file mode 100644 index 0000000..827b0f2 --- /dev/null +++ b/pig/test-domains/root_a/dir_x/first_duplicated_domain_name.ion @@ -0,0 +1,4 @@ + +// The first instance of domain_dup should not result in an error +(define domain_dup (domain (product foo))) + diff --git a/pig/test-domains/root_a/dir_x/include_b.ion b/pig/test-domains/root_a/dir_x/include_b.ion new file mode 100644 index 0000000..11553e2 --- /dev/null +++ b/pig/test-domains/root_a/dir_x/include_b.ion @@ -0,0 +1,3 @@ + +// include file with an aboslute path +(include_file "dir_z/universe_b.ion") diff --git a/pig/test-domains/root_a/dir_x/universe_a.ion b/pig/test-domains/root_a/dir_x/universe_a.ion new file mode 100644 index 0000000..922ce36 --- /dev/null +++ b/pig/test-domains/root_a/dir_x/universe_a.ion @@ -0,0 +1,3 @@ + +(define domain_a (domain (product foo))) +(include_file "include_b.ion") diff --git a/pig/test-domains/root_a/same_file_name.ion b/pig/test-domains/root_a/same_file_name.ion new file mode 100644 index 0000000..058937e --- /dev/null +++ b/pig/test-domains/root_a/same_file_name.ion @@ -0,0 +1,4 @@ +// Another file of the same name resides in `root_b`, which comes later in the `-I` include search order. That file +// should be ignored and this file should be included. +(define domain_f (domain (product a))) + diff --git a/pig/test-domains/root_b/dir_z/second_duplicated_domain_name.ion b/pig/test-domains/root_b/dir_z/second_duplicated_domain_name.ion new file mode 100644 index 0000000..bf8cc3a --- /dev/null +++ b/pig/test-domains/root_b/dir_z/second_duplicated_domain_name.ion @@ -0,0 +1,4 @@ + +// the second instance of domain_dup should result in an error +(define domain_dup (domain (product foo))) + diff --git a/pig/test-domains/root_b/dir_z/universe_b.ion b/pig/test-domains/root_b/dir_z/universe_b.ion new file mode 100644 index 0000000..a8487bd --- /dev/null +++ b/pig/test-domains/root_b/dir_z/universe_b.ion @@ -0,0 +1,3 @@ + +(define domain_b (domain (product foo))) + diff --git a/pig/test-domains/root_b/same_file_name.ion b/pig/test-domains/root_b/same_file_name.ion new file mode 100644 index 0000000..16d8df1 --- /dev/null +++ b/pig/test-domains/root_b/same_file_name.ion @@ -0,0 +1,4 @@ +// Another file of the same name resides in `root_a`, which comes earlier in the `-I` include search order. That file +// should be included and this one should be ignored. +(define this_domain_should_not_be_included (domain (product a))) + diff --git a/pig/test-domains/sibling-of-main.ion b/pig/test-domains/sibling-of-main.ion new file mode 100644 index 0000000..d01e632 --- /dev/null +++ b/pig/test-domains/sibling-of-main.ion @@ -0,0 +1,5 @@ + + +(define domain_s (domain (product foo))) + + diff --git a/pig/test/org/partiql/pig/cmdline/CommandLineParserTests.kt b/pig/test/org/partiql/pig/cmdline/CommandLineParserTests.kt index 1bb6362..d2f8d43 100644 --- a/pig/test/org/partiql/pig/cmdline/CommandLineParserTests.kt +++ b/pig/test/org/partiql/pig/cmdline/CommandLineParserTests.kt @@ -18,7 +18,7 @@ package org.partiql.pig.cmdline import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource -import java.io.File +import java.nio.file.Paths class CommandLineParserTests { @@ -39,77 +39,113 @@ class CommandLineParserTests { @JvmStatic @Suppress("unused") - fun parametersForTests() = listOf( - // Help - TestCase(Command.ShowHelp, "-h"), - TestCase(Command.ShowHelp, "--help"), - - //////////////////////////////////////////////////////// - // Missing parameters required for all language targets - //////////////////////////////////////////////////////// - // No --universe - TestCase( - Command.InvalidCommandLineArguments("Missing required option(s) [u/universe]"), - "--target=kotlin", "--output=output.kt", "--namespace=some.package"), - - // No --target - TestCase( - Command.InvalidCommandLineArguments("Missing required option(s) [t/target]"), - "--universe=input.ion", "--output=output.kt", "--namespace=some.package"), - - // No the --output argument - TestCase( - Command.InvalidCommandLineArguments("Missing required option(s) [o/output]"), - "--universe=input.ion", "--target=kotlin", "--namespace=some.package"), - - //////////////////////////////////////////////////////// - // Kotlin target - //////////////////////////////////////////////////////// - // long parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.kt"), TargetLanguage.Kotlin("some.package")), - "--universe=input.ion", "--target=kotlin", "--output=output.kt", "--namespace=some.package"), - - // short parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.kt"), TargetLanguage.Kotlin("some.package")), - "-u=input.ion", "-t=kotlin", "-o=output.kt", "-n=some.package"), - - // missing the --namespace argument - TestCase( - Command.InvalidCommandLineArguments("The selected language target requires the --namespace argument"), - "-u=input.ion", "-t=kotlin", "-o=output.kt"), - - //////////////////////////////////////////////////////// - // Html target - //////////////////////////////////////////////////////// - // long parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.html"), TargetLanguage.Html), - "--universe=input.ion", "--target=html", "--output=output.html"), - - // short parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.html"), TargetLanguage.Html), - "-u=input.ion", "-target=html", "--output=output.html"), - - //////////////////////////////////////////////////////// - // Custom target - //////////////////////////////////////////////////////// - // long parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.txt"), TargetLanguage.Custom(File("template.ftl"))), - "--universe=input.ion", "--target=custom", "--output=output.txt", "--template=template.ftl"), - - // short parameter names - TestCase( - Command.Generate(File("input.ion"), File("output.txt"), TargetLanguage.Custom(File("template.ftl"))), - "-u=input.ion", "-t=custom", "-o=output.txt", "-e=template.ftl"), - - // missing the --template argument - TestCase( - Command.InvalidCommandLineArguments("The selected language target requires the --template argument"), - "-u=input.ion", "-t=custom", "-o=output.kt") + fun parametersForTests(): List { + + return listOf( + // Help + TestCase(Command.ShowHelp, "-h"), + TestCase(Command.ShowHelp, "--help"), + + //////////////////////////////////////////////////////// + // Missing parameters required for all language targets + //////////////////////////////////////////////////////// + // No --universe + TestCase( + Command.InvalidCommandLineArguments("Missing required option(s) [u/universe]"), + "--target=kotlin", "--output=output.kt", "--namespace=some.package"), + + // No --target + TestCase( + Command.InvalidCommandLineArguments("Missing required option(s) [t/target]"), + "--universe=input.ion", "--output=output.kt", "--namespace=some.package"), + + // No --output argument + TestCase( + Command.InvalidCommandLineArguments("Missing required option(s) [o/output]"), + "--universe=input.ion", "--target=kotlin", "--namespace=some.package"), + + //////////////////////////////////////////////////////// + // Kotlin target + //////////////////////////////////////////////////////// + // long parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Kotlin("some.package"), "dir_a", "dir_b"), + "--universe=input.ion", + "--target=kotlin", + "--output=output.any", + "--include=dir_a", + "--include=dir_b", + "--namespace=some.package"), + + // short parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Kotlin("some.package"), "dir_a", "dir_b"), + "-u=input.ion", + "-t=kotlin", + "-o=output.any", + "-I=dir_a", + "-I=dir_b", + "-n=some.package" + ), + + // no include directories + TestCase( + createExpectedGenerateCommand(TargetLanguage.Kotlin("some.package")), + "-u=input.ion", + "-t=kotlin", + "-o=output.any", + "-n=some.package" + ), + + // missing the --namespace argument + TestCase( + Command.InvalidCommandLineArguments("The selected language target requires the --namespace argument"), + "-u=input.ion", "-t=kotlin", "-o=output.any"), + + //////////////////////////////////////////////////////// + // Html target + //////////////////////////////////////////////////////// + // long parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Html), + "--universe=input.ion", "--target=html", "--output=output.any"), + + // short parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Html), + "-u=input.ion", "-target=html", "--output=output.any"), + + //////////////////////////////////////////////////////// + // Custom target + //////////////////////////////////////////////////////// + // long parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Custom(Paths.get("template.ftl").toAbsolutePath())), + "--universe=input.ion", "--target=custom", "--output=output.any", "--template=template.ftl"), + + // short parameter names + TestCase( + createExpectedGenerateCommand(TargetLanguage.Custom(Paths.get("template.ftl").toAbsolutePath())), + "-u=input.ion", "-t=custom", "-o=output.any", "-e=template.ftl"), + + // missing the --template argument + TestCase( + Command.InvalidCommandLineArguments("The selected language target requires the --template argument"), + "-u=input.ion", "-t=custom", "-o=output.any") + ) + } + + private fun createExpectedGenerateCommand( + target: TargetLanguage, + vararg additionalIncludePaths: String + ) = Command.Generate( + typeUniverseFilePath = Paths.get("input.ion").toAbsolutePath(), + outputFilePath = Paths.get("output.any").toAbsolutePath(), + includePaths = listOf( + Paths.get(".").toAbsolutePath().normalize(), + *additionalIncludePaths.map { Paths.get(it).toAbsolutePath() }.toTypedArray() + ), + target = target ) } } \ No newline at end of file diff --git a/pig/test/org/partiql/pig/domain/PermuteDomainTests.kt b/pig/test/org/partiql/pig/domain/PermuteDomainTests.kt index e4c34ac..a3570f6 100644 --- a/pig/test/org/partiql/pig/domain/PermuteDomainTests.kt +++ b/pig/test/org/partiql/pig/domain/PermuteDomainTests.kt @@ -15,14 +15,13 @@ package org.partiql.pig.domain -import com.amazon.ion.system.IonReaderBuilder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.partiql.pig.domain.model.Arity import org.partiql.pig.domain.model.DataType import org.partiql.pig.domain.model.TypeUniverse -import org.partiql.pig.domain.parser.parseTypeUniverse +import org.partiql.pig.util.parseTypeUniverseString class PermuteDomainTests { /** @@ -59,7 +58,7 @@ class PermuteDomainTests { (e b::symbol))))) """ - val td: TypeUniverse = IonReaderBuilder.standard().build(typeUniverseWithExtensions).use { parseTypeUniverse(it) } + val td: TypeUniverse = parseTypeUniverseString(typeUniverseWithExtensions) val concretes = td.computeTypeDomains() diff --git a/pig/test/org/partiql/pig/domain/TypeDomainSemanticCheckerTests.kt b/pig/test/org/partiql/pig/domain/TypeDomainSemanticCheckerTests.kt index 2aeb03b..3d7ab37 100644 --- a/pig/test/org/partiql/pig/domain/TypeDomainSemanticCheckerTests.kt +++ b/pig/test/org/partiql/pig/domain/TypeDomainSemanticCheckerTests.kt @@ -15,14 +15,24 @@ package org.partiql.pig.domain +import com.amazon.ionelement.api.IonTextLocation import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import org.partiql.pig.domain.model.SemanticErrorContext -import org.partiql.pig.domain.parser.parseTypeUniverse +import org.partiql.pig.domain.parser.SourceLocation +import org.partiql.pig.errors.ErrorContext import org.partiql.pig.errors.PigError import org.partiql.pig.errors.PigException +import org.partiql.pig.util.FAKE_ROOT_FILE +import org.partiql.pig.util.MAIN_DOMAINS_DIR +import org.partiql.pig.util.ROOT_B +import org.partiql.pig.util.parseTypeUniverseString +import org.partiql.pig.util.parseWithTestRoots +import java.io.File +import java.nio.file.Paths class TypeDomainSemanticCheckerTests { @@ -43,7 +53,7 @@ class TypeDomainSemanticCheckerTests { fun nameErrorsTest2(tc: TestCase) = runTest(tc) private fun runTest(tc: TestCase) { - val u = parseTypeUniverse(tc.typeUniverseText) + val u = parseTypeUniverseString(tc.typeUniverseText) val ex = assertThrows { u.computeTypeDomains() } assertEquals(tc.expectedError, ex.error) } @@ -210,4 +220,28 @@ class TypeDomainSemanticCheckerTests { TestCase("(define some_domain (domain (record some_record)))", makeErr(1, 29, SemanticErrorContext.EmptyRecord))) } + + /** + * This ensures that we can still detect duplicate domains even if they reside in different files. + * + * To be totally clear here, [org.partiql.pig.domain.model.TypeDomainSemanticChecker] knows absolutely nothing + * of the `include_file` statement: all `TypeDomainSemanticChecker` sees is a list of domains and does not care one + * whit what file the domain was defined in: if a duplicate name was in the list of domains, it will issue an + * error. This test only exists to cover the unlikely scenario that this assumption changes in the future. + */ + @Test + fun `duplicate domain detection works across include_file`() { + val u = parseWithTestRoots("test-domains/duplicate_domains.ion") + val ex = assertThrows { u.computeTypeDomains() } + val expectedError = + PigError( + SourceLocation( + ROOT_B.resolve("dir_z/second_duplicated_domain_name.ion").toString(), + IonTextLocation(3, 20) + ), + SemanticErrorContext.DuplicateTypeDomainName("domain_dup") + ) + assertEquals(expectedError, ex.error) + } + } diff --git a/pig/test/org/partiql/pig/domain/Util.kt b/pig/test/org/partiql/pig/domain/Util.kt index 6bff621..bd49301 100644 --- a/pig/test/org/partiql/pig/domain/Util.kt +++ b/pig/test/org/partiql/pig/domain/Util.kt @@ -29,21 +29,28 @@ import org.partiql.pig.domain.model.Transform import org.partiql.pig.domain.model.TupleType import org.partiql.pig.domain.model.TypeDomain import org.partiql.pig.domain.model.TypeUniverse +import org.partiql.pig.domain.parser.SourceLocation import org.partiql.pig.errors.ErrorContext import org.partiql.pig.errors.PigError +import org.partiql.pig.util.FAKE_ROOT_FILE +import java.io.File fun makeErr(line: Int, col: Int, errorContext: ErrorContext) = - PigError(IonTextLocation(line.toLong(), col.toLong()), errorContext) + PigError( + SourceLocation( + File(FAKE_ROOT_FILE).canonicalPath, + IonTextLocation(line.toLong(), col.toLong()) + ), + errorContext + ) fun makeErr(errorContext: ErrorContext) = PigError(null, errorContext) - /* * The [toIonElement] functions below generate an s-expression representation of a [TypeUniverse]. */ - fun TypeUniverse.toIonElement(): IonElement = ionSexpOf( ionSymbol("universe"), diff --git a/pig/test/org/partiql/pig/domain/parser/TypeDomainParserIncludeFileTests.kt b/pig/test/org/partiql/pig/domain/parser/TypeDomainParserIncludeFileTests.kt new file mode 100644 index 0000000..259b259 --- /dev/null +++ b/pig/test/org/partiql/pig/domain/parser/TypeDomainParserIncludeFileTests.kt @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser + +import com.amazon.ionelement.api.IonTextLocation +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.partiql.pig.domain.model.TypeDomain +import org.partiql.pig.errors.PigError +import org.partiql.pig.errors.PigException +import org.partiql.pig.util.MAIN_DOMAINS_DIR +import org.partiql.pig.util.ROOT_A +import org.partiql.pig.util.ROOT_B +import org.partiql.pig.util.parseWithTestRoots +import java.nio.file.Paths + +class TypeDomainParserIncludeFileTests { + @Test + fun `include happy case`() { + val universe = parseWithTestRoots("test-domains/main.ion") + val allDomains = universe.statements.filterIsInstance() + + // If the following 6 domains are loaded, then we deal with relative paths and circular references correctly. + // see documentation at top of test-domains/main.ion + assertEquals( + setOf( + "domain_a", + "domain_b", + "domain_c", + "domain_d", + "domain_f", + "domain_s" + ), + allDomains.map { it.tag }.toSet() + ) + } + + + + @Test + fun `include sad case - missing include - absolute path`() { + val includeeFile = "dir_x/does-not-exist.ion" + testMissingInclude( + mainUniverseFile = "test-domains/include-missing.ion", + includeeFile = includeeFile, + pathsSearched = listOf( + "${MAIN_DOMAINS_DIR}/$includeeFile", + "${ROOT_A.resolve("dir_x/does-not-exist.ion").toAbsolutePath()}", + "${ROOT_B.resolve("dir_x/does-not-exist.ion").toAbsolutePath()}" + ) + ) + } + + private fun testMissingInclude( + mainUniverseFile: String, + includeeFile: String, + pathsSearched: List + ) { + val ex = assertThrows { parseWithTestRoots(mainUniverseFile) } + assertEquals( + PigError( + SourceLocation(Paths.get(mainUniverseFile).toAbsolutePath().toString(), IonTextLocation(4L, 1L)), + ParserErrorContext.IncludeFileNotFound( + includeeFile, + pathsSearched + ) + ), + ex.error + ) + } +} diff --git a/pig/test/org/partiql/pig/domain/TypeDomainParserTests.kt b/pig/test/org/partiql/pig/domain/parser/TypeDomainParserTests.kt similarity index 93% rename from pig/test/org/partiql/pig/domain/TypeDomainParserTests.kt rename to pig/test/org/partiql/pig/domain/parser/TypeDomainParserTests.kt index 3ae4863..25ccc14 100644 --- a/pig/test/org/partiql/pig/domain/TypeDomainParserTests.kt +++ b/pig/test/org/partiql/pig/domain/parser/TypeDomainParserTests.kt @@ -13,9 +13,8 @@ * permissions and limitations under the License. */ -package org.partiql.pig.domain +package org.partiql.pig.domain.parser -import com.amazon.ion.system.IonReaderBuilder import com.amazon.ionelement.api.IonElementLoaderOptions import com.amazon.ionelement.api.createIonElementLoader import com.amazon.ionelement.api.ionSexpOf @@ -23,7 +22,8 @@ import com.amazon.ionelement.api.ionSymbol import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow -import org.partiql.pig.domain.parser.parseTypeUniverse +import org.partiql.pig.domain.toIonElement +import org.partiql.pig.util.parseTypeUniverseString class TypeDomainParserTests { private val loader = createIonElementLoader(IonElementLoaderOptions(includeLocationMeta = true)) @@ -36,6 +36,7 @@ class TypeDomainParserTests { (product foo a::string b::(* int 2)) (product bar a::bat b::(? baz) c::(* blargh 10)))) """) + @Test fun testTransform() = runTestCase("(transform domain_a domain_b)") @@ -88,10 +89,9 @@ class TypeDomainParserTests { val expected = assertDoesNotThrow("loading the expected type universe") { loader.loadSingleElement(tc) } + val parsed = assertDoesNotThrow("parsing type universe") { - IonReaderBuilder.standard().build(tc).use { - parseTypeUniverse(it) - } + parseTypeUniverseString(tc) } assertEquals( diff --git a/pig/test/org/partiql/pig/domain/parser/include/IncludeResolverTests.kt b/pig/test/org/partiql/pig/domain/parser/include/IncludeResolverTests.kt new file mode 100644 index 0000000..17aaa71 --- /dev/null +++ b/pig/test/org/partiql/pig/domain/parser/include/IncludeResolverTests.kt @@ -0,0 +1,110 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.domain.parser.include + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.nio.file.FileSystems +import java.nio.file.Path +import java.nio.file.Paths + +class IncludeResolverTests { + companion object { + private val PATH_TO_MAIN = Paths.get("./test-domains/main.ion") + + class HappyCase(val includeePath: String, expectedResolved: String) { + val includerPath: Path = PATH_TO_MAIN + val expectedResolvedPath: Path = Paths.get(expectedResolved).toAbsolutePath().normalize() + } + + @JvmStatic @Suppress("UNUSED") + fun happyCases() = listOf( + HappyCase( + includeePath = "sibling-of-main.ion", + expectedResolved = "./test-domains/sibling-of-main.ion" + ), + HappyCase( + includeePath = "/dir_x/universe_a.ion", + expectedResolved = "./test-domains/root_a/dir_x/universe_a.ion" + ), + HappyCase( + includeePath = "/dir_z/universe_b.ion", + expectedResolved = "./test-domains/root_b/dir_z/universe_b.ion" + ) + ) + + class NotFoundCase(includee: String, val expectedConsdieredPaths: List) { + val includee: Path = Paths.get(includee) + } + + @JvmStatic @Suppress("UNUSED") + fun notFoundCases() = listOf( + NotFoundCase( + includee = "does-not-exist.ion", + expectedConsdieredPaths = expectedConsideredPaths + ), + NotFoundCase( + includee = "/does-not-exist.ion", + expectedConsdieredPaths = expectedConsideredPaths + ) + ) + + private val expectedConsideredPaths + get() = listOf( + Paths.get("test-domains/does-not-exist.ion").toAbsolutePath().toString(), + Paths.get("test-domains/root_a/does-not-exist.ion").toAbsolutePath().toString(), + Paths.get("test-domains/root_b/does-not-exist.ion").toAbsolutePath().toString() + ) + } + + private val resolver = IncludeResolver( + listOf( + Paths.get("./test-domains/root_a"), + Paths.get("./test-domains/root_b") + ), + // These tests refer to actual files in the source tree--here we should use the default FileSystem. + FileSystems.getDefault() + ) + + @ParameterizedTest + @MethodSource("happyCases") + fun `test happy resolution cases`(tc: HappyCase) { + val actualResolvedPath = resolver.resolve(Paths.get(tc.includeePath), tc.includerPath) + assertEquals(tc.expectedResolvedPath, actualResolvedPath) + } + + @ParameterizedTest + @MethodSource("notFoundCases") + fun `test not found resolution cases`(tc: NotFoundCase) { + val ex = assertThrows { + resolver.resolve(tc.includee, PATH_TO_MAIN) + } + assertEquals(tc.expectedConsdieredPaths, ex.consideredFilePaths) + } + + @Test + fun `test invalid include search path`() { + assertThrows { + IncludeResolver( + listOf(Paths.get("/this/dir/does/not/exist")), + FileSystems.getDefault() + ) + } + } +} diff --git a/pig/test/org/partiql/pig/domain/TypeDomainParserErrorsTest.kt b/pig/test/org/partiql/pig/domain/parser/include/TypeDomainParserErrorsTest.kt similarity index 72% rename from pig/test/org/partiql/pig/domain/TypeDomainParserErrorsTest.kt rename to pig/test/org/partiql/pig/domain/parser/include/TypeDomainParserErrorsTest.kt index 0bb3135..615957c 100644 --- a/pig/test/org/partiql/pig/domain/TypeDomainParserErrorsTest.kt +++ b/pig/test/org/partiql/pig/domain/parser/include/TypeDomainParserErrorsTest.kt @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package org.partiql.pig.domain +package org.partiql.pig.domain.parser.include import com.amazon.ionelement.api.ElementType import com.amazon.ionelement.api.IonElementLoaderException @@ -22,10 +22,13 @@ import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.assertThrows import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource +import org.partiql.pig.domain.makeErr import org.partiql.pig.domain.parser.ParserErrorContext -import org.partiql.pig.domain.parser.parseTypeUniverse +import org.partiql.pig.domain.toIonElement import org.partiql.pig.errors.PigError import org.partiql.pig.errors.PigException +import org.partiql.pig.util.makeFakePath +import org.partiql.pig.util.parseTypeUniverseString class TypeDomainParserErrorsTest { @@ -35,7 +38,7 @@ class TypeDomainParserErrorsTest { @MethodSource("parametersForErrorsTest") fun errorsTest(tc: TestCase) { val ex = assertThrows { - val oops = parseTypeUniverse(tc.typeUniverseText) + val oops = parseTypeUniverseString(tc.typeUniverseText) println("this was erroneously parsed: ${oops.toIonElement()}") } assertEquals(tc.expectedError, ex.error) @@ -105,9 +108,37 @@ class TypeDomainParserErrorsTest { TestCase( // Covers second place in parser this can be thrown "(define huh (domain (product huh x::int y::42)))", - makeErr(1, 41, ParserErrorContext.ExpectedSymbolOrSexp(ElementType.INT))) + makeErr(1, 41, ParserErrorContext.ExpectedSymbolOrSexp(ElementType.INT))), + + TestCase( + "(include_file \"some-non-existing-file.ion\")", + makeErr(1, 1, + ParserErrorContext.IncludeFileNotFound( + "some-non-existing-file.ion", + listOf(makeFakePath("some-non-existing-file.ion")) + ))), + TestCase( + "(include_file \"some-sub-dir/some-non-existing-file.ion\")", + makeErr(1, 1, + ParserErrorContext.IncludeFileNotFound( + "some-sub-dir/some-non-existing-file.ion", + listOf(makeFakePath("some-sub-dir/some-non-existing-file.ion")) + ))), + TestCase( + "(include_file \"../doesntmatter.ion\")", + makeErr(1, 15, ParserErrorContext.IncludeFilePathContainsParentDirectory)), + TestCase( + "(include_file \"c:/windows/drive/letter/is/bad.ion\")", + makeErr(1, 15, ParserErrorContext.IncludeFilePathContainsIllegalCharacter(':'))), + TestCase( + """(include_file "\\windows\\path\\separator")""", + makeErr(1, 15, ParserErrorContext.IncludeFilePathContainsIllegalCharacter('\\'))), + TestCase( + "(include_file \"space in name\")", + makeErr(1, 15, ParserErrorContext.IncludeFilePathContainsIllegalCharacter(' '))) ) } } + diff --git a/pig/test/org/partiql/pig/util/ParseHelpers.kt b/pig/test/org/partiql/pig/util/ParseHelpers.kt new file mode 100644 index 0000000..adf3b3b --- /dev/null +++ b/pig/test/org/partiql/pig/util/ParseHelpers.kt @@ -0,0 +1,62 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package org.partiql.pig.util + +import com.google.common.collect.ImmutableList +import com.google.common.jimfs.Configuration +import com.google.common.jimfs.Jimfs +import org.partiql.pig.domain.model.TypeUniverse +import org.partiql.pig.domain.parser.parseMainTypeUniverse +import java.nio.charset.StandardCharsets +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + + +internal const val FAKE_ROOT_DIR = "/fake-directory" +internal fun makeFakePath(fileName: String) = "$FAKE_ROOT_DIR/$fileName" + +/** The name of the "fake" root file used by unit tests. */ +internal val FAKE_ROOT_FILE = makeFakePath("root.ion") + + +/** + * For unit tests only. Parses the type universe specified in [topUnvierseText]. + * + * Accomplishes this by using Jimfs to create an in-memory file system, and then + * writing [topUnvierseText] to [FAKE_ROOT_FILE] within it. + */ +internal fun parseTypeUniverseString(topUnvierseText: String): TypeUniverse { + val build = Configuration.unix().toBuilder().setWorkingDirectory("/").build() + val fs: FileSystem = Jimfs.newFileSystem(build) + Files.createDirectory(fs.getPath(FAKE_ROOT_DIR)) + val rootPath: Path = fs.getPath(FAKE_ROOT_FILE) + + Files.write(rootPath, ImmutableList.of(topUnvierseText), StandardCharsets.UTF_8) + + return parseMainTypeUniverse(rootPath, listOf()) +} + + +internal val MAIN_DOMAINS_DIR = Paths.get("test-domains").toAbsolutePath() +internal val ROOT_A = MAIN_DOMAINS_DIR.resolve("root_a").toAbsolutePath() +internal val ROOT_B = MAIN_DOMAINS_DIR.resolve("root_b").toAbsolutePath() + +internal fun parseWithTestRoots(universeFile: String): TypeUniverse { + val includeSearchRoots = listOf(ROOT_A, ROOT_B) + return parseMainTypeUniverse(Paths.get(universeFile), includeSearchRoots) +}