From 207984da1a67caa88d424fd7ec88ffa00a0d5375 Mon Sep 17 00:00:00 2001 From: Gary O'Neall Date: Tue, 20 Feb 2018 17:43:39 -0800 Subject: [PATCH 1/2] Make license expressions completely case insensitive Signed-off-by: Gary O'Neall --- .../license/TestLicenseInfoFactory.java | 19 +++++++++++++++-- .../spdx/rdfparser/SpdxDocumentContainer.java | 10 ++++----- .../license/LicenseExpressionParser.java | 13 +++++------- .../rdfparser/license/ListedLicenses.java | 21 ++++++++++--------- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/Test/org/spdx/rdfparser/license/TestLicenseInfoFactory.java b/Test/org/spdx/rdfparser/license/TestLicenseInfoFactory.java index 43c3e564..8bbe7ace 100644 --- a/Test/org/spdx/rdfparser/license/TestLicenseInfoFactory.java +++ b/Test/org/spdx/rdfparser/license/TestLicenseInfoFactory.java @@ -39,8 +39,6 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; -import junit.framework.Assert; - import org.apache.jena.graph.Node; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -427,6 +425,23 @@ public void testParseSPDXLicenseString() throws InvalidLicenseStringException { } } + @Test + public void testParseSPDXLicenseStringMixedCase() throws InvalidLicenseStringException { + String parseString = COMPLEX_LICENSE.toString(); + StringBuilder mixedCase = new StringBuilder(); + for (int i = 0; i < parseString.length(); i++) { + if (i % 2 == 0) { + mixedCase.append(parseString.substring(i, i+1).toUpperCase()); + } else { + mixedCase.append(parseString.substring(i, i+1).toLowerCase()); + } + } + AnyLicenseInfo li = LicenseInfoFactory.parseSPDXLicenseString(mixedCase.toString()); + if (!li.equals(COMPLEX_LICENSE)) { + fail("Parsed license does not equal"); + } + } + @Test public void testSpecialLicenses() throws InvalidLicenseStringException, InvalidSPDXAnalysisException { // NONE diff --git a/src/org/spdx/rdfparser/SpdxDocumentContainer.java b/src/org/spdx/rdfparser/SpdxDocumentContainer.java index 4986687c..8d492189 100644 --- a/src/org/spdx/rdfparser/SpdxDocumentContainer.java +++ b/src/org/spdx/rdfparser/SpdxDocumentContainer.java @@ -83,7 +83,7 @@ public class SpdxDocumentContainer implements IModelContainer, SpdxRdfConstants Set spdxRefs = Sets.newHashSet(); /** - * Map of license ID to extracted license info + * Map of lower case license ID to extracted license info */ Map licenseIdToExtractedLicense = Maps.newHashMap(); @@ -526,7 +526,7 @@ public void addNewExtractedLicenseInfo(ExtractedLicenseInfo license) throws Inva Property p = model.getProperty(SPDX_NAMESPACE, PROP_SPDX_EXTRACTED_LICENSES); Resource s = getResource(getSpdxDocNode()); s.addProperty(p, license.createResource(this)); - this.licenseIdToExtractedLicense.put(license.getLicenseId(), license); + this.licenseIdToExtractedLicense.put(license.getLicenseId().toLowerCase(), license); } /** @@ -535,7 +535,7 @@ public void addNewExtractedLicenseInfo(ExtractedLicenseInfo license) throws Inva * @throws InvalidSPDXAnalysisException */ public boolean extractedLicenseExists(String id) throws InvalidSPDXAnalysisException { - return this.licenseIdToExtractedLicense.containsKey(id); + return this.licenseIdToExtractedLicense.containsKey(id.toLowerCase()); } /** @@ -544,7 +544,7 @@ public boolean extractedLicenseExists(String id) throws InvalidSPDXAnalysisExcep * @throws InvalidSPDXAnalysisException */ public ExtractedLicenseInfo getExtractedLicense(String id) throws InvalidSPDXAnalysisException { - return this.licenseIdToExtractedLicense.get(id); + return this.licenseIdToExtractedLicense.get(id.toLowerCase()); } /** @@ -563,7 +563,7 @@ public void getExtractedLicenseInfosFromModel() throws InvalidSPDXAnalysisExcept throw new InvalidSPDXAnalysisException("Invalid type for extracted license infos: " + extractedAnyLicenseInfo[i]); } ExtractedLicenseInfo lic = (ExtractedLicenseInfo)extractedAnyLicenseInfo[i]; - this.licenseIdToExtractedLicense.put(lic.getLicenseId(), lic); + this.licenseIdToExtractedLicense.put(lic.getLicenseId().toLowerCase(), lic); } } diff --git a/src/org/spdx/rdfparser/license/LicenseExpressionParser.java b/src/org/spdx/rdfparser/license/LicenseExpressionParser.java index 51423fe0..56295d5d 100644 --- a/src/org/spdx/rdfparser/license/LicenseExpressionParser.java +++ b/src/org/spdx/rdfparser/license/LicenseExpressionParser.java @@ -48,9 +48,6 @@ enum Operator { static { OPERATOR_MAP.put("+", Operator.OR_LATER); - OPERATOR_MAP.put("AND", Operator.AND); - OPERATOR_MAP.put("OR", Operator.OR); - OPERATOR_MAP.put("WITH", Operator.WITH); OPERATOR_MAP.put("and", Operator.AND); OPERATOR_MAP.put("or", Operator.OR); OPERATOR_MAP.put("with", Operator.WITH); @@ -68,9 +65,9 @@ static AnyLicenseInfo parseLicenseExpression(String expression, SpdxDocumentCont throw(new LicenseParserException("Empty license expression")); } String[] tokens = tokenizeExpression(expression); - if (tokens.length == 1 && tokens[0].equals(SpdxRdfConstants.NOASSERTION_VALUE)) { + if (tokens.length == 1 && tokens[0].toLowerCase().equals(SpdxRdfConstants.NOASSERTION_VALUE.toLowerCase())) { return new SpdxNoAssertionLicense(); - } else if (tokens.length == 1 && tokens[0].equals(SpdxRdfConstants.NONE_VALUE)) { + } else if (tokens.length == 1 && tokens[0].toLowerCase().equals(SpdxRdfConstants.NONE_VALUE.toLowerCase())) { return new SpdxNoneLicense(); } else { try { @@ -129,7 +126,7 @@ private static AnyLicenseInfo parseLicenseExpression(String[] tokens, SpdxDocume throw(new LicenseParserException("Expected license expression")); } Stack operandStack = new Stack(); - Stack operatorStack = new Stack(); + Stack operatorStack = new Stack(); int tokenIndex = 0; String token; while (tokenIndex < tokens.length) { @@ -143,10 +140,10 @@ private static AnyLicenseInfo parseLicenseExpression(String[] tokens, SpdxDocume String[] nestedTokens = Arrays.copyOfRange(tokens, tokenIndex, rightParenIndex); operandStack.push(parseLicenseExpression(nestedTokens, container)); tokenIndex = rightParenIndex + 1; - } else if (OPERATOR_MAP.get(token) == null) { // assumed to be a simple licensing type + } else if (OPERATOR_MAP.get(token.toLowerCase()) == null) { // assumed to be a simple licensing type operandStack.push(parseSimpleLicenseToken(token, container)); } else { - Operator operator = OPERATOR_MAP.get(token); + Operator operator = OPERATOR_MAP.get(token.toLowerCase()); if (operator == Operator.WITH) { // special processing here since With must be with an exception, not a licenseInfo if (!operatorStack.isEmpty() && Operator.OR_LATER.equals(operatorStack.peek())) { diff --git a/src/org/spdx/rdfparser/license/ListedLicenses.java b/src/org/spdx/rdfparser/license/ListedLicenses.java index df02f6b0..9054bb20 100644 --- a/src/org/spdx/rdfparser/license/ListedLicenses.java +++ b/src/org/spdx/rdfparser/license/ListedLicenses.java @@ -24,7 +24,6 @@ import java.nio.charset.Charset; import java.util.Map; import java.util.Properties; -import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -45,7 +44,6 @@ import org.spdx.rdfparser.model.IRdfModel; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import net.rootdev.javardfa.jena.RDFaReader.HTMLRDFaReader; /** @@ -67,7 +65,10 @@ public class ListedLicenses implements IModelContainer { private Model listedLicenseModel = null; - Set listdLicenseIds = null; + /** + * Map of lowercase listed license ID to proper cased listed license ID + */ + Map listdLicenseIds = null; Map listedLicenseCache = null; Map> listedLicenseNodeCache = Maps.newHashMap(); @@ -401,7 +402,7 @@ private void loadListedLicenseIDs() { listedLicenseModificationLock.writeLock().lock(); try { listedLicenseCache = Maps.newHashMap(); // clear the cache - listdLicenseIds = Sets.newHashSet(); //Clear the listed license IDs to avoid stale licenses. + listdLicenseIds = Maps.newHashMap(); //Clear the listed license IDs to avoid stale licenses. //TODO: Can the keys of listedLicenseCache be used instead of this set? //NOTE: THis includes deprecated licenses - should this be changed to only return non-deprecated licenses? Model stdLicenseModel = getListedLicenseModel(); @@ -410,7 +411,7 @@ private void loadListedLicenseIDs() { ExtendedIterator tripleIter = stdLicenseModel.getGraph().find(m); while (tripleIter.hasNext()) { Triple t = tripleIter.next(); - listdLicenseIds.add(t.getObject().toString(false)); + listdLicenseIds.put(t.getObject().toString(false).toLowerCase(),t.getObject().toString(false)); } p = stdLicenseModel.getProperty(SpdxRdfConstants.SPDX_NAMESPACE, SpdxRdfConstants.PROP_LICENSE_LIST_VERSION).asNode(); m = Triple.createMatch(null, p, null); @@ -435,7 +436,7 @@ private void loadListedLicenseIDs() { public boolean isSpdxListedLicenseID(String licenseID) { try { listedLicenseModificationLock.readLock().lock(); - return listdLicenseIds.contains(licenseID); + return listdLicenseIds.containsKey(licenseID.toLowerCase()); } finally { listedLicenseModificationLock.readLock().unlock(); } @@ -481,7 +482,7 @@ private static Properties loadLicenseProperties() { public String[] getSpdxListedLicenseIds() { listedLicenseModificationLock.readLock().lock(); try { - return listdLicenseIds.toArray(new String[listdLicenseIds.size()]); + return listdLicenseIds.values().toArray(new String[listdLicenseIds.size()]); } finally { listedLicenseModificationLock.readLock().unlock(); } @@ -501,7 +502,7 @@ public String getLicenseListVersion() { * @throws InvalidSPDXAnalysisException */ public SpdxListedLicense getListedLicenseById(String licenseId)throws InvalidSPDXAnalysisException { - return getLicenseFromUri(LISTED_LICENSE_URI_PREFIX + licenseId); + return getLicenseFromUri(LISTED_LICENSE_URI_PREFIX + this.listdLicenseIds.get(licenseId.toLowerCase())); } /** @@ -567,7 +568,7 @@ public synchronized String getNextSpdxElementRef() { */ @Override public boolean spdxElementRefExists(String elementRef) { - return(listdLicenseIds.contains(elementRef)); + return(listdLicenseIds.values().contains(elementRef)); } /* (non-Javadoc) @@ -575,7 +576,7 @@ public boolean spdxElementRefExists(String elementRef) { */ @Override public void addSpdxElementRef(String elementRef) { - listdLicenseIds.add(elementRef); + listdLicenseIds.put(elementRef.toLowerCase(),elementRef); } /* (non-Javadoc) From 38ac42e73ae9e7af8a398e72ddc35c510750cd94 Mon Sep 17 00:00:00 2001 From: Gary O'Neall Date: Sat, 25 Apr 2020 22:05:38 -0700 Subject: [PATCH 2/2] Fix merge errors Signed-off-by: Gary O'Neall --- resources/log4j2.xml | 13 +++++++++++++ src/org/spdx/rdfparser/license/LicenseJsonTOC.java | 9 +++++---- src/org/spdx/rdfparser/license/ListedLicenses.java | 5 +++-- 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 resources/log4j2.xml diff --git a/resources/log4j2.xml b/resources/log4j2.xml new file mode 100644 index 00000000..8f7a1675 --- /dev/null +++ b/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/org/spdx/rdfparser/license/LicenseJsonTOC.java b/src/org/spdx/rdfparser/license/LicenseJsonTOC.java index d7034828..c4791dfe 100644 --- a/src/org/spdx/rdfparser/license/LicenseJsonTOC.java +++ b/src/org/spdx/rdfparser/license/LicenseJsonTOC.java @@ -15,9 +15,10 @@ */ package org.spdx.rdfparser.license; +import java.util.Map; import java.util.Set; -import com.google.common.collect.Sets; +import com.google.common.collect.Maps; /** * Table of Contents for the listed license list as represented as a JSON index file @@ -106,13 +107,13 @@ public LicenseJson[] getLicenses() { return licenses; } - public Set getLicenseIds() { - Set retval = Sets.newHashSet(); + public Map getLicenseIds() { + Map retval = Maps.newHashMap(); if (licenses == null) { return retval; } for (LicenseJson license:licenses) { - retval.add(license.licenseId); + retval.put(license.licenseId.toLowerCase(), license.licenseId); } return retval; } diff --git a/src/org/spdx/rdfparser/license/ListedLicenses.java b/src/org/spdx/rdfparser/license/ListedLicenses.java index c1bb43b5..9ca3ee21 100644 --- a/src/org/spdx/rdfparser/license/ListedLicenses.java +++ b/src/org/spdx/rdfparser/license/ListedLicenses.java @@ -43,7 +43,6 @@ import org.spdx.rdfparser.model.IRdfModel; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.gson.Gson; /** @@ -387,6 +386,8 @@ private void loadListedLicenseIDs() { } catch (IOException e) { logger.warn("Unable to close JSON TOC input stream"); } + } + } } finally { listedLicenseModificationLock.writeLock().unlock(); } @@ -466,7 +467,7 @@ public String getLicenseListVersion() { * @throws InvalidSPDXAnalysisException */ public SpdxListedLicense getListedLicenseById(String licenseId)throws InvalidSPDXAnalysisException { - SpdxListedLicense retval = getLicenseFromUri(LISTED_LICENSE_URI_PREFIX + licenseId + JSONLD_URL_SUFFIX); + SpdxListedLicense retval = getLicenseFromUri(LISTED_LICENSE_URI_PREFIX + listdLicenseIds.get(licenseId.toLowerCase()) + JSONLD_URL_SUFFIX); if (retval != null) { retval = (SpdxListedLicense)retval.clone(); // We need to clone the license to remove the references to the model in the cache }