From fb426479646005f8db638cee73a68cda059ba6c8 Mon Sep 17 00:00:00 2001
From: Nikolas Komonen <nikolaskomonen@gmail.com>
Date: Wed, 16 Jan 2019 11:12:46 -0500
Subject: [PATCH] Fixes DTD completion

Fixes #232

Signed-off-by: Nikolas Komonen <nikolaskomonen@gmail.com>
---
 .../org/eclipse/lsp4xml/dom/DOMDocument.java  | 43 +++++++++
 .../lsp4xml/dom/parser/XMLScanner.java        | 22 +++--
 .../lsp4xml/services/XMLCompletions.java      | 96 ++++++++++++-------
 .../java/org/eclipse/lsp4xml/XMLAssert.java   |  2 +
 .../DTDCompletionExtensionsTest.java          | 70 +++++++++++++-
 5 files changed, 188 insertions(+), 45 deletions(-)

diff --git a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/DOMDocument.java b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/DOMDocument.java
index 8f858a8384..5d4dba4bb7 100644
--- a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/DOMDocument.java
+++ b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/DOMDocument.java
@@ -26,6 +26,7 @@
 import org.eclipse.lsp4xml.uriresolver.URIResolverExtensionManager;
 import org.eclipse.lsp4xml.utils.DOMUtils;
 import org.eclipse.lsp4xml.utils.StringUtils;
+import org.eclipse.lsp4xml.utils.XMLPositionUtility;
 import org.w3c.dom.CDATASection;
 import org.w3c.dom.DOMConfiguration;
 import org.w3c.dom.DOMException;
@@ -760,6 +761,48 @@ public boolean isDTD() {
 		return false;
 	}
 
+	/**
+	 * Returns true if 'offset' is within an internal DOCTYPE dtd.
+	 * Else false.
+	 * @param offset
+	 * @return 
+	 */
+	public boolean isWithinInternalDTD(int offset) {
+		DOMDocumentType doctype = this.getDoctype();
+		if(doctype != null && doctype.internalSubset != null) {
+			return offset > doctype.internalSubset.start && offset < doctype.internalSubset.end;
+		}
+		return false;
+	}
+
+	public Range getTrimmedRange(Range range) {
+		if(range != null) {
+			return getTrimmedRange(range.getStart().getCharacter(), range.getEnd().getCharacter());
+		}
+		return null;
+
+	}
+
+	public Range getTrimmedRange(int start, int end) {
+		String text = getText();
+		char c = text.charAt(start);
+		while(Character.isWhitespace(c)) {
+			start++;
+			c = text.charAt(start);
+		}
+		if(start == end) {
+			return null;
+		}
+		end--;
+		c = text.charAt(end);
+		while(Character.isWhitespace(c)) {
+			end--;
+			c = text.charAt(end);
+		}
+		end++;
+		return XMLPositionUtility.createRange(start, end, this);
+	}
+
 	/**
 	 * Returns the DTD Attribute list for the given element name and empty
 	 * otherwise.
diff --git a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/parser/XMLScanner.java b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/parser/XMLScanner.java
index 1b59f1e452..bd067dd779 100644
--- a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/parser/XMLScanner.java
+++ b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/dom/parser/XMLScanner.java
@@ -446,9 +446,9 @@ TokenType internalScan() {
 				isInsideDTDContent = false;
 				return finishToken(offset, TokenType.DTDEndInternalSubset);
 			}
-
+			boolean startsWithLessThanBracket = false;
 			if (stream.advanceIfChar(_LAN)) { // <
-				
+				startsWithLessThanBracket = true;
 				if (!stream.eos() && stream.peekChar() == _EXL) { // !
 					isDeclCompleted = false;
 					if (stream.advanceIfChars(_EXL, _EVL, _LVL, _EVL, _MVL, _EVL, _NVL, _TVL)) { // !ELEMENT
@@ -469,15 +469,23 @@ TokenType internalScan() {
 						return finishToken(offset, TokenType.StartCommentTag);
 					}
 				}
-				if (stream.advanceUntilCharOrNewTag(_RAN)) { // >
-					stream.advanceIfChar(_RAN); // >
-					return finishToken(offset, TokenType.Content);
-				}
 			}
 			if(isDTDFile) {
-				stream.advanceUntilChar(_LAN); // <
+				if(startsWithLessThanBracket) {
+					if(stream.advanceUntilCharOrNewTag(_RAN)){ // >
+						if(stream.peekChar() == _RAN) {
+							stream.advance(1); //consume '>'
+						}
+					}
+				}
+				else {
+					stream.advanceUntilAnyOfChars(_LAN); // <
+				}
 			} else {
 				stream.advanceUntilAnyOfChars(_RAN, _LAN, _CSB); // > || < || ]
+				if(startsWithLessThanBracket && stream.peekChar() == _RAN) {
+					stream.advance(1); //consume '>'
+				}
 			}
 			return finishToken(offset, TokenType.Content);
 			
diff --git a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/services/XMLCompletions.java b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/services/XMLCompletions.java
index e1139da317..790dad486b 100644
--- a/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/services/XMLCompletions.java
+++ b/org.eclipse.lsp4xml/src/main/java/org/eclipse/lsp4xml/services/XMLCompletions.java
@@ -201,6 +201,13 @@ public CompletionList doComplete(DOMDocument xmlDocument, Position position, Com
 				}
 				break;
 			case Content:
+				if(completionRequest.getXMLDocument().isDTD() || completionRequest.getXMLDocument().isWithinInternalDTD(offset)) {
+					if (scanner.getTokenOffset() <= offset) {
+						collectInsideDTDContent(completionRequest, completionResponse, true);
+						return completionResponse;
+					}
+					break;
+				}
 				if (offset <= scanner.getTokenEnd()) {
 					collectInsideContent(completionRequest, completionResponse);
 					return completionResponse;
@@ -248,6 +255,12 @@ public CompletionList doComplete(DOMDocument xmlDocument, Position position, Com
 				break;
 			}
 			// DTD
+			case DTDAttlistAttributeName:
+			case DTDAttlistAttributeType:
+			case DTDAttlistAttributeValue:
+			case DTDStartAttlist:
+			case DTDStartElement:
+			case DTDStartEntity:
 			case DTDEndTag:
 			case DTDStartInternalSubset:
 			case DTDEndInternalSubset: {
@@ -691,21 +704,40 @@ private void collectAttributeValueSuggestions(int valueStart, int valueEnd, Comp
 	}
 
 	private void collectInsideDTDContent(CompletionRequest request, CompletionResponse response) {
+		collectInsideDTDContent(request,response, false);
+	}
+	private void collectInsideDTDContent(CompletionRequest request, CompletionResponse response, boolean isContent) {
 		// Insert DTD Element Declaration
 		// see https://www.w3.org/TR/REC-xml/#dt-eldecl
+		boolean isSnippetsSupported = request.getCompletionSettings().isCompletionSnippetsSupported();
+		InsertTextFormat insertFormat = isSnippetsSupported ? InsertTextFormat.Snippet : InsertTextFormat.PlainText;
 		CompletionItem elementDecl = new CompletionItem();
 		elementDecl.setLabel("Insert DTD Element declaration");
 		elementDecl.setKind(CompletionItemKind.EnumMember);
-		elementDecl.setFilterText("<!ELEMENT");
-		elementDecl.setInsertTextFormat(InsertTextFormat.Snippet);
+		elementDecl.setFilterText("<!ELEMENT ");
+		elementDecl.setInsertTextFormat(insertFormat);
 		int startOffset = request.getOffset();
+		Range editRange = null;
+		DOMDocument document = request.getXMLDocument();
+		DOMNode node = document.findNodeAt(startOffset);
 		try {
-			Range editRange = getReplaceRange(startOffset, startOffset, request);
-			elementDecl.setTextEdit(new TextEdit(editRange, "<!ELEMENT ${0:element-name} (${1:#PCDATA})>"));
-			elementDecl.setDocumentation("<!ELEMENT element-name (#PCDATA)>");
+			if(node.isDoctype()) {
+				editRange = getReplaceRange(startOffset, startOffset, request);
+			} else {
+				if(isContent) {
+					editRange = document.getTrimmedRange(node.getStart(), node.getEnd());
+				} 
+				if(editRange == null) {
+					editRange = getReplaceRange(node.getStart(), node.getEnd(), request);
+				}
+			}
 		} catch (BadLocationException e) {
-			LOGGER.log(Level.SEVERE, "While performing getReplaceRange for DTD !ELEMENT completion.", e);
+			LOGGER.log(Level.SEVERE, "While performing getReplaceRange for DTD completion.", e);
 		}
+		String textEdit = isSnippetsSupported ? "<!ELEMENT ${1:element-name} (${2:#PCDATA})>" : "<!ELEMENT element-name (#PCDATA)>";
+		elementDecl.setTextEdit(new TextEdit(editRange, textEdit));
+		elementDecl.setDocumentation("<!ELEMENT element-name (#PCDATA)>");
+		
 		response.addCompletionItem(elementDecl);
 
 		// Insert DTD AttrList Declaration
@@ -713,17 +745,15 @@ private void collectInsideDTDContent(CompletionRequest request, CompletionRespon
 		CompletionItem attrListDecl = new CompletionItem();
 		attrListDecl.setLabel("Insert DTD Attributes list declaration");
 		attrListDecl.setKind(CompletionItemKind.EnumMember);
-		attrListDecl.setFilterText("<!ATTLIST");
-		attrListDecl.setInsertTextFormat(InsertTextFormat.Snippet);
+		attrListDecl.setFilterText("<!ATTLIST ");
+		attrListDecl.setInsertTextFormat(insertFormat);
 		startOffset = request.getOffset();
-		try {
-			Range editRange = getReplaceRange(startOffset, startOffset, request);
-			attrListDecl.setTextEdit(new TextEdit(editRange,
-					"<!ATTLIST ${0:element-name} ${1:attribute-name} ${2:ID} ${3:#REQUIRED} >"));
-			attrListDecl.setDocumentation("<!ATTLIST element-name attribute-name ID #REQUIRED >");
-		} catch (BadLocationException e) {
-			LOGGER.log(Level.SEVERE, "While performing getReplaceRange for DTD !ATTLIST completion.", e);
-		}
+		
+		textEdit = isSnippetsSupported ? "<!ATTLIST ${1:element-name} ${2:attribute-name} ${3:ID} ${4:#REQUIRED}>" 
+										: "<!ATTLIST element-name attribute-name ID #REQUIRED>";
+		attrListDecl.setTextEdit(new TextEdit(editRange, textEdit));
+		attrListDecl.setDocumentation("<!ATTLIST element-name attribute-name ID #REQUIRED>");
+		
 		response.addCompletionItem(attrListDecl);
 
 		// Insert Internal DTD Entity Declaration
@@ -731,16 +761,14 @@ private void collectInsideDTDContent(CompletionRequest request, CompletionRespon
 		CompletionItem internalEntity = new CompletionItem();
 		internalEntity.setLabel("Insert Internal DTD Entity declaration");
 		internalEntity.setKind(CompletionItemKind.EnumMember);
-		internalEntity.setFilterText("<!ENTITY");
-		internalEntity.setInsertTextFormat(InsertTextFormat.Snippet);
+		internalEntity.setFilterText("<!ENTITY ");
+		internalEntity.setInsertTextFormat(insertFormat);
 		startOffset = request.getOffset();
-		try {
-			Range editRange = getReplaceRange(startOffset, startOffset, request);
-			internalEntity.setTextEdit(new TextEdit(editRange, "<!ENTITY ${0:entity-name} \"${1:entity-value}\" >"));
-			internalEntity.setDocumentation("<!ENTITY entity-name \"entity-value\" >");
-		} catch (BadLocationException e) {
-			LOGGER.log(Level.SEVERE, "While performing getReplaceRange for DTD !ENTITY completion.", e);
-		}
+		
+		textEdit = isSnippetsSupported ? "<!ENTITY ${1:entity-name} \"${2:entity-value}\">" : "<!ENTITY entity-name \"entity-value\">";
+		internalEntity.setTextEdit(new TextEdit(editRange, textEdit));
+		internalEntity.setDocumentation("<!ENTITY entity-name \"entity-value\">");
+		
 		response.addCompletionItem(internalEntity);
 
 		// Insert External DTD Entity Declaration
@@ -748,17 +776,15 @@ private void collectInsideDTDContent(CompletionRequest request, CompletionRespon
 		CompletionItem externalEntity = new CompletionItem();
 		externalEntity.setLabel("Insert External DTD Entity declaration");
 		externalEntity.setKind(CompletionItemKind.EnumMember);
-		externalEntity.setFilterText("<!ENTITY");
-		externalEntity.setInsertTextFormat(InsertTextFormat.Snippet);
+		externalEntity.setFilterText("<!ENTITY ");
+		externalEntity.setInsertTextFormat(insertFormat);
 		startOffset = request.getOffset();
-		try {
-			Range editRange = getReplaceRange(startOffset, startOffset, request);
-			externalEntity
-					.setTextEdit(new TextEdit(editRange, "<!ENTITY ${0:entity-name} SYSTEM \"${1:entity-value}\" >"));
-			externalEntity.setDocumentation("<!ENTITY entity-name SYSTEM \"entity-value\" >");
-		} catch (BadLocationException e) {
-			LOGGER.log(Level.SEVERE, "While performing getReplaceRange for DTD !ENTITY completion.", e);
-		}
+		
+		textEdit = isSnippetsSupported ? "<!ENTITY ${1:entity-name} SYSTEM \"${2:entity-value}\">" : "<!ENTITY entity-name SYSTEM \"entity-value\">";
+		externalEntity
+				.setTextEdit(new TextEdit(editRange, textEdit));
+		externalEntity.setDocumentation("<!ENTITY entity-name SYSTEM \"entity-value\">");
+		
 		response.addCompletionItem(externalEntity);
 	}
 
diff --git a/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/XMLAssert.java b/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/XMLAssert.java
index ceacd42a5f..3fc22f6d0a 100644
--- a/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/XMLAssert.java
+++ b/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/XMLAssert.java
@@ -112,6 +112,8 @@ public static void testCompletionFor(XMLLanguageService xmlLanguageService, Stri
 				new CompletionSettings(autoCloseTags), expectedItems);
 	}
 
+	
+
 	public static void testCompletionFor(XMLLanguageService xmlLanguageService, String value, String catalogPath,
 			Consumer<XMLLanguageService> customConfiguration, String fileURI, Integer expectedCount,
 			CompletionSettings completionSettings, CompletionItem... expectedItems) throws BadLocationException {
diff --git a/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/extensions/contentmodel/DTDCompletionExtensionsTest.java b/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/extensions/contentmodel/DTDCompletionExtensionsTest.java
index eb481a33b2..c63ff84a30 100644
--- a/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/extensions/contentmodel/DTDCompletionExtensionsTest.java
+++ b/org.eclipse.lsp4xml/src/test/java/org/eclipse/lsp4xml/extensions/contentmodel/DTDCompletionExtensionsTest.java
@@ -83,15 +83,79 @@ public void externalDTDCompletionAttribute() throws BadLocationException {
 				c("Friend", te(11, 9, 11, 9, "Friend=\"\""), "Friend"), //
 				c("Likes", te(11, 9, 11, 9, "Likes=\"\""), "Likes"));
 	}
+
+	@Test
+	public void externalDTDCompletionElementDecl() throws BadLocationException {
+		// completion on <|
+		String xml = "<?xml version = \"1.0\"?>\r\n" + //
+				"<!DOCTYPE Folks [\r\n" + //
+				"	<!ELEMENT Folks (Person*)>\r\n" + //
+				"	<!ELEMENT|\r\n" + //
+				"]>\r\n" + //
+				"<Folks>\r\n" + //
+				"	" + //
+				"</Folks>";
+		testCompletionFor(xml, c("Insert DTD Element declaration", te(3, 1, 3, 11, "<!ELEMENT ${1:element-name} (${2:#PCDATA})>"), "<!ELEMENT "));
+	}
+
+	@Test
+	public void externalDTDCompletionElementDecl2() throws BadLocationException {
+		// completion on <|
+		String xml = "<?xml version = \"1.0\"?>\r\n" + //
+				"<!DOCTYPE Folks [\r\n" + //
+				"	<!ELEMENT Folks (Person*)>\r\n" + //
+				"	<!ELEM|\r\n" + //
+				"]>\r\n" + //
+				"<Folks>\r\n" + //
+				"	" + //
+				"</Folks>";
+		testCompletionFor(xml, c("Insert DTD Element declaration", te(3, 1, 3, 7, "<!ELEMENT ${1:element-name} (${2:#PCDATA})>"), "<!ELEMENT "));
+	}
+	@Test
+	public void externalDTDCompletionAllDecls() throws BadLocationException {
+		// completion on <|
+		String xml = "<?xml version = \"1.0\"?>\r\n" + //
+				"<!DOCTYPE Folks [\r\n" + //
+				"	<!ELEMENT Folks (Person*)>\r\n" + //
+				"	|\r\n" + //
+				"]>\r\n" + //
+				"<Folks>\r\n" + //
+				"	" + //
+				"</Folks>";
+		testCompletionFor(xml, true, 4, c("Insert DTD Element declaration", te(3, 1, 3, 1, "<!ELEMENT ${1:element-name} (${2:#PCDATA})>"), "<!ELEMENT ")
+		,c("Insert Internal DTD Entity declaration", te(3, 1, 3, 1, "<!ENTITY ${1:entity-name} \"${2:entity-value}\">"), "<!ENTITY ")
+		,c("Insert DTD Attributes list declaration", te(3, 1, 3, 1, "<!ATTLIST ${1:element-name} ${2:attribute-name} ${3:ID} ${4:#REQUIRED}>"), "<!ATTLIST ")
+		,c("Insert External DTD Entity declaration", te(3, 1, 3, 1, "<!ENTITY ${1:entity-name} SYSTEM \"${2:entity-value}\">"), "<!ENTITY "));
+	}
+
+	@Test
+	public void externalDTDCompletionAllDeclsSnippetsNotSupported() throws BadLocationException {
+		// completion on <|
+		String xml = "<?xml version = \"1.0\"?>\r\n" + //
+				"<!DOCTYPE Folks [\r\n" + //
+				"	<!ELEMENT Folks (Person*)>\r\n" + //
+				"	|\r\n" + //
+				"]>\r\n" + //
+				"<Folks>\r\n" + //
+				"	" + //
+				"</Folks>";
+		testCompletionFor(xml, false, 4, c("Insert DTD Element declaration", te(3, 1, 3, 1, "<!ELEMENT element-name (#PCDATA)>"), "<!ELEMENT ")
+		,c("Insert Internal DTD Entity declaration", te(3, 1, 3, 1, "<!ENTITY entity-name \"entity-value\">"), "<!ENTITY ")
+		,c("Insert DTD Attributes list declaration", te(3, 1, 3, 1, "<!ATTLIST element-name attribute-name ID #REQUIRED>"), "<!ATTLIST ")
+		,c("Insert External DTD Entity declaration", te(3, 1, 3, 1, "<!ENTITY entity-name SYSTEM \"entity-value\">"), "<!ENTITY "));
+	}
 	
 	private void testCompletionFor(String xml, CompletionItem... expectedItems) throws BadLocationException {
+		testCompletionFor(xml,true, null, expectedItems);
+	}
+
+	private void testCompletionFor(String xml, boolean isSnippetsSupported, Integer expectedCount, CompletionItem... expectedItems) throws BadLocationException {
 		CompletionSettings completionSettings = new CompletionSettings();
 		CompletionCapabilities completionCapabilities = new CompletionCapabilities();
-		CompletionItemCapabilities completionItem = new CompletionItemCapabilities(true); // activate snippets
+		CompletionItemCapabilities completionItem = new CompletionItemCapabilities(isSnippetsSupported); // activate snippets
 		completionCapabilities.setCompletionItem(completionItem);
 		completionSettings.setCapabilities(completionCapabilities);
 		XMLAssert.testCompletionFor(new XMLLanguageService(), xml, "src/test/resources/catalogs/catalog.xml", null,
-				null, null, completionSettings, expectedItems);
-		;
+				null, expectedCount, completionSettings, expectedItems);
 	}
 }