[Java.Interop.Tools.JavaSource] Improve <code> parsing (#1125)

Fixes: #1071

The latest API docs update contained a couple dozen parsing issues
due to `<code/>` parsing, including:

  * Closing element doesn't match opening element: `<code>null</null>`
  * Content including `@`: `<code>android:label="@string/resolve_title"</code>`
  * Closing element is actually an opening element:
    `<code>Activity.RESULT_OK<code>`
  * Improper element nesting: `<code><pre><p>content</code></pre></p>`
  * Use of attributes: `<code class=prettyprint>content<code>`

Fix this by replacing `CodeElementDeclaration` to use a new
`CodeElementContentTerm` terminal, which is a "greedy regex" which
grabs `<code` until one of:

  * `</code>`
  * `</null>`
  * `<code>`

The result of `CodeElementDeclaration` is the end of the `<code>`
element until the beginning of one of the above terminators:

  * `<code>null</null>` becomes `<c>null</c>`
  * `<code>android:label="@string/resolve_title"</code>` becomes
    `<c>android:label="@string/resolve_title"</c>`.`
  * `<code>Activity.RESULT_OK<code>` becomes `<c>Activity.RESULT_OK</c>`.
  * `<code><pre><p>content</code></pre></p>` becomes the mess
    `<c>&lt;pre&gt;&lt;p&gt;some content</c>&lt;/pre&gt;&lt;/p&gt;` 🤷‍♂️
  * `<code class=prettyprint>content<code>` becomes `<c>content</c>`.`

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs

@@ -131,9 +131,17 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 					}
 				};
 
-				CodeElementDeclaration.Rule = CreateStartElement ("code", grammar) + InlineDeclarations + CreateEndElement ("code", grammar);
+				CodeElementDeclaration.Rule = CodeElementContentTerm;
 				CodeElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
-					var target = parseNode.ChildNodes [1].AstNode;
+					// Parse the entire <code> element captured in the token
+					var codeElementText = parseNode.ChildNodes [0].Token.Text;
+					int startIndex = codeElementText.IndexOf ('>');
+					int stopIndex = codeElementText.LastIndexOf ('<');
+					if (startIndex == -1 || stopIndex == -1) {
+						parseNode.AstNode = new XText (codeElementText);
+						return;
+					}
+					var target = codeElementText.Substring (startIndex + 1, stopIndex - startIndex - 1);
 					parseNode.AstNode = new XElement ("c", target);
 				};
 			}
@@ -232,6 +240,12 @@ static string GetChildNodesAsString (ParseTreeNode parseNode)
 			public  readonly    NonTerminal InlineHyperLinkDeclaration  = new NonTerminal (nameof (InlineHyperLinkDeclaration), ConcatChildNodes);
 			public  readonly    NonTerminal CodeElementDeclaration      = new NonTerminal (nameof (CodeElementDeclaration), ConcatChildNodes);
 
+			public  readonly    Terminal    CodeElementContentTerm      = new RegexBasedTerminal ("<code>", $@"(?i)<code\s*[^>]*>(.|\s)*?(<\/code>|<\/null>|<code>)") {
+				AstConfig = new AstNodeConfig {
+					NodeCreator = (context, parseNode) => parseNode.AstNode = "",
+				},
+			};
+
 			public  readonly    Terminal    InlineHyperLinkOpenTerm     = new RegexBasedTerminal ("<a attr=", @"(?i)<a\s*.*=") {
 				AstConfig = new AstNodeConfig {
 					NodeCreator = (context, parseNode) => parseNode.AstNode = "",

tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs

@@ -101,7 +101,38 @@ public void CodeElementDeclaration ()
 			var r = p.Parse ("<code>input.position()</code>");
 			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
 			Assert.AreEqual ("<c>input.position()</c>", r.Root.AstNode.ToString ());
-		}
 
+			r = p.Parse ("<code>null</null>");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ("<c>null</c>", r.Root.AstNode.ToString ());
+
+			r = p.Parse ("<code>android:label=\"@string/resolve_title\"</code>");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ("<c>android:label=\"@string/resolve_title\"</c>", r.Root.AstNode.ToString ());
+
+			r = p.Parse ("<code>Activity.RESULT_OK<code>");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ("<c>Activity.RESULT_OK</c>", r.Root.AstNode.ToString ());
+
+			r = p.Parse ("<code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code>");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ("<c>format.setString(MediaFormat.KEY_FRAME_RATE, null)</c>", r.Root.AstNode.ToString ());
+
+			r = p.Parse (@"<code>
+<p> [ 0,  0,  0,  0,  0  ]
+<p> [ 0,  0,  0,  0,  0  ]
+<p> [ 0,  0,  1,  0,  0  ]
+<p> [ 0,  0,  0,  0,  0  ]
+<p> [ 0,  0,  0,  0,  0  ]
+</code>");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual (@"<c>
+&lt;p&gt; [ 0,  0,  0,  0,  0  ]
+&lt;p&gt; [ 0,  0,  0,  0,  0  ]
+&lt;p&gt; [ 0,  0,  1,  0,  0  ]
+&lt;p&gt; [ 0,  0,  0,  0,  0  ]
+&lt;p&gt; [ 0,  0,  0,  0,  0  ]
+</c>", r.Root.AstNode.ToString ());
+		}
 	}
 }

tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs

@@ -144,16 +144,36 @@ more description here.</para>
 </member>",
 			},
 			new ParseResult {
-				Javadoc = "Something {@link #method}: description, \"<code>declaration</code>\" or \"<code>another declaration</code>\".\n\n@apiSince 1\n",
+				Javadoc = "Something {@link #method}: description, \"<code>declaration</code>\" or <code><pre><p>some content</code></pre></p>.\n\n@apiSince 1\n",
 				FullXml = @"<member>
-  <summary>Something <c>#method</c>: description, ""<c>declaration</c>"" or ""<c>another declaration</c>"".</summary>
+  <summary>Something <c>#method</c>: description, ""<c>declaration</c>"" or <c>&lt;pre&gt;&lt;p&gt;some content</c>&lt;/pre&gt;&lt;/p&gt;.</summary>
   <remarks>
-    <para>Something <c>#method</c>: description, ""<c>declaration</c>"" or ""<c>another declaration</c>"".</para>
+    <para>Something <c>#method</c>: description, ""<c>declaration</c>"" or <c>&lt;pre&gt;&lt;p&gt;some content</c>&lt;/pre&gt;&lt;/p&gt;.</para>
     <para>Added in API level 1.</para>
   </remarks>
 </member>",
 				IntelliSenseXml = @"<member>
-  <summary>Something <c>#method</c>: description, ""<c>declaration</c>"" or ""<c>another declaration</c>"".</summary>
+  <summary>Something <c>#method</c>: description, ""<c>declaration</c>"" or <c>&lt;pre&gt;&lt;p&gt;some content</c>&lt;/pre&gt;&lt;/p&gt;.</summary>
+</member>",
+			},
+			new ParseResult {
+				Javadoc = @"The result code will be <code>Activity.RESULT_OK<code> for success,
+  or one of these errors:
+  <code>RESULT_ERROR_GENERIC_FAILURE</code>",
+				FullXml = @"<member>
+  <summary>The result code will be <c>Activity.RESULT_OK</c> for success,
+  or one of these errors:
+  <c>RESULT_ERROR_GENERIC_FAILURE</c></summary>
+  <remarks>
+    <para>The result code will be <c>Activity.RESULT_OK</c> for success,
+  or one of these errors:
+  <c>RESULT_ERROR_GENERIC_FAILURE</c></para>
+  </remarks>
+</member>",
+				IntelliSenseXml = @"<member>
+  <summary>The result code will be <c>Activity.RESULT_OK</c> for success,
+  or one of these errors:
+  <c>RESULT_ERROR_GENERIC_FAILURE</c></summary>
 </member>",
 			},
 			new ParseResult {