Ingest RANLP 2019

bin/latex_to_unicode.py

@@ -23,6 +23,13 @@
     # Entry(r'\textsc', None, 'sc', 'unary', False),
     # Entry(r'\sc', None, 'sc', 'setter', False),
     Entry(r"\url", None, "url", "unary", True),
+    Entry(r"\footnote", None, "fn", "unary", False),
+    Entry(r"\href", None, "delete", "unary", False),
+    # \cite and friends. There are a lot more, but these are the most common.
+    Entry(r"\cite", None, "cite", "unary", False),
+    Entry(r"\newcite", None, "cite", "unary", False),
+    Entry(r"\citet", None, "cite", "unary", False),
+    Entry(r"\citep", None, "cite", "unary", False),
     Entry(r"\fixedcase", None, "fixed-case", "unary", False),  # for our internal use
     Entry(r"", None, None, "trivial", False),
 ]
@@ -185,7 +192,7 @@ def latex_to_unicode(s):
     s = s.replace(r"\}", "}")
 
     def repl(s):
-        logging.warning("discarding control sequence {}".format(s.group(0)))
+        logging.warning(f"discarding control sequence '{s.group(0)}' from '{s.string}'")
         return ""
 
     s = re.sub(r"\\[A-Za-z]+ |\\.", repl, s)
@@ -223,6 +230,27 @@ def visit(node):
     return node
 
 
+def remove_notes(node):
+    def visit(node, outparent):
+        if isinstance(node, str):
+            outparent.append(node)
+        elif isinstance(node, list):
+            if openers[node[0].rstrip()].tag in ["fn", "delete"]:
+                return  # without copying
+            elif openers[node[0].rstrip()].tag == "cite":
+                outparent.append("(CITATION)")
+            else:
+                outnode = []
+                for child in node:
+                    visit(child, outnode)
+                outparent.append(outnode)
+
+    outroot = []
+    visit(node, outroot)
+    assert len(outroot) == 1
+    return outroot[0]
+
+
 def append_text(xnode, text):
     if len(xnode) == 0:
         xnode.text = (xnode.text or "") + text
@@ -293,6 +321,7 @@ def latex_to_xml(s, fixed_case=False, trivial_math=False):
         tree = find_fixed_case(tree, conservative=True)
     if trivial_math:
         tree = flatten_trivial_math(tree)
+    tree = remove_notes(tree)
     tree = latextree_to_xml(tree)
     tree = xml_to_unicode(tree)
     return tree

bin/normalize_anth.py

@@ -174,7 +174,11 @@ def normalize(oldnode, informat):
                     )
                 )
             oldtext = "".join(oldnode.itertext())
-            newnode = latex_to_xml(oldtext, trivial_math=True, fixed_case=True)
+            newnode = latex_to_xml(
+                oldtext,
+                trivial_math=True,
+                fixed_case=oldnode.tag in ["title", "booktitle"],
+            )
             newnode.tag = oldnode.tag
             newnode.attrib.update(oldnode.attrib)
             replace_node(oldnode, newnode)

data/yaml/venues_joint_map.yaml

@@ -727,3 +727,6 @@ D19-65: [WS,DiscoMT]
 D19-66: WS
 D19-67: WS
 D19-68: WS
+W19-87: RANLP
+W19-89: RANLP
+W19-90: RANLP