Merge pull request #1123 from eregon/javaNewlineFlag

Set newline flag for Java nodes by walking the AST after deserialization

bin/insns

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ruby --dump=insns "$@"

config.yml

@@ -940,6 +940,7 @@ nodes:
         kind: StatementsNode
       - name: end_keyword_loc
         type: location?
+    newline: false
     comment: |
       Represents an `else` clause in a `case`, `if`, or `unless` statement.
 
@@ -1178,6 +1179,7 @@ nodes:
         type: node?
       - name: end_keyword_loc
         type: location?
+    newline: false
     comment: |
       Represents the use of the `if` keyword, either in the block form or the modifier form.
 
@@ -1287,6 +1289,7 @@ nodes:
         type: location
       - name: flags
         type: uint32
+    newline: false
     comment: |
       Represents a regular expression literal that contains interpolation.
 
@@ -1300,6 +1303,7 @@ nodes:
         type: node[]
       - name: closing_loc
         type: location?
+    newline: false
     comment: |
       Represents a string literal that contains interpolation.
 
@@ -1313,6 +1317,7 @@ nodes:
         type: node[]
       - name: closing_loc
         type: location?
+    newline: false
     comment: |
       Represents a symbol literal that contains interpolation.
 
@@ -1326,6 +1331,7 @@ nodes:
         type: node[]
       - name: closing_loc
         type: location
+    newline: false
     comment: |
       Represents an xstring literal that contains interpolation.
 
@@ -1622,8 +1628,9 @@ nodes:
         type: location
       - name: closing_loc
         type: location
+    newline: false
     comment: |
-      Represents a parentesized expression
+      Represents a parenthesized expression
 
           (10 + 34)
           ^^^^^^^^^
@@ -1694,6 +1701,7 @@ nodes:
       - name: statements
         type: node
         kind: StatementsNode
+    newline: false
     comment: The top level node of any parse tree.
   - name: RangeNode
     child_nodes:
@@ -1898,6 +1906,7 @@ nodes:
     child_nodes:
       - name: body
         type: node[]
+    newline: false
     comment: |
       Represents a set of statements contained within some scope.
 
@@ -2007,6 +2016,7 @@ nodes:
         kind: ElseNode
       - name: end_keyword_loc
         type: location?
+    newline: false
     comment: |
       Represents the use of the `unless` keyword, either in the block form or the modifier form.
 

templates/java/org/yarp/Loader.java.erb

@@ -7,8 +7,8 @@ import java.nio.ByteOrder;
 // @formatter:off
 public class Loader {
 
-    public static Nodes.Node load(byte[] source, byte[] serialized) {
-        return new Loader(serialized).load(source);
+    public static Nodes.Node load(byte[] serialized, Nodes.Source source) {
+        return new Loader(serialized, source).load();
     }
 
     private static final class ConstantPool {
@@ -42,12 +42,16 @@ public class Loader {
 
     private final ByteBuffer buffer;
     private ConstantPool constantPool;
+    private final Nodes.Source source;
+    private final boolean[] newlineMarked;
 
-    private Loader(byte[] serialized) {
-        buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder());
+    private Loader(byte[] serialized, Nodes.Source source) {
+        this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder());
+        this.source = source;
+        this.newlineMarked = new boolean[1 + source.getLineCount()];
     }
 
-    private Nodes.Node load(byte[] source) {
+    private Nodes.Node load() {
         expect((byte) 'Y');
         expect((byte) 'A');
         expect((byte) 'R');
@@ -65,15 +69,17 @@ public class Loader {
 
         int constantPoolBufferOffset = buffer.getInt();
         int constantPoolLength = loadVarInt();
-        this.constantPool = new ConstantPool(source, constantPoolBufferOffset, constantPoolLength);
+        this.constantPool = new ConstantPool(source.bytes, constantPoolBufferOffset, constantPoolLength);
 
         Nodes.Node node = loadNode();
-        int left = constantPoolBufferOffset - buffer.position();
 
+        int left = constantPoolBufferOffset - buffer.position();
         if (left != 0) {
             throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left");
         }
 
+        node.setNewLineFlag(this.source, newlineMarked);
+
         return node;
     }
 

templates/java/org/yarp/Nodes.java.erb

@@ -77,12 +77,41 @@ public abstract class Nodes {
         }
     }
 
+    public static final class Source {
+        public final byte[] bytes;
+        private final int[] lineOffsets;
+
+        public Source(byte[] bytes, int[] lineOffsets) {
+            assert lineOffsets[0] == 0;
+            this.bytes = bytes;
+            this.lineOffsets = lineOffsets;
+        }
+
+        public int line(int byteOffset) {
+            assert byteOffset >= 0 && byteOffset < bytes.length : byteOffset;
+            int index = Arrays.binarySearch(lineOffsets, byteOffset);
+            int line;
+            if (index < 0) {
+                line = -index - 1;
+            } else {
+                line = index + 1;
+            }
+            assert line >= 1 && line <= getLineCount() : line;
+            return line;
+        }
+
+        public int getLineCount() {
+            return lineOffsets.length;
+        }
+    }
+
     public static abstract class Node {
 
         public static final Node[] EMPTY_ARRAY = {};
 
         public final int startOffset;
         public final int length;
+        private boolean newLineFlag = false;
 
         public Node(int startOffset, int length) {
             this.startOffset = startOffset;
@@ -97,6 +126,18 @@ public abstract class Nodes {
             return startOffset + length;
         }
 
+        public boolean hasNewLineFlag() {
+            return newLineFlag;
+        }
+
+        public void setNewLineFlag(Source source, boolean[] newlineMarked) {
+            int line = source.line(this.startOffset);
+            if (!newlineMarked[line]) {
+                newlineMarked[line] = true;
+                this.newLineFlag = true;
+            }
+        }
+
         public abstract <T> T accept(AbstractNodeVisitor<T> visitor);
 
         public abstract Node[] childNodes();
@@ -108,7 +149,11 @@ public abstract class Nodes {
 
         private String toString(String indent) {
             StringBuilder builder = new StringBuilder();
-            builder.append(indent).append(this.getClass().getSimpleName()).append('\n');
+            builder.append(indent).append(this.getClass().getSimpleName());
+            if (hasNewLineFlag()) {
+                builder.append("[Li]");
+            }
+            builder.append('\n');
             for (Node child : childNodes()) {
                 if (child != null) {
                     builder.append(child.toString(indent + "  "));
@@ -144,6 +189,28 @@ public abstract class Nodes {
         <%- end -%>
         }
 
+        <%- unless node.newline -%>
+        public void setNewLineFlag(Source source, boolean[] newlineMarked) {
+          <%- node.params.each do |param| -%>
+            <%- case param -%>
+            <%- when NodeListParam -%>
+            for (Nodes.Node node : this.<%= param.name %>) {
+                node.setNewLineFlag(source, newlineMarked);
+            }
+            <%- when NodeParam -%>
+            this.<%= param.name %>.setNewLineFlag(source, newlineMarked);
+            <%- when OptionalNodeParam -%>
+            if (this.<%= param.name %> != null) {
+                this.<%= param.name %>.setNewLineFlag(source, newlineMarked);
+            }
+            <%- when -> _ { !param.java_type.include?("Node") } -%>
+            <%- else -%>
+            <%- raise "Param type #{param.class} not handled in #{__FILE__ }" -%>
+            <%- end -%>
+          <%- end -%>
+        }
+        <%- end -%>
+
         public Node[] childNodes() {
           <%- if node.params.none?(NodeListParam) and node.params.none?(SingleNodeParam) -%>
             return EMPTY_ARRAY;

templates/template.rb

@@ -118,7 +118,7 @@ def java_type = "int"
 # YAML format. It contains information about the name of the node and the
 # various child nodes it contains.
 class NodeType
-  attr_reader :name, :type, :human, :params, :comment
+  attr_reader :name, :type, :human, :params, :newline, :comment
 
   def initialize(config)
     @name = config.fetch("name")
@@ -131,6 +131,7 @@ def initialize(config)
                    raise("Unknown param type: #{param["type"].inspect}")
       param_type.new(**param.transform_keys(&:to_sym))
     end
+    @newline = config.fetch("newline", true)
     @comment = config.fetch("comment")
   end