Merge branch 'dev'

antlr · Feb 19, 2023 · 6763f58 · 6763f58
2 parents a156a06 + 46445ea
commit 6763f58
Show file tree

Hide file tree

Showing 282 changed files with 5,779 additions and 2,637 deletions.
diff --git a/.github/workflows/hosted.yml b/.github/workflows/hosted.yml
@@ -1,5 +1,9 @@
 name: antlr4
 
+concurrency:
+  group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches: [ master, dev, hostedci ]
@@ -17,16 +21,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [
-          macos-latest,
-          ubuntu-latest,
-          windows-latest
+          macos-11,
+          ubuntu-20.04,
+          windows-2022
         ]
         compiler: [ clang, gcc ]
         exclude:
-          - os: windows-latest
+          - os: windows-2022
             compiler: gcc
         include:
-          - os: windows-latest
+          - os: windows-2022
             compiler: cl
 
     steps:
@@ -49,11 +53,11 @@ jobs:
         cygwin: 0
 
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Use ccache
       if: startswith(matrix.os, 'macos') || startswith(matrix.os, 'ubuntu')
-      uses: hendrikmuhs/ccache-action@v1
+      uses: hendrikmuhs/ccache-action@v1.2
       with:
         key: ${{ matrix.os }}-${{ matrix.compiler }}
 
@@ -112,10 +116,10 @@ jobs:
           echo 'CC=clang' >> $GITHUB_ENV
           echo 'CXX=clang++' >> $GITHUB_ENV
         else
-          export CC=gcc-9
-          export CXX=g++-9
-          echo 'CC=gcc-9' >> $GITHUB_ENV
-          echo 'CXX=g++-9' >> $GITHUB_ENV
+          export CC=gcc
+          export CXX=g++
+          echo 'CC=gcc' >> $GITHUB_ENV
+          echo 'CXX=g++' >> $GITHUB_ENV
         fi
 
         env
@@ -141,7 +145,7 @@ jobs:
 
     - name: Archive artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: antlr_${{ matrix.os }}_${{ matrix.compiler }}
         path: antlr_${{ matrix.os }}_${{ matrix.compiler }}.tgz
@@ -154,9 +158,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [
-          macos-latest,
-          ubuntu-latest,
-          windows-latest
+          macos-11,
+          ubuntu-20.04,
+          windows-2022
         ]
         target: [
           tool,
@@ -166,13 +170,14 @@ jobs:
           go,
           java,
           javascript,
+          typescript,
           php,
           python2,
           python3,
-          swift,
+          # swift,
         ]
         exclude:
-          - os: windows-latest
+          - os: windows-2022
             target: swift
 
     steps:
@@ -181,15 +186,23 @@ jobs:
       # dependencies, for instance, the setup-java actually parses
       # **/pom.xml files to decide what to cache.
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Checkout antlr PHP runtime
       if: matrix.target == 'php'
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         repository: antlr/antlr-php-runtime
         path: runtime/PHP
 
+    - name: Install dependencies
+      env:
+        COMPOSER_CACHE_DIR: ${{ github.workspace }}/.cache
+      if: matrix.target == 'php'
+      run: |-
+        cd runtime/PHP
+        composer install --no-progress --no-interaction --prefer-dist --optimize-autoloader
+
     - name: Install dependencies (Ubuntu)
       if: startswith(matrix.os, 'ubuntu')
       run: |
@@ -210,7 +223,7 @@ jobs:
 
     - name: Set up Maven
       if: steps.setup-java.outputs.cache-hit != 'true'
-      uses: stCarolas/setup-maven@v4.4
+      uses: stCarolas/setup-maven@v4.5
       with:
         maven-version: 3.8.5
 
@@ -232,15 +245,15 @@ jobs:
         python-version: '3.x'
         architecture: 'x64'
 
-    - name: Set up Node 14
-      if: matrix.target == 'javascript'
-      uses: actions/setup-node@v3
+    - name: Set up Node 16
+      if: (matrix.target == 'javascript') || (matrix.target == 'typescript')
+      uses: actions/setup-node@v3.6.0
       with:
-        node-version: '14'
+        node-version: '16'
 
     - name: Setup Dotnet
       if: matrix.target == 'csharp'
-      uses: actions/setup-dotnet@v2
+      uses: actions/setup-dotnet@v3.0.3
       with:
         dotnet-version: '6.0.x'
 
@@ -252,27 +265,27 @@ jobs:
 
     - name: Setup Go 1.19
       if: matrix.target == 'go'
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v3.3.1
       with:
         go-version: '^1.19'
 
     - name: Setup PHP 8.2
       if: matrix.target == 'php'
-      uses: shivammathur/setup-php@v2
+      uses: shivammathur/setup-php@2.22.0
       with:
         php-version: '8.2'
         extensions: mbstring
         tools: composer
 
     - name: Setup Swift
       if: matrix.target == 'swift'
-      uses: swift-actions/setup-swift@v1
+      uses: swift-actions/setup-swift@v1.19.0
       with:
         swift-version: '5.2'
 
     - name: Use ccache
       if: (startswith(matrix.os, 'macos') || startswith(matrix.os, 'ubuntu')) && (matrix.target == 'cpp')
-      uses: hendrikmuhs/ccache-action@v1
+      uses: hendrikmuhs/ccache-action@v1.2
       with:
         key: ${{ matrix.os }}-${{ matrix.target }}
 
@@ -318,7 +331,6 @@ jobs:
         case ${{ matrix.target }} in
           python2) mvn -X '-Dantlr-python2-exec=${{ env.pythonLocation }}/bin/python' '-Dtest=python2.**' test ;;
           python3) mvn -X '-Dantlr-python3-exec=${{ env.pythonLocation }}/bin/python' '-Dtest=python3.**' test ;;
-          swift) mvn -X -e '-Dtest=python3.**' test ;;
           *) mvn -X '-Dtest=${{ matrix.target }}.**' test ;;
         esac
 
@@ -331,7 +343,7 @@ jobs:
 
     - name: Archive artifacts
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: antlr_${{ matrix.os }}_${{ matrix.target }}
         path: antlr_${{ matrix.os }}_${{ matrix.target }}.tgz
diff --git a/Package.swift b/Package.swift
@@ -7,7 +7,10 @@ let package = Package(
     products: [
         .library(
             name: "Antlr4",
-            type: .dynamic,
+            targets: ["Antlr4"]),
+        .library(
+            name: "Antlr4Static",
+            type: .static,
             targets: ["Antlr4"]),
     ],
     targets: [

diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml
@@ -8,7 +8,7 @@
   <parent>
     <groupId>org.antlr</groupId>
     <artifactId>antlr4-master</artifactId>
-    <version>4.11.2-SNAPSHOT</version>
+    <version>4.12.0-SNAPSHOT</version>
   </parent>
   <artifactId>antlr4-maven-plugin</artifactId>
   <packaging>maven-plugin</packaging>
@@ -143,6 +143,7 @@
       <plugin>
          <groupId>io.takari.maven.plugins</groupId>
          <artifactId>takari-lifecycle-plugin</artifactId>
+         <version>2.0.7</version>         
          <extensions>true</extensions>
          <executions>
            <execution>

diff --git a/doc/actions.md b/doc/actions.md
@@ -65,7 +65,7 @@ Most of the time you access the attributes of the token, but sometimes it is use
 |text|String|The text matched for the token; translates to a call to getText. Example: $ID.text.|
 |type|int|The token type (nonzero positive integer) of the token such as INT; translates to a call to getType. Example: $ID.type.|
 |line|int|The line number on which the token occurs, counting from 1; translates to a call to getLine. Example: $ID.line.|
-|pos|int|The character position within the line at which the token’s first character occurs counting from zero; translates to a call togetCharPositionInLine. Example: $ID.pos.|
+|pos|int|The character position within the line at which the token’s first character occurs counting from zero; translates to a call to getCharPositionInLine. Example: $ID.pos.|
 |index|int|The overall index of this token in the token stream, counting from zero; translates to a call to getTokenIndex. Example: $ID.index.|
 |channel|int|The token’s channel number. The parser tunes to only one channel, effectively ignoring off-channel tokens. The default channel is 0 (Token.DEFAULT_CHANNEL), and the default hidden channel is Token.HIDDEN_CHANNEL. Translates to a call to getChannel. Example: $ID.channel.|
 |int|int|The integer value of the text held by this token; it assumes that the text is a valid numeric string. Handy for building calculators and so on. Translates to Integer.valueOf(text-of-token). Example: $INT.int.|

diff --git a/doc/antlr-project-testing.md b/doc/antlr-project-testing.md
@@ -98,7 +98,7 @@ From the `runtime-testsuite` dir
 ```bash
 $ cd runtime-testsuite
 $ export MAVEN_OPTS="-Xmx1G"     # don't forget this on linux
-$ mvn -Dtest=java.** test
+$ mvn -Dtest='java.**' test
 -------------------------------------------------------
  T E S T S
 -------------------------------------------------------

diff --git a/doc/creating-a-language-target.md b/doc/creating-a-language-target.md
@@ -45,3 +45,37 @@ $ mvn compile
 ```
 
 That should proceed with success. See [Building ANTLR](building-antlr.md) for more details.
+
+## Comparing your target's parsing decisionmaking with Java's
+
+ANTLR's power comes from it's dynamic parsing strategy, but that means each target
+must implement that complicated algorithm. You should compare your target's debug
+output for ParserATNSimulator with Java's.
+
+Run this so we get right jars before trying this script:
+
+```
+cd ANTLR-ROOT-DIR
+mvn install -DskipTests=true
+cd runtime-tests 
+mvn install -DskipTests=true # yes do it again
+```
+
+Run the script from `runtime-tests` dir with
+
+```
+../scripts/traceatn.sh /tmp/JSON.g4 json -target Go /tmp/foo.json
+```
+
+or whatever your test grammar, start rule, target, test input are.
+
+### Debugging the PHP target
+
+Because the PHP target is hosted in a separate repository, you will need to clone the [antlr/php-antlr-runtime](https://github.com/antlr/antlr-php-runtime)
+repository into the `runtime/PHP` and install the dependencies with `composer install` before you can run the tests.
+
+```
+git clone -b dev https://github.com/antlr/antlr-php-runtime.git runtime/PHP
+cd runtime/PHP
+composer install
+```
diff --git a/doc/getting-started.md b/doc/getting-started.md
@@ -14,10 +14,10 @@ That command creates `antlr4` and `antlr4-parse` executables that, if necessary,
 
 ```bash
 $ antlr4 
-Downloading antlr4-4.11.1-complete.jar
+Downloading antlr4-4.12.0-complete.jar
 ANTLR tool needs Java to run; install Java JRE 11 yes/no (default yes)? y
 Installed Java in /Users/parrt/.jre/jdk-11.0.15+10-jre; remove that dir to uninstall
-ANTLR Parser Generator  Version 4.11.1
+ANTLR Parser Generator  Version 4.12.0
  -o ___              specify output directory where all output is generated
  -lib ___            specify location of grammars, tokens files
 ...
@@ -130,7 +130,7 @@ ExprBaseListener.h    ExprLexer.h           ExprListener.h        ExprParser.h
 
 ANTLR is really two things: a tool written in Java that translates your grammar to a parser/lexer in Java (or other target language) and the runtime library needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library. 
 
-The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers.  In what follows, I talk about antlr-4.11.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
+The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers.  In what follows, I talk about antlr-4.12.0-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3).
 
 If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see [Integrating ANTLR into Development Systems](https://github.com/antlr/antlr4/blob/master/doc/IDEs.md).
 
@@ -140,38 +140,38 @@ If you are going to integrate ANTLR into your existing build system using mvn, a
 1. Download
 ```
 $ cd /usr/local/lib
-$ curl -O https://www.antlr.org/download/antlr-4.11.1-complete.jar
+$ curl -O https://www.antlr.org/download/antlr-4.12.0-complete.jar
 ```
 Or just download in browser from website:
     [https://www.antlr.org/download.html](https://www.antlr.org/download.html)
 and put it somewhere rational like `/usr/local/lib`.
 
-if you are using lower version jdk, just download from [website download](https://github.com/antlr/website-antlr4/tree/gh-pages/download) for previous version, and antlr version before 4.11.1 support jdk 1.8  
+if you are using lower version jdk, just download from [website download](https://github.com/antlr/website-antlr4/tree/gh-pages/download) for previous version, and antlr version before 4.12.0 support jdk 1.8  
 
-2. Add `antlr-4.11.1-complete.jar` to your `CLASSPATH`:
+2. Add `antlr-4.12.0-complete.jar` to your `CLASSPATH`:
 ```
-$ export CLASSPATH=".:/usr/local/lib/antlr-4.11.1-complete.jar:$CLASSPATH"
+$ export CLASSPATH=".:/usr/local/lib/antlr-4.12.0-complete.jar:$CLASSPATH"
 ```
 It's also a good idea to put this in your `.bash_profile` or whatever your startup script is.
 
 3. Create aliases for the ANTLR Tool, and `TestRig`.
 ```
-$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.11.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
-$ alias grun='java -Xmx500M -cp "/usr/local/lib/antlr-4.11.1-complete.jar:$CLASSPATH" org.antlr.v4.gui.TestRig'
+$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.12.0-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
+$ alias grun='java -Xmx500M -cp "/usr/local/lib/antlr-4.12.0-complete.jar:$CLASSPATH" org.antlr.v4.gui.TestRig'
 ```
 
 ### WINDOWS
 
 (*Thanks to Graham Wideman*)
 
 0. Install Java (version 1.7 or higher)
-1. Download antlr-4.11.1-complete.jar (or whatever version) from [https://www.antlr.org/download.html](https://www.antlr.org/download.html)
+1. Download antlr-4.12.0-complete.jar (or whatever version) from [https://www.antlr.org/download.html](https://www.antlr.org/download.html)
 Save to your directory for 3rd party Java libraries, say `C:\Javalib`
-2. Add `antlr-4.11.1-complete.jar` to CLASSPATH, either:
+2. Add `antlr-4.12.0-complete.jar` to CLASSPATH, either:
   * Permanently: Using System Properties dialog > Environment variables > Create or append to `CLASSPATH` variable
   * Temporarily, at command line:
 ```
-SET CLASSPATH=.;C:\Javalib\antlr-4.11.1-complete.jar;%CLASSPATH%
+SET CLASSPATH=.;C:\Javalib\antlr-4.12.0-complete.jar;%CLASSPATH%
 ```
 3. Create short convenient commands for the ANTLR Tool, and TestRig, using batch files or doskey commands:
   * Batch files (in directory in system PATH) antlr4.bat and grun.bat
@@ -197,7 +197,7 @@ Either launch org.antlr.v4.Tool directly:
 
 ```
 $ java org.antlr.v4.Tool
-ANTLR Parser Generator Version 4.11.1
+ANTLR Parser Generator Version 4.12.0
 -o ___ specify output directory where all output is generated
 -lib ___ specify location of .tokens files
 ...
@@ -206,8 +206,8 @@ ANTLR Parser Generator Version 4.11.1
 or use -jar option on java:
 
 ```
-$ java -jar /usr/local/lib/antlr-4.11.1-complete.jar
-ANTLR Parser Generator Version 4.11.1
+$ java -jar /usr/local/lib/antlr-4.12.0-complete.jar
+ANTLR Parser Generator Version 4.12.0
 -o ___ specify output directory where all output is generated
 -lib ___ specify location of .tokens files
 ...