chore: decouple CLI from ROWKEY

fixes: confluentinc#4928 The server response when describing a source schema now includes `fieldType='SYSTEM'` or `fieldType='KEY'` to differentiate system columns, i.e. ROWTIME, and key columns from value column. The default is no field type. This design decision was taken as the `FieldInfo` pojo is used not just to describe the columns in the schema, but also fields in structs and a fieldType of `VALUE` wouldn't make much sense for a struct field. Where as no field type kind of works.
big-andy-coates · Mar 30, 2020 · 18b7c79 · 18b7c79
1 parent e2ee7e8
commit 18b7c79
Show file tree

Hide file tree

Showing 20 changed files with 120 additions and 64 deletions.
diff --git a/docs-md/developer-guide/create-a-stream.md b/docs-md/developer-guide/create-a-stream.md
@@ -92,7 +92,7 @@ Name                 : PAGEVIEWS
  Field    | Type
 --------------------------------------
  ROWTIME  | BIGINT           (system)
- ROWKEY   | VARCHAR(STRING)  (system)
+ ROWKEY   | VARCHAR(STRING)  (key)
  VIEWTIME | BIGINT
  USERID   | VARCHAR(STRING)
  PAGEID   | VARCHAR(STRING)

diff --git a/docs-md/developer-guide/create-a-table.md b/docs-md/developer-guide/create-a-table.md
@@ -98,7 +98,7 @@ Name                 : USERS
  Field        | Type
 ------------------------------------------
  ROWTIME      | BIGINT           (system)
- ROWKEY       | VARCHAR(STRING)  (system)
+ ROWKEY       | VARCHAR(STRING)  (key)
  REGISTERTIME | BIGINT
  USERID       | VARCHAR(STRING)
  GENDER       | VARCHAR(STRING)

diff --git a/docs-md/developer-guide/ksqldb-reference/describe.md b/docs-md/developer-guide/ksqldb-reference/describe.md
@@ -56,7 +56,7 @@ Your output should resemble:
  Field   | Type
 -------------------------------------
  ROWTIME | BIGINT           (system)
- ROWKEY  | VARCHAR(STRING)  (system)
+ ROWKEY  | VARCHAR(STRING)  (key)
  IP      | VARCHAR(STRING)  (key)
  KBYTES  | BIGINT
 -------------------------------------
@@ -83,7 +83,7 @@ Kafka output topic   : IP_SUM (partitions: 4, replication: 1)
  Field   | Type
 -------------------------------------
  ROWTIME | BIGINT           (system)
- ROWKEY  | VARCHAR(STRING)  (system)
+ ROWKEY  | VARCHAR(STRING)  (key)
  IP      | VARCHAR(STRING)  (key)
  KBYTES  | BIGINT
 -------------------------------------

diff --git a/docs-md/developer-guide/query-with-structured-data.md b/docs-md/developer-guide/query-with-structured-data.md
@@ -276,7 +276,7 @@ Name                 : TYPE_1
  Field         | Type
 -------------------------------------------
  ROWTIME       | BIGINT           (system)
- ROWKEY        | VARCHAR(STRING)  (system)
+ ROWKEY        | VARCHAR(STRING)  (key)
  DATA__field-a | INTEGER
  DATA__field-b | VARCHAR(STRING)
 -------------------------------------------
@@ -296,7 +296,7 @@ Name                 : TYPE_2
  Field         | Type
 -------------------------------------------
  ROWTIME       | BIGINT           (system)
- ROWKEY        | VARCHAR(STRING)  (system)
+ ROWKEY        | VARCHAR(STRING)  (key)
  DATA__field-a | INTEGER
  DATA__field-c | INTEGER
  DATA__field-d | VARCHAR(STRING)

diff --git a/docs-md/developer-guide/test-and-debug/generate-custom-test-data.md b/docs-md/developer-guide/test-and-debug/generate-custom-test-data.md
@@ -131,7 +131,7 @@ Name                 : ORDERS_RAW
  Field     | Type                                                                 
 ----------------------------------------------------------------------------------
  ROWTIME   | BIGINT           (system)                                            
- ROWKEY    | VARCHAR(STRING)  (system)                                            
+ ROWKEY    | VARCHAR(STRING)  (key)
  ITEMID    | VARCHAR(STRING)                                                      
  PRICE     | DOUBLE                                                               
  LOCATION  | STRUCT<CITY VARCHAR(STRING), STATE VARCHAR(STRING), ZIPCODE INTEGER> 
@@ -187,7 +187,7 @@ Name                 : USERS_ORIGINAL
  Field        | Type                      
 ------------------------------------------
  ROWTIME      | BIGINT           (system) 
- ROWKEY       | VARCHAR(STRING)  (system) 
+ ROWKEY       | VARCHAR(STRING)  (key)
  REGISTERTIME | BIGINT                    
  GENDER       | VARCHAR(STRING)           
  REGIONID     | VARCHAR(STRING)           
@@ -243,7 +243,7 @@ Name                 : USERS_EXTENDED
  Field        | Type                         
 ---------------------------------------------
  ROWTIME      | BIGINT           (system)    
- ROWKEY       | VARCHAR(STRING)  (system)    
+ ROWKEY       | VARCHAR(STRING)  (key)
  REGISTERTIME | BIGINT                       
  GENDER       | VARCHAR(STRING)              
  REGIONID     | VARCHAR(STRING)              
@@ -297,7 +297,7 @@ Name                 : PAGEVIEWS_ORIGINAL
  Field    | Type                      
 --------------------------------------
  ROWTIME  | BIGINT           (system) 
- ROWKEY   | VARCHAR(STRING)  (system) 
+ ROWKEY   | VARCHAR(STRING)  (key)
  VIEWTIME | BIGINT                    
  USERID   | VARCHAR(STRING)           
  PAGEID   | VARCHAR(STRING)           

diff --git a/docs-md/developer-guide/test-and-debug/processing-log.md b/docs-md/developer-guide/test-and-debug/processing-log.md
@@ -243,7 +243,7 @@ Name                 : PROCESSING_LOG
 Field   | Type
 ---------------------------------------------------------------------------------------------------------------------------
  ROWTIME | BIGINT           (system)
- ROWKEY  | VARCHAR(STRING)  (system)
+ ROWKEY  | VARCHAR(STRING)  (key)
  LOGGER  | VARCHAR(STRING)
  LEVEL   | VARCHAR(STRING)
  TIME    | BIGINT

diff --git a/docs-md/tutorials/basics-docker.md b/docs-md/tutorials/basics-docker.md
@@ -725,7 +725,7 @@ EMIT CHANGES;
  Field    | Type                                              
 --------------------------------------------------------------
  ROWTIME  | BIGINT           (system)                         
- ROWKEY   | VARCHAR(STRING)  (system) (Window type: TUMBLING) 
+ ROWKEY   | VARCHAR(STRING)  (key) (Window type: TUMBLING)
  GENDER   | VARCHAR(STRING)                                   
  REGIONID | VARCHAR(STRING)                                   
  NUMUSERS | BIGINT                                            

diff --git a/docs/developer-guide/create-a-stream.rst b/docs/developer-guide/create-a-stream.rst
@@ -86,7 +86,7 @@ Your output should resemble:
      Field    | Type
     --------------------------------------
      ROWTIME  | BIGINT           (system)
-     ROWKEY   | VARCHAR(STRING)  (system)
+     ROWKEY   | VARCHAR(STRING)  (key)
      VIEWTIME | BIGINT
      USERID   | VARCHAR(STRING)
      PAGEID   | VARCHAR(STRING)

diff --git a/docs/developer-guide/create-a-table.rst b/docs/developer-guide/create-a-table.rst
@@ -93,7 +93,7 @@ Your output should resemble:
      Field        | Type
     ------------------------------------------
      ROWTIME      | BIGINT           (system)
-     ROWKEY       | VARCHAR(STRING)  (system)
+     ROWKEY       | VARCHAR(STRING)  (key)
      REGISTERTIME | BIGINT
      USERID       | VARCHAR(STRING)
      GENDER       | VARCHAR(STRING)

diff --git a/docs/developer-guide/processing-log.rst b/docs/developer-guide/processing-log.rst
@@ -206,7 +206,7 @@ When you start KSQL, you should see the stream in your list of streams:
     Field   | Type
     ---------------------------------------------------------------------------------------------------------------------------
      ROWTIME | BIGINT           (system)
-     ROWKEY  | VARCHAR(STRING)  (system)
+     ROWKEY  | VARCHAR(STRING)  (key)
      LOGGER  | VARCHAR(STRING)
      LEVEL   | VARCHAR(STRING)
      TIME    | BIGINT

diff --git a/docs/developer-guide/query-with-structured-data.rst b/docs/developer-guide/query-with-structured-data.rst
@@ -269,7 +269,7 @@ Your output should resemble:
      Field         | Type
     -------------------------------------------
      ROWTIME       | BIGINT           (system)
-     ROWKEY        | VARCHAR(STRING)  (system)
+     ROWKEY        | VARCHAR(STRING)  (key)
      DATA__field-a | INTEGER
      DATA__field-b | VARCHAR(STRING)
     -------------------------------------------
@@ -289,7 +289,7 @@ Your output should resemble:
      Field         | Type
     -------------------------------------------
      ROWTIME       | BIGINT           (system)
-     ROWKEY        | VARCHAR(STRING)  (system)
+     ROWKEY        | VARCHAR(STRING)  (key)
      DATA__field-a | INTEGER
      DATA__field-c | INTEGER
      DATA__field-d | VARCHAR(STRING)

diff --git a/docs/developer-guide/syntax-reference.rst b/docs/developer-guide/syntax-reference.rst
@@ -1064,7 +1064,7 @@ Your output should resemble:
      Field   | Type
     -------------------------------------
      ROWTIME | BIGINT           (system)
-     ROWKEY  | VARCHAR(STRING)  (system)
+     ROWKEY  | VARCHAR(STRING)  (key)
      IP      | VARCHAR(STRING)  (key)
      KBYTES  | BIGINT
     -------------------------------------
@@ -1090,7 +1090,7 @@ Your output should resemble:
      Field   | Type
     -------------------------------------
      ROWTIME | BIGINT           (system)
-     ROWKEY  | VARCHAR(STRING)  (system)
+     ROWKEY  | VARCHAR(STRING)  (key)
      IP      | VARCHAR(STRING)  (key)
      KBYTES  | BIGINT
     -------------------------------------

diff --git a/docs/includes/ksql-includes.rst b/docs/includes/ksql-includes.rst
@@ -436,7 +436,7 @@ the latest offset.
         Field    | Type
         --------------------------------------
         ROWTIME  | BIGINT           (system)
-        ROWKEY   | VARCHAR(STRING)  (system)
+        ROWKEY   | VARCHAR(STRING)  (key)
         GENDER   | VARCHAR(STRING)
         REGIONID | VARCHAR(STRING)
         NUMUSERS | BIGINT
@@ -563,7 +563,7 @@ Your output should resemble:
      Field      | Type
     ----------------------------------------------------------------------------------
      ROWTIME    | BIGINT           (system)
-     ROWKEY     | VARCHAR(STRING)  (system)
+     ROWKEY     | VARCHAR(STRING)  (key)
      ORDERTIME  | BIGINT
      ORDERID    | INTEGER
      ITEMID     | VARCHAR(STRING)
@@ -869,7 +869,7 @@ Your output should resemble:
      Field      | Type
     ----------------------------------------------------------------------------------
      ROWTIME    | BIGINT           (system)
-     ROWKEY     | VARCHAR(STRING)  (system)
+     ROWKEY     | VARCHAR(STRING)  (key)
      SRC        | VARCHAR(STRING)
      ORDERTIME  | BIGINT
      ORDERID    | INTEGER

diff --git a/docs/tutorials/generate-custom-test-data.rst b/docs/tutorials/generate-custom-test-data.rst
@@ -133,7 +133,7 @@ Your output should resemble:
     Field     | Type                                                                 
    ----------------------------------------------------------------------------------
     ROWTIME   | BIGINT           (system)                                            
-    ROWKEY    | VARCHAR(STRING)  (system)                                            
+    ROWKEY    | VARCHAR(STRING)  (key)
     ITEMID    | VARCHAR(STRING)                                                      
     PRICE     | DOUBLE                                                               
     LOCATION  | STRUCT<CITY VARCHAR(STRING), STATE VARCHAR(STRING), ZIPCODE INTEGER> 
@@ -188,7 +188,7 @@ Your output should resemble:
     Field        | Type                      
    ------------------------------------------
     ROWTIME      | BIGINT           (system) 
-    ROWKEY       | VARCHAR(STRING)  (system) 
+    ROWKEY       | VARCHAR(STRING)  (key)
     REGISTERTIME | BIGINT                    
     GENDER       | VARCHAR(STRING)           
     REGIONID     | VARCHAR(STRING)           
@@ -243,7 +243,7 @@ Your output should resemble:
     Field        | Type                         
    ---------------------------------------------
     ROWTIME      | BIGINT           (system)    
-    ROWKEY       | VARCHAR(STRING)  (system)    
+    ROWKEY       | VARCHAR(STRING)  (key)
     REGISTERTIME | BIGINT                       
     GENDER       | VARCHAR(STRING)              
     REGIONID     | VARCHAR(STRING)              
@@ -295,7 +295,7 @@ Your output should resemble:
     Field    | Type                      
    --------------------------------------
     ROWTIME  | BIGINT           (system) 
-    ROWKEY   | VARCHAR(STRING)  (system) 
+    ROWKEY   | VARCHAR(STRING)  (key)
     VIEWTIME | BIGINT                    
     USERID   | VARCHAR(STRING)           
     PAGEID   | VARCHAR(STRING)           

diff --git a/ksqldb-cli/src/main/java/io/confluent/ksql/cli/console/Console.java b/ksqldb-cli/src/main/java/io/confluent/ksql/cli/console/Console.java
@@ -54,6 +54,7 @@
 import io.confluent.ksql.rest.entity.ErrorEntity;
 import io.confluent.ksql.rest.entity.ExecutionPlan;
 import io.confluent.ksql.rest.entity.FieldInfo;
+import io.confluent.ksql.rest.entity.FieldInfo.FieldType;
 import io.confluent.ksql.rest.entity.FunctionDescriptionList;
 import io.confluent.ksql.rest.entity.FunctionInfo;
 import io.confluent.ksql.rest.entity.FunctionNameList;
@@ -85,7 +86,6 @@
 import io.confluent.ksql.util.HandlerMaps.ClassHandlerMap1;
 import io.confluent.ksql.util.HandlerMaps.Handler1;
 import io.confluent.ksql.util.KsqlException;
-import io.confluent.ksql.util.SchemaUtil;
 import io.confluent.ksql.util.TabularRow;
 import java.io.Closeable;
 import java.io.File;
@@ -455,16 +455,18 @@ private static String formatFieldType(
       final Optional<WindowType> windowType,
       final String keyField
   ) {
-    if (field.getName().equals(SchemaUtil.ROWTIME_NAME.text())) {
+    final FieldType possibleFieldType = field.getType().orElse(null);
+
+    if (possibleFieldType == FieldType.SYSTEM) {
       return String.format("%-16s %s", field.getSchema().toTypeString(), "(system)");
     }
 
-    if (field.getName().equals(SchemaUtil.ROWKEY_NAME.text())) {
+    if (possibleFieldType == FieldType.KEY) {
       final String wt = windowType
           .map(v -> " (Window type: " + v + ")")
           .orElse("");
 
-      return String.format("%-16s %s%s", field.getSchema().toTypeString(), "(system)", wt);
+      return String.format("%-16s %s%s", field.getSchema().toTypeString(), "(key)", wt);
     }
 
     if (keyField != null && keyField.contains("." + field.getName())) {

diff --git a/ksqldb-cli/src/test/java/io/confluent/ksql/cli/console/ConsoleTest.java b/ksqldb-cli/src/test/java/io/confluent/ksql/cli/console/ConsoleTest.java
@@ -404,14 +404,16 @@ public void testPrintSourceDescription() {
           + "        \"type\" : \"BIGINT\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"SYSTEM\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"ROWKEY\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
           + "        \"type\" : \"STRING\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"KEY\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"f_0\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
@@ -505,7 +507,7 @@ public void testPrintSourceDescription() {
           + " Field   | Type                      " + NEWLINE
           + "-------------------------------------" + NEWLINE
           + " ROWTIME | BIGINT           (system) " + NEWLINE
-          + " ROWKEY  | VARCHAR(STRING)  (system) " + NEWLINE
+          + " ROWKEY  | VARCHAR(STRING)  (key)    " + NEWLINE
           + " f_0     | BOOLEAN                   " + NEWLINE
           + " f_1     | INTEGER                   " + NEWLINE
           + " f_2     | BIGINT                    " + NEWLINE
@@ -607,14 +609,16 @@ public void testPrintConnectorDescription() {
           + "        \"type\" : \"BIGINT\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"SYSTEM\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"ROWKEY\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
           + "        \"type\" : \"STRING\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"KEY\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"f_0\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
@@ -915,7 +919,7 @@ public void shouldPrintTypesList() {
             "typeA", new SchemaInfo(
                 SqlBaseType.STRUCT,
                 ImmutableList.of(
-                    new FieldInfo("f1", new SchemaInfo(SqlBaseType.STRING, null, null))),
+                    new FieldInfo("f1", new SchemaInfo(SqlBaseType.STRING, null, null), Optional.empty())),
                 null)
         ))
     ));
@@ -1059,14 +1063,16 @@ public void shouldPrintTopicDescribeExtended() {
           + "        \"type\" : \"BIGINT\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"SYSTEM\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"ROWKEY\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
           + "        \"type\" : \"STRING\"," + NEWLINE
           + "        \"fields\" : null," + NEWLINE
           + "        \"memberSchema\" : null" + NEWLINE
-          + "      }" + NEWLINE
+          + "      }," + NEWLINE
+          + "      \"type\" : \"KEY\"" + NEWLINE
           + "    }, {" + NEWLINE
           + "      \"name\" : \"f_0\"," + NEWLINE
           + "      \"schema\" : {" + NEWLINE
@@ -1104,7 +1110,7 @@ public void shouldPrintTopicDescribeExtended() {
           + " Field   | Type                      " + NEWLINE
           + "-------------------------------------" + NEWLINE
           + " ROWTIME | BIGINT           (system) " + NEWLINE
-          + " ROWKEY  | VARCHAR(STRING)  (system) " + NEWLINE
+          + " ROWKEY  | VARCHAR(STRING)  (key)    " + NEWLINE
           + " f_0     | VARCHAR(STRING)           " + NEWLINE
           + "-------------------------------------" + NEWLINE
           + "" + NEWLINE