Merge branch 'apache:dev' into dev

.github/workflows/backend.yml

@@ -306,6 +306,37 @@ jobs:
           npm set strict-ssl false
           npm install
           npm run build
+
+  seatunnel-ui:
+    if: needs.changes.outputs.api == 'true'
+    needs: [ changes, sanity-check ]
+    name: Build SeaTunnel UI
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 20.x
+
+      - name: Install Dependencies and Check Code Style
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm install
+          npm run lint
+
+      - name: Run unit tests
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm run test:unit
+
+      - name: Build SeaTunnel UI
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm run build
+
   unit-test:
     needs: [ changes, sanity-check ]
     if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '')

.gitignore

@@ -55,3 +55,8 @@ seatunnel-examples
 
 /lib/*
 version.properties
+node/
+
+dist/
+
+seatunnel-engine/seatunnel-engine-server/**/ui/*

docs/en/connector-v2/source/Doris.md

@@ -13,27 +13,21 @@
 - [x] [batch](../../concept/connector-v2-features.md)
 - [ ] [stream](../../concept/connector-v2-features.md)
 - [ ] [exactly-once](../../concept/connector-v2-features.md)
-- [x] [schema projection](../../concept/connector-v2-features.md)
+- [x] [column projection](../../concept/connector-v2-features.md)
 - [x] [parallelism](../../concept/connector-v2-features.md)
 - [x] [support user-defined split](../../concept/connector-v2-features.md)
+- [x] [support multiple table read](../../concept/connector-v2-features.md)
 
 ## Description
 
-Used to read data from Doris.
-Doris Source will send a SQL to FE, FE will parse it into an execution plan, send it to BE, and BE will
-directly return the data
+Used to read data from Apache Doris.
 
 ## Supported DataSource Info
 
 | Datasource |          Supported versions          | Driver | Url | Maven |
 |------------|--------------------------------------|--------|-----|-------|
 | Doris      | Only Doris2.0 or later is supported. | -      | -   | -     |
 
-## Database Dependency
-
-> Please download the support list corresponding to 'Maven' and copy it to the '$SEATNUNNEL_HOME/plugins/jdbc/lib/'
-> working directory<br/>
-
 ## Data Type Mapping
 
 |           Doris Data type            |                                                                 SeaTunnel Data type                                                                 |
@@ -54,29 +48,40 @@ directly return the data
 
 ## Source Options
 
+Base configuration:
+
 |               Name               |  Type  | Required |  Default   |                                             Description                                             |
 |----------------------------------|--------|----------|------------|-----------------------------------------------------------------------------------------------------|
 | fenodes                          | string | yes      | -          | FE address, the format is `"fe_host:fe_http_port"`                                                  |
 | username                         | string | yes      | -          | User username                                                                                       |
 | password                         | string | yes      | -          | User password                                                                                       |
+| doris.request.retries            | int    | no       | 3          | Number of retries to send requests to Doris FE.                                                     |
+| doris.request.read.timeout.ms    | int    | no       | 30000      |                                                                                                     |
+| doris.request.connect.timeout.ms | int    | no       | 30000      |                                                                                                     |
+| query-port                       | string | no       | 9030       | Doris QueryPort                                                                                     |
+| doris.request.query.timeout.s    | int    | no       | 3600       | Timeout period of Doris scan data, expressed in seconds.                                            |
+| table_list                       | string | 否       | -          | table list                                                                                          |
+
+Table list configuration:
+
+|               Name               |  Type  | Required |  Default   |                                             Description                                             |
+|----------------------------------|--------|----------|------------|-----------------------------------------------------------------------------------------------------|
 | database                         | string | yes      | -          | The name of Doris database                                                                          |
 | table                            | string | yes      | -          | The name of Doris table                                                                             |
 | doris.read.field                 | string | no       | -          | Use the 'doris.read.field' parameter to select the doris table columns to read                      |
-| query-port                       | string | no       | 9030       | Doris QueryPort                                                                                     |
 | doris.filter.query               | string | no       | -          | Data filtering in doris. the format is "field = value",example : doris.filter.query = "F_ID > 2"    |
 | doris.batch.size                 | int    | no       | 1024       | The maximum value that can be obtained by reading Doris BE once.                                    |
-| doris.request.query.timeout.s    | int    | no       | 3600       | Timeout period of Doris scan data, expressed in seconds.                                            |
 | doris.exec.mem.limit             | long   | no       | 2147483648 | Maximum memory that can be used by a single be scan request. The default memory is 2G (2147483648). |
-| doris.request.retries            | int    | no       | 3          | Number of retries to send requests to Doris FE.                                                     |
-| doris.request.read.timeout.ms    | int    | no       | 30000      |                                                                                                     |
-| doris.request.connect.timeout.ms | int    | no       | 30000      |                                                                                                     |
+
+Note: When this configuration corresponds to a single table, you can flatten the configuration items in table_list to the outer layer.
 
 ### Tips
 
 > It is not recommended to modify advanced parameters at will
 
-## Task Example
+## Example
 
+### single table
 > This is an example of reading a Doris table and writing to Console.
 
 ```
@@ -159,4 +164,49 @@ sink {
     Console {}
 }
 ```
+### Multiple table
+```
+env{
+  parallelism = 1
+  job.mode = "BATCH"
+}
 
+source{
+  Doris {
+      fenodes = "xxxx:8030"
+      username = root
+      password = ""
+      table_list = [
+          {
+            database = "st_source_0"
+            table = "doris_table_0"
+            doris.read.field = "F_ID,F_INT,F_BIGINT,F_TINYINT"
+            doris.filter.query = "F_ID >= 50"
+          },
+          {
+            database = "st_source_1"
+            table = "doris_table_1"
+          }
+      ]
+  }
+}
+
+transform {}
+
+sink{
+  Doris {
+      fenodes = "xxxx:8030"
+      schema_save_mode = "RECREATE_SCHEMA"
+      username = root
+      password = ""
+      database = "st_sink"
+      table = "${table_name}"
+      sink.enable-2pc = "true"
+      sink.label-prefix = "test_json"
+      doris.config = {
+          format="json"
+          read_json_by_line="true"
+      }
+  }
+}
+```

docs/en/seatunnel-engine/web-ui.md

@@ -0,0 +1,48 @@
+# Apache SeaTunnel Web UI Documentation
+
+## Access
+
+Before accessing the web ui we need to enable the http rest api. first you need to configure it in the `seatunnel.yaml` configuration file
+
+```
+seatunnel:
+  engine:
+    http:
+      enable-http: true
+      port: 8080
+
+```
+
+Then visit `http://ip:8080/#/overview`
+
+## Overview
+
+The Web UI of Apache SeaTunnel offers a user-friendly interface for monitoring and managing SeaTunnel jobs. Through the Web UI, users can view real-time information on currently running jobs, finished jobs, and the status of worker and master nodes within the cluster. The main functional modules include Jobs, Workers, and Master, each providing detailed status information and operational options to help users efficiently manage and optimize their data processing workflows.
+![overview.png](../../images/ui/overview.png)
+
+## Jobs
+
+### Running Jobs
+
+The "Running Jobs" section lists all SeaTunnel jobs that are currently in execution. Users can view basic information for each job, including Job ID, submission time, status, execution time, and more. By clicking on a specific job, users can access detailed information such as task distribution, resource utilization, and log outputs, allowing for real-time monitoring of job progress and timely handling of potential issues.
+![running.png](../../images/ui/running.png)
+![detail.png](../../images/ui/detail.png)
+
+### Finished Jobs
+
+The "Finished Jobs" section displays all SeaTunnel jobs that have either successfully completed or failed. This section provides execution results, completion times, durations, and failure reasons (if any) for each job. Users can review past job records through this module to analyze job performance, troubleshoot issues, or rerun specific jobs as needed.
+![finished.png](../../images/ui/finished.png)
+
+## Workers
+
+### Workers Information
+
+The "Workers" section displays detailed information about all worker nodes in the cluster, including each worker's address, running status, CPU and memory usage, number of tasks being executed, and more. Through this module, users can monitor the health of each worker node, promptly identify and address resource bottlenecks or node failures, ensuring the stable operation of the SeaTunnel cluster.
+![workers.png](../../images/ui/workers.png)
+
+## Master
+
+### Master Information
+
+The "Master" section provides the status and configuration information of the master node in the SeaTunnel cluster. Users can view the master's address, running status, job scheduling responsibilities, and overall resource allocation within the cluster. This module helps users gain a comprehensive understanding of the cluster's core management components, facilitating cluster configuration optimization and troubleshooting.
+![master.png](../../images/ui/master.png)

docs/sidebars.js

@@ -206,7 +206,8 @@ const sidebars = {
                 "seatunnel-engine/rest-api-v2",
                 "seatunnel-engine/user-command",
                 "seatunnel-engine/logging",
-                "seatunnel-engine/telemetry"
+                "seatunnel-engine/telemetry",
+                "seatunnel-engine/web-ui"
             ]
         },
         {