Output a warning on mulitple paths being tested in the same file. (#452)

* Output a warning on mulitple paths being tested in the same file. Signed-off-by: dblock <[email protected]> * Support disabling warnings. Signed-off-by: dblock <[email protected]> * Fixed prologue warnings. Signed-off-by: dblock <[email protected]> --------- Signed-off-by: dblock <[email protected]>
opensearch-project · Aug 1, 2024 · b67104c · b67104c
1 parent 659a3ce
commit b67104c
Show file tree

Hide file tree

Showing 36 changed files with 386 additions and 164 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -57,7 +57,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `--opensearch-version` to `merger` that excludes schema elements per semver ([#428](https://github.com/opensearch-project/opensearch-api-specification/pull/428))
 - Added `retry` to `tester` to support asynchronous tasks ([453](https://github.com/opensearch-project/opensearch-api-specification/pull/453))
 - Added passing OPENSEARCH_JAVA_OPTS into the docker container used for tests ([#454](https://github.com/opensearch-project/opensearch-api-specification/pull/454))
-
+- Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452))
+
 ### Changed
 
 - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))

diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
@@ -6,6 +6,9 @@
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
     - [Managing Versions](#managing-versions)
     - [Waiting for Tasks](#waiting-for-tasks)
+    - [Warnings](#warnings)
+      - [multiple-paths-detected](#multiple-paths-detected)
+      - [Suppressing Warnings](#suppressing-warnings)
 <!-- TOC -->
 
 # Spec Testing Guide
@@ -175,3 +178,34 @@ For example, an ML task returns `CREATED` when created, and `COMPLETED` when it'
       count: 3
       wait: 30000
 ```
+### Warnings
+
+#### multiple-paths-detected
+
+The test runner expects all tests in the same file to be variation of the same path in order to keep tests well-organized. Otherwise, a warning will be emitted.
+
+```
+WARNING Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.
+  /_component_template/{name}
+  /_index_template/{name}
+  /{index}
+```
+
+#### Suppressing Warnings
+
+The test runner may generate warnings that can be suppressed with `warnings:`. For example, to suppress the multiple paths detected warning.
+
+```yaml
+- synopsis: Create an index.
+  method: PUT
+  path: /{index}
+  parameters:
+    index: movies
+- synopsis: Search the index to make sure it has been created.
+  method: POST
+  warnings:
+    multiple-paths-detected: false
+  path: /{index}/_search
+  parameters:
+    index: movies
+```
diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
@@ -34,6 +34,8 @@ definitions:
             description: A brief description of the chapter.
           response:
             $ref: '#/definitions/ExpectedResponse'
+          warnings:
+            $ref: '#/definitions/Warnings'
         required: [synopsis]
 
   ReadChapter:
@@ -180,3 +182,12 @@ definitions:
       - type: string
       - type: number
       - type: boolean
+
+  Warnings:
+    type: object
+    properties:
+      multiple-paths-detected:
+        type: boolean
+        default: true
+        description: Enable/disable warnings about multiple paths being tested in the same story.
+    additionalProperties: false
diff --git a/tests/_core/reindex/pipeline.yaml b/tests/_core/reindex/pipeline.yaml
@@ -54,11 +54,15 @@ chapters:
     response:
       status: 200
   - synopsis: Refresh the index.
+    warnings:
+      multiple-paths-detected: false
     path: /{index}/_refresh
     method: POST
     parameters:
       index: videos
   - synopsis: Get all videos.
+    warnings:
+      multiple-paths-detected: false
     path: /{index}/_search
     method: POST
     parameters:

diff --git a/tests/cat/pit_segments/all.yaml b/tests/cat/pit_segments/all.yaml
@@ -0,0 +1,36 @@
+$schema: ../../../json_schemas/test_story.schema.yaml
+
+description: Test cat/pit_segments/_all endpoints.
+version: '>= 2.4'
+epilogues:
+  - path: /games
+    method: DELETE
+    status: [200, 404]
+prologues:
+  - path: /games/_doc
+    method: POST
+    parameters:
+      refresh: true
+    request_body:
+      payload:
+        title: Monopoly
+    status: [201]
+  - id: create_pit
+    path: /{index}/_search/point_in_time
+    method: POST
+    parameters:
+      index:
+        - games
+      keep_alive: 1m
+    output:
+      pit_id: payload.pit_id
+chapters:
+  - synopsis: Cat _all with a json response.
+    path: /_cat/pit_segments/_all
+    method: GET
+    parameters:
+      format: json
+    response:
+      status: 200
+      payload:
+        - index: games
diff --git a/tests/cat/pit_segments.yaml → tests/cat/pit_segments/pit_segments.yaml b/tests/cat/pit_segments.yaml → tests/cat/pit_segments/pit_segments.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/pit_segments endpoints.
 version: '>= 2.4'
@@ -15,9 +15,7 @@ prologues:
       payload:
         title: Monopoly
     status: [201]
-chapters:
-  - synopsis: Create a PIT.
-    id: create_pit
+  - id: create_pit
     path: /{index}/_search/point_in_time
     method: POST
     parameters:
@@ -26,15 +24,7 @@ chapters:
       keep_alive: 1m
     output:
       pit_id: payload.pit_id
-  - synopsis: Cat _all with a json response.
-    path: /_cat/pit_segments/_all
-    method: GET
-    parameters:
-      format: json
-    response:
-      status: 200
-      payload:
-        - index: games
+chapters:
   - synopsis: Cat pit_segments/pit_id with a json response.
     path: /_cat/pit_segments
     method: GET

diff --git a/tests/cat/snapshots.yaml b/tests/cat/snapshots.yaml
@@ -2,22 +2,18 @@ $schema: ../../json_schemas/test_story.schema.yaml
 
 description: Test cat/snapshots endpoints.
 epilogues:
-  - path: /_snapshot/{repository}
+  - path: /_snapshot/my-fs-repository
     method: DELETE
     status: [200, 404]
-    parameters:
-      repository: my-fs-repository
-chapters:
-  - synopsis: Create a snapshot repository.
-    path: /_snapshot/{repository}
+prologues:
+  - path: /_snapshot/my-fs-repository
     method: PUT
-    parameters:
-      repository: my-fs-repository
     request_body:
       payload:
         type: fs
         settings:
           location: /tmp/opensearch/repo
+chapters:
   - synopsis: Cat with a json response.
     path: /_cat/snapshots/{repository}
     method: GET

diff --git a/tests/indices/alias/alias.yaml b/tests/indices/alias/alias.yaml
@@ -0,0 +1,68 @@
+$schema: ../../../json_schemas/test_story.schema.yaml
+
+description: Test alias endpoints.
+epilogues:
+  - path: /games
+    method: DELETE
+    status: [200, 404]
+prologues:
+  - path: /{index}
+    method: PUT
+    parameters:
+      index: games
+chapters:
+  - synopsis: Create an alias.
+    path: /{index}/_alias/{name}
+    method: PUT
+    parameters:
+      index: games
+      name: jeux
+  - synopsis: Get an alias from index.
+    method: GET
+    path: /{index}/_alias/{name}
+    parameters:
+      index: games
+      name: jeux
+    response:
+      status: 200
+      content_type: application/json
+      payload:
+        games:
+          aliases:
+            jeux: {}
+  - synopsis: Delete an alias.
+    path: /{index}/_alias/{name}
+    method: DELETE
+    parameters:
+      index: games
+      name: jeux
+  - synopsis: Create an alias by Create or Update alias endpoint.
+    path: /{index}/_alias/{name}
+    method: PUT
+    parameters:
+      index: games
+      name: jeux
+  - synopsis: Create an alias with custom settings by Create or Update alias endpoint.
+    path: /{index}/_alias/{name}
+    method: PUT
+    parameters:
+      index: games
+      name: jeux
+    request_body:
+      payload:
+        index_routing: test1
+        search_routing: test2
+        routing: test
+        is_write_index: true
+        filter:
+          match_all: {}
+  - synopsis: Create an alias with is_hidden by Create or Update alias endpoint.
+    version: '>= 2.16'
+    path: /{index}/_alias/{name}
+    method: PUT
+    parameters:
+      index: games
+      name: jeux
+    request_body:
+      payload:
+        is_hidden: true
diff --git a/tests/indices/aliases/aliases.yaml b/tests/indices/aliases/aliases.yaml
@@ -6,42 +6,15 @@ epilogues:
     method: DELETE
     status: [200, 404]
 prologues:
-  - path: /{index}
+  - path: games
     method: PUT
-    parameters:
-      index: games
 chapters:
   - synopsis: Create an alias.
     path: /{index}/_aliases/{name}
     method: PUT
     parameters:
       index: games
       name: jeux
-  - synopsis: Get an alias from index.
-    method: GET
-    path: /{index}/_alias/{name}
-    parameters:
-      index: games
-      name: jeux
-    response:
-      status: 200
-      content_type: application/json
-      payload:
-        games:
-          aliases:
-            jeux: {}
-  - synopsis: Get an alias from _aliases.
-    method: GET
-    path: /_alias/{name}
-    parameters:
-      name: jeux
-    response:
-      status: 200
-      content_type: application/json
-      payload:
-        games:
-          aliases:
-            jeux: {}
   - synopsis: Multiple alias operations.
     path: /_aliases
     method: POST
@@ -61,45 +34,9 @@ chapters:
       status: 200
       payload:
         acknowledged: true
-  - synopsis: Delete an alias from _aliases.
+  - synopsis: Delete an alias.
     path: /{index}/_aliases/{name}
     method: DELETE
     parameters:
       index: games
       name: plays1
-  - synopsis: Delete an alias from _alias.
-    path: /{index}/_alias/{name}
-    method: DELETE
-    parameters:
-      index: games
-      name: plays2
-  - synopsis: Create an alias by Create or Update alias endpoint.
-    path: /{index}/_alias/{name}
-    method: PUT
-    parameters:
-      index: games
-      name: jeux
-  - synopsis: Create an alias with custom settings by Create or Update alias endpoint.
-    path: /{index}/_alias/{name}
-    method: PUT
-    parameters:
-      index: games
-      name: jeux
-    request_body:
-      payload:
-        index_routing: test1
-        search_routing: test2
-        routing: test
-        is_write_index: true
-        filter:
-          match_all: {}
-  - synopsis: Create an alias with is_hidden by Create or Update alias endpoint.
-    version: '>= 2.16'
-    path: /{index}/_alias/{name}
-    method: PUT
-    parameters:
-      index: games
-      name: jeux
-    request_body:
-      payload:
-        is_hidden: true
diff --git a/tests/indices/clone.yaml b/tests/indices/clone.yaml
@@ -4,6 +4,12 @@ description: Test cloning an index.
 prologues:
   - path: /movies
     method: PUT
+  - path: /movies/_settings
+    method: PUT
+    request_body:
+      payload:
+        settings:
+          index.blocks.write: true
 epilogues:
   - path: /movies
     method: DELETE
@@ -18,15 +24,6 @@ epilogues:
     method: DELETE
     status: [200, 404]
 chapters:
-  - synopsis: Block writes to the source index.
-    path: /{index}/_settings
-    method: PUT
-    parameters:
-      index: movies
-    request_body:
-      payload:
-        settings:
-          index.blocks.write: true
   - synopsis: Clone an index.
     path: /{index}/_clone/{target}
     method: POST