CLOUDP-273211: improve error handling and documentation for postman collection generation

tools/postman/Makefile

@@ -1,6 +1,7 @@
 export OPENAPI_FOLDER=./openapi
 export TMP_FOLDER=./tmp
 export FULL_OPENAPI_FOLDER=../../openapi/v2/
+ENV_FILE = ./local.env
 
 default: build
 
@@ -44,3 +45,16 @@ compare_and_send_forks: compare_forks send_forks
 .PHONY: clean
 clean:
 	rm ./openapi/*; rm ./tmp/*
+
+# Command to load local environment variables from file
+load-env:
+	@if [ -f $(ENV_FILE) ]; then \
+		echo "Loading environment variables from $(ENV_FILE)..."; \
+		for var in $$(grep -v '^#' $(ENV_FILE) | cut -d= -f1); do \
+			echo "Loaded: $$var"; \
+		done; \
+		export $$(grep -v '^#' $(ENV_FILE) | xargs); \
+	else \
+		echo "Error: $(ENV_FILE) file not found."; \
+		exit 1; \
+	fi

tools/postman/README.md

@@ -47,3 +47,41 @@ flowchart TD
    for upload.
 
 5. **Upload Collection to Postman**: Use the Postman API to upload the Collection to Postman.
+
+## Local setup and validation
+
+### Prerequisites
+
+To push to your own workspace via Postman API, [create a private workspace](https://learning.postman.com/docs/collaborating-in-postman/using-workspaces/use-workspaces/) and retrieve the workspace id
+
+Then, [create a Postman API Key](https://learning.postman.com/docs/developer/postman-api/authentication/) that has access to this workspace
+
+### Env var setup
+
+To test the scripts locally, you should set up the required environment variables in `tools/postman/scripts` by creating a `local.env` file populated with the following (at the minimum):
+
+```
+BASE_URL=""
+WORKSPACE_ID=""
+POSTMAN_API_KEY=""
+```
+
+Run `make load-env` to set your required and overriding env vars.
+
+The OpenAPI path for Postman generation and configured feature flags can also be overriden based on provided env vars.
+
+### Running locally
+
+Once env vars are configured, the setup scripts can be run locally using the Make following commands:
+- `make fetch_openapi`
+- `make convert_to_collection`
+- `make transform_collection`
+- `make upload_collection`
+
+
+### Limitations
+
+Our Postman collection generation has several limitations, meaning some manual user actions may be necessary during setup:
+
+-  Only a single content-type header is supported generation, meaning users may have to manually update this header if the API supports more than one; see docs links on endpoint level
+- Only a single auth scheme is supported during generation, meaning users may have to update their auth on the collection level or endpoint level if not selecting the default

tools/postman/scripts/convert-to-collection.sh

@@ -21,7 +21,7 @@ jq 'del(.. | select(type == "object") | .pattern?)' "$OPENAPI_FOLDER"/"$OPENAPI_
 echo "Installing openapi-to-postmanv2"
 npm install
 
-echo "Converting from OpenAPI to PostmanV2"
+echo "Converting $OPENAPI_FOLDER/$OPENAPI_FILE_NAME from OpenAPI to PostmanV2"
 ./node_modules/.bin/openapi2postmanv2 -s "$TMP_FOLDER"/tmp.json -o "$TMP_FOLDER"/"$COLLECTION_FILE_NAME" -O folderStrategy=Tags
 
 rm "$TMP_FOLDER"/tmp.json

tools/postman/scripts/transform-for-api.sh

@@ -9,10 +9,11 @@ set -euo pipefail
 #   OPENAPI_FILE_NAME - name of the openapi specification file
 #   OPENAPI_FOLDER - folder where openapi file is saved
 #   TMP_FOLDER - folder for temporary files during transformations
-#   TOGGLE_USE_ENVIRONMENT_AUTH - bool for if auth variables are stored at the environment or collection level
-#   TOGGLE_INCLUDE_BODY - bool for if generated bodies should be removed or kept
 #   VERSION_FILE_NAME - name of the file where the current version is stored
 #   DESCRIPTION_FILE - name for the markdown description file
+#   TOGGLE_INCLUDE_BODY - bool for if generated bodies should be removed or kept
+#   TOGGLE_ADD_DOCS_LINKS - updates requests with corresponding docs links
+#   TOKEN_URL_ENV - client credentials auth path to set at the environment level, will not be set if unpopulated
 #   BASE_URL - the default base url the Postman Collection will use
 #########################################################
 
@@ -22,17 +23,17 @@ OPENAPI_FILE_NAME=${OPENAPI_FILE_NAME:-"atlas-api.json"}
 OPENAPI_FOLDER=${OPENAPI_FOLDER:-"../openapi"}
 TMP_FOLDER=${TMP_FOLDER:-"../tmp"}
 VERSION_FILE_NAME=${VERSION_FILE_NAME:-"version.txt"}
-
 DESCRIPTION_FILE=${DESCRIPTION_FILE:-"../collection-description.md"}
 
-TOGGLE_USE_ENVIRONMENT_AUTH=${TOGGLE_USE_ENVIRONMENT_AUTH:-true}
 TOGGLE_INCLUDE_BODY=${TOGGLE_INCLUDE_BODY:-true}
+TOGGLE_ADD_DOCS_LINKS=${TOGGLE_ADD_DOCS_LINKS:-true}
+TOKEN_URL_ENV=${TOKEN_URL_ENV:-""}
 
 current_api_revision=$(<"$OPENAPI_FOLDER/$VERSION_FILE_NAME")
 
 pushd "${TMP_FOLDER}"
 
-echo "Wrapping Collection in \"collection\" tag"
+echo "Wrapping Collection $COLLECTION_FILE_NAME in \"collection\" tag"
 jq '{"collection": .}' "$COLLECTION_FILE_NAME" > intermediateCollectionWrapped.json
 
 echo "Disabling query params by default"
@@ -47,48 +48,52 @@ jq 'del(.collection.info._postman_id)' \
 echo "Removing circular references"
 sed 's/\\"value\\": \\"<Circular reference to #[^>"]* detected>\\"//g' intermediateCollectionNoPostmanID.json > intermediateCollectionNoCircular.json
 
-echo "Updating name with version"
+echo "Updating name with version $current_api_revision"
 jq --arg api_version "$current_api_revision" \
   '.collection.info.name = ("MongoDB Atlas Administration API " + $api_version)' \
   intermediateCollectionNoCircular.json >  intermediateCollectionWithName.json
 
-echo "Adding Collection description"
+echo "Adding Collection description to $DESCRIPTION_FILE"
 description=$(<"$DESCRIPTION_FILE")
 jq --arg desc "$description" \
   '.collection.info.description.content = $desc' \
   intermediateCollectionWithName.json >  intermediateCollectionWithDescription.json
 
-echo "Updating baseUrl"
+echo "Adding baseUrl env $BASE_URL"
 jq --arg base_url "$BASE_URL" \
   '.collection.variable[0].value = $base_url' \
   intermediateCollectionWithDescription.json > intermediateCollectionWithBaseURL.json
 
-echo "Adding links to docs"
-cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json
+if [ "$TOGGLE_ADD_DOCS_LINKS" = "true" ]; then
+  echo "Adding links to docs for each request"
+  cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json
 
-# Store all paths to requests. The summary field is the same as the title in the collection
-paths=$(jq 'path(.. | objects | select(has("summary"))) | @sh' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME")
-declare -a paths_array="($paths)"
+  # Store all paths to requests. The summary field is the same as the title in the collection
+  paths=$(jq 'path(.. | objects | select(has("summary"))) | @sh' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME")
+  declare -a paths_array="($paths)"
 
-for path in "${paths_array[@]}"; do
-  declare -a single_path_array="($path)"
-  path_json=$(jq -n '$ARGS.positional' --args "${single_path_array[@]}")
+  for path in "${paths_array[@]}"; do
+    declare -a single_path_array="($path)"
+    path_json=$(jq -n '$ARGS.positional' --args "${single_path_array[@]}")
 
-  # Use the path to get all the information about this request without searching
-  requestInfo=$(jq --argjson path "$path_json" 'getpath($path)' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME")
+    # Use the path to get all the information about this request without searching
+    requestInfo=$(jq --argjson path "$path_json" 'getpath($path)' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME")
 
-  title=$(echo "$requestInfo" | jq -r '.summary')
-  operationId=$(echo "$requestInfo" | jq -r '.operationId')
-  tag=$(echo "$requestInfo" | jq -r '.tags[0]' | tr " " "-")
+    title=$(echo "$requestInfo" | jq -r '.summary')
+    operationId=$(echo "$requestInfo" | jq -r '.operationId')
+    tag=$(echo "$requestInfo" | jq -r '.tags[0]' | tr " " "-")
 
-  url="https://mongodb.com/docs/atlas/reference/api-resources-spec/v2/#tag/${tag}/operation/$operationId"
+    url="https://mongodb.com/docs/atlas/reference/api-resources-spec/v2/#tag/${tag}/operation/$operationId"
 
-  # Search the collection for the request with the matching name. Add the link to its description 
-  jq --arg title "$title" --arg url "$url" \
-    'first(.collection.item[].item[].request |  select(.name == $title).description.content) += "\n\nFind out more at " + $url' \
-    intermediateCollectionWithLinks.json > tmp.json && mv tmp.json intermediateCollectionWithLinks.json
+    # Search the collection for the request with the matching name. Add the link to its description 
+    jq --arg title "$title" --arg url "$url" \
+      'first(.collection.item[].item[].request |  select(.name == $title).description.content) += "\n\nFind out more at " + $url' \
+      intermediateCollectionWithLinks.json > tmp.json && mv tmp.json intermediateCollectionWithLinks.json
 
-done
+  done
+else
+  cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json
+fi
 
 # Togglable features 
 if [ "$TOGGLE_INCLUDE_BODY" = "false" ]; then
@@ -107,11 +112,10 @@ else
   cp intermediateCollectionWithLinks.json intermediateCollectionPostBody.json
 fi
 
-if [ "$TOGGLE_USE_ENVIRONMENT_AUTH" = "false" ]; then
-  echo "Adding auth variables"
-  jq '.collection.variable += [{"key": "digestAuthUsername", "value": "<string>"},
-  {"key": "digestAuthPassword", "value": "<string>"},
-  {"key": "realm", "value": "<string>"}]' intermediateCollectionPostBody.json > "$COLLECTION_TRANSFORMED_FILE_NAME"
+if [ "$TOKEN_URL_ENV" != "" ]; then
+  echo "Adding client credentials auth url variable $TOKEN_URL_ENV"
+  jq --arg token_url "$TOKEN_URL_ENV" '.collection.variable += [{"key": "clientCredentialsTokenUrl", "value": $token_url}]' \
+  intermediateCollectionPostBody.json > "$COLLECTION_TRANSFORMED_FILE_NAME"
 else
   cp intermediateCollectionPostBody.json "$COLLECTION_TRANSFORMED_FILE_NAME"
 fi

tools/postman/scripts/upload-collection.sh

@@ -28,6 +28,9 @@ pushd "${OPENAPI_FOLDER}"
 current_collection_name="MongoDB Atlas Administration API ${current_api_revision}"
 
 echo "Fetching list of current collections"
+echo "curl -o ${COLLECTIONS_LIST_FILE} 
+     --location 'https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}'
+     --header 'X-API-Key: **********'"
 curl --show-error --fail --silent -o "${COLLECTIONS_LIST_FILE}" \
      --location "https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}" \
      --header "X-API-Key: ${POSTMAN_API_KEY}"
@@ -37,15 +40,25 @@ collection_exists=$(jq '.collections | any(.name=="'"${current_collection_name}"
 if [  "$collection_exists" = "false" ]; then
   # Create new collection
   echo "Creating new remote collection ${current_collection_name}"
+  echo "curl -o ${COLLECTIONS_LIST_FILE}
+     --location 'https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}'
+     --header 'Content-Type: application/json'
+     --header 'X-API-Key: **********'
+     --data ${collection_transformed_path}"
   curl --show-error --fail --retry 5 --retry-all-errors --silent \
        --location "https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}" \
        --header "Content-Type: application/json" \
        --header "X-API-Key: ${POSTMAN_API_KEY}" \
-       --data "@${collection_transformed_path}"
+       --data "@${collection_transformed_path}" \
 
 else
   # Find collection ID and update collection
   echo "Updating remote collection ${current_collection_name}"
+  echo "curl --request PUT
+     --location 'https://api.getpostman.com/collections/${collection_id}'
+     --header 'Content-Type: application/json'
+     --header 'X-API-Key: **********'
+     --data ${collection_transformed_path}"
   collection_id=$(jq -r '.collections | map(select(.name=="'"${current_collection_name}"'").id)[0]' "${COLLECTIONS_LIST_FILE}")
   curl --show-error --fail --retry 5 --retry-all-errors --silent --request PUT \
        --location "https://api.getpostman.com/collections/${collection_id}" \