🐙 octavia-cli: generate open api client (#9277)

.pre-commit-config.yaml

@@ -7,19 +7,16 @@ repos:
     hooks:
       - id: licenseheaders
         args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"]
-
   - repo: https://github.com/ambv/black
     rev: 21.11b1
     hooks:
       - id: black
-
   - repo: https://github.com/timothycrosley/isort
     rev: 5.10.1
     hooks:
       - id: isort
         args: ["--dont-follow-links", "--jobs=-1"]
         additional_dependencies: ["colorama"]
-
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.0
     hooks:
@@ -29,7 +26,8 @@ repos:
           (?x)^.*(
               .github/|
               source_specs.yaml|
-              destination_specs.yaml
+              destination_specs.yaml|
+              .gitlab-ci.yml
           ).?$
 
   - repo: https://github.com/csachs/pyproject-flake8
@@ -38,12 +36,14 @@ repos:
       - id: pyproject-flake8
         additional_dependencies: ["mccabe"]
         alias: flake8
-
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.910-1
     hooks:
       - id: mypy
-
+        exclude: |
+          (?x)^.*(
+              octavia-cli/unit_tests/|
+          ).?$
   - repo: local
     hooks:
       - id: spec-linter

build.gradle

@@ -340,6 +340,7 @@ subprojects {
             source = fileTree(dir: projectDir)
                     .include("**/*.py")
                     .exclude(".venv/**/*.py")
+                    .exclude("**/airbyte_api_client/**/*.py")
                     .exclude("**/__init__.py")
             strictCheck = true
         }

octavia-cli/.dockerignore

@@ -1,3 +1,4 @@
 build
+!build/airbyte_api_client
 .venv
 octavia_cli.egg-info

octavia-cli/README.md

@@ -14,9 +14,9 @@ The project is under development: readers can refer to our [tech spec deck](http
 We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation. 
 The project is under development: we have not yet published any docker image to our Docker registry.
 
-1. Build the image locally:
+1. Build the project locally (from the root of the repo):
 ```bash
-docker build -t octavia-cli:dev --rm .
+SUB_BUILD=OCTAVIA_CLI ./gradlew build #from the root of the repo
 ```
 2. Run the CLI from docker:
 ```bash
@@ -34,10 +34,11 @@ Octavia is currently under development.
 You can find a detailed and updated execution plan [here](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0).
 We welcome community contributions!
 
-Summary of achievements:
+**Summary of achievements**:
 
 | Date       | Milestone                           |
 |------------|-------------------------------------|
+| 2022-01-06 | Generate an API Python client from our Open API spec |
 | 2021-12-22 | Bootstrapping the project's code base |
 
 # Developing locally

octavia-cli/build.gradle

@@ -1,4 +1,7 @@
+import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
+
 plugins {
+    id "org.openapi.generator" version "5.3.1"
     id 'airbyte-python'
     id 'airbyte-docker'
 }
@@ -7,3 +10,16 @@ airbytePython {
     moduleDirectory 'octavia_cli'
 }
 
+
+task generateApiClient(type: GenerateTask) {
+    inputSpec =  "$rootDir.absolutePath/airbyte-api/src/main/openapi/config.yaml"
+    outputDir = "$buildDir/airbyte_api_client"
+
+    generatorName = "python"
+    packageName = "airbyte_api_client"
+}
+
+blackFormat.dependsOn generateApiClient
+isortFormat.dependsOn generateApiClient
+flakeCheck.dependsOn generateApiClient
+installReqs.dependsOn generateApiClient

octavia-cli/octavia_cli/entrypoint.py

@@ -2,41 +2,56 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import airbyte_api_client
 import click
+from airbyte_api_client.api import workspace_api
 
 
 @click.group()
 @click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.")
-def octavia(airbyte_url):
-    # TODO: check if the airbyte_url is reachable
-    click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}")
+@click.pass_context
+def octavia(ctx: click.Context, airbyte_url: str) -> None:
+    ctx.ensure_object(dict)
+    client_configuration = airbyte_api_client.Configuration(host=f"{airbyte_url}/api")
+    api_client = airbyte_api_client.ApiClient(client_configuration)
+    # TODO alafanechere workspace check might deserve its own function
+    api_instance = workspace_api.WorkspaceApi(api_client)
+    # open-api-generator consider non-required field as not nullable
+    # This will break validation of WorkspaceRead object for firstCompletedSync and feedbackDone fields
+    # This is why we bypass _check_return_type
+    api_response = api_instance.list_workspaces(_check_return_type=False)
+    # TODO alafanechere prompt user to chose a workspace if multiple workspaces exist
+    workspace_id = api_response.workspaces[0]["workspaceId"]
+    click.echo(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url} on workspace {workspace_id}")
+    ctx.obj["API_CLIENT"] = api_client
+    ctx.obj["WORKSPACE_ID"] = workspace_id
 
 
 @octavia.command(help="Scaffolds a local project directories.")
-def init():
+def init() -> None:
     raise click.ClickException("The init command is not yet implemented.")
 
 
 @octavia.command(name="list", help="List existing resources on the Airbyte instance.")
-def _list():
-    raise click.ClickException("The init command is not yet implemented.")
+def _list() -> None:
+    raise click.ClickException("The list command is not yet implemented.")
 
 
 @octavia.command(name="import", help="Import an existing resources from the Airbyte instance.")
-def _import():
-    raise click.ClickException("The init command is not yet implemented.")
+def _import() -> None:
+    raise click.ClickException("The import command is not yet implemented.")
 
 
 @octavia.command(help="Generate a YAML configuration file to manage a resource.")
-def create():
-    raise click.ClickException("The init command is not yet implemented.")
+def create() -> None:
+    raise click.ClickException("The create command is not yet implemented.")
 
 
 @octavia.command(help="Create or update resources according to YAML configurations.")
-def apply():
-    raise click.ClickException("The init command is not yet implemented.")
+def apply() -> None:
+    raise click.ClickException("The apply command is not yet implemented.")
 
 
 @octavia.command(help="Delete resources")
-def delete():
-    raise click.ClickException("The init command is not yet implemented.")
+def delete() -> None:
+    raise click.ClickException("The delete command is not yet implemented.")

octavia-cli/setup.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import os
 import pathlib
 
 from setuptools import find_packages, setup
@@ -39,8 +40,8 @@
         "Source": "https://github.com/airbytehq/airbyte",
         "Tracker": "https://github.com/airbytehq/airbyte/issues",
     },
-    packages=find_packages(exclude=("tests", "docs")),
-    install_requires=["click~=8.0.3"],
+    packages=find_packages(exclude=("unit_tests", "docs")),
+    install_requires=["click~=8.0.3", f"airbyte_api_client @ file://{os.getcwd()}/build/airbyte_api_client"],
     python_requires=">=3.8.12",
     extras_require={
         "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"],

octavia-cli/unit_tests/test_entrypoint.py

@@ -2,16 +2,38 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+from unittest import mock
+
+import click
 import pytest
 from click.testing import CliRunner
 from octavia_cli import entrypoint
 
 
-def test_octavia():
+@click.command()
+@click.pass_context
+def dumb(ctx):
+    pass
+
+
+def test_octavia(mocker):
+    mocker.patch.object(entrypoint, "workspace_api")
+    mocker.patch.object(entrypoint, "airbyte_api_client")
+
+    context_object = {}
+    mock_api_instance = entrypoint.workspace_api.WorkspaceApi.return_value
+    mock_api_instance.list_workspaces.return_value = mock.MagicMock(workspaces=[{"workspaceId": "expected_workspace_id"}])
+
+    entrypoint.octavia.add_command(dumb)
     runner = CliRunner()
-    result = runner.invoke(entrypoint.octavia)
+    result = runner.invoke(entrypoint.octavia, ["--airbyte-url", "test-airbyte-url", "dumb"], obj=context_object)
+    entrypoint.airbyte_api_client.Configuration.assert_called_with(host="test-airbyte-url/api")
+    entrypoint.airbyte_api_client.ApiClient.assert_called_with(entrypoint.airbyte_api_client.Configuration.return_value)
+    entrypoint.workspace_api.WorkspaceApi.assert_called_with(entrypoint.airbyte_api_client.ApiClient.return_value)
+    mock_api_instance.list_workspaces.assert_called_once()
+    assert context_object["API_CLIENT"] == entrypoint.airbyte_api_client.ApiClient.return_value
+    assert context_object["WORKSPACE_ID"] == "expected_workspace_id"
     assert result.exit_code == 0
-    assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...")
 
 
 @pytest.mark.parametrize(

tools/python/.flake8

@@ -5,6 +5,7 @@ exclude =
     .eggs # python libraries"
     .tox
     build
+    airbyte_api_client # generated api client in octavia-cli
 extend-ignore =
   E203,  # whitespace before ':' (conflicts with Black)
   E231,  # Bad trailing comma (conflicts with Black)