From d65b8f986b29e7ca9232eebc29495f34479a9948 Mon Sep 17 00:00:00 2001 From: Augustin Date: Mon, 17 Jan 2022 09:27:14 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=99=20octavia-cli:=20generate=20open?= =?UTF-8?q?=20api=20client=20(#9277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 12 +++---- build.gradle | 1 + octavia-cli/.dockerignore | 1 + octavia-cli/README.md | 7 ++-- octavia-cli/build.gradle | 16 +++++++++ octavia-cli/octavia_cli/entrypoint.py | 43 +++++++++++++++-------- octavia-cli/setup.py | 5 +-- octavia-cli/unit_tests/test_entrypoint.py | 28 +++++++++++++-- tools/python/.flake8 | 1 + 9 files changed, 86 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bf284964259..e62b52145420 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,19 +7,16 @@ repos: hooks: - id: licenseheaders args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"] - - repo: https://github.com/ambv/black rev: 21.11b1 hooks: - id: black - - repo: https://github.com/timothycrosley/isort rev: 5.10.1 hooks: - id: isort args: ["--dont-follow-links", "--jobs=-1"] additional_dependencies: ["colorama"] - - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.5.0 hooks: @@ -29,7 +26,8 @@ repos: (?x)^.*( .github/| source_specs.yaml| - destination_specs.yaml + destination_specs.yaml| + .gitlab-ci.yml ).?$ - repo: https://github.com/csachs/pyproject-flake8 @@ -38,12 +36,14 @@ repos: - id: pyproject-flake8 additional_dependencies: ["mccabe"] alias: flake8 - - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.910-1 hooks: - id: mypy - + exclude: | + (?x)^.*( + octavia-cli/unit_tests/| + ).?$ - repo: local hooks: - id: spec-linter diff --git a/build.gradle b/build.gradle index ad8535bbc29f..0c86b69da188 100644 --- a/build.gradle +++ b/build.gradle @@ -340,6 +340,7 @@ subprojects { source = fileTree(dir: projectDir) .include("**/*.py") .exclude(".venv/**/*.py") + .exclude("**/airbyte_api_client/**/*.py") .exclude("**/__init__.py") strictCheck = true } diff --git a/octavia-cli/.dockerignore b/octavia-cli/.dockerignore index 89b498ce9934..1a4780ba9ba3 100644 --- a/octavia-cli/.dockerignore +++ b/octavia-cli/.dockerignore @@ -1,3 +1,4 @@ build +!build/airbyte_api_client .venv octavia_cli.egg-info diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 4bcc11dac808..3589ea1512f8 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -14,9 +14,9 @@ The project is under development: readers can refer to our [tech spec deck](http We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation. The project is under development: we have not yet published any docker image to our Docker registry. -1. Build the image locally: +1. Build the project locally (from the root of the repo): ```bash -docker build -t octavia-cli:dev --rm . +SUB_BUILD=OCTAVIA_CLI ./gradlew build #from the root of the repo ``` 2. Run the CLI from docker: ```bash @@ -34,10 +34,11 @@ Octavia is currently under development. You can find a detailed and updated execution plan [here](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0). We welcome community contributions! -Summary of achievements: +**Summary of achievements**: | Date | Milestone | |------------|-------------------------------------| +| 2022-01-06 | Generate an API Python client from our Open API spec | | 2021-12-22 | Bootstrapping the project's code base | # Developing locally diff --git a/octavia-cli/build.gradle b/octavia-cli/build.gradle index 5ebd064d689b..ea29a91ffa36 100644 --- a/octavia-cli/build.gradle +++ b/octavia-cli/build.gradle @@ -1,4 +1,7 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + plugins { + id "org.openapi.generator" version "5.3.1" id 'airbyte-python' id 'airbyte-docker' } @@ -7,3 +10,16 @@ airbytePython { moduleDirectory 'octavia_cli' } + +task generateApiClient(type: GenerateTask) { + inputSpec = "$rootDir.absolutePath/airbyte-api/src/main/openapi/config.yaml" + outputDir = "$buildDir/airbyte_api_client" + + generatorName = "python" + packageName = "airbyte_api_client" +} + +blackFormat.dependsOn generateApiClient +isortFormat.dependsOn generateApiClient +flakeCheck.dependsOn generateApiClient +installReqs.dependsOn generateApiClient diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index 3d82bc32f5ea..b0de9afb5517 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -2,41 +2,56 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import airbyte_api_client import click +from airbyte_api_client.api import workspace_api @click.group() @click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.") -def octavia(airbyte_url): - # TODO: check if the airbyte_url is reachable - click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}") +@click.pass_context +def octavia(ctx: click.Context, airbyte_url: str) -> None: + ctx.ensure_object(dict) + client_configuration = airbyte_api_client.Configuration(host=f"{airbyte_url}/api") + api_client = airbyte_api_client.ApiClient(client_configuration) + # TODO alafanechere workspace check might deserve its own function + api_instance = workspace_api.WorkspaceApi(api_client) + # open-api-generator consider non-required field as not nullable + # This will break validation of WorkspaceRead object for firstCompletedSync and feedbackDone fields + # This is why we bypass _check_return_type + api_response = api_instance.list_workspaces(_check_return_type=False) + # TODO alafanechere prompt user to chose a workspace if multiple workspaces exist + workspace_id = api_response.workspaces[0]["workspaceId"] + click.echo(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url} on workspace {workspace_id}") + ctx.obj["API_CLIENT"] = api_client + ctx.obj["WORKSPACE_ID"] = workspace_id @octavia.command(help="Scaffolds a local project directories.") -def init(): +def init() -> None: raise click.ClickException("The init command is not yet implemented.") @octavia.command(name="list", help="List existing resources on the Airbyte instance.") -def _list(): - raise click.ClickException("The init command is not yet implemented.") +def _list() -> None: + raise click.ClickException("The list command is not yet implemented.") @octavia.command(name="import", help="Import an existing resources from the Airbyte instance.") -def _import(): - raise click.ClickException("The init command is not yet implemented.") +def _import() -> None: + raise click.ClickException("The import command is not yet implemented.") @octavia.command(help="Generate a YAML configuration file to manage a resource.") -def create(): - raise click.ClickException("The init command is not yet implemented.") +def create() -> None: + raise click.ClickException("The create command is not yet implemented.") @octavia.command(help="Create or update resources according to YAML configurations.") -def apply(): - raise click.ClickException("The init command is not yet implemented.") +def apply() -> None: + raise click.ClickException("The apply command is not yet implemented.") @octavia.command(help="Delete resources") -def delete(): - raise click.ClickException("The init command is not yet implemented.") +def delete() -> None: + raise click.ClickException("The delete command is not yet implemented.") diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index e5ad552eb3d9..b7056db32bfb 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -2,6 +2,7 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import os import pathlib from setuptools import find_packages, setup @@ -39,8 +40,8 @@ "Source": "https://github.com/airbytehq/airbyte", "Tracker": "https://github.com/airbytehq/airbyte/issues", }, - packages=find_packages(exclude=("tests", "docs")), - install_requires=["click~=8.0.3"], + packages=find_packages(exclude=("unit_tests", "docs")), + install_requires=["click~=8.0.3", f"airbyte_api_client @ file://{os.getcwd()}/build/airbyte_api_client"], python_requires=">=3.8.12", extras_require={ "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"], diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index c8effc674c65..50d5ad68af11 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -2,16 +2,38 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from unittest import mock + +import click import pytest from click.testing import CliRunner from octavia_cli import entrypoint -def test_octavia(): +@click.command() +@click.pass_context +def dumb(ctx): + pass + + +def test_octavia(mocker): + mocker.patch.object(entrypoint, "workspace_api") + mocker.patch.object(entrypoint, "airbyte_api_client") + + context_object = {} + mock_api_instance = entrypoint.workspace_api.WorkspaceApi.return_value + mock_api_instance.list_workspaces.return_value = mock.MagicMock(workspaces=[{"workspaceId": "expected_workspace_id"}]) + + entrypoint.octavia.add_command(dumb) runner = CliRunner() - result = runner.invoke(entrypoint.octavia) + result = runner.invoke(entrypoint.octavia, ["--airbyte-url", "test-airbyte-url", "dumb"], obj=context_object) + entrypoint.airbyte_api_client.Configuration.assert_called_with(host="test-airbyte-url/api") + entrypoint.airbyte_api_client.ApiClient.assert_called_with(entrypoint.airbyte_api_client.Configuration.return_value) + entrypoint.workspace_api.WorkspaceApi.assert_called_with(entrypoint.airbyte_api_client.ApiClient.return_value) + mock_api_instance.list_workspaces.assert_called_once() + assert context_object["API_CLIENT"] == entrypoint.airbyte_api_client.ApiClient.return_value + assert context_object["WORKSPACE_ID"] == "expected_workspace_id" assert result.exit_code == 0 - assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...") @pytest.mark.parametrize( diff --git a/tools/python/.flake8 b/tools/python/.flake8 index a270aeef0863..b07e01f847ba 100644 --- a/tools/python/.flake8 +++ b/tools/python/.flake8 @@ -5,6 +5,7 @@ exclude = .eggs # python libraries" .tox build + airbyte_api_client # generated api client in octavia-cli extend-ignore = E203, # whitespace before ':' (conflicts with Black) E231, # Bad trailing comma (conflicts with Black)