From 78b83793374fa742e9161ac84998d801a2a87e7c Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 09:30:31 +0200 Subject: [PATCH 01/13] connectors-ci: make source-file testable in airbyte-ci --- .../connectors/source-file/Dockerfile | 2 +- .../source-file/integration_tests/conftest.py | 13 ++++++++++++- .../integration_tests/docker-compose.yml | 4 ++-- .../connectors/source-file/metadata.yaml | 2 +- .../connectors/source-file/setup.py | 2 +- docs/integrations/sources/file.md | 1 + .../pipelines/pipelines/connectors.py | 1 - .../pipelines/tests/python_connectors.py | 1 + 8 files changed, 19 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index 480930cc2177..3c8d231bfd17 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -17,5 +17,5 @@ COPY source_file ./source_file ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.9 +LABEL io.airbyte.version=0.3.10 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py index 054f3ae3442c..cb8041a4f396 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -6,6 +6,7 @@ import json import os import random +import shutil import socket import string import tempfile @@ -81,7 +82,17 @@ def is_ssh_ready(ip, port): @pytest.fixture(scope="session") -def ssh_service(docker_ip, docker_services): +def move_sample_files_to_tmp(): + """Copy sample files to /tmp so that they can be accessed by the dockerd service in the context of Dagger test runs. + The sample files are mounted to the SSH service from the container under test (container) following instructions of docker-compose.yml in this directory.""" + sample_files = Path(HERE / "sample_files") + shutil.copytree(sample_files, "/tmp/s3_sample_files") + yield True + shutil.rmtree("/tmp/s3_sample_files") + + +@pytest.fixture(scope="session") +def ssh_service(move_sample_files_to_tmp, docker_ip, docker_services): """Ensure that SSH service is up and responsive.""" # `port_for` takes a container port and returns the corresponding host port port = docker_services.port_for("ssh", 22) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml index 67306a1e102e..60414c5910a5 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml +++ b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml @@ -1,9 +1,9 @@ -version: '3' +version: "3" services: ssh: image: atmoz/sftp ports: - "2222:22" volumes: - - ./sample_files:/home/user1/files + - /tmp/s3_sample_files:/home/user1/files command: user1:abc123@456#:1001 diff --git a/airbyte-integrations/connectors/source-file/metadata.yaml b/airbyte-integrations/connectors/source-file/metadata.yaml index c816ab4309cc..1fdffd405ff1 100644 --- a/airbyte-integrations/connectors/source-file/metadata.yaml +++ b/airbyte-integrations/connectors/source-file/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.9 + dockerImageTag: 0.3.10 dockerRepository: airbyte/source-file githubIssueLabel: source-file icon: file.svg diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index ee23ed829cb7..694ed8764c72 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1", "docker-compose"] setup( name="source_file", diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index cefbe549b08c..271dc9cf84e4 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -191,6 +191,7 @@ In order to read large files from a remote location, this connector uses the [sm | Version | Date | Pull Request | Subject | |:----------|:-------------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.3.10 | 2023-06-07 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Make source-file testable in our new airbyte-ci pipelines | | 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | add ParserError handling | | 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | | 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py index ec9fc303b1ff..8cdf1ffbfefb 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py @@ -32,6 +32,5 @@ async def run_connectors_pipelines( for context in contexts: context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") context.dockerd_service = dockerd_service - tg.start_soon(connector_pipeline, context, semaphore, *args) return contexts diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py index 5b82cd53e2e7..26a0da152f47 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py @@ -100,6 +100,7 @@ async def _run(self, connector_under_test: Container) -> StepResult: Returns: StepResult: Failure or success of the integration tests with stdout and stdout. """ + connector_under_test = environments.with_bound_docker_host(self.context, connector_under_test) connector_under_test_with_secrets = connector_under_test.with_directory("secrets", self.context.secrets_dir) return await self._run_tests_in_directory(connector_under_test_with_secrets, "integration_tests") From 1984f1da13edf796b48c1a052b5790e3c590fdc4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 18:24:23 +0200 Subject: [PATCH 02/13] make it work --- .../connectors/source-file-secure/Dockerfile | 24 +++++- .../source-file-secure/requirements.txt | 1 - .../connectors/source-file-secure/setup.py | 13 +++ .../source_file_secure/source.py | 25 ++---- .../pipelines/actions/environments.py | 85 ++++++++++++++----- .../ci_connector_ops/pipelines/bases.py | 4 +- .../pipelines/tests/common.py | 2 + 7 files changed, 106 insertions(+), 48 deletions(-) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 77b342fbf959..98aa59bedfa3 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -1,10 +1,26 @@ -FROM airbyte/source-file:0.3.9 +### WARNING ### +# This Dockerfile will soon be deprecated. +# It is not used to build the connector image we publish to DockerHub. +# The new logic to build the connector image is declared with Dagger here: +# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L771 +# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. +# Please reach out to the Connectors Operations team if you have any question. +FROM python:3.9-slim as base +FROM base as builder + +RUN apt-get update WORKDIR /airbyte/integration_code -COPY source_file_secure ./source_file_secure -COPY main.py ./ COPY setup.py ./ -RUN pip install . +RUN pip install --prefix=/install . + +FROM base +WORKDIR /airbyte/integration_code +COPY --from=builder /install /usr/local + +COPY main.py ./ +COPY source_file_secure ./source_file_secure + ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] diff --git a/airbyte-integrations/connectors/source-file-secure/requirements.txt b/airbyte-integrations/connectors/source-file-secure/requirements.txt index add0ec3bd480..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-file-secure/requirements.txt +++ b/airbyte-integrations/connectors/source-file-secure/requirements.txt @@ -1,3 +1,2 @@ -e ../../bases/connector-acceptance-test --e ../source-file -e . diff --git a/airbyte-integrations/connectors/source-file-secure/setup.py b/airbyte-integrations/connectors/source-file-secure/setup.py index 4fc2474a58c7..8956c853d13f 100644 --- a/airbyte-integrations/connectors/source-file-secure/setup.py +++ b/airbyte-integrations/connectors/source-file-secure/setup.py @@ -2,9 +2,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os +from pathlib import Path from setuptools import find_packages, setup + +def local_dependency(name: str) -> str: + """Returns a path to a local package.""" + if os.environ.get("DAGGER_BUILD"): + return f"{name} @ file:///local_dependencies/{name}" + else: + return f"{name} @ file://{Path.cwd().parent / name}" + + MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "gcsfs==2022.7.1", @@ -21,8 +32,10 @@ "xlrd==2.0.1", "openpyxl==3.0.10", "pyxlsb==1.0.9", + local_dependency("source-file"), ] + TEST_REQUIREMENTS = ["boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] setup( diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py index 95c8d328d87f..efc59cfbdceb 100644 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py @@ -4,7 +4,8 @@ import json import os -import sys + +import source_file # some integration tests doesn't setup dependences from # requirements.txt file and Python can return a exception. @@ -12,26 +13,10 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import ConnectorSpecification -try: - import source_file.source -except ModuleNotFoundError: - current_dir = os.path.dirname(os.path.abspath(__file__)) - parent_source_local = os.path.join(current_dir, "../../source-file") - if os.path.isdir(parent_source_local): - sys.path.append(parent_source_local) - else: - raise RuntimeError("not found parent source folder") - import source_file.source - -# import original classes of the native Source File -from source_file import SourceFile as ParentSourceFile -from source_file.client import Client -from source_file.client import URLFile as ParentURLFile - LOCAL_STORAGE_NAME = "local" -class URLFileSecure(ParentURLFile): +class URLFileSecure(source_file.client.URLFile): """Updating of default logic: This connector shouldn't work with local files. """ @@ -43,7 +28,7 @@ def __init__(self, url: str, provider: dict, binary=None, encoding=None): super().__init__(url, provider, binary, encoding) -class SourceFileSecure(ParentSourceFile): +class SourceFileSecure(source_file.SourceFile): """Updating of default source logic This connector shouldn't work with local files. The base logic of this connector are implemented in the "source-file" connector. @@ -52,7 +37,7 @@ class SourceFileSecure(ParentSourceFile): @property def client_class(self): # replace a standard class variable to the new one - class ClientSecure(Client): + class ClientSecure(source_file.client.Client): reader_class = URLFileSecure return ClientSecure diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py index 8d6adb7d888a..df7067ebd132 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py @@ -7,8 +7,10 @@ from __future__ import annotations import importlib.util +import re import uuid -from typing import TYPE_CHECKING, List, Optional +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional, Tuple from ci_connector_ops.pipelines import consts from ci_connector_ops.pipelines.consts import ( @@ -48,7 +50,7 @@ def with_python_base(context: PipelineContext, python_image_name: str = "python: .from_(python_image_name) .with_mounted_cache("/root/.cache/pip", pip_cache) .with_mounted_directory("/tools", context.get_repo_dir("tools", include=["ci_credentials", "ci_common_utils"])) - .with_exec(["pip", "install", "--upgrade", "pip"]) + .with_exec(["pip", "install", "pip==23.1.2"]) ) return base_container @@ -99,6 +101,37 @@ def with_python_package( return container +async def find_local_python_dependencies(context: PipelineContext, package_source_code_path: str) -> Tuple[List[str], List[str]]: + python_environment = with_python_base(context) + container = with_python_package(context, python_environment, package_source_code_path) + + # Find local dependencies in setup.py + setup_dependency_paths = [] + if await get_file_contents(container, "setup.py"): + container_with_egg_info = container.with_exec(["python", "setup.py", "egg_info"]) + egg_info_output = await container_with_egg_info.stdout() + for line in egg_info_output.split("\n"): + if line.startswith("writing requirements to"): + requires_txt_path = line.replace("writing requirements to", "").strip() + requires_txt = await container_with_egg_info.file(requires_txt_path).contents() + for line in requires_txt.split("\n"): + if "file://" in line: + match = re.search(r"file:///(.+)", line) + if match: + setup_dependency_paths.append(match.group(1)) + break + + # Find local dependencies in requirements.txt + requirements_dependency_paths = [] + if requirements_txt := await get_file_contents(container, "requirements.txt"): + for line in requirements_txt.split("\n"): + if line.startswith("-e ."): + local_dependency_path = Path(package_source_code_path + "/" + line[3:]).resolve() + local_dependency_path = str(local_dependency_path.relative_to(Path.cwd())) + requirements_dependency_paths.append(local_dependency_path) + return setup_dependency_paths, requirements_dependency_paths + + async def with_installed_python_package( context: PipelineContext, python_environment: Container, @@ -118,20 +151,19 @@ async def with_installed_python_package( Returns: Container: A python environment container with the python package installed. """ - install_local_requirements_cmd = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + install_requirements_cmd = ["python", "-m", "pip", "install", "-r", "requirements.txt"] install_connector_package_cmd = ["python", "-m", "pip", "install", "."] container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude) - if requirements_txt := await get_file_contents(container, "requirements.txt"): - for line in requirements_txt.split("\n"): - if line.startswith("-e ."): - local_dependency_path = package_source_code_path + "/" + line[3:] - container = container.with_mounted_directory( - "/" + local_dependency_path, context.get_repo_dir(local_dependency_path, exclude=DEFAULT_PYTHON_EXCLUDE) - ) - container = container.with_exec(install_local_requirements_cmd) - container = container.with_exec(install_connector_package_cmd) + setup_dependencies, requirements_dependencies = await find_local_python_dependencies(context, package_source_code_path) + for dependency_directory in setup_dependencies + requirements_dependencies: + container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) + + if await get_file_contents(container, "setup.py"): + container = container.with_exec(install_connector_package_cmd) + if await get_file_contents(container, "requirements.txt"): + container = container.with_exec(install_requirements_cmd) if additional_dependency_groups: container = container.with_exec( @@ -677,6 +709,9 @@ async def get_cdk_version_from_python_connector(python_connector: Container) -> async def with_airbyte_python_connector(context: ConnectorContext, build_platform: Platform) -> Container: + if context.connector.technical_name == "source-file-secure": + return await with_airbyte_python_connector_full_dagger(context, build_platform) + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") connector_container = ( context.dagger_client.container(platform=build_platform) @@ -732,28 +767,35 @@ async def finalize_build(context: ConnectorContext, connector_container: Contain return connector_container.with_entrypoint(original_entrypoint) -# This function is not used at the moment as we decided to use Python connectors dockerfile instead of building it with dagger. -# Some python connectors use alpine base image, other use debian... We should unify this. -def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_platform: Platform) -> Container: +async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_platform: Platform) -> Container: + setup_dependencies_to_mount, _ = await find_local_python_dependencies(context, str(context.connector.code_directory)) + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - base = context.dagger_client.container(platform=build_platform).from_("python:3.9.11-alpine3.15") + base = context.dagger_client.container(platform=build_platform).from_("python:3.9-slim") snake_case_name = context.connector.technical_name.replace("-", "_") entrypoint = ["python", "/airbyte/integration_code/main.py"] builder = ( base.with_workdir("/airbyte/integration_code") - .with_exec(["apk", "--no-cache", "upgrade"]) + .with_env_variable("DAGGER_BUILD", "True") + .with_exec(["apt-get", "update"]) .with_mounted_cache("/root/.cache/pip", pip_cache) .with_exec(["pip", "install", "--upgrade", "pip"]) - .with_exec(["apk", "--no-cache", "add", "tzdata", "build-base"]) + .with_exec(["apt-get", "install", "-y", "tzdata"]) .with_file("setup.py", context.get_connector_dir(include="setup.py").file("setup.py")) - .with_exec(["pip", "install", "--prefix=/install", "."]) ) - return ( + + for dependency_path in setup_dependencies_to_mount: + in_container_dependency_path = f"/local_dependencies/{Path(dependency_path).name}" + builder = builder.with_mounted_directory(in_container_dependency_path, context.get_repo_dir(dependency_path)) + + builder = builder.with_exec(["pip", "install", "--prefix=/install", "."]) + + connector_container = ( base.with_workdir("/airbyte/integration_code") .with_directory("/usr/local", builder.directory("/install")) .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC")) .with_new_file("/etc/timezone", "Etc/UTC") - .with_exec(["apk", "--no-cache", "add", "bash"]) + .with_exec(["apt-get", "install", "-y", "bash"]) .with_file("main.py", context.get_connector_dir(include="main.py").file("main.py")) .with_directory(snake_case_name, context.get_connector_dir(include=snake_case_name).directory(snake_case_name)) .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) @@ -761,6 +803,7 @@ def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_p .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) .with_label("io.airbyte.name", context.metadata["dockerRepository"]) ) + return await finalize_build(context, connector_container) def with_crane( diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py index 82a021b35bf0..ebc163dad189 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py @@ -94,7 +94,6 @@ class Step(ABC): title: ClassVar[str] started_at: ClassVar[datetime] - retry: ClassVar[bool] = False max_retries: ClassVar[int] = 0 def __init__(self, context: ConnectorContext) -> None: # noqa D107 @@ -112,7 +111,7 @@ async def run(self, *args, **kwargs) -> StepResult: self.started_at = datetime.utcnow() try: result = await self._run(*args, **kwargs) - if result.status is StepStatus.FAILURE and self.retry_count <= self.max_retries: + if result.status is StepStatus.FAILURE and self.retry_count <= self.max_retries and self.max_retries > 0: self.retry_count += 1 await anyio.sleep(10) self.context.logger.warn( @@ -209,6 +208,7 @@ async def _run_tests_in_directory(self, connector_under_test: Container, test_di """ test_config = "pytest.ini" if await check_path_in_workdir(connector_under_test, "pytest.ini") else "/" + PYPROJECT_TOML_FILE_PATH if await check_path_in_workdir(connector_under_test, test_directory): + tester = connector_under_test.with_exec( [ "python", diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py index 3fba7b41f094..49a00a90c346 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py @@ -155,12 +155,14 @@ async def _run(self) -> StepResult: filtered_repo = self.context.get_repo_dir( include=include, ) + ci_connector_ops = ci_connector_ops.with_mounted_directory("/airbyte", filtered_repo).with_workdir("/airbyte").with_exec(["ls"]) qa_checks = ( ci_connector_ops.with_mounted_directory("/airbyte", filtered_repo) .with_workdir("/airbyte") .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) ) + return await self.get_step_result(qa_checks) From a84f280ca91a9140f68c667a20ffde51cd2b4d93 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 18:28:21 +0200 Subject: [PATCH 03/13] revert source-file-secure Dockerfile --- .../connectors/source-file-secure/Dockerfile | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 98aa59bedfa3..4829b37511fb 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -6,21 +6,13 @@ # If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. # Please reach out to the Connectors Operations team if you have any question. -FROM python:3.9-slim as base -FROM base as builder +FROM airbyte/source-file:0.3.9 -RUN apt-get update WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install --prefix=/install . - -FROM base -WORKDIR /airbyte/integration_code -COPY --from=builder /install /usr/local - -COPY main.py ./ COPY source_file_secure ./source_file_secure - +COPY main.py ./ +COPY setup.py ./ +RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] From 5ea0d9af1bd1612a8f56ac2053608b22bc4a498d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 20:56:45 +0200 Subject: [PATCH 04/13] bump versions --- airbyte-integrations/connectors/source-file-secure/Dockerfile | 4 ++-- .../connectors/source-file-secure/metadata.yaml | 2 +- docs/integrations/sources/file.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 4829b37511fb..9b5aca2da99c 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -6,7 +6,7 @@ # If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. # Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/source-file:0.3.9 +FROM airbyte/source-file:0.3.10 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure @@ -17,5 +17,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.9 +LABEL io.airbyte.version=0.3.10 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/metadata.yaml b/airbyte-integrations/connectors/source-file-secure/metadata.yaml index d88de8ae1f31..01f929b0c3af 100644 --- a/airbyte-integrations/connectors/source-file-secure/metadata.yaml +++ b/airbyte-integrations/connectors/source-file-secure/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.9 + dockerImageTag: 0.3.10 dockerRepository: airbyte/source-file-secure githubIssueLabel: source-file icon: file.svg diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 271dc9cf84e4..1d9a740001b1 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -191,7 +191,7 @@ In order to read large files from a remote location, this connector uses the [sm | Version | Date | Pull Request | Subject | |:----------|:-------------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| -| 0.3.10 | 2023-06-07 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Make source-file testable in our new airbyte-ci pipelines | +| 0.3.10 | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107) | Make source-file testable in our new airbyte-ci pipelines | | 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | add ParserError handling | | 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | | 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | From 3a80a31cf6eb7da2bfa7557e9af7f8fc7e28f9ae Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 21:02:12 +0200 Subject: [PATCH 05/13] add docstring --- .../ci_connector_ops/pipelines/actions/environments.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py index df7067ebd132..7ceef451d7f9 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py @@ -102,6 +102,16 @@ def with_python_package( async def find_local_python_dependencies(context: PipelineContext, package_source_code_path: str) -> Tuple[List[str], List[str]]: + """Retrieve the list of local dependencies of a python package source code. + Returns both the list of local dependencies found in setup.py and requirements.txt. + + Args: + context (PipelineContext): The current pipeline context. + package_source_code_path (str): Path to the package source code in airbyte repo . + + Returns: + Tuple[List[str], List[str]]: A tuple containing the list of local dependencies found in setup.py and requirements.txt. + """ python_environment = with_python_base(context) container = with_python_package(context, python_environment, package_source_code_path) From d3f4d8048052ae603c95d1829fa26d23fd14b0ef Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 20:26:26 +0000 Subject: [PATCH 06/13] Automated Commit - Format and Process Resources Changes --- .../utils/backward_compatibility.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py index 36061dec7029..b877c570fb65 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py @@ -3,16 +3,15 @@ # from abc import ABC, abstractmethod -from deepdiff import DeepDiff from enum import Enum -import jsonschema - -from hypothesis import HealthCheck, Verbosity, given, settings -from hypothesis_jsonschema import from_schema +from typing import Any, Dict +import jsonschema from airbyte_cdk.models import ConnectorSpecification from connector_acceptance_test.utils import SecretDict -from typing import Dict, Any +from deepdiff import DeepDiff +from hypothesis import HealthCheck, Verbosity, given, settings +from hypothesis_jsonschema import from_schema class BackwardIncompatibilityContext(Enum): @@ -206,8 +205,8 @@ def remove_date_time_pattern_format(schema: Dict[str, Any]) -> Dict[str, Any]: if isinstance(schema, dict): for key, value in schema.items(): if isinstance(value, dict): - if value.get('format') == 'date-time' and 'pattern' in value: - del value['format'] + if value.get("format") == "date-time" and "pattern" in value: + del value["format"] remove_date_time_pattern_format(value) return schema @@ -219,6 +218,7 @@ def validate_previous_configs( 1. Generate fake previous config with the previous connector specification json schema. 2. Validate a fake previous config against the actual connector specification json schema.""" prev_con_spec = previous_connector_spec.dict()["connectionSpecification"] + @given(from_schema(remove_date_time_pattern_format(prev_con_spec))) @settings( max_examples=number_of_configs_to_generate, From b6c9c47da857148e2881e552d0a19c067175a4a4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 7 Jun 2023 22:51:31 +0200 Subject: [PATCH 07/13] handle missing secrets --- .../connector_acceptance_test/tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index 17a1ff606cc1..a08827a443e9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -580,6 +580,7 @@ class TestDiscovery(BaseTest): ({"string"}, "timestamp_with_timezone"), ({"string"}, "timestamp_without_timezone"), ({"string", "null"}, "timestamp_with_timezone"), + ({"string", "null"}, "timestamp_without_timezone"), ({"integer"}, "integer"), ({"integer", "null"}, "integer"), ({"number"}, "integer"), From 93661425ebbb31bec2f97e8b8f1742f6851e6fb0 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 00:46:21 +0200 Subject: [PATCH 08/13] handle missing secrets --- .../ci_connector_ops/pipelines/actions/secrets.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py index c9e5a78aca8c..c1590a550013 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py @@ -10,6 +10,7 @@ import anyio from ci_connector_ops.pipelines.actions import environments +from ci_connector_ops.pipelines.utils import get_file_contents from dagger import Directory if TYPE_CHECKING: @@ -22,11 +23,11 @@ async def mask_secrets_in_gha_logs(ci_credentials_with_downloaded_secrets: Conta We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. """ - secrets_to_mask = await ci_credentials_with_downloaded_secrets.file("/tmp/secrets_to_mask.txt").contents() - for secret_to_mask in secrets_to_mask.splitlines(): - # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" - # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked - print(f"::add-mask::{secret_to_mask}") + if secrets_to_mask := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): + for secret_to_mask in secrets_to_mask.splitlines(): + # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" + # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked + print(f"::add-mask::{secret_to_mask}") async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Directory: From 20f75e1822e52a5eb18a6d4b9edfca75c8622aa8 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 00:47:06 +0200 Subject: [PATCH 09/13] Revert "handle missing secrets" This reverts commit b6c9c47da857148e2881e552d0a19c067175a4a4. --- .../connector_acceptance_test/tests/test_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index a08827a443e9..17a1ff606cc1 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -580,7 +580,6 @@ class TestDiscovery(BaseTest): ({"string"}, "timestamp_with_timezone"), ({"string"}, "timestamp_without_timezone"), ({"string", "null"}, "timestamp_with_timezone"), - ({"string", "null"}, "timestamp_without_timezone"), ({"integer"}, "integer"), ({"integer", "null"}, "integer"), ({"number"}, "integer"), From afe9318a081b84eec7aa7e9a77ff92f707612f21 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 15:30:12 +0200 Subject: [PATCH 10/13] source-file-secure based on source-file:0.3.9 --- airbyte-integrations/connectors/source-file-secure/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 9b5aca2da99c..17f6af27883e 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -6,7 +6,7 @@ # If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. # Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/source-file:0.3.10 +FROM airbyte/source-file:0.3.9 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure From 6a62ada15671f139df7956ebf336633d7ca67250 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 15:59:11 +0200 Subject: [PATCH 11/13] source-file-secure: remove explicit dependency to source file in docker build --- airbyte-integrations/connectors/source-file-secure/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-file-secure/setup.py b/airbyte-integrations/connectors/source-file-secure/setup.py index 8956c853d13f..93347ef4d274 100644 --- a/airbyte-integrations/connectors/source-file-secure/setup.py +++ b/airbyte-integrations/connectors/source-file-secure/setup.py @@ -32,9 +32,10 @@ def local_dependency(name: str) -> str: "xlrd==2.0.1", "openpyxl==3.0.10", "pyxlsb==1.0.9", - local_dependency("source-file"), ] +if not os.environ.get("DOCKER_BUILD"): + MAIN_REQUIREMENTS.append(local_dependency("source-file")) TEST_REQUIREMENTS = ["boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] From 3dc35e1ec3309f1c783184ee9e5c952bf3a6dd93 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 15:59:25 +0200 Subject: [PATCH 12/13] source-file-secure: remove explicit dependency to source file in docker build --- airbyte-integrations/connectors/source-file-secure/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 17f6af27883e..6743c32215e8 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -12,6 +12,7 @@ WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure COPY main.py ./ COPY setup.py ./ +ENV DOCKER_BUILD=True RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" From 8f2b428261891d09549ba6535f1002797a48f2da Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 8 Jun 2023 16:36:35 +0200 Subject: [PATCH 13/13] source-file-secure FROM source-file:0:3.10 --- airbyte-integrations/connectors/source-file-secure/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 6743c32215e8..4245582feb02 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -6,7 +6,7 @@ # If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. # Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/source-file:0.3.9 +FROM airbyte/source-file:0.3.10 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure