From ca1ac784e53ad58b980b6d05878ba58f6806813e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 19 Nov 2020 01:06:45 +0100 Subject: [PATCH] Adds support for Connection discovery from providers This PR extends providers discovery with the mechanism of retrieving mapping of connections from type to hook. Fixes #12465 --- airflow/__init__.py | 1 - airflow/models/connection.py | 79 ++---- airflow/provider.yaml.schema.json | 45 ++- airflow/providers/amazon/provider.yaml | 178 ++++++------ .../providers/apache/cassandra/provider.yaml | 13 +- airflow/providers/apache/druid/provider.yaml | 10 +- airflow/providers/apache/hdfs/provider.yaml | 10 +- airflow/providers/apache/hive/provider.yaml | 34 ++- airflow/providers/apache/kylin/provider.yaml | 6 +- airflow/providers/apache/livy/provider.yaml | 9 +- airflow/providers/apache/pig/provider.yaml | 13 +- airflow/providers/apache/pinot/provider.yaml | 5 +- airflow/providers/apache/spark/provider.yaml | 17 +- airflow/providers/apache/sqoop/provider.yaml | 7 +- airflow/providers/celery/provider.yaml | 5 +- airflow/providers/cloudant/provider.yaml | 11 +- .../providers/cncf/kubernetes/provider.yaml | 16 +- airflow/providers/databricks/provider.yaml | 7 +- airflow/providers/datadog/provider.yaml | 6 +- airflow/providers/dingding/provider.yaml | 7 +- airflow/providers/discord/provider.yaml | 7 +- airflow/providers/docker/provider.yaml | 14 +- airflow/providers/elasticsearch/provider.yaml | 10 +- airflow/providers/exasol/provider.yaml | 11 +- airflow/providers/facebook/provider.yaml | 5 +- airflow/providers/ftp/provider.yaml | 6 +- airflow/providers/google/provider.yaml | 261 +++++++++--------- airflow/providers/grpc/provider.yaml | 12 +- airflow/providers/hashicorp/provider.yaml | 5 +- airflow/providers/http/provider.yaml | 8 +- airflow/providers/imap/provider.yaml | 11 +- airflow/providers/jdbc/provider.yaml | 12 +- airflow/providers/jenkins/provider.yaml | 7 +- airflow/providers/jira/provider.yaml | 13 +- .../providers/microsoft/azure/provider.yaml | 60 ++-- .../providers/microsoft/mssql/provider.yaml | 7 +- .../providers/microsoft/winrm/provider.yaml | 6 +- airflow/providers/mongo/provider.yaml | 11 +- airflow/providers/mysql/provider.yaml | 18 +- airflow/providers/odbc/provider.yaml | 14 +- airflow/providers/openfaas/provider.yaml | 5 +- airflow/providers/opsgenie/provider.yaml | 7 +- airflow/providers/oracle/provider.yaml | 16 +- airflow/providers/pagerduty/provider.yaml | 5 +- airflow/providers/papermill/provider.yaml | 5 +- airflow/providers/plexus/provider.yaml | 7 +- airflow/providers/postgres/provider.yaml | 11 +- airflow/providers/presto/provider.yaml | 10 +- airflow/providers/qubole/provider.yaml | 14 +- airflow/providers/redis/provider.yaml | 15 +- airflow/providers/salesforce/provider.yaml | 15 +- airflow/providers/samba/provider.yaml | 4 +- airflow/providers/segment/provider.yaml | 6 +- airflow/providers/sendgrid/provider.yaml | 2 + airflow/providers/sftp/provider.yaml | 8 +- airflow/providers/singularity/provider.yaml | 5 +- airflow/providers/slack/provider.yaml | 11 +- airflow/providers/snowflake/provider.yaml | 15 +- airflow/providers/sqlite/provider.yaml | 11 +- airflow/providers/ssh/provider.yaml | 6 +- airflow/providers/vertica/provider.yaml | 11 +- airflow/providers/yandex/provider.yaml | 8 +- airflow/providers/zendesk/provider.yaml | 4 +- airflow/providers_manager.py | 147 ++++++++-- .../pre_commit_check_provider_yaml_files.py | 9 +- tests/core/test_providers_manager.py | 42 +++ tests/models/test_connection.py | 14 - 67 files changed, 902 insertions(+), 488 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index 35e755ece0e04..ca7281e933b97 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -79,7 +79,6 @@ def __getattr__(name): from airflow.models.dag import DAG from airflow.exceptions import AirflowException - if not PY37: from pep562 import Pep562 diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 2d15d3adcc264..2d3406a4ab3fc 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -18,8 +18,9 @@ import json import warnings +from functools import lru_cache from json import JSONDecodeError -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from urllib.parse import parse_qsl, quote, unquote, urlencode, urlparse from sqlalchemy import Boolean, Column, Integer, String @@ -30,69 +31,22 @@ from airflow.exceptions import AirflowException, AirflowNotFoundException from airflow.models.base import ID_LEN, Base from airflow.models.crypto import get_fernet +from airflow.providers_manager import get_providers_manager from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.module_loading import import_string -# A map that assigns a connection type to a tuple that contains -# the path of the class and the name of the conn_id key parameter. -# PLEASE KEEP BELOW LIST IN ALPHABETICAL ORDER. -CONN_TYPE_TO_HOOK = { - "azure_batch": ( - "airflow.providers.microsoft.azure.hooks.azure_batch.AzureBatchHook", - "azure_batch_conn_id", - ), - "azure_cosmos": ( - "airflow.providers.microsoft.azure.hooks.azure_cosmos.AzureCosmosDBHook", - "azure_cosmos_conn_id", - ), - "azure_data_lake": ( - "airflow.providers.microsoft.azure.hooks.azure_data_lake.AzureDataLakeHook", - "azure_data_lake_conn_id", - ), - "cassandra": ("airflow.providers.apache.cassandra.hooks.cassandra.CassandraHook", "cassandra_conn_id"), - "cloudant": ("airflow.providers.cloudant.hooks.cloudant.CloudantHook", "cloudant_conn_id"), - "dataprep": ("airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook", "dataprep_default"), - "docker": ("airflow.providers.docker.hooks.docker.DockerHook", "docker_conn_id"), - "elasticsearch": ( - "airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchHook", - "elasticsearch_conn_id", - ), - "exasol": ("airflow.providers.exasol.hooks.exasol.ExasolHook", "exasol_conn_id"), - "gcpcloudsql": ( - "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLDatabaseHook", - "gcp_cloudsql_conn_id", - ), - "gcpssh": ( - "airflow.providers.google.cloud.hooks.compute_ssh.ComputeEngineSSHHook", - "gcp_conn_id", - ), - "google_cloud_platform": ( - "airflow.providers.google.cloud.hooks.bigquery.BigQueryHook", - "bigquery_conn_id", - ), - "grpc": ("airflow.providers.grpc.hooks.grpc.GrpcHook", "grpc_conn_id"), - "hive_cli": ("airflow.providers.apache.hive.hooks.hive.HiveCliHook", "hive_cli_conn_id"), - "hiveserver2": ("airflow.providers.apache.hive.hooks.hive.HiveServer2Hook", "hiveserver2_conn_id"), - "imap": ("airflow.providers.imap.hooks.imap.ImapHook", "imap_conn_id"), - "jdbc": ("airflow.providers.jdbc.hooks.jdbc.JdbcHook", "jdbc_conn_id"), - "jira": ("airflow.providers.jira.hooks.jira.JiraHook", "jira_conn_id"), - "kubernetes": ("airflow.providers.cncf.kubernetes.hooks.kubernetes.KubernetesHook", "kubernetes_conn_id"), - "mongo": ("airflow.providers.mongo.hooks.mongo.MongoHook", "conn_id"), - "mssql": ("airflow.providers.odbc.hooks.odbc.OdbcHook", "odbc_conn_id"), - "mysql": ("airflow.providers.mysql.hooks.mysql.MySqlHook", "mysql_conn_id"), - "odbc": ("airflow.providers.odbc.hooks.odbc.OdbcHook", "odbc_conn_id"), - "oracle": ("airflow.providers.oracle.hooks.oracle.OracleHook", "oracle_conn_id"), - "pig_cli": ("airflow.providers.apache.pig.hooks.pig.PigCliHook", "pig_cli_conn_id"), - "postgres": ("airflow.providers.postgres.hooks.postgres.PostgresHook", "postgres_conn_id"), - "presto": ("airflow.providers.presto.hooks.presto.PrestoHook", "presto_conn_id"), - "redis": ("airflow.providers.redis.hooks.redis.RedisHook", "redis_conn_id"), - "snowflake": ("airflow.providers.snowflake.hooks.snowflake.SnowflakeHook", "snowflake_conn_id"), - "sqlite": ("airflow.providers.sqlite.hooks.sqlite.SqliteHook", "sqlite_conn_id"), - "tableau": ("airflow.providers.salesforce.hooks.tableau.TableauHook", "tableau_conn_id"), - "vertica": ("airflow.providers.vertica.hooks.vertica.VerticaHook", "vertica_conn_id"), - "wasb": ("airflow.providers.microsoft.azure.hooks.wasb.WasbHook", "wasb_conn_id"), -} -# PLEASE KEEP ABOVE LIST IN ALPHABETICAL ORDER. +cache = lru_cache(maxsize=None) + + +@cache +def get_conn_type_to_hook() -> Dict[str, Tuple[str, str]]: + """ + Return A map that assigns a connection type to a tuple that contains + the path of the class and the name of the conn_id key parameter. + + :return: The map of connection type to hook + """ + return get_providers_manager().connections def parse_netloc_to_hostname(*args, **kwargs): @@ -326,7 +280,8 @@ def rotate_fernet_key(self): def get_hook(self): """Return hook based on conn_type.""" - hook_class_name, conn_id_param = CONN_TYPE_TO_HOOK.get(self.conn_type, (None, None)) + hook_class_name, conn_id_param = get_conn_type_to_hook().get(self.conn_type, (None, None)) + if not hook_class_name: raise AirflowException(f'Unknown hook type "{self.conn_type}"') hook_class = import_string(hook_class_name) diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index 219a5e5cce476..81275076d0295 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -17,6 +17,11 @@ "type": "string" } }, + "provider-package": { + "description": "Package of the provider. All classes and resources of this provider must reside within this package.", + "type": "string" + }, + "integrations": { "type": "array", "items": { @@ -75,10 +80,12 @@ "description": "Integration name. It must have a matching item in the 'integration' section of any provider." }, "python-modules": { + "description": "Name of the module - relative to the provider package. Must start with .", "type": "array", "items": { "type": "string" - } + }, + "pattern": "^\\..*$" } }, "additionalProperties": false, @@ -98,9 +105,11 @@ "description": "Integration name. It must have a matching item in the 'integration' section of any provider." }, "python-modules": { + "description": "Name of the module - relative to the provider package. Must start with .", "type": "array", "items": { - "type": "string" + "type": "string", + "pattern": "^\\..*$" } } }, @@ -121,9 +130,11 @@ "description": "Integration name. It must have a matching item in the 'integration' section of any provider." }, "python-modules": { + "description": "Name of the module - relative to the provider package. Must start with .", "type": "array", "items": { - "type": "string" + "type": "string", + "pattern": "^\\..*$" } } }, @@ -153,7 +164,8 @@ }, "python-module": { "type": "string", - "description": "Source integration name. It must have a matching item in the 'integration' section of any provider." + "description": "Name of the module - relative to the provider package. Must start with .", + "pattern": "^\\..*$" } }, "additionalProperties": false, @@ -163,12 +175,35 @@ "python-module" ] } + }, + "connections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "connection-type": { + "description": "Type of connection that the provider has", + "type": "string" + }, + "hook-class": { + "description": "Path of the class that the provider adds. Must start with . and be relative to the provider package defined in providerPackage field.", + "type": "string", + "pattern": "^\\..*$" + }, + "connection-id-parameter-name": { + "description": "Name of the parameter to pass to the hook.", + "type": "string" + } + }, + "required": ["connection-type", "hook-class", "connection-id-parameter-name"] + } } }, "additionalProperties": false, "required": [ "package-name", "description", - "versions" + "versions", + "provider-package" ] } diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index bc852af008f4e..a5f6e9dae83f8 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.amazon + integrations: - integration-name: Amazon Athena external-doc-url: https://aws.amazon.com/athena/ @@ -108,219 +110,219 @@ integrations: operators: - integration-name: Amazon Athena python-modules: - - airflow.providers.amazon.aws.operators.athena + - .aws.operators.athena - integration-name: AWS Batch python-modules: - - airflow.providers.amazon.aws.operators.batch + - .aws.operators.batch - integration-name: Amazon CloudFormation python-modules: - - airflow.providers.amazon.aws.operators.cloud_formation + - .aws.operators.cloud_formation - integration-name: Amazon DataSync python-modules: - - airflow.providers.amazon.aws.operators.datasync + - .aws.operators.datasync - integration-name: Amazon EC2 python-modules: - - airflow.providers.amazon.aws.operators.ec2_start_instance - - airflow.providers.amazon.aws.operators.ec2_stop_instance + - .aws.operators.ec2_start_instance + - .aws.operators.ec2_stop_instance - integration-name: Amazon ECS python-modules: - - airflow.providers.amazon.aws.operators.ecs + - .aws.operators.ecs - integration-name: Amazon EMR python-modules: - - airflow.providers.amazon.aws.operators.emr_add_steps - - airflow.providers.amazon.aws.operators.emr_create_job_flow - - airflow.providers.amazon.aws.operators.emr_modify_cluster - - airflow.providers.amazon.aws.operators.emr_terminate_job_flow + - .aws.operators.emr_add_steps + - .aws.operators.emr_create_job_flow + - .aws.operators.emr_modify_cluster + - .aws.operators.emr_terminate_job_flow - integration-name: Amazon Glacier python-modules: - - airflow.providers.amazon.aws.operators.glacier + - .aws.operators.glacier - integration-name: AWS Glue python-modules: - - airflow.providers.amazon.aws.operators.glue + - .aws.operators.glue - integration-name: Amazon Simple Storage Service (S3) python-modules: - - airflow.providers.amazon.aws.operators.s3_bucket - - airflow.providers.amazon.aws.operators.s3_copy_object - - airflow.providers.amazon.aws.operators.s3_delete_objects - - airflow.providers.amazon.aws.operators.s3_file_transform - - airflow.providers.amazon.aws.operators.s3_list + - .aws.operators.s3_bucket + - .aws.operators.s3_copy_object + - .aws.operators.s3_delete_objects + - .aws.operators.s3_file_transform + - .aws.operators.s3_list - integration-name: Amazon SageMaker python-modules: - - airflow.providers.amazon.aws.operators.sagemaker_base - - airflow.providers.amazon.aws.operators.sagemaker_endpoint - - airflow.providers.amazon.aws.operators.sagemaker_endpoint_config - - airflow.providers.amazon.aws.operators.sagemaker_model - - airflow.providers.amazon.aws.operators.sagemaker_processing - - airflow.providers.amazon.aws.operators.sagemaker_training - - airflow.providers.amazon.aws.operators.sagemaker_transform - - airflow.providers.amazon.aws.operators.sagemaker_tuning + - .aws.operators.sagemaker_base + - .aws.operators.sagemaker_endpoint + - .aws.operators.sagemaker_endpoint_config + - .aws.operators.sagemaker_model + - .aws.operators.sagemaker_processing + - .aws.operators.sagemaker_training + - .aws.operators.sagemaker_transform + - .aws.operators.sagemaker_tuning - integration-name: Amazon Simple Notification Service (SNS) python-modules: - - airflow.providers.amazon.aws.operators.sns + - .aws.operators.sns - integration-name: Amazon Simple Queue Service (SQS) python-modules: - - airflow.providers.amazon.aws.operators.sqs + - .aws.operators.sqs - integration-name: AWS Step Functions python-modules: - - airflow.providers.amazon.aws.operators.step_function_get_execution_output - - airflow.providers.amazon.aws.operators.step_function_start_execution + - .aws.operators.step_function_get_execution_output + - .aws.operators.step_function_start_execution sensors: - integration-name: Amazon Athena python-modules: - - airflow.providers.amazon.aws.sensors.athena + - .aws.sensors.athena - integration-name: Amazon Athena python-modules: - - airflow.providers.amazon.aws.sensors.athena + - .aws.sensors.athena - integration-name: Amazon CloudFormation python-modules: - - airflow.providers.amazon.aws.sensors.cloud_formation + - .aws.sensors.cloud_formation - integration-name: Amazon EC2 python-modules: - - airflow.providers.amazon.aws.sensors.ec2_instance_state + - .aws.sensors.ec2_instance_state - integration-name: Amazon EMR python-modules: - - airflow.providers.amazon.aws.sensors.emr_base - - airflow.providers.amazon.aws.sensors.emr_job_flow - - airflow.providers.amazon.aws.sensors.emr_step + - .aws.sensors.emr_base + - .aws.sensors.emr_job_flow + - .aws.sensors.emr_step - integration-name: Amazon Glacier python-modules: - - airflow.providers.amazon.aws.sensors.glacier + - .aws.sensors.glacier - integration-name: AWS Glue python-modules: - - airflow.providers.amazon.aws.sensors.glue - - airflow.providers.amazon.aws.sensors.glue_catalog_partition + - .aws.sensors.glue + - .aws.sensors.glue_catalog_partition - integration-name: Amazon Redshift python-modules: - - airflow.providers.amazon.aws.sensors.redshift + - .aws.sensors.redshift - integration-name: Amazon Simple Storage Service (S3) python-modules: - - airflow.providers.amazon.aws.sensors.s3_key - - airflow.providers.amazon.aws.sensors.s3_keys_unchanged - - airflow.providers.amazon.aws.sensors.s3_prefix + - .aws.sensors.s3_key + - .aws.sensors.s3_keys_unchanged + - .aws.sensors.s3_prefix - integration-name: Amazon SageMaker python-modules: - - airflow.providers.amazon.aws.sensors.sagemaker_base - - airflow.providers.amazon.aws.sensors.sagemaker_endpoint - - airflow.providers.amazon.aws.sensors.sagemaker_training - - airflow.providers.amazon.aws.sensors.sagemaker_transform - - airflow.providers.amazon.aws.sensors.sagemaker_tuning + - .aws.sensors.sagemaker_base + - .aws.sensors.sagemaker_endpoint + - .aws.sensors.sagemaker_training + - .aws.sensors.sagemaker_transform + - .aws.sensors.sagemaker_tuning - integration-name: Amazon Simple Queue Service (SQS) python-modules: - - airflow.providers.amazon.aws.sensors.sqs + - .aws.sensors.sqs - integration-name: AWS Step Functions python-modules: - - airflow.providers.amazon.aws.sensors.step_function_execution + - .aws.sensors.step_function_execution hooks: - integration-name: Amazon Athena python-modules: - - airflow.providers.amazon.aws.hooks.athena + - .aws.hooks.athena - integration-name: Amazon DynamoDB python-modules: - - airflow.providers.amazon.aws.hooks.dynamodb - - airflow.providers.amazon.aws.hooks.aws_dynamodb + - .aws.hooks.dynamodb + - .aws.hooks.aws_dynamodb - integration-name: Amazon Web Services python-modules: - - airflow.providers.amazon.aws.hooks.base_aws + - .aws.hooks.base_aws - integration-name: AWS Batch python-modules: - - airflow.providers.amazon.aws.hooks.batch_client - - airflow.providers.amazon.aws.hooks.batch_waiters + - .aws.hooks.batch_client + - .aws.hooks.batch_waiters - integration-name: Amazon CloudFormation python-modules: - - airflow.providers.amazon.aws.hooks.cloud_formation + - .aws.hooks.cloud_formation - integration-name: Amazon DataSync python-modules: - - airflow.providers.amazon.aws.hooks.datasync + - .aws.hooks.datasync - integration-name: Amazon EC2 python-modules: - - airflow.providers.amazon.aws.hooks.ec2 + - .aws.hooks.ec2 - integration-name: Amazon ElastiCache python-modules: - - airflow.providers.amazon.aws.hooks.elasticache_replication_group + - .aws.hooks.elasticache_replication_group - integration-name: Amazon EMR python-modules: - - airflow.providers.amazon.aws.hooks.emr + - .aws.hooks.emr - integration-name: Amazon Glacier python-modules: - - airflow.providers.amazon.aws.hooks.glacier + - .aws.hooks.glacier - integration-name: AWS Glue python-modules: - - airflow.providers.amazon.aws.hooks.glue - - airflow.providers.amazon.aws.hooks.glue_catalog + - .aws.hooks.glue + - .aws.hooks.glue_catalog - integration-name: Amazon Kinesis Data Firehose python-modules: - - airflow.providers.amazon.aws.hooks.kinesis + - .aws.hooks.kinesis - integration-name: AWS Lambda python-modules: - - airflow.providers.amazon.aws.hooks.lambda_function + - .aws.hooks.lambda_function - integration-name: Amazon CloudWatch Logs python-modules: - - airflow.providers.amazon.aws.hooks.logs + - .aws.hooks.logs - integration-name: Amazon Redshift python-modules: - - airflow.providers.amazon.aws.hooks.redshift + - .aws.hooks.redshift - integration-name: Amazon Simple Storage Service (S3) python-modules: - - airflow.providers.amazon.aws.hooks.s3 + - .aws.hooks.s3 - integration-name: Amazon SageMaker python-modules: - - airflow.providers.amazon.aws.hooks.sagemaker + - .aws.hooks.sagemaker - integration-name: Amazon Simple Email Service (SES) python-modules: - - airflow.providers.amazon.aws.hooks.ses + - .aws.hooks.ses - integration-name: Amazon SecretsManager python-modules: - - airflow.providers.amazon.aws.hooks.secrets_manager + - .aws.hooks.secrets_manager - integration-name: Amazon Simple Notification Service (SNS) python-modules: - - airflow.providers.amazon.aws.hooks.sns + - .aws.hooks.sns - integration-name: Amazon Simple Queue Service (SQS) python-modules: - - airflow.providers.amazon.aws.hooks.sqs + - .aws.hooks.sqs - integration-name: AWS Step Functions python-modules: - - airflow.providers.amazon.aws.hooks.step_function + - .aws.hooks.step_function transfers: - source-integration-name: Amazon DynamoDB target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3 + python-module: .aws.transfers.dynamodb_to_s3 - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3 + python-module: .aws.transfers.gcs_to_s3 - source-integration-name: Amazon Glacier target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/amazon/aws/transfer/glacier_to_gcs.rst - python-module: airflow.providers.amazon.aws.transfers.glacier_to_gcs + python-module: .aws.transfers.glacier_to_gcs - source-integration-name: Google target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/howto/operator/amazon/aws/google_api_to_s3_transfer.rst - python-module: airflow.providers.amazon.aws.transfers.google_api_to_s3 + python-module: .aws.transfers.google_api_to_s3 - source-integration-name: Apache Hive target-integration-name: Amazon DynamoDB - python-module: airflow.providers.amazon.aws.transfers.hive_to_dynamodb + python-module: .aws.transfers.hive_to_dynamodb - source-integration-name: Internet Message Access Protocol (IMAP) target-integration-name: Amazon Simple Storage Service (S3) how-to-guide: /docs/howto/operator/amazon/aws/imap_attachment_to_s3.rst - python-module: airflow.providers.amazon.aws.transfers.imap_attachment_to_s3 + python-module: .aws.transfers.imap_attachment_to_s3 - source-integration-name: MongoDB target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3 + python-module: .aws.transfers.mongo_to_s3 - source-integration-name: MySQL target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.mysql_to_s3 + python-module: .aws.transfers.mysql_to_s3 - source-integration-name: Amazon Redshift target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3 + python-module: .aws.transfers.redshift_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Amazon Redshift how-to-guide: /docs/howto/operator/amazon/aws/s3_to_redshift.rst - python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift + python-module: .aws.transfers.s3_to_redshift - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: SSH File Transfer Protocol (SFTP) - python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp + python-module: .aws.transfers.s3_to_sftp - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Amazon Simple Storage Service (S3) - python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3 + python-module: .aws.transfers.sftp_to_s3 diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml index a17345deb68e4..23212a972e104 100644 --- a/airflow/providers/apache/cassandra/provider.yaml +++ b/airflow/providers/apache/cassandra/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.apache.cassandra + integrations: - integration-name: Apache Cassandra external-doc-url: http://cassandra.apache.org/ @@ -33,10 +35,15 @@ integrations: sensors: - integration-name: Apache Cassandra python-modules: - - airflow.providers.apache.cassandra.sensors.record - - airflow.providers.apache.cassandra.sensors.table + - .sensors.record + - .sensors.table hooks: - integration-name: Apache Cassandra python-modules: - - airflow.providers.apache.cassandra.hooks.cassandra + - .hooks.cassandra + +connections: + - connection-type: cassandra + hook-class: .hooks.cassandra.CassandraHook + connection-id-parameter-name: cassandra_conn_id diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index b9c2783a24f3a..e8e1472f85e01 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.apache.druid + integrations: - integration-name: Apache Druid external-doc-url: https://druid.apache.org/ @@ -31,15 +33,15 @@ integrations: operators: - integration-name: Apache Druid python-modules: - - airflow.providers.apache.druid.operators.druid - - airflow.providers.apache.druid.operators.druid_check + - .operators.druid + - .operators.druid_check hooks: - integration-name: Apache Druid python-modules: - - airflow.providers.apache.druid.hooks.druid + - .hooks.druid transfers: - source-integration-name: Apache Hive target-integration-name: Apache Druid - python-module: airflow.providers.apache.druid.transfers.hive_to_druid + python-module: .transfers.hive_to_druid diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml index b764f191360d2..e82755be4c2b9 100644 --- a/airflow/providers/apache/hdfs/provider.yaml +++ b/airflow/providers/apache/hdfs/provider.yaml @@ -24,6 +24,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.apache.hdfs + integrations: - integration-name: Hadoop Distributed File System (HDFS) external-doc-url: https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html @@ -35,15 +37,15 @@ integrations: sensors: - integration-name: Hadoop Distributed File System (HDFS) python-modules: - - airflow.providers.apache.hdfs.sensors.hdfs + - .sensors.hdfs - integration-name: WebHDFS python-modules: - - airflow.providers.apache.hdfs.sensors.web_hdfs + - .sensors.web_hdfs hooks: - integration-name: Hadoop Distributed File System (HDFS) python-modules: - - airflow.providers.apache.hdfs.hooks.hdfs + - .hooks.hdfs - integration-name: WebHDFS python-modules: - - airflow.providers.apache.hdfs.hooks.webhdfs + - .hooks.webhdfs diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 9234075652647..699c59224e9df 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.apache.hive + integrations: - integration-name: Apache Hive external-doc-url: https://hive.apache.org/ @@ -31,37 +33,45 @@ integrations: operators: - integration-name: Apache Hive python-modules: - - airflow.providers.apache.hive.operators.hive - - airflow.providers.apache.hive.operators.hive_stats + - .operators.hive + - .operators.hive_stats sensors: - integration-name: Apache Hive python-modules: - - airflow.providers.apache.hive.sensors.hive_partition - - airflow.providers.apache.hive.sensors.metastore_partition - - airflow.providers.apache.hive.sensors.named_hive_partition + - .sensors.hive_partition + - .sensors.metastore_partition + - .sensors.named_hive_partition hooks: - integration-name: Apache Hive python-modules: - - airflow.providers.apache.hive.hooks.hive + - .hooks.hive transfers: - source-integration-name: Vertica target-integration-name: Apache Hive - python-module: airflow.providers.apache.hive.transfers.vertica_to_hive + python-module: .transfers.vertica_to_hive - source-integration-name: Apache Hive target-integration-name: MySQL - python-module: airflow.providers.apache.hive.transfers.hive_to_mysql + python-module: .transfers.hive_to_mysql - source-integration-name: Apache Hive target-integration-name: Samba - python-module: airflow.providers.apache.hive.transfers.hive_to_samba + python-module: .transfers.hive_to_samba - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Apache Hive - python-module: airflow.providers.apache.hive.transfers.s3_to_hive + python-module: .transfers.s3_to_hive - source-integration-name: MySQL target-integration-name: Apache Hive - python-module: airflow.providers.apache.hive.transfers.mysql_to_hive + python-module: .transfers.mysql_to_hive - source-integration-name: Microsoft SQL Server (MSSQL) target-integration-name: Apache Hive - python-module: airflow.providers.apache.hive.transfers.mssql_to_hive + python-module: .transfers.mssql_to_hive + +connections: + - connection-type: hive_cli + hook-class: .hooks.hive.HiveCliHook + connection-id-parameter-name: hive_cli_conn_id + - connection-type: hiveserver2 + hook-class: .hooks.hive.HiveServer2Hook + connection-id-parameter-name: hiveserver2_conn_id diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml index 80fb84e96d688..eb6680a0aaf4a 100644 --- a/airflow/providers/apache/kylin/provider.yaml +++ b/airflow/providers/apache/kylin/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.apache.kylin + integrations: - integration-name: Apache Kylin external-doc-url: https://kylin.apache.org/ @@ -31,9 +33,9 @@ integrations: operators: - integration-name: Apache Kylin python-modules: - - airflow.providers.apache.kylin.operators.kylin_cube + - .operators.kylin_cube hooks: - integration-name: Apache Kylin python-modules: - - airflow.providers.apache.kylin.hooks.kylin + - .hooks.kylin diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index 3a4088b408762..4f316e903bb93 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.apache.livy + integrations: - integration-name: Apache Livy external-doc-url: https://livy.apache.org/ @@ -30,14 +33,14 @@ integrations: operators: - integration-name: Apache Livy python-modules: - - airflow.providers.apache.livy.operators.livy + - .operators.livy sensors: - integration-name: Apache Livy python-modules: - - airflow.providers.apache.livy.sensors.livy + - .sensors.livy hooks: - integration-name: Apache Livy python-modules: - - airflow.providers.apache.livy.hooks.livy + - .hooks.livy diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index aec15b182a4b6..77dee1f2c3eb6 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.apache.pig + integrations: - integration-name: Apache Pig external-doc-url: https://pig.apache.org/ @@ -30,8 +33,14 @@ integrations: operators: - integration-name: Apache Pig python-modules: - - airflow.providers.apache.pig.operators.pig + - .operators.pig + hooks: - integration-name: Apache Pig python-modules: - - airflow.providers.apache.pig.hooks.pig + - .hooks.pig + +connections: + - connection-type: pig_cli + hook-class: .hooks.pig.PigCliHook + connection-id-parameter-name: pig_cli_conn_id diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index a3e1e9075fea2..a848b566120c5 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.apache.pinot + integrations: - integration-name: Apache Pinot external-doc-url: https://pinot.apache.org/ @@ -30,4 +33,4 @@ integrations: hooks: - integration-name: Apache Pinot python-modules: - - airflow.providers.apache.pinot.hooks.pinot + - .hooks.pinot diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 18a4ce8a7d5eb..6a8edc419ef98 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.apache.spark + integrations: - integration-name: Apache Spark external-doc-url: https://spark.apache.org/ @@ -32,14 +35,14 @@ integrations: operators: - integration-name: Apache Spark python-modules: - - airflow.providers.apache.spark.operators.spark_jdbc - - airflow.providers.apache.spark.operators.spark_sql - - airflow.providers.apache.spark.operators.spark_submit + - .operators.spark_jdbc + - .operators.spark_sql + - .operators.spark_submit hooks: - integration-name: Apache Spark python-modules: - - airflow.providers.apache.spark.hooks.spark_jdbc - - airflow.providers.apache.spark.hooks.spark_jdbc_script - - airflow.providers.apache.spark.hooks.spark_sql - - airflow.providers.apache.spark.hooks.spark_submit + - .hooks.spark_jdbc + - .hooks.spark_jdbc_script + - .hooks.spark_sql + - .hooks.spark_submit diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index 840406d549436..7f05d6ec5751e 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.apache.sqoop + integrations: - integration-name: Apache Sqoop external-doc-url: https://sqoop.apache.org/ @@ -30,9 +33,9 @@ integrations: operators: - integration-name: Apache Sqoop python-modules: - - airflow.providers.apache.sqoop.operators.sqoop + - .operators.sqoop hooks: - integration-name: Apache Sqoop python-modules: - - airflow.providers.apache.sqoop.hooks.sqoop + - .hooks.sqoop diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 8d3a6c0c48418..010b257e6ce16 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.celery + integrations: - integration-name: Celery external-doc-url: http://www.celeryproject.org/ @@ -30,4 +33,4 @@ integrations: sensors: - integration-name: Celery python-modules: - - airflow.providers.celery.sensors.celery_queue + - .sensors.celery_queue diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 56d8e2d4f2f04..1852b5f2f3b95 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -22,6 +22,10 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.cloudant + + integrations: - integration-name: IBM Cloudant external-doc-url: https://www.ibm.com/cloud/cloudant @@ -30,4 +34,9 @@ integrations: hooks: - integration-name: IBM Cloudant python-modules: - - airflow.providers.cloudant.hooks.cloudant + - .hooks.cloudant + +connections: + - connection-type: cloudant + hook-class: .hooks.cloudant.CloudantHook + connection-id-parameter-name: cloudant_conn_id diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index bc6d01dd35408..6651d205b6264 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.cncf.kubernetes + integrations: - integration-name: Kubernetes external-doc-url: https://kubernetes.io/ @@ -35,15 +38,20 @@ integrations: operators: - integration-name: Kubernetes python-modules: - - airflow.providers.cncf.kubernetes.operators.kubernetes_pod - - airflow.providers.cncf.kubernetes.operators.spark_kubernetes + - .operators.kubernetes_pod + - .operators.spark_kubernetes sensors: - integration-name: Kubernetes python-modules: - - airflow.providers.cncf.kubernetes.sensors.spark_kubernetes + - .sensors.spark_kubernetes hooks: - integration-name: Kubernetes python-modules: - - airflow.providers.cncf.kubernetes.hooks.kubernetes + - .hooks.kubernetes + +connections: + - connection-type: kubernetes + hook-class: .hooks.kubernetes.KubernetesHook + connection-id-parameter-name: kubernetes_conn_id diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 00bd293a5f634..6a7a37b404172 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.databricks + integrations: - integration-name: Databricks external-doc-url: https://databricks.com/ @@ -31,9 +34,9 @@ integrations: operators: - integration-name: Databricks python-modules: - - airflow.providers.databricks.operators.databricks + - .operators.databricks hooks: - integration-name: Databricks python-modules: - - airflow.providers.databricks.hooks.databricks + - .hooks.databricks diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml index 78262444bd815..4e671bcb53264 100644 --- a/airflow/providers/datadog/provider.yaml +++ b/airflow/providers/datadog/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.datadog + integrations: - integration-name: Datadog external-doc-url: https://www.datadoghq.com/ @@ -31,9 +33,9 @@ integrations: sensors: - integration-name: Datadog python-modules: - - airflow.providers.datadog.sensors.datadog + - .sensors.datadog hooks: - integration-name: Datadog python-modules: - - airflow.providers.datadog.hooks.datadog + - .hooks.datadog diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml index 8fc729c0ede2e..81cc3646bd85e 100644 --- a/airflow/providers/dingding/provider.yaml +++ b/airflow/providers/dingding/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.dingding + integrations: - integration-name: Dingding external-doc-url: https://oapi.dingtalk.com @@ -32,9 +35,9 @@ integrations: operators: - integration-name: Dingding python-modules: - - airflow.providers.dingding.operators.dingding + - .operators.dingding hooks: - integration-name: IBM Cloudant python-modules: - - airflow.providers.dingding.hooks.dingding + - .hooks.dingding diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml index 4e72a9c2aaa10..4edde760a4e20 100644 --- a/airflow/providers/discord/provider.yaml +++ b/airflow/providers/discord/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.discord + integrations: - integration-name: Discord external-doc-url: https://discordapp.com @@ -30,9 +33,9 @@ integrations: operators: - integration-name: Discord python-modules: - - airflow.providers.discord.operators.discord_webhook + - .operators.discord_webhook hooks: - integration-name: Discord python-modules: - - airflow.providers.discord.hooks.discord_webhook + - .hooks.discord_webhook diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml index 7549bf068786d..7ba6b440a78e3 100644 --- a/airflow/providers/docker/provider.yaml +++ b/airflow/providers/docker/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.docker + integrations: - integration-name: Docker external-doc-url: https://docs.docker.com/ @@ -33,12 +36,17 @@ integrations: operators: - integration-name: Docker python-modules: - - airflow.providers.docker.operators.docker + - .operators.docker - integration-name: Docker Swarm python-modules: - - airflow.providers.docker.operators.docker_swarm + - .operators.docker_swarm hooks: - integration-name: Docker python-modules: - - airflow.providers.docker.hooks.docker + - .hooks.docker + +connections: + - connection-type: docker + hook-class: .hooks.docker.DockerHook + connection-id-parameter-name: docker_conn_id diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index a3b49944293aa..6d454ad537a41 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.elasticsearch + integrations: - integration-name: Elasticsearch external-doc-url: https://www.elastic.co/elasticsearch @@ -30,4 +33,9 @@ integrations: hooks: - integration-name: Elasticsearch python-modules: - - airflow.providers.elasticsearch.hooks.elasticsearch + - .hooks.elasticsearch + +connections: + - connection-type: elasticsearch + hook-class: .hooks.elasticsearch.ElasticsearchHook + connection-id-parameter-name: elasticsearch_conn_id diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index fbec0f3715a03..58fc3f6166f7e 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.exasol + integrations: - integration-name: Exasol external-doc-url: https://docs.exasol.com/home.htm @@ -31,9 +33,14 @@ integrations: operators: - integration-name: Exasol python-modules: - - airflow.providers.exasol.operators.exasol + - .operators.exasol hooks: - integration-name: Exasol python-modules: - - airflow.providers.exasol.hooks.exasol + - .hooks.exasol + +connections: + - connection-type: exasol + hook-class: .hooks.exasol.ExasolHook + connection-id-parameter-name: exasol_conn_id diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml index 067e35fdc2ad5..56b3409bb6345 100644 --- a/airflow/providers/facebook/provider.yaml +++ b/airflow/providers/facebook/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.facebook + integrations: - integration-name: Facebook Ads external-doc-url: http://business.facebook.com @@ -30,4 +33,4 @@ integrations: hooks: - integration-name: Facebook Ads python-modules: - - airflow.providers.facebook.ads.hooks.ads + - .ads.hooks.ads diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index 2dad415a286bd..fe7ae344b1db8 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.ftp + integrations: - integration-name: File Transfer Protocol (FTP) external-doc-url: https://tools.ietf.org/html/rfc114 @@ -31,9 +33,9 @@ integrations: sensors: - integration-name: File Transfer Protocol (FTP) python-modules: - - airflow.providers.ftp.sensors.ftp + - .sensors.ftp hooks: - integration-name: File Transfer Protocol (FTP) python-modules: - - airflow.providers.ftp.hooks.ftp + - .hooks.ftp diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 51d2abdff3273..99b37d134ab51 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -29,6 +29,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.google + integrations: - integration-name: Google Analytics360 external-doc-url: https://analytics.google.com/ @@ -254,381 +256,392 @@ integrations: operators: - integration-name: Google Ads python-modules: - - airflow.providers.google.ads.operators.ads + - .ads.operators.ads - integration-name: Google AutoML python-modules: - - airflow.providers.google.cloud.operators.automl + - .cloud.operators.automl - integration-name: Google BigQuery python-modules: - - airflow.providers.google.cloud.operators.bigquery + - .cloud.operators.bigquery - integration-name: Google BigQuery Data Transfer Service python-modules: - - airflow.providers.google.cloud.operators.bigquery_dts + - .cloud.operators.bigquery_dts - integration-name: Google Bigtable python-modules: - - airflow.providers.google.cloud.operators.bigtable + - .cloud.operators.bigtable - integration-name: Google Cloud Build python-modules: - - airflow.providers.google.cloud.operators.cloud_build + - .cloud.operators.cloud_build - integration-name: Google Cloud Memorystore python-modules: - - airflow.providers.google.cloud.operators.cloud_memorystore + - .cloud.operators.cloud_memorystore - integration-name: Google Cloud SQL python-modules: - - airflow.providers.google.cloud.operators.cloud_sql + - .cloud.operators.cloud_sql - integration-name: Google Cloud Storage Transfer Service python-modules: - - airflow.providers.google.cloud.operators.cloud_storage_transfer_service + - .cloud.operators.cloud_storage_transfer_service - integration-name: Google Compute Engine python-modules: - - airflow.providers.google.cloud.operators.compute + - .cloud.operators.compute - integration-name: Google Data Catalog python-modules: - - airflow.providers.google.cloud.operators.datacatalog + - .cloud.operators.datacatalog - integration-name: Google Dataflow python-modules: - - airflow.providers.google.cloud.operators.dataflow + - .cloud.operators.dataflow - integration-name: Google Data Fusion python-modules: - - airflow.providers.google.cloud.operators.datafusion + - .cloud.operators.datafusion - integration-name: Google Dataprep python-modules: - - airflow.providers.google.cloud.operators.dataprep + - .cloud.operators.dataprep - integration-name: Google Dataproc python-modules: - - airflow.providers.google.cloud.operators.dataproc + - .cloud.operators.dataproc - integration-name: Google Datastore python-modules: - - airflow.providers.google.cloud.operators.datastore + - .cloud.operators.datastore - integration-name: Google Cloud Data Loss Prevention (DLP) python-modules: - - airflow.providers.google.cloud.operators.dlp + - .cloud.operators.dlp - integration-name: Google Cloud Functions python-modules: - - airflow.providers.google.cloud.operators.functions + - .cloud.operators.functions - integration-name: Google Cloud Storage (GCS) python-modules: - - airflow.providers.google.cloud.operators.gcs + - .cloud.operators.gcs - integration-name: Google Kubernetes Engine python-modules: - - airflow.providers.google.cloud.operators.kubernetes_engine + - .cloud.operators.kubernetes_engine - integration-name: Google Cloud Life Sciences python-modules: - - airflow.providers.google.cloud.operators.life_sciences + - .cloud.operators.life_sciences - integration-name: Google Machine Learning Engine python-modules: - - airflow.providers.google.cloud.operators.mlengine + - .cloud.operators.mlengine - integration-name: Google Cloud Natural Language python-modules: - - airflow.providers.google.cloud.operators.natural_language + - .cloud.operators.natural_language - integration-name: Google Cloud Pub/Sub python-modules: - - airflow.providers.google.cloud.operators.pubsub + - .cloud.operators.pubsub - integration-name: Google Cloud Spanner python-modules: - - airflow.providers.google.cloud.operators.spanner + - .cloud.operators.spanner - integration-name: Google Cloud Speech-to-Text python-modules: - - airflow.providers.google.cloud.operators.speech_to_text + - .cloud.operators.speech_to_text - integration-name: Google Cloud Stackdriver python-modules: - - airflow.providers.google.cloud.operators.stackdriver + - .cloud.operators.stackdriver - integration-name: Google Cloud Tasks python-modules: - - airflow.providers.google.cloud.operators.tasks + - .cloud.operators.tasks - integration-name: Google Cloud Text-to-Speech python-modules: - - airflow.providers.google.cloud.operators.text_to_speech - - airflow.providers.google.cloud.operators.translate_speech + - .cloud.operators.text_to_speech + - .cloud.operators.translate_speech - integration-name: Google Cloud Translation python-modules: - - airflow.providers.google.cloud.operators.translate - - airflow.providers.google.cloud.operators.translate_speech + - .cloud.operators.translate + - .cloud.operators.translate_speech - integration-name: Google Cloud Video Intelligence python-modules: - - airflow.providers.google.cloud.operators.video_intelligence + - .cloud.operators.video_intelligence - integration-name: Google Cloud Vision python-modules: - - airflow.providers.google.cloud.operators.vision + - .cloud.operators.vision - integration-name: Google Cloud Firestore python-modules: - - airflow.providers.google.firebase.operators.firestore + - .firebase.operators.firestore - integration-name: Google Analytics360 python-modules: - - airflow.providers.google.marketing_platform.operators.analytics + - .marketing_platform.operators.analytics - integration-name: Google Campaign Manager python-modules: - - airflow.providers.google.marketing_platform.operators.campaign_manager + - .marketing_platform.operators.campaign_manager - integration-name: Google Display&Video 360 python-modules: - - airflow.providers.google.marketing_platform.operators.display_video + - .marketing_platform.operators.display_video - integration-name: Google Search Ads 360 python-modules: - - airflow.providers.google.marketing_platform.operators.search_ads + - .marketing_platform.operators.search_ads - integration-name: Google Spreadsheet python-modules: - - airflow.providers.google.suite.operators.sheets + - .suite.operators.sheets sensors: - integration-name: Google BigQuery python-modules: - - airflow.providers.google.cloud.sensors.bigquery + - .cloud.sensors.bigquery - integration-name: Google BigQuery Data Transfer Service python-modules: - - airflow.providers.google.cloud.sensors.bigquery_dts + - .cloud.sensors.bigquery_dts - integration-name: Google Bigtable python-modules: - - airflow.providers.google.cloud.sensors.bigtable + - .cloud.sensors.bigtable - integration-name: Google Cloud Storage Transfer Service python-modules: - - airflow.providers.google.cloud.sensors.cloud_storage_transfer_service + - .cloud.sensors.cloud_storage_transfer_service - integration-name: Google Dataflow python-modules: - - airflow.providers.google.cloud.sensors.dataflow + - .cloud.sensors.dataflow - integration-name: Google Dataproc python-modules: - - airflow.providers.google.cloud.sensors.dataproc + - .cloud.sensors.dataproc - integration-name: Google Cloud Storage (GCS) python-modules: - - airflow.providers.google.cloud.sensors.gcs + - .cloud.sensors.gcs - integration-name: Google Cloud Pub/Sub python-modules: - - airflow.providers.google.cloud.sensors.pubsub + - .cloud.sensors.pubsub - integration-name: Google Campaign Manager python-modules: - - airflow.providers.google.marketing_platform.sensors.campaign_manager - - integration-name: Google Dataflow - python-modules: - - airflow.providers.google.cloud.sensors.dataflow + - .marketing_platform.sensors.campaign_manager - integration-name: Google Display&Video 360 python-modules: - - airflow.providers.google.marketing_platform.sensors.display_video + - .marketing_platform.sensors.display_video - integration-name: Google Search Ads 360 python-modules: - - airflow.providers.google.marketing_platform.sensors.search_ads + - .marketing_platform.sensors.search_ads hooks: - integration-name: Google Ads python-modules: - - airflow.providers.google.ads.hooks.ads + - .ads.hooks.ads - integration-name: Google AutoML python-modules: - - airflow.providers.google.cloud.hooks.automl + - .cloud.hooks.automl - integration-name: Google BigQuery python-modules: - - airflow.providers.google.cloud.hooks.bigquery + - .cloud.hooks.bigquery - integration-name: Google BigQuery Data Transfer Service python-modules: - - airflow.providers.google.cloud.hooks.bigquery_dts + - .cloud.hooks.bigquery_dts - integration-name: Google Bigtable python-modules: - - airflow.providers.google.cloud.hooks.bigtable + - .cloud.hooks.bigtable - integration-name: Google Cloud Build python-modules: - - airflow.providers.google.cloud.hooks.cloud_build + - .cloud.hooks.cloud_build - integration-name: Google Cloud Memorystore python-modules: - - airflow.providers.google.cloud.hooks.cloud_memorystore + - .cloud.hooks.cloud_memorystore - integration-name: Google Cloud SQL python-modules: - - airflow.providers.google.cloud.hooks.cloud_sql + - .cloud.hooks.cloud_sql - integration-name: Google Cloud Storage Transfer Service python-modules: - - airflow.providers.google.cloud.hooks.cloud_storage_transfer_service + - .cloud.hooks.cloud_storage_transfer_service - integration-name: Google Compute Engine python-modules: - - airflow.providers.google.cloud.hooks.compute - - airflow.providers.google.cloud.hooks.compute_ssh + - .cloud.hooks.compute + - .cloud.hooks.compute_ssh - integration-name: Google Data Catalog python-modules: - - airflow.providers.google.cloud.hooks.datacatalog + - .cloud.hooks.datacatalog - integration-name: Google Dataflow python-modules: - - airflow.providers.google.cloud.hooks.dataflow + - .cloud.hooks.dataflow - integration-name: Google Data Fusion python-modules: - - airflow.providers.google.cloud.hooks.datafusion + - .cloud.hooks.datafusion - integration-name: Google Dataprep python-modules: - - airflow.providers.google.cloud.hooks.dataprep + - .cloud.hooks.dataprep - integration-name: Google Dataproc python-modules: - - airflow.providers.google.cloud.hooks.dataproc + - .cloud.hooks.dataproc - integration-name: Google Datastore python-modules: - - airflow.providers.google.cloud.hooks.datastore + - .cloud.hooks.datastore - integration-name: Google Cloud Data Loss Prevention (DLP) python-modules: - - airflow.providers.google.cloud.hooks.dlp + - .cloud.hooks.dlp - integration-name: Google Cloud Functions python-modules: - - airflow.providers.google.cloud.hooks.functions + - .cloud.hooks.functions - integration-name: Google Cloud Storage (GCS) python-modules: - - airflow.providers.google.cloud.hooks.gcs + - .cloud.hooks.gcs - integration-name: Google Deployment Manager python-modules: - - airflow.providers.google.cloud.hooks.gdm + - .cloud.hooks.gdm - integration-name: Google Cloud Key Management Service (KMS) python-modules: - - airflow.providers.google.cloud.hooks.kms + - .cloud.hooks.kms - integration-name: Google Kubernetes Engine python-modules: - - airflow.providers.google.cloud.hooks.kubernetes_engine + - .cloud.hooks.kubernetes_engine - integration-name: Google Cloud Life Sciences python-modules: - - airflow.providers.google.cloud.hooks.life_sciences + - .cloud.hooks.life_sciences - integration-name: Google Machine Learning Engine python-modules: - - airflow.providers.google.cloud.hooks.mlengine + - .cloud.hooks.mlengine - integration-name: Google Cloud Natural Language python-modules: - - airflow.providers.google.cloud.hooks.natural_language + - .cloud.hooks.natural_language - integration-name: Google Cloud OS Login python-modules: - - airflow.providers.google.cloud.hooks.os_login + - .cloud.hooks.os_login - integration-name: Google Cloud Pub/Sub python-modules: - - airflow.providers.google.cloud.hooks.pubsub + - .cloud.hooks.pubsub - integration-name: Google Cloud Secret Manager python-modules: - - airflow.providers.google.cloud.hooks.secret_manager + - .cloud.hooks.secret_manager - integration-name: Google Cloud Spanner python-modules: - - airflow.providers.google.cloud.hooks.spanner + - .cloud.hooks.spanner - integration-name: Google Cloud Speech-to-Text python-modules: - - airflow.providers.google.cloud.hooks.speech_to_text + - .cloud.hooks.speech_to_text - integration-name: Google Cloud Stackdriver python-modules: - - airflow.providers.google.cloud.hooks.stackdriver + - .cloud.hooks.stackdriver - integration-name: Google Cloud Tasks python-modules: - - airflow.providers.google.cloud.hooks.tasks + - .cloud.hooks.tasks - integration-name: Google Cloud Text-to-Speech python-modules: - - airflow.providers.google.cloud.hooks.text_to_speech + - .cloud.hooks.text_to_speech - integration-name: Google Cloud Translation python-modules: - - airflow.providers.google.cloud.hooks.translate + - .cloud.hooks.translate - integration-name: Google Cloud Video Intelligence python-modules: - - airflow.providers.google.cloud.hooks.video_intelligence + - .cloud.hooks.video_intelligence - integration-name: Google Cloud Vision python-modules: - - airflow.providers.google.cloud.hooks.vision + - .cloud.hooks.vision - integration-name: Google python-modules: - - airflow.providers.google.common.hooks.base_google + - .common.hooks.base_google - integration-name: Google Discovery API python-modules: - - airflow.providers.google.common.hooks.discovery_api + - .common.hooks.discovery_api - integration-name: Google Cloud Firestore python-modules: - - airflow.providers.google.firebase.hooks.firestore + - .firebase.hooks.firestore - integration-name: Google Analytics360 python-modules: - - airflow.providers.google.marketing_platform.hooks.analytics + - .marketing_platform.hooks.analytics - integration-name: Google Campaign Manager python-modules: - - airflow.providers.google.marketing_platform.hooks.campaign_manager + - .marketing_platform.hooks.campaign_manager - integration-name: Google Display&Video 360 python-modules: - - airflow.providers.google.marketing_platform.hooks.display_video + - .marketing_platform.hooks.display_video - integration-name: Google Search Ads 360 python-modules: - - airflow.providers.google.marketing_platform.hooks.search_ads + - .marketing_platform.hooks.search_ads - integration-name: Google Drive python-modules: - - airflow.providers.google.suite.hooks.drive + - .suite.hooks.drive - integration-name: Google Spreadsheet python-modules: - - airflow.providers.google.suite.hooks.sheets + - .suite.hooks.sheets transfers: - source-integration-name: Presto target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/presto_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.presto_to_gcs + python-module: .cloud.transfers.presto_to_gcs - source-integration-name: SQL target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.sql_to_gcs + python-module: .cloud.transfers.sql_to_gcs - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Google Drive how-to-guide: /docs/howto/operator/google/transfer/gcs_to_gdrive.rst - python-module: airflow.providers.google.suite.transfers.gcs_to_gdrive + python-module: .suite.transfers.gcs_to_gdrive - source-integration-name: Microsoft SQL Server (MSSQL) target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.mssql_to_gcs + python-module: .cloud.transfers.mssql_to_gcs - source-integration-name: Microsoft Azure FileShare target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/azure_fileshare_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs + python-module: .cloud.transfers.azure_fileshare_to_gcs - source-integration-name: Apache Cassandra target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.cassandra_to_gcs + python-module: .cloud.transfers.cassandra_to_gcs - source-integration-name: Google Spreadsheet target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/sheets_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.sheets_to_gcs + python-module: .cloud.transfers.sheets_to_gcs - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/s3_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.s3_to_gcs + python-module: .cloud.transfers.s3_to_gcs - source-integration-name: Google Cloud Storage (GCS) target-integration-name: SSH File Transfer Protocol (SFTP) how-to-guide: /docs/howto/operator/google/transfer/gcs_to_sftp.rst - python-module: airflow.providers.google.cloud.transfers.gcs_to_sftp + python-module: .cloud.transfers.gcs_to_sftp - source-integration-name: PostgreSQL target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.postgres_to_gcs + python-module: .cloud.transfers.postgres_to_gcs - source-integration-name: Google BigQuery target-integration-name: MySQL - python-module: airflow.providers.google.cloud.transfers.bigquery_to_mysql + python-module: .cloud.transfers.bigquery_to_mysql - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Google BigQuery - python-module: airflow.providers.google.cloud.transfers.gcs_to_bigquery + python-module: .cloud.transfers.gcs_to_bigquery - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/gcs_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.gcs_to_gcs + python-module: .cloud.transfers.gcs_to_gcs - source-integration-name: Facebook Ads target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/facebook_ads_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.facebook_ads_to_gcs + python-module: .cloud.transfers.facebook_ads_to_gcs - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/sftp_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.sftp_to_gcs + python-module: .cloud.transfers.sftp_to_gcs - source-integration-name: Microsoft Azure Data Lake Storage target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.adls_to_gcs + python-module: .cloud.transfers.adls_to_gcs - source-integration-name: Google BigQuery target-integration-name: Google BigQuery - python-module: airflow.providers.google.cloud.transfers.bigquery_to_bigquery + python-module: .cloud.transfers.bigquery_to_bigquery - source-integration-name: MySQL target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.mysql_to_gcs + python-module: .cloud.transfers.mysql_to_gcs how-to-guide: /docs/howto/operator/google/transfer/mysql_to_gcs.rst - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Google Spreadsheet how-to-guide: /docs/howto/operator/google/transfer/gcs_to_sheets.rst - python-module: airflow.providers.google.suite.transfers.gcs_to_sheets + python-module: .suite.transfers.gcs_to_sheets - source-integration-name: Local target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/local_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.local_to_gcs + python-module: .cloud.transfers.local_to_gcs - source-integration-name: Google BigQuery target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.cloud.transfers.bigquery_to_gcs + python-module: .cloud.transfers.bigquery_to_gcs - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Local how-to-guide: /docs/howto/operator/google/transfer/gcs_to_local.rst - python-module: airflow.providers.google.cloud.transfers.gcs_to_local + python-module: .cloud.transfers.gcs_to_local - source-integration-name: Salesforce target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/google/transfer/salesforce_to_gcs.rst - python-module: airflow.providers.google.cloud.transfers.salesforce_to_gcs + python-module: .cloud.transfers.salesforce_to_gcs - source-integration-name: Google Ads target-integration-name: Google Cloud Storage (GCS) - python-module: airflow.providers.google.ads.transfers.ads_to_gcs + python-module: .ads.transfers.ads_to_gcs + +connections: + - connection-type: dataprep + hook-class: .cloud.hooks.dataprep.GoogleDataprepHook + connection-id-parameter-name: dataprep_default + - connection-type: gcpcloudsql + hook-class: .cloud.hooks.cloud_sql.CloudSQLDatabaseHook + connection-id-parameter-name: gcp_cloudsql_conn_id + - connection-type: gcpssh + hook-class: .cloud.hooks.compute_ssh.ComputeEngineSSHHook + connection-id-parameter-name: gcp_conn_id + - connection-type: google_cloud_platform + hook-class: .cloud.hooks.bigquery.BigQueryHook + connection-id-parameter-name: bigquery_conn_id diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml index 65465ee0e6752..45f5b5b8b7385 100644 --- a/airflow/providers/grpc/provider.yaml +++ b/airflow/providers/grpc/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.grpc + integrations: - integration-name: gRPC external-doc-url: https://grpc.io/ @@ -30,9 +33,14 @@ integrations: operators: - integration-name: gRPC python-modules: - - airflow.providers.grpc.operators.grpc + - .operators.grpc hooks: - integration-name: gRPC python-modules: - - airflow.providers.grpc.hooks.grpc + - .hooks.grpc + +connections: + - connection-type: grpc + hook-class: .hooks.grpc.GrpcHook + connection-id-parameter-name: grpc_conn_id diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index 959df236abc82..700127e7bfe93 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.hashicorp + integrations: - integration-name: Hashicorp Vault external-doc-url: https://www.vaultproject.io/ @@ -30,4 +33,4 @@ integrations: hooks: - integration-name: Hashicorp Vault python-modules: - - airflow.providers.hashicorp.hooks.vault + - .hooks.vault diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 444984958a716..96a69c325107c 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.http + integrations: - integration-name: Hypertext Transfer Protocol (HTTP) external-doc-url: https://www.w3.org/Protocols/ @@ -33,13 +35,13 @@ integrations: operators: - integration-name: Hypertext Transfer Protocol (HTTP) python-modules: - - airflow.providers.http.operators.http + - .operators.http sensors: - integration-name: Hypertext Transfer Protocol (HTTP) python-modules: - - airflow.providers.http.sensors.http + - .sensors.http hooks: - integration-name: Hypertext Transfer Protocol (HTTP) python-modules: - - airflow.providers.http.hooks.http + - .hooks.http diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index 6f6290326a249..b65399ddd3a01 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.imap + integrations: - integration-name: Internet Message Access Protocol (IMAP) external-doc-url: https://tools.ietf.org/html/rfc3501 @@ -31,9 +33,14 @@ integrations: sensors: - integration-name: Internet Message Access Protocol (IMAP) python-modules: - - airflow.providers.imap.sensors.imap_attachment + - .sensors.imap_attachment hooks: - integration-name: Internet Message Access Protocol (IMAP) python-modules: - - airflow.providers.imap.hooks.imap + - .hooks.imap + +connections: + - connection-type: imap + hook-class: .hooks.imap.ImapHook + connection-id-parameter-name: imap_conn_id diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index 4cb9b1cb98654..1c3e04bc29cc5 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.jdbc + integrations: - integration-name: Java Database Connectivity (JDBC) external-doc-url: https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/ @@ -32,8 +35,13 @@ integrations: operators: - integration-name: Java Database Connectivity (JDBC) python-modules: - - airflow.providers.jdbc.operators.jdbc + - .operators.jdbc hooks: - integration-name: Java Database Connectivity (JDBC) python-modules: - - airflow.providers.jdbc.hooks.jdbc + - .hooks.jdbc + +connections: + - connection-type: jdbc + hook-class: .hooks.jdbc.JdbcHook + connection-id-parameter-name: jdbc_conn_id diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml index da1c1d125c5ab..1bae327ce6316 100644 --- a/airflow/providers/jenkins/provider.yaml +++ b/airflow/providers/jenkins/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.jenkins + integrations: - integration-name: Jenkins external-doc-url: https://jenkins.io/ @@ -30,8 +33,8 @@ integrations: operators: - integration-name: Jenkins python-modules: - - airflow.providers.jenkins.operators.jenkins_job_trigger + - .operators.jenkins_job_trigger hooks: - integration-name: Jenkins python-modules: - - airflow.providers.jenkins.hooks.jenkins + - .hooks.jenkins diff --git a/airflow/providers/jira/provider.yaml b/airflow/providers/jira/provider.yaml index 6e040cb6197a4..314b974674ccf 100644 --- a/airflow/providers/jira/provider.yaml +++ b/airflow/providers/jira/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.jira + integrations: - integration-name: Atlassian Jira external-doc-url: https://www.atlassian.com/pl/software/jira @@ -31,14 +33,19 @@ integrations: operators: - integration-name: Atlassian Jira python-modules: - - airflow.providers.jira.operators.jira + - .operators.jira sensors: - integration-name: Atlassian Jira python-modules: - - airflow.providers.jira.sensors.jira + - .sensors.jira hooks: - integration-name: Atlassian Jira python-modules: - - airflow.providers.jira.hooks.jira + - .hooks.jira + +connections: + - connection-type: jira + hook-class: .hooks.jira.JiraHook + connection-id-parameter-name: jira_conn_id diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index e937f0d9dd6e0..f5def4a51395a 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.microsoft.azure + integrations: - integration-name: Microsoft Azure Batch external-doc-url: https://azure.microsoft.com/en-us/services/batch/ @@ -55,71 +57,85 @@ integrations: operators: - integration-name: Microsoft Azure Data Lake Storage python-modules: - - airflow.providers.microsoft.azure.operators.adls_list + - .operators.adls_list - integration-name: Microsoft Azure Data Explorer python-modules: - - airflow.providers.microsoft.azure.operators.adx + - .operators.adx - integration-name: Microsoft Azure Batch python-modules: - - airflow.providers.microsoft.azure.operators.azure_batch + - .operators.azure_batch - integration-name: Microsoft Azure Container Instances python-modules: - - airflow.providers.microsoft.azure.operators.azure_container_instances + - .operators.azure_container_instances - integration-name: Microsoft Azure Cosmos DB python-modules: - - airflow.providers.microsoft.azure.operators.azure_cosmos + - .operators.azure_cosmos - integration-name: Microsoft Azure Blob Storage python-modules: - - airflow.providers.microsoft.azure.operators.wasb_delete_blob + - .operators.wasb_delete_blob sensors: - integration-name: Microsoft Azure Cosmos DB python-modules: - - airflow.providers.microsoft.azure.sensors.azure_cosmos + - .sensors.azure_cosmos - integration-name: Microsoft Azure Blob Storage python-modules: - - airflow.providers.microsoft.azure.sensors.wasb + - .sensors.wasb hooks: - integration-name: Microsoft Azure Container Instances python-modules: - - airflow.providers.microsoft.azure.hooks.azure_container_volume - - airflow.providers.microsoft.azure.hooks.azure_container_registry - - airflow.providers.microsoft.azure.hooks.azure_container_instance + - .hooks.azure_container_volume + - .hooks.azure_container_registry + - .hooks.azure_container_instance - integration-name: Microsoft Azure Data Explorer python-modules: - - airflow.providers.microsoft.azure.hooks.adx + - .hooks.adx - integration-name: Microsoft Azure FileShare python-modules: - - airflow.providers.microsoft.azure.hooks.azure_fileshare + - .hooks.azure_fileshare - integration-name: Microsoft Azure python-modules: - - airflow.providers.microsoft.azure.hooks.base_azure + - .hooks.base_azure - integration-name: Microsoft Azure Batch python-modules: - - airflow.providers.microsoft.azure.hooks.azure_batch + - .hooks.azure_batch - integration-name: Microsoft Azure Data Lake Storage python-modules: - - airflow.providers.microsoft.azure.hooks.azure_data_lake + - .hooks.azure_data_lake - integration-name: Microsoft Azure Cosmos DB python-modules: - - airflow.providers.microsoft.azure.hooks.azure_cosmos + - .hooks.azure_cosmos - integration-name: Microsoft Azure Blob Storage python-modules: - - airflow.providers.microsoft.azure.hooks.wasb + - .hooks.wasb transfers: - source-integration-name: Local target-integration-name: Microsoft Azure Data Lake Storage how-to-guide: /docs/howto/operator/microsoft/transfer/local_to_adls.rst - python-module: airflow.providers.microsoft.azure.transfers.local_to_adls + python-module: .transfers.local_to_adls - source-integration-name: Oracle target-integration-name: Microsoft Azure Data Lake Storage - python-module: airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake + python-module: .transfers.oracle_to_azure_data_lake - source-integration-name: Local target-integration-name: Microsoft Azure Blob Storage - python-module: airflow.providers.microsoft.azure.transfers.file_to_wasb + python-module: .transfers.file_to_wasb - source-integration-name: Microsoft Azure Blob Storage target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/howto/operator/microsoft/transfer/blob_storage_to_gcs.rst - python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs + python-module: .transfers.azure_blob_to_gcs + +connections: + - connection-type: azure_batch + hook-class: .hooks.azure_batch.AzureBatchHook + connection-id-parameter-name: azure_batch_conn_id + - connection-type: azure_cosmos + hook-class: .hooks.azure_cosmos.AzureCosmosDBHook + connection-id-parameter-name: azure_cosmos_conn_id + - connection-type: azure_data_lake + hook-class: .hooks.azure_data_lake.AzureDataLakeHook + connection-id-parameter-name: azure_data_lake_conn_id + - connection-type: wasb + hook-class: .hooks.wasb.WasbHook + connection-id-parameter-name: wasb_conn_id diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 79719cc97c05d..cf7e200bbed39 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.microsoft.mssql + integrations: - integration-name: Microsoft SQL Server (MSSQL) external-doc-url: https://www.microsoft.com/en-us/sql-server/sql-server-downloads @@ -30,9 +33,9 @@ integrations: operators: - integration-name: Microsoft SQL Server (MSSQL) python-modules: - - airflow.providers.microsoft.mssql.operators.mssql + - .operators.mssql hooks: - integration-name: Microsoft SQL Server (MSSQL) python-modules: - - airflow.providers.microsoft.mssql.hooks.mssql + - .hooks.mssql diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml index 7f84bb42daa49..24fd4ed878d23 100644 --- a/airflow/providers/microsoft/winrm/provider.yaml +++ b/airflow/providers/microsoft/winrm/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.microsoft.winrm + integrations: - integration-name: Windows Remote Management (WinRM) external-doc-url: https://docs.microsoft.com/en-us/windows/win32/winrm/portal @@ -31,9 +33,9 @@ integrations: operators: - integration-name: Windows Remote Management (WinRM) python-modules: - - airflow.providers.microsoft.winrm.operators.winrm + - .operators.winrm hooks: - integration-name: Windows Remote Management (WinRM) python-modules: - - airflow.providers.microsoft.winrm.hooks.winrm + - .hooks.winrm diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml index 7bdad89d6233e..9b951d17bdb50 100644 --- a/airflow/providers/mongo/provider.yaml +++ b/airflow/providers/mongo/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.mongo + integrations: - integration-name: MongoDB external-doc-url: https://www.mongodb.com/what-is-mongodb @@ -31,8 +33,13 @@ integrations: sensors: - integration-name: MongoDB python-modules: - - airflow.providers.mongo.sensors.mongo + - .sensors.mongo hooks: - integration-name: MongoDB python-modules: - - airflow.providers.mongo.hooks.mongo + - .hooks.mongo + +connections: + - connection-type: mongo + hook-class: .hooks.mongo.MongoHook + connection-id-parameter-name: conn_id diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index a7390f7ada9f7..a51637d941935 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.mysql + integrations: - integration-name: MySQL external-doc-url: https://www.mysql.com/ @@ -33,20 +36,25 @@ operators: - integration-name: MySQL python-modules: - - airflow.providers.mysql.operators.mysql + - .operators.mysql hooks: - integration-name: MySQL python-modules: - - airflow.providers.mysql.hooks.mysql + - .hooks.mysql transfers: - source-integration-name: Vertica target-integration-name: MySQL - python-module: airflow.providers.mysql.transfers.vertica_to_mysql + python-module: .transfers.vertica_to_mysql - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: MySQL - python-module: airflow.providers.mysql.transfers.s3_to_mysql + python-module: .transfers.s3_to_mysql - source-integration-name: Snowflake target-integration-name: MySQL - python-module: airflow.providers.mysql.transfers.presto_to_mysql + python-module: .transfers.presto_to_mysql + +connections: + - connection-type: mysql + hook-class: .hooks.mysql.MySqlHook + connection-id-parameter-name: mysql_conn_id diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 26c275f9e9567..531bd97875b71 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.odbc + integrations: - integration-name: ODBC external-doc-url: https://github.com/mkleehammer/pyodbc/wiki @@ -30,4 +33,13 @@ integrations: hooks: - integration-name: ODBC python-modules: - - airflow.providers.odbc.hooks.odbc + - .hooks.odbc + +connections: + - connection-type: mssql + hook-class: .hooks.odbc.OdbcHook + connection-id-parameter-name: odbc_conn_id + + - connection-type: odbc + hook-class: .hooks.odbc.OdbcHook + connection-id-parameter-name: odbc_conn_id diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml index 0731477f1220f..a5f1e66e18376 100644 --- a/airflow/providers/openfaas/provider.yaml +++ b/airflow/providers/openfaas/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.openfaas + integrations: - integration-name: OpenFaaS external-doc-url: https://www.openfaas.com/ @@ -30,4 +33,4 @@ integrations: hooks: - integration-name: OpenFaaS python-modules: - - airflow.providers.openfaas.hooks.openfaas + - .hooks.openfaas diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml index f2e0e2ae412c0..a94c4da004ed8 100644 --- a/airflow/providers/opsgenie/provider.yaml +++ b/airflow/providers/opsgenie/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.opsgenie + integrations: - integration-name: Opsgenie external-doc-url: https://www.opsgenie.com/ @@ -30,9 +33,9 @@ integrations: operators: - integration-name: Opsgenie python-modules: - - airflow.providers.opsgenie.operators.opsgenie_alert + - .operators.opsgenie_alert hooks: - integration-name: Opsgenie python-modules: - - airflow.providers.opsgenie.hooks.opsgenie_alert + - .hooks.opsgenie_alert diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 6afd04d8cf1f3..302d77a7798df 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.oracle + integrations: - integration-name: Oracle external-doc-url: https://www.oracle.com/en/database/ @@ -30,12 +33,19 @@ integrations: operators: - integration-name: Oracle python-modules: - - airflow.providers.oracle.operators.oracle + - .operators.oracle + hooks: - integration-name: Oracle python-modules: - - airflow.providers.oracle.hooks.oracle + - .hooks.oracle + transfers: - source-integration-name: Oracle target-integration-name: Oracle - python-module: airflow.providers.oracle.transfers.oracle_to_oracle + python-module: .transfers.oracle_to_oracle + +connections: + - connection-type: oracle + hook-class: .hooks.oracle.OracleHook + connection-id-parameter-name: oracle_conn_id diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml index 49cc86e83c9fa..2b56aa821a62d 100644 --- a/airflow/providers/pagerduty/provider.yaml +++ b/airflow/providers/pagerduty/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.pagerduty + integrations: - integration-name: Pagerduty external-doc-url: https://www.pagerduty.com/ @@ -31,4 +34,4 @@ integrations: hooks: - integration-name: Pagerduty python-modules: - - airflow.providers.pagerduty.hooks.pagerduty + - .hooks.pagerduty diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index 81722e312b779..2e0b0b95f1b4c 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.papermill + integrations: - integration-name: Papermill external-doc-url: https://github.com/nteract/papermill @@ -32,4 +35,4 @@ integrations: operators: - integration-name: Papermill python-modules: - - airflow.providers.papermill.operators.papermill + - .operators.papermill diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index eff6c991462d7..84170e538b702 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.plexus + integrations: - integration-name: Plexus external-doc-url: https://plexus.corescientific.com/ @@ -30,8 +33,8 @@ integrations: operators: - integration-name: Plexus python-modules: - - airflow.providers.plexus.operators.job + - .operators.job hooks: - integration-name: Plexus python-modules: - - airflow.providers.plexus.hooks.plexus + - .hooks.plexus diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 48915ea88247c..bab65d5b0e1c9 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.postgres + integrations: - integration-name: PostgreSQL external-doc-url: https://www.postgresql.org/ @@ -31,9 +33,14 @@ integrations: operators: - integration-name: PostgreSQL python-modules: - - airflow.providers.postgres.operators.postgres + - .operators.postgres hooks: - integration-name: PostgreSQL python-modules: - - airflow.providers.postgres.hooks.postgres + - .hooks.postgres + +connections: + - connection-type: postgres + hook-class: .hooks.postgres.PostgresHook + connection-id-parameter-name: postgres_conn_id diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 882dc957e524f..ca12231b86732 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -22,6 +22,9 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.presto + integrations: - integration-name: Presto external-doc-url: http://prestodb.github.io/ @@ -30,4 +33,9 @@ integrations: hooks: - integration-name: Presto python-modules: - - airflow.providers.presto.hooks.presto + - .hooks.presto + +connections: + - connection-type: presto + hook-class: .hooks.presto.PrestoHook + connection-id-parameter-name: presto_conn_id diff --git a/airflow/providers/qubole/provider.yaml b/airflow/providers/qubole/provider.yaml index 837ac36f9361a..b3ef32dea26c8 100644 --- a/airflow/providers/qubole/provider.yaml +++ b/airflow/providers/qubole/provider.yaml @@ -23,24 +23,26 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.qubole + integrations: - integration-name: Qubole - external-doc-url: https://www.qubole.com/ + external-doc-url: https://www.qbole.com/ tags: [service] operators: - integration-name: Qubole python-modules: - - airflow.providers.qubole.operators.qubole - - airflow.providers.qubole.operators.qubole_check + - .operators.qubole + - .operators.qubole_check sensors: - integration-name: Qubole python-modules: - - airflow.providers.qubole.sensors.qubole + - .sensors.qubole hooks: - integration-name: Qubole python-modules: - - airflow.providers.qubole.hooks.qubole - - airflow.providers.qubole.hooks.qubole_check + - .hooks.qubole + - .hooks.qubole_check diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index b9ab7078b7bb4..9c517aeb86a9e 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.redis + integrations: - integration-name: Redis external-doc-url: https://redis.io/ @@ -31,15 +33,20 @@ integrations: operators: - integration-name: Redis python-modules: - - airflow.providers.redis.operators.redis_publish + - .operators.redis_publish sensors: - integration-name: Redis python-modules: - - airflow.providers.redis.sensors.redis_key - - airflow.providers.redis.sensors.redis_pub_sub + - .sensors.redis_key + - .sensors.redis_pub_sub hooks: - integration-name: Redis python-modules: - - airflow.providers.redis.hooks.redis + - .hooks.redis + +connections: + - connection-type: redis + hook-class: .hooks.redis.RedisHook + connection-id-parameter-name: redis_conn_id diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index ea78fa43c6177..a743ffcfd5ed0 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.salesforce + integrations: - integration-name: Salesforce external-doc-url: https://www.salesforce.com/ @@ -31,15 +33,20 @@ integrations: operators: - integration-name: Salesforce python-modules: - - airflow.providers.salesforce.operators.tableau_refresh_workbook + - .operators.tableau_refresh_workbook sensors: - integration-name: Salesforce python-modules: - - airflow.providers.salesforce.sensors.tableau_job_status + - .sensors.tableau_job_status hooks: - integration-name: Salesforce python-modules: - - airflow.providers.salesforce.hooks.salesforce - - airflow.providers.salesforce.hooks.tableau + - .hooks.salesforce + - .hooks.tableau + +connections: + - connection-type: tableau + hook-class: .hooks.tableau.TableauHook + connection-id-parameter-name: tableau_conn_id diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml index 8a3a0c1db9e9b..831fc5b33cc5d 100644 --- a/airflow/providers/samba/provider.yaml +++ b/airflow/providers/samba/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.samba + integrations: - integration-name: Samba external-doc-url: https://www.samba.org/ @@ -31,4 +33,4 @@ integrations: hooks: - integration-name: Samba python-modules: - - airflow.providers.samba.hooks.samba + - .hooks.samba diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml index aa0edf981685c..8d178a1825780 100644 --- a/airflow/providers/segment/provider.yaml +++ b/airflow/providers/segment/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.segment + integrations: - integration-name: Segment external-doc-url: https://oapi.dingtalk.com @@ -31,9 +33,9 @@ integrations: operators: - integration-name: Segment python-modules: - - airflow.providers.segment.operators.segment_track_event + - .operators.segment_track_event hooks: - integration-name: Segment python-modules: - - airflow.providers.segment.hooks.segment + - .hooks.segment diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml index 3f2f3bad768e0..f22e6a27f709e 100644 --- a/airflow/providers/sendgrid/provider.yaml +++ b/airflow/providers/sendgrid/provider.yaml @@ -22,3 +22,5 @@ description: | versions: - 1.0.0b2 + +provider-package: airflow.providers.sendgrid diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index 277845f7fef84..cd6f19e27633e 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.sftp + integrations: - integration-name: SSH File Transfer Protocol (SFTP) external-doc-url: https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/ @@ -31,14 +33,14 @@ integrations: operators: - integration-name: SSH File Transfer Protocol (SFTP) python-modules: - - airflow.providers.sftp.operators.sftp + - .operators.sftp sensors: - integration-name: SSH File Transfer Protocol (SFTP) python-modules: - - airflow.providers.sftp.sensors.sftp + - .sensors.sftp hooks: - integration-name: SSH File Transfer Protocol (SFTP) python-modules: - - airflow.providers.sftp.hooks.sftp + - .hooks.sftp diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml index 73b1c95bfdd41..0660c798dc064 100644 --- a/airflow/providers/singularity/provider.yaml +++ b/airflow/providers/singularity/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.singularity + integrations: - integration-name: Singularity external-doc-url: https://sylabs.io/guides/latest/user-guide/ @@ -31,4 +33,5 @@ integrations: operators: - integration-name: Singularity python-modules: - - airflow.providers.singularity.operators.singularity + - .operators.singularity + - .operators.singularity diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index 2742d7e6333fb..b14ab334fdeb3 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -23,6 +23,9 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.slack + + integrations: - integration-name: Slack external-doc-url: https://slack.com/ @@ -31,11 +34,11 @@ integrations: operators: - integration-name: Slack python-modules: - - airflow.providers.slack.operators.slack - - airflow.providers.slack.operators.slack_webhook + - .operators.slack + - .operators.slack_webhook hooks: - integration-name: Slack python-modules: - - airflow.providers.slack.hooks.slack - - airflow.providers.slack.hooks.slack_webhook + - .hooks.slack + - .hooks.slack_webhook diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 7a16a85f2dba8..062dbd4deabce 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.snowflake + integrations: - integration-name: Snowflake external-doc-url: https://snowflake.com/ @@ -33,17 +35,22 @@ integrations: operators: - integration-name: Snowflake python-modules: - - airflow.providers.snowflake.operators.snowflake + - .operators.snowflake hooks: - integration-name: Snowflake python-modules: - - airflow.providers.snowflake.hooks.snowflake + - .hooks.snowflake transfers: - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Snowflake - python-module: airflow.providers.snowflake.transfers.s3_to_snowflake + python-module: .transfers.s3_to_snowflake - source-integration-name: Snowflake target-integration-name: Slack - python-module: airflow.providers.snowflake.transfers.snowflake_to_slack + python-module: .transfers.snowflake_to_slack + +connections: + - connection-type: snowflake + hook-class: .hooks.snowflake.SnowflakeHook + connection-id-parameter-name: snowflake_conn_id diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index b1afc3c3ca27c..c334564e243aa 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.sqlite + integrations: - integration-name: SQLite external-doc-url: https://www.sqlite.org/index.html @@ -31,9 +33,14 @@ integrations: operators: - integration-name: SQLite python-modules: - - airflow.providers.sqlite.operators.sqlite + - .operators.sqlite hooks: - integration-name: SQLite python-modules: - - airflow.providers.sqlite.hooks.sqlite + - .hooks.sqlite + +connections: + - connection-type: sqlite + hook-class: .hooks.sqlite.SqliteHook + connection-id-parameter-name: sqlite_conn_id diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml index 7fc1804f0378c..86e8d0caa3fe5 100644 --- a/airflow/providers/ssh/provider.yaml +++ b/airflow/providers/ssh/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.ssh + integrations: - integration-name: Secure Shell (SSH) external-doc-url: https://tools.ietf.org/html/rfc4251 @@ -31,9 +33,9 @@ integrations: operators: - integration-name: Secure Shell (SSH) python-modules: - - airflow.providers.ssh.operators.ssh + - .operators.ssh hooks: - integration-name: Secure Shell (SSH) python-modules: - - airflow.providers.ssh.hooks.ssh + - .hooks.ssh diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 516c52089eb3a..63928b249648f 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.vertica + integrations: - integration-name: Vertica external-doc-url: https://www.vertica.com/ @@ -31,9 +33,14 @@ integrations: operators: - integration-name: Vertica python-modules: - - airflow.providers.vertica.operators.vertica + - .operators.vertica hooks: - integration-name: Vertica python-modules: - - airflow.providers.vertica.hooks.vertica + - .hooks.vertica + +connections: + - connection-type: vertica + hook-class: .hooks.vertica.VerticaHook + connection-id-parameter-name: vertica_conn_id diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index 51029c80bcabc..2d0437229d3a7 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.yandex + integrations: - integration-name: Yandex.Cloud external-doc-url: https://cloud.yandex.com/ @@ -37,12 +39,12 @@ integrations: operators: - integration-name: Yandex.Cloud Dataproc python-modules: - - airflow.providers.yandex.operators.yandexcloud_dataproc + - .operators.yandexcloud_dataproc hooks: - integration-name: Yandex.Cloud python-modules: - - airflow.providers.yandex.hooks.yandex + - .hooks.yandex - integration-name: Yandex.Cloud python-modules: - - airflow.providers.yandex.hooks.yandexcloud_dataproc + - .hooks.yandexcloud_dataproc diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml index db8ab80e12448..24e9b6befbdab 100644 --- a/airflow/providers/zendesk/provider.yaml +++ b/airflow/providers/zendesk/provider.yaml @@ -23,6 +23,8 @@ description: | versions: - 1.0.0b2 +provider-package: airflow.providers.zendesk + integrations: - integration-name: Zendesk external-doc-url: https://www.zendesk.com/ @@ -31,4 +33,4 @@ integrations: hooks: - integration-name: Zendesk python-modules: - - airflow.providers.zendesk.hooks.zendesk + - .hooks.zendesk diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index e09b5d8a8db5c..07ffe40cb0298 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -21,6 +21,8 @@ import logging import pkgutil import traceback +from functools import lru_cache +from typing import Dict, Tuple import jsonschema import yaml @@ -29,10 +31,7 @@ import importlib.resources as importlib_resources except ImportError: # Try backported to PY<37 `importlib_resources`. - import importlib_resources - - -log = logging.getLogger(__name__) + import importlib_resources # noqa def _create_validator(): @@ -46,42 +45,134 @@ class ProvidersManager: """Manages all provider packages.""" def __init__(self): + self.__log = logging.getLogger(__name__) self._provider_directory = {} try: from airflow import providers except ImportError as e: - log.warning("No providers are present or error when importing them! :%s", e) + self.__log.warning("No providers are present or error when importing them! :%s", e) return - self._validator = _create_validator() + self.__connections: Dict[str, Tuple[str, str]] = {} + self.__validator = _create_validator() self.__find_all_providers(providers.__path__) + self.__retrieve_connections() - def __find_all_providers(self, paths: str): - def onerror(_): - exception_string = traceback.format_exc() - log.warning(exception_string) + @staticmethod + def get_full_class_name(provider_package: str, class_path: str): + """ + Return full class name - if the class starts with ., it adds package_name in front. + :param provider_package: Package name + :param class_path: class path (might be relative if starts with .) + :return: full class name + """ + if class_path.startswith('.'): + return f"{provider_package}{class_path}" + return class_path - for module_info in pkgutil.walk_packages(paths, prefix="airflow.providers.", onerror=onerror): - try: - imported_module = importlib.import_module(module_info.name) - except Exception as e: # noqa pylint: disable=broad-except - log.warning("Error when importing %s:%s", module_info.name, e) - continue + def __find_all_providers(self, paths: str) -> None: + """ + Finds all providers installed and stores then in the directory of the providers. + :param paths: Path where to look for providers + """ + import warnings + + old_warn = warnings.warn + + def warn(*args, **kwargs): + if "deprecated" not in args[0]: + old_warn(*args, **kwargs) + + warnings.warn = warn + try: + + def onerror(_): + exception_string = traceback.format_exc() + self.__log.warning(exception_string) + + for module_info in pkgutil.walk_packages(paths, prefix="airflow.providers.", onerror=onerror): + try: + imported_module = importlib.import_module(module_info.name) + except Exception as e: # noqa pylint: disable=broad-except + self.__log.warning("Error when importing %s:%s", module_info.name, e) + continue + self.__read_provider_metadata(imported_module, module_info) + finally: + warnings.warn = old_warn + + def __read_provider_metadata(self, imported_module, module_info): + try: + provider = importlib_resources.read_text(imported_module, 'provider.yaml') + provider_info = yaml.safe_load(provider) + self.__validator.validate(provider_info) + package_name = provider_info['package-name'] + provider_package = provider_info['provider-package'] + if provider_package.startswith("apache.airflow"): + expected_package_name = provider_package.replace('.', '.') + if expected_package_name != package_name: + raise Exception( + f"Expected package name: {expected_package_name}" f" but it was {package_name}" + ) + self._provider_directory[package_name] = provider_info + except FileNotFoundError: + # This is OK - this is not a provider package + pass + except TypeError as e: + if "is not a package" not in str(e): + self.__log.warning("Error when loading 'provider.yaml' file from %s:%s}", module_info.name, e) + # Otherwise this is OK - this is likely a module + except Exception as e: # noqa pylint: disable=broad-except + self.__log.warning("Error when loading 'provider.yaml' file from %s:%s", module_info.name, e) + + def __retrieve_connections(self) -> None: + """Retrieves all connections defined in the providers""" + for provider in self._provider_directory.values(): + provider_package = provider['provider-package'] + if provider.get("connections"): + for connection in provider["connections"]: + self.__add_connection_to_table(connection, provider_package) + + def __add_connection_to_table(self, connection, provider_package): + hook_class_name = connection['hook-class'] + if not hook_class_name.startswith('.'): + self.__log.warning( + "The %s in provider.yaml when trying %s " + "provider should start with '.' being relative to the package", + hook_class_name, + provider_package, + ) + else: + full_class_name = f"{provider_package}{hook_class_name}" try: - provider = importlib_resources.read_text(imported_module, 'provider.yaml') - provider_info = yaml.safe_load(provider) - self._validator.validate(provider_info) - self._provider_directory[provider_info['package-name']] = provider_info - except FileNotFoundError: - # This is OK - this is not a provider package - pass - except TypeError as e: - if "is not a package" not in str(e): - log.warning("Error when loading 'provider.yaml' file from %s:%s}", module_info.name, e) - # Otherwise this is OK - this is likely a module + module, class_name = full_class_name.rsplit('.', maxsplit=1) + getattr(importlib.import_module(module), class_name) except Exception as e: # noqa pylint: disable=broad-except - log.warning("Error when loading 'provider.yaml' file from %s:%s", module_info.name, e) + self.__log.warning( + "The %s in provider.yaml when trying %s " "could not be imported: %s", + full_class_name, + provider_package, + e, + ) + return + self.__connections[connection['connection-type']] = ( + full_class_name, + connection['connection-id-parameter-name'], + ) @property def providers(self): """Returns information about available providers.""" return self._provider_directory + + @property + def connections(self): + """Returns dictionary of connection-to-hook mapping""" + return self.__connections + + +cache = lru_cache(maxsize=None) + + +@cache +def get_providers_manager() -> ProvidersManager: # noqa + """Returns singleton instance of ProvidersManager""" + return ProvidersManager() diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py index fd748016119ac..62b69100b7b6b 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py @@ -136,9 +136,13 @@ def check_completeness_of_list_of_hooks_sensors_hooks(yaml_files: Dict[str, Dict ) expected_modules = {_filepath_to_module(f) for f in py_files if not f.endswith("/__init__.py")} + provider_package = provider_data['provider-package'] + resource_data = provider_data.get(resource_type, []) - current_modules = {i for r in resource_data for i in r.get('python-modules', [])} + current_modules = { + f"{provider_package}{i}" for r in resource_data for i in r.get('python-modules', []) + } try: assert_sets_equal(set(expected_modules), set(current_modules)) except AssertionError as ex: @@ -167,8 +171,9 @@ def check_completeness_of_list_of_transfers(yaml_files: Dict[str, Dict]): expected_modules = {_filepath_to_module(f) for f in py_files if not f.endswith("/__init__.py")} resource_data = provider_data.get(resource_type, []) + provider_package = provider_data['provider-package'] - current_modules = {r.get('python-module') for r in resource_data} + current_modules = {provider_package + r.get('python-module') for r in resource_data} try: assert_sets_equal(set(expected_modules), set(current_modules)) except AssertionError as ex: diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py index c18566e8dd6e5..de73ad0003a78 100644 --- a/tests/core/test_providers_manager.py +++ b/tests/core/test_providers_manager.py @@ -82,6 +82,42 @@ 'apache-airflow-providers-zendesk', ] +CONNECTIONS_LIST = [ + 'azure_batch', + 'azure_cosmos', + 'azure_data_lake', + 'cassandra', + 'cloudant', + 'dataprep', + 'docker', + 'elasticsearch', + 'exasol', + 'gcpcloudsql', + 'gcpssh', + 'google_cloud_platform', + 'grpc', + 'hive_cli', + 'hiveserver2', + 'imap', + 'jdbc', + 'jira', + 'kubernetes', + 'mongo', + 'mssql', + 'mysql', + 'odbc', + 'oracle', + 'pig_cli', + 'postgres', + 'presto', + 'redis', + 'snowflake', + 'sqlite', + 'tableau', + 'vertica', + 'wasb', +] + class TestProviderManager(unittest.TestCase): def test_providers_are_loaded(self): @@ -89,3 +125,9 @@ def test_providers_are_loaded(self): provider_list = list(provider_manager.providers.keys()) provider_list.sort() self.assertEqual(ALL_PROVIDERS, provider_list) + + def test_connections(self): + provider_manager = ProvidersManager() + connections_list = list(provider_manager.connections.keys()) + connections_list.sort() + self.assertEqual(CONNECTIONS_LIST, connections_list) diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py index fbd275d01164c..c43c85c8a915e 100644 --- a/tests/models/test_connection.py +++ b/tests/models/test_connection.py @@ -28,9 +28,7 @@ from airflow import AirflowException from airflow.hooks.base_hook import BaseHook from airflow.models import Connection, crypto -from airflow.models.connection import CONN_TYPE_TO_HOOK from airflow.providers.sqlite.hooks.sqlite import SqliteHook -from airflow.utils.module_loading import import_string from tests.test_utils.config import conf_vars ConnectionParts = namedtuple("ConnectionParts", ["conn_type", "login", "password", "host", "port", "schema"]) @@ -549,15 +547,3 @@ def test_connection_mixed(self): ), ): Connection(conn_id="TEST_ID", uri="mysql://", schema="AAA") - - -class TestConnTypeToHook(unittest.TestCase): - def test_enforce_alphabetical_order(self): - current_keys = list(CONN_TYPE_TO_HOOK.keys()) - expected_keys = sorted(current_keys) - - self.assertEqual(expected_keys, current_keys) - - def test_hooks_importable(self): - for hook_path, _ in CONN_TYPE_TO_HOOK.values(): - self.assertTrue(issubclass(import_string(hook_path), BaseHook))