forked from apache/seatunnel
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #11 from zilliztech/nianliuu
nianliuu
- Loading branch information
Showing
35 changed files
with
1,629 additions
and
44 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <!-- | ||
| Licensed to the Apache Software Foundation (ASF) under one or more | ||
| contributor license agreements. See the NOTICE file distributed with | ||
| this work for additional information regarding copyright ownership. | ||
| The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| (the "License"); you may not use this file except in compliance with | ||
| the License. You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| --> | ||
| <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <parent> | ||
| <groupId>org.apache.seatunnel</groupId> | ||
| <artifactId>seatunnel-connectors-v2</artifactId> | ||
| <version>${revision}</version> | ||
| </parent> | ||
|
|
||
| <artifactId>connector-pinecone</artifactId> | ||
| <name>SeaTunnel : Connectors V2 : Pinecone</name> | ||
|
|
||
| <dependencies> | ||
| <dependency> | ||
| <groupId>io.pinecone</groupId> | ||
| <artifactId>pinecone-client</artifactId> | ||
| <version>2.1.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.google.code.gson</groupId> | ||
| <artifactId>gson</artifactId> | ||
| <version>2.10.1</version> | ||
| </dependency> | ||
| </dependencies> | ||
|
|
||
|
|
||
| </project> |
20 changes: 20 additions & 0 deletions
20
...e/src/main/java/org/apache/seatunnel/connectors/pinecone/config/PineconeSourceConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package org.apache.seatunnel.connectors.pinecone.config; | ||
|
|
||
| import org.apache.seatunnel.api.configuration.Option; | ||
| import org.apache.seatunnel.api.configuration.Options; | ||
|
|
||
| public class PineconeSourceConfig { | ||
| public static final String CONNECTOR_IDENTITY = "Pinecone"; | ||
|
|
||
| public static final Option<String> API_KEY = | ||
| Options.key("api_key") | ||
| .stringType() | ||
| .noDefaultValue() | ||
| .withDescription("Pinecone token for authentication"); | ||
|
|
||
| public static final Option<String> INDEX = | ||
| Options.key("index") | ||
| .stringType() | ||
| .noDefaultValue() | ||
| .withDescription("Pinecone index name"); | ||
| } |
18 changes: 18 additions & 0 deletions
18
.../java/org/apache/seatunnel/connectors/pinecone/exception/PineconeConnectionErrorCode.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package org.apache.seatunnel.connectors.pinecone.exception; | ||
|
|
||
| import lombok.Getter; | ||
| import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; | ||
|
|
||
| @Getter | ||
| public enum PineconeConnectionErrorCode implements SeaTunnelErrorCode { | ||
| SOURCE_TABLE_SCHEMA_IS_NULL("PINECONE-01", "Source table schema is null"), | ||
| READ_DATA_FAIL("PINECONE-02", "Read data fail"); | ||
|
|
||
| private final String code; | ||
| private final String description; | ||
|
|
||
| PineconeConnectionErrorCode(String code, String description) { | ||
| this.code = code; | ||
| this.description = description; | ||
| } | ||
| } |
14 changes: 14 additions & 0 deletions
14
...n/java/org/apache/seatunnel/connectors/pinecone/exception/PineconeConnectorException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package org.apache.seatunnel.connectors.pinecone.exception; | ||
|
|
||
| import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; | ||
| import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; | ||
|
|
||
| public class PineconeConnectorException extends SeaTunnelRuntimeException { | ||
| public PineconeConnectorException(SeaTunnelErrorCode seaTunnelErrorCode) { | ||
| super(seaTunnelErrorCode, seaTunnelErrorCode.getErrorMessage()); | ||
| } | ||
|
|
||
| public PineconeConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) { | ||
| super(seaTunnelErrorCode, seaTunnelErrorCode.getErrorMessage(), cause); | ||
| } | ||
| } |
92 changes: 92 additions & 0 deletions
92
...inecone/src/main/java/org/apache/seatunnel/connectors/pinecone/source/PineconeSource.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package org.apache.seatunnel.connectors.pinecone.source; | ||
|
|
||
| import org.apache.seatunnel.api.configuration.ReadonlyConfig; | ||
| import org.apache.seatunnel.api.source.*; | ||
| import org.apache.seatunnel.api.table.catalog.CatalogTable; | ||
| import org.apache.seatunnel.api.table.catalog.TablePath; | ||
| import org.apache.seatunnel.api.table.type.SeaTunnelRow; | ||
| import org.apache.seatunnel.connectors.pinecone.config.PineconeSourceConfig; | ||
| import org.apache.seatunnel.connectors.pinecone.utils.PineconeUtils; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| public class PineconeSource implements SeaTunnelSource<SeaTunnelRow, PineconeSourceSplit, PineconeSourceState>, | ||
| SupportParallelism, | ||
| SupportColumnProjection { | ||
| private final ReadonlyConfig config; | ||
| private final Map<TablePath, CatalogTable> sourceTables; | ||
|
|
||
| public PineconeSource(ReadonlyConfig config) { | ||
| this.config = config; | ||
| PineconeUtils pineconeUtils = new PineconeUtils(config); | ||
| this.sourceTables = pineconeUtils.getSourceTables(); | ||
| } | ||
|
|
||
| /** | ||
| * Get the boundedness of this source. | ||
| * | ||
| * @return the boundedness of this source. | ||
| */ | ||
| @Override | ||
| public Boundedness getBoundedness() { | ||
| return null; | ||
| } | ||
|
|
||
| /** | ||
| * Create source reader, used to produce data. | ||
| * | ||
| * @param readerContext reader context. | ||
| * @return source reader. | ||
| * @throws Exception when create reader failed. | ||
| */ | ||
| @Override | ||
| public SourceReader<SeaTunnelRow, PineconeSourceSplit> createReader(SourceReader.Context readerContext) throws Exception { | ||
| return new PineconeSourceReader(readerContext, config, sourceTables); | ||
| } | ||
|
|
||
| @Override | ||
| public List<CatalogTable> getProducedCatalogTables() { | ||
| return new ArrayList<>(sourceTables.values()); | ||
| } | ||
|
|
||
| /** | ||
| * Create source split enumerator, used to generate splits. This method will be called only once | ||
| * when start a source. | ||
| * | ||
| * @param enumeratorContext enumerator context. | ||
| * @return source split enumerator. | ||
| * @throws Exception when create enumerator failed. | ||
| */ | ||
| @Override | ||
| public SourceSplitEnumerator<PineconeSourceSplit, PineconeSourceState> createEnumerator(SourceSplitEnumerator.Context<PineconeSourceSplit> enumeratorContext) throws Exception { | ||
| return new PineconeSourceSplitEnumertor(enumeratorContext, config, sourceTables, null); | ||
| } | ||
|
|
||
| /** | ||
| * Create source split enumerator, used to generate splits. This method will be called when | ||
| * restore from checkpoint. | ||
| * | ||
| * @param enumeratorContext enumerator context. | ||
| * @param checkpointState checkpoint state. | ||
| * @return source split enumerator. | ||
| * @throws Exception when create enumerator failed. | ||
| */ | ||
| @Override | ||
| public SourceSplitEnumerator<PineconeSourceSplit, PineconeSourceState> restoreEnumerator(SourceSplitEnumerator.Context<PineconeSourceSplit> enumeratorContext, PineconeSourceState checkpointState) throws Exception { | ||
| return new PineconeSourceSplitEnumertor(enumeratorContext, config, sourceTables, checkpointState); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a unique identifier among same factory interfaces. | ||
| * | ||
| * <p>For consistency, an identifier should be declared as one lower case word (e.g. {@code | ||
| * kafka}). If multiple factories exist for different versions, a version should be appended | ||
| * using "-" (e.g. {@code elasticsearch-7}). | ||
| */ | ||
| @Override | ||
| public String getPluginName() { | ||
| return PineconeSourceConfig.CONNECTOR_IDENTITY; | ||
| } | ||
| } |
43 changes: 43 additions & 0 deletions
43
.../src/main/java/org/apache/seatunnel/connectors/pinecone/source/PineconeSourceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package org.apache.seatunnel.connectors.pinecone.source; | ||
|
|
||
| import com.google.auto.service.AutoService; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.apache.seatunnel.api.configuration.util.OptionRule; | ||
| import org.apache.seatunnel.api.source.SeaTunnelSource; | ||
| import org.apache.seatunnel.api.source.SourceSplit; | ||
| import org.apache.seatunnel.api.table.connector.TableSource; | ||
| import org.apache.seatunnel.api.table.factory.Factory; | ||
| import org.apache.seatunnel.api.table.factory.TableSourceFactory; | ||
| import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; | ||
| import org.apache.seatunnel.connectors.pinecone.config.PineconeSourceConfig; | ||
|
|
||
| import java.io.Serializable; | ||
|
|
||
| @Slf4j | ||
| @AutoService(Factory.class) | ||
| public class PineconeSourceFactory implements TableSourceFactory { | ||
|
|
||
| @Override | ||
| public <T, SplitT extends SourceSplit, StateT extends Serializable> | ||
| TableSource<T, SplitT, StateT> createSource(TableSourceFactoryContext context) { | ||
| return () -> (SeaTunnelSource<T, SplitT, StateT>) new PineconeSource(context.getOptions()); | ||
| } | ||
|
|
||
| @Override | ||
| public OptionRule optionRule() { | ||
| return OptionRule.builder() | ||
| .required(PineconeSourceConfig.API_KEY) | ||
| .optional() | ||
| .build(); | ||
| } | ||
|
|
||
| @Override | ||
| public Class<? extends SeaTunnelSource> getSourceClass() { | ||
| return PineconeSource.class; | ||
| } | ||
|
|
||
| @Override | ||
| public String factoryIdentifier() { | ||
| return PineconeSourceConfig.CONNECTOR_IDENTITY; | ||
| } | ||
| } |
Oops, something went wrong.