-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Implement concurrency and relations resolving (#91)
- Loading branch information
Showing
7 changed files
with
142 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
lib/src/main/java/io/cloudquery/memdb/Table2ChildData.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package io.cloudquery.memdb; | ||
|
|
||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
|
|
||
| @Builder | ||
| @Getter | ||
| public class Table2ChildData { | ||
| private String name; | ||
| } |
3 changes: 0 additions & 3 deletions
3
lib/src/main/java/io/cloudquery/scheduler/OnResourceResolved.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 46 additions & 27 deletions
73
lib/src/main/java/io/cloudquery/scheduler/SchedulerTableOutputStream.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,52 +1,71 @@ | ||
| package io.cloudquery.scheduler; | ||
|
|
||
| import com.google.protobuf.ByteString; | ||
| import io.cloudquery.plugin.TableOutputStream; | ||
| import io.cloudquery.plugin.v3.Sync; | ||
| import io.cloudquery.schema.ClientMeta; | ||
| import io.cloudquery.schema.Column; | ||
| import io.cloudquery.schema.Resource; | ||
| import io.cloudquery.schema.Table; | ||
| import io.cloudquery.transformers.TransformerException; | ||
| import io.grpc.stub.StreamObserver; | ||
| import lombok.Builder; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.TimeUnit; | ||
| import lombok.NonNull; | ||
| import org.apache.logging.log4j.Logger; | ||
|
|
||
| @Builder | ||
| public class SchedulerTableOutputStream implements TableOutputStream { | ||
| private static final int RESOURCE_RESOLVE_CONCURRENCY = 100; | ||
| private static final int RESOURCE_RESOLVE_TIMEOUT_MINUTES = 10; | ||
| @NonNull private final Table table; | ||
| private final Resource parent; | ||
| @NonNull private final ClientMeta client; | ||
| @NonNull private final Logger logger; | ||
| @NonNull private final StreamObserver<io.cloudquery.plugin.v3.Sync.Response> syncStream; | ||
|
|
||
| private List<Resource> resources = new ArrayList<Resource>(); | ||
|
|
||
| private ExecutorService executor; | ||
|
|
||
| public SchedulerTableOutputStream( | ||
| @NonNull Table table, Resource parent, @NonNull ClientMeta client, @NonNull Logger logger) { | ||
| this.table = table; | ||
| this.parent = parent; | ||
| this.client = client; | ||
| this.logger = logger; | ||
| this.executor = Executors.newFixedThreadPool(RESOURCE_RESOLVE_CONCURRENCY); | ||
| } | ||
|
|
||
| @Override | ||
| public void write(Object data) { | ||
| Resource resource = Resource.builder().table(table).parent(parent).item(data).build(); | ||
| for (Column column : table.getColumns()) { | ||
| try { | ||
| logger.info("resolving column: {}", column.getName()); | ||
| if (column.getResolver() == null) { | ||
| logger.error("no resolver for column: {}", column.getName()); | ||
| continue; | ||
| } | ||
| column.getResolver().resolve(client, resource, column); | ||
| logger.info("resolved column: {}", column.getName()); | ||
| } catch (TransformerException e) { | ||
| logger.error("Failed to resolve column: {}", column.getName(), e); | ||
| return; | ||
| } | ||
| executor.submit( | ||
| new Runnable() { | ||
| @Override | ||
| public void run() { | ||
| try { | ||
| logger.debug("resolving column: {}", column.getName()); | ||
| if (column.getResolver() == null) { | ||
| logger.error("no resolver for column: {}", column.getName()); | ||
| return; | ||
| } | ||
| column.getResolver().resolve(client, resource, column); | ||
| logger.debug("resolved column: {}", column.getName()); | ||
| return; | ||
| } catch (TransformerException e) { | ||
| logger.error("Failed to resolve column: {}", column.getName(), e); | ||
| return; | ||
| } | ||
| } | ||
| }); | ||
| } | ||
| resources.add(resource); | ||
| } | ||
|
|
||
| try { | ||
| ByteString record = resource.encode(); | ||
| Sync.MessageInsert insert = Sync.MessageInsert.newBuilder().setRecord(record).build(); | ||
| Sync.Response response = Sync.Response.newBuilder().setInsert(insert).build(); | ||
| syncStream.onNext(response); | ||
| } catch (Exception e) { | ||
| logger.error("Failed to encode resource: {}", resource, e); | ||
| return; | ||
| } | ||
| public List<Resource> getResources() throws InterruptedException { | ||
| // TODO: Optimize this to not wait for all resources to complete | ||
| executor.shutdown(); | ||
| executor.awaitTermination(RESOURCE_RESOLVE_TIMEOUT_MINUTES, TimeUnit.MINUTES); | ||
| return this.resources; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters