Skip to content

Commit

Permalink
[Ingest Manager] Support updated package output structure (#69864) (#…
Browse files Browse the repository at this point in the history
…70263)

* Update EPM package registry typings to reflect registry changes

* Change `dataset.id` references to `dataset.name`

* Fix RegistryStream

* Fix packageToConfigDatasourceInputs service

* Fix assignPackageStream service

* Fix validateDatasource service

* Fix configure data source components

* Fix variable

* Fix stream template installation

* Add support for `input[].dataset.type` and change `stream.dataset` mapping to be object containing `name` with instead of just a string

* Nest package information under `meta` in agent config yaml

* Move `dataset.type` to stream level instead of input level

* Make single call to fetch registry package information instead of doing it per stream

* Fix type issues

* Update endpoint test assertion

Co-authored-by: Elastic Machine <[email protected]>

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
jen-huang and elasticmachine authored Jun 29, 2020
1 parent f0002f4 commit 2e07eb4
Show file tree
Hide file tree
Showing 27 changed files with 461 additions and 248 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const CONFIG_KEYS_ORDER = [
'inputs',
'enabled',
'use_output',
'package',
'meta',
'input',
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
{
id: 'test-logs-foo',
enabled: true,
dataset: 'foo',
dataset: { name: 'foo', type: 'logs' },
vars: {
fooVar: { value: 'foo-value' },
fooVar2: { value: [1, 2] },
Expand All @@ -52,7 +52,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
{
id: 'test-logs-bar',
enabled: true,
dataset: 'bar',
dataset: { name: 'bar', type: 'logs' },
vars: {
barVar: { value: 'bar-value' },
barVar2: { value: [1, 2] },
Expand Down Expand Up @@ -101,23 +101,41 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
});

it('returns agent inputs', () => {
expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([
expect(
storedDatasourcesToAgentInputs([
{
...mockDatasource,
package: {
name: 'mock-package',
title: 'Mock package',
version: '0.0.0',
},
inputs: [mockInput],
},
])
).toEqual([
{
id: 'some-uuid',
name: 'mock-datasource',
type: 'test-logs',
dataset: { namespace: 'default' },
use_output: 'default',
meta: {
package: {
name: 'mock-package',
version: '0.0.0',
},
},
streams: [
{
id: 'test-logs-foo',
dataset: { name: 'foo' },
dataset: { name: 'foo', type: 'logs' },
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
{
id: 'test-logs-bar',
dataset: { name: 'bar' },
dataset: { name: 'bar', type: 'logs' },
},
],
},
Expand Down Expand Up @@ -147,7 +165,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
streams: [
{
id: 'test-logs-foo',
dataset: { name: 'foo' },
dataset: { name: 'foo', type: 'logs' },
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ export const storedDatasourcesToAgentInputs = (
id: datasource.id || datasource.name,
name: datasource.name,
type: input.type,
dataset: { namespace: datasource.namespace || 'default' },
dataset: {
namespace: datasource.namespace || 'default',
},
use_output: DEFAULT_OUTPUT.name,
...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
Expand All @@ -35,7 +37,7 @@ export const storedDatasourcesToAgentInputs = (
.map((stream) => {
const fullStream: FullAgentConfigInputStream = {
id: stream.id,
dataset: { name: stream.dataset },
dataset: stream.dataset,
...stream.agent_stream,
...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
Expand All @@ -50,9 +52,11 @@ export const storedDatasourcesToAgentInputs = (
};

if (datasource.package) {
fullInput.package = {
name: datasource.package.name,
version: datasource.package.version,
fullInput.meta = {
package: {
name: datasource.package.name,
version: datasource.package.version,
},
};
}

Expand Down
Loading

0 comments on commit 2e07eb4

Please sign in to comment.