[ML] Fix file upload when pipline is used (elastic#194273)

Fixes bug introduced in elastic#193744

If a file is uploaded with an ingest pipeline, the pipeline is created,
but not used when uploading the data.

Adds tests to check that the data exists in the index and the expected
fields exist and are populated.

x-pack/plugins/file_upload/server/import_data.ts

@@ -49,6 +49,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
         createdPipelineId = pipelineId;
       } else {
         createdIndex = index;
+        createdPipelineId = pipelineId;
       }
 
       let failures: ImportFailure[] = [];

x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts

@@ -85,6 +85,24 @@ export default function ({ getService }: FtrProviderContext) {
             docCountFormatted: '19 (100%)',
           },
         ],
+        allFields: [
+          '@timestamp',
+          'http',
+          'http.request',
+          'http.request.method',
+          'http.response',
+          'http.response.body',
+          'http.response.body.bytes',
+          'http.response.status_code',
+          'http.version',
+          'message',
+          'source',
+          'source.address',
+          'url',
+          'url.original',
+          'user_agent',
+          'user_agent.original',
+        ],
         visibleMetricFieldsCount: 3,
         totalMetricFieldsCount: 3,
         populatedFieldsCount: 9,
@@ -127,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) {
             exampleCount: 7,
           },
         ],
+        allFields: ['Coordinates', 'Location'],
         visibleMetricFieldsCount: 0,
         totalMetricFieldsCount: 0,
         populatedFieldsCount: 3,
@@ -171,6 +190,7 @@ export default function ({ getService }: FtrProviderContext) {
             exampleCount: 3,
           },
         ],
+        allFields: ['description', 'title', 'value'],
         visibleMetricFieldsCount: 0,
         totalMetricFieldsCount: 0,
         populatedFieldsCount: 3,
@@ -207,6 +227,41 @@ export default function ({ getService }: FtrProviderContext) {
             exampleCount: 11,
           },
         ],
+        allFields: [
+          'AvgTicketPrice',
+          'Cancelled',
+          'Carrier',
+          'Dest',
+          'DestAirportID',
+          'DestCityName',
+          'DestCountry',
+          'DestLocation',
+          'DestLocation.lat',
+          'DestLocation.lat.keyword',
+          'DestLocation.lon',
+          'DestLocation.lon.keyword',
+          'DestRegion',
+          'DestWeather',
+          'DistanceKilometers',
+          'FlightDelayMin',
+          'FlightDelayType',
+          'FlightNum',
+          'FlightTimeHour',
+          'FlightTimeMin',
+          'Origin',
+          'OriginAirportID',
+          'OriginCityName',
+          'OriginCountry',
+          'OriginLocation',
+          'OriginLocation.lat',
+          'OriginLocation.lat.keyword',
+          'OriginLocation.lon',
+          'OriginLocation.lon.keyword',
+          'OriginRegion',
+          'OriginWeather',
+          'dayOfWeek',
+          'timestamp',
+        ],
         visibleMetricFieldsCount: 0,
         totalMetricFieldsCount: 0,
         populatedFieldsCount: 3,
@@ -349,6 +404,16 @@ export default function ({ getService }: FtrProviderContext) {
 
           await ml.testExecution.logTestStep('closes filebeat config');
           await ml.dataVisualizerFileBased.closeCreateFilebeatConfig();
+
+          await ml.dataVisualizerFileBased.assertDocCountInIndex(
+            testData.indexName,
+            testData.expected.ingestedDocCount
+          );
+
+          await ml.dataVisualizerFileBased.assertFieldsFromIndex(
+            testData.indexName,
+            testData.expected.allFields
+          );
         });
       });
     }

x-pack/test/functional/services/ml/data_visualizer_file_based.ts

@@ -16,6 +16,7 @@ export function MachineLearningDataVisualizerFileBasedProvider(
   { getService, getPageObjects }: FtrProviderContext,
   mlCommonUI: MlCommonUI
 ) {
+  const es = getService('es');
   const log = getService('log');
   const retry = getService('retry');
   const testSubjects = getService('testSubjects');
@@ -177,5 +178,46 @@ export function MachineLearningDataVisualizerFileBasedProvider(
       await testSubjects.click('fileBeatConfigFlyoutCloseButton');
       await testSubjects.missingOrFail('fileDataVisFilebeatConfigPanel');
     },
+
+    async assertDocCountInIndex(index: string, expectedCount: number) {
+      await retry.tryForTime(60 * 1000, async () => {
+        const count = await this.getDocCountFromIndex(index);
+        expect(count).to.eql(
+          expectedCount,
+          `Expected document count in index '${index}' to be '${expectedCount}' (got '${count}')`
+        );
+      });
+    },
+
+    async getDocCountFromIndex(index: string) {
+      const resp = await es.search({
+        index,
+        body: {
+          size: 0,
+          query: {
+            match_all: {},
+          },
+        },
+      });
+      // @ts-expect-error incorrect type definition
+      return resp.hits.total?.value;
+    },
+
+    async assertFieldsFromIndex(index: string, fields: string[]) {
+      await retry.tryForTime(60 * 1000, async () => {
+        const sortedFields = fields.sort();
+        const fieldCaps = await es.fieldCaps({
+          index,
+          fields: '*',
+          filters: '-metadata',
+          include_empty_fields: false,
+        });
+        const fieldsFromIndex = Object.keys(fieldCaps.fields).sort();
+        expect(fieldsFromIndex).to.eql(
+          sortedFields,
+          `Expected fields to be ${sortedFields} (got ${fieldsFromIndex})`
+        );
+      });
+    },
   };
 }