Code review updates and naming changes.

api/src/main/java/marquez/api/SearchResource.java

@@ -42,7 +42,7 @@
 import org.opensearch.client.opensearch.core.search.Hit;
 
 @Slf4j
-@Path("/api/v1/search")
+@Path("/api/v2beta/search")
 public class SearchResource {
   private static final String YYYY_MM_DD = "^\\d{4}-\\d{2}-\\d{2}$";
   private static final String DEFAULT_SORT = "name";
@@ -109,16 +109,16 @@ private Response formatEsResponse(SearchResponse<ObjectNode> response) {
     List<Map<String, List<String>>> highlights =
         response.hits().hits().stream().map(Hit::highlight).collect(Collectors.toList());
 
-    return Response.ok(new EsResult(hits, highlights)).build();
+    return Response.ok(new OpenSearchResult(hits, highlights)).build();
   }
 
   @ToString
-  public static final class EsResult {
+  public static final class OpenSearchResult {
     @Getter private final List<ObjectNode> hits;
     @Getter private final List<Map<String, List<String>>> highlights;
 
     @JsonCreator
-    public EsResult(
+    public OpenSearchResult(
         @NonNull List<ObjectNode> hits, @NonNull List<Map<String, List<String>>> highlights) {
       this.hits = hits;
       this.highlights = highlights;

api/src/main/java/marquez/service/SearchService.java

@@ -39,6 +39,30 @@
 @Slf4j
 public class SearchService {
 
+  String[] DATASET_FIELDS = {
+    "run_id",
+    "name",
+    "namespace",
+    "facets.schema.fields.name",
+    "facets.schema.fields.type",
+    "facets.columnLineage.fields.*.inputFields.name",
+    "facets.columnLineage.fields.*.inputFields.namespace",
+    "facets.columnLineage.fields.*.inputFields.field",
+    "facets.columnLineage.fields.*.transformationDescription",
+    "facets.columnLineage.fields.*.transformationType"
+  };
+
+  String[] JOB_FIELDS = {
+    "facets.sql.query",
+    "facets.sourceCode.sourceCode",
+    "facets.sourceCode.language",
+    "runFacets.processing_engine.name",
+    "run_id",
+    "name",
+    "namespace",
+    "type"
+  };
+
   private final OpenSearchClient openSearchClient;
 
   public SearchService(SearchConfig searchConfig) {
@@ -74,18 +98,6 @@ public OpenSearchClient getClient() {
   }
 
   public SearchResponse<ObjectNode> searchDatasets(String query) throws IOException {
-    String[] fields = {
-      "run_id",
-      "name",
-      "namespace",
-      "facets.schema.fields.name",
-      "facets.schema.fields.type",
-      "facets.columnLineage.fields.*.inputFields.name",
-      "facets.columnLineage.fields.*.inputFields.namespace",
-      "facets.columnLineage.fields.*.inputFields.field",
-      "facets.columnLineage.fields.*.transformationDescription",
-      "facets.columnLineage.fields.*.transformationType"
-    };
     return this.openSearchClient.search(
         s ->
             s.index("datasets")
@@ -95,11 +107,11 @@ public SearchResponse<ObjectNode> searchDatasets(String query) throws IOExceptio
                             m ->
                                 m.query(query)
                                     .type(TextQueryType.PhrasePrefix)
-                                    .fields(Arrays.stream(fields).toList())
+                                    .fields(Arrays.stream(DATASET_FIELDS).toList())
                                     .operator(Operator.Or)))
                 .highlight(
                     hl -> {
-                      for (String field : fields) {
+                      for (String field : DATASET_FIELDS) {
                         hl.fields(
                             field,
                             f ->
@@ -113,16 +125,6 @@ public SearchResponse<ObjectNode> searchDatasets(String query) throws IOExceptio
   }
 
   public SearchResponse<ObjectNode> searchJobs(String query) throws IOException {
-    String[] fields = {
-      "facets.sql.query",
-      "facets.sourceCode.sourceCode",
-      "facets.sourceCode.language",
-      "runFacets.processing_engine.name",
-      "run_id",
-      "name",
-      "namespace",
-      "type"
-    };
     return this.openSearchClient.search(
         s -> {
           s.index("jobs")
@@ -132,11 +134,11 @@ public SearchResponse<ObjectNode> searchJobs(String query) throws IOException {
                           m ->
                               m.query(query)
                                   .type(TextQueryType.PhrasePrefix)
-                                  .fields(Arrays.stream(fields).toList())
+                                  .fields(Arrays.stream(JOB_FIELDS).toList())
                                   .operator(Operator.Or)));
           s.highlight(
               hl -> {
-                for (String field : fields) {
+                for (String field : JOB_FIELDS) {
                   hl.fields(
                       field,
                       f ->
@@ -152,7 +154,7 @@ public SearchResponse<ObjectNode> searchJobs(String query) throws IOException {
 
   public void indexEvent(@Valid @NotNull LineageEvent event) {
     UUID runUuid = runUuidFromEvent(event.getRun());
-    log.info("Indexing event {}", event);
+    log.debug("Indexing event {}", event);
 
     if (event.getInputs() != null) {
       indexDatasets(event.getInputs(), runUuid, event);
@@ -231,7 +233,7 @@ private void index(IndexRequest<Map<String, Object>> request) {
     try {
       this.openSearchClient.index(request);
     } catch (IOException e) {
-      log.info("Failed to index event OpenSearch not available.", e);
+      log.error("Failed to index event OpenSearch not available.", e);
     }
   }
 }

docker-compose.search.yml

@@ -120,6 +120,10 @@ services:
     networks:
       - opensearch-net
 
+  seed_marquez:
+    depends_on:
+      - opensearch
+
 volumes:
   data:
   opensearch-data:

web/jest.config.js

@@ -16,6 +16,7 @@ module.exports = {
   testEnvironment: 'jsdom',
   globals: {
     __API_URL__: '/api/v1',
+    __API_BETA_URL__: '/api/v2beta',
     __FEEDBACK_FORM_URL__: 'https://forms.gle/f3tTSrZ8wPj3sHTA7',
     __REACT_APP_ADVANCED_SEARCH__: true,
     __API_DOCS_URL__: 'https://marquezproject.github.io/marquez/openapi.html',

web/src/components/search/Search.tsx

@@ -11,7 +11,7 @@ import { useLocation } from 'react-router'
 import BaseSearch from './base-search/BaseSearch'
 import CircularProgress from '@mui/material/CircularProgress/CircularProgress'
 import ClickAwayListener from '@mui/material/ClickAwayListener'
-import EsSearch from './es-search/EsSearch'
+import OpenSearch from './open-search/OpenSearch'
 import IconButton from '@mui/material/IconButton'
 import React, { useEffect, useRef, useState } from 'react'
 import SearchPlaceholder from './SearchPlaceholder'
@@ -188,7 +188,7 @@ const Search: React.FC = ({ isLoading }: StateProps) => {
                   maxHeight={`calc(100vh - ${HEADER_HEIGHT}px - 24px)`}
                 >
                   {REACT_APP_ADVANCED_SEARCH ? (
-                    <EsSearch search={search} />
+                    <OpenSearch search={search} />
                   ) : (
                     <BaseSearch search={search} />
                   )}
@@ -203,7 +203,7 @@ const Search: React.FC = ({ isLoading }: StateProps) => {
 }
 
 const mapStateToProps = (state: IState) => ({
-  isLoading: state.esSearchJobs.isLoading || state.esSearchDatasets.isLoading,
+  isLoading: state.openSearchJobs.isLoading || state.openSearchDatasets.isLoading,
 })
 
 export default connect(mapStateToProps)(Search)

web/src/components/search/es-search/EsSearch.tsx → web/src/components/search/open-search/OpenSearch.tsx

@@ -4,8 +4,8 @@
 import * as Redux from 'redux'
 import { Chip, Divider } from '@mui/material'
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
-import { IEsSearchDatasetsState } from '../../../store/reducers/esSearchDatasets'
-import { IEsSearchJobsState } from '../../../store/reducers/esSearch'
+import { IOpenSearchDatasetsState } from '../../../store/reducers/openSearchDatasets'
+import { IOpenSearchJobsState } from '../../../store/reducers/openSearch'
 import { IState } from '../../../store/reducers'
 import { Nullable } from '../../../types/util/Nullable'
 import { bindActionCreators } from 'redux'
@@ -14,7 +14,7 @@ import { debounce } from 'lodash'
 import { encodeNode, eventTypeColor } from '../../../helpers/nodes'
 import { faCog } from '@fortawesome/free-solid-svg-icons/faCog'
 import { faDatabase } from '@fortawesome/free-solid-svg-icons'
-import { fetchEsSearchDatasets, fetchEsSearchJobs } from '../../../store/actionCreators'
+import { fetchOpenSearchDatasets, fetchOpenSearchJobs } from '../../../store/actionCreators'
 import { theme } from '../../../helpers/theme'
 import { truncateText } from '../../../helpers/text'
 import { useNavigate } from 'react-router-dom'
@@ -28,13 +28,13 @@ import airflow_logo from './airlfow-logo.svg'
 import spark_logo from './spark-logo.svg'
 
 interface StateProps {
-  esSearchJobs: IEsSearchJobsState
-  esSearchDatasets: IEsSearchDatasetsState
+  openSearchJobs: IOpenSearchJobsState
+  openSearchDatasets: IOpenSearchDatasetsState
 }
 
 interface DispatchProps {
-  fetchEsSearchJobs: typeof fetchEsSearchJobs
-  fetchEsSearchDatasets: typeof fetchEsSearchDatasets
+  fetchOpenSearchJobs: typeof fetchOpenSearchJobs
+  fetchOpenSearchDatasets: typeof fetchOpenSearchDatasets
 }
 
 interface Props {
@@ -86,12 +86,12 @@ const useArrowKeys = (callback: (key: 'up' | 'down' | 'enter') => void) => {
 const FIELDS_TO_PRINT = 5
 const DEBOUNCE_TIME_MS = 200
 
-const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
+const OpenSearch: React.FC<StateProps & DispatchProps & Props> = ({
   search,
-  fetchEsSearchJobs,
-  fetchEsSearchDatasets,
-  esSearchJobs,
-  esSearchDatasets,
+  fetchOpenSearchJobs,
+  fetchOpenSearchDatasets,
+  openSearchJobs,
+  openSearchDatasets,
 }) => {
   const [selectedIndex, setSelectedIndex] = React.useState<Nullable<number>>(null)
   const [isDebouncing, setIsDebouncing] = React.useState<boolean>(true)
@@ -106,23 +106,23 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
           ? 0
           : Math.min(
               selectedIndex + 1,
-              esSearchJobs.data.hits.length + esSearchDatasets.data.hits.length - 1
+              openSearchJobs.data.hits.length + openSearchDatasets.data.hits.length - 1
             )
       )
     } else if (selectedIndex !== null) {
-      if (selectedIndex < esSearchJobs.data.hits.length) {
-        const jobHit = esSearchJobs.data.hits[selectedIndex]
+      if (selectedIndex < openSearchJobs.data.hits.length) {
+        const jobHit = openSearchJobs.data.hits[selectedIndex]
         navigate(`/lineage/${encodeNode('JOB', jobHit.namespace, jobHit.name)}`)
       } else {
-        const datasetHit = esSearchDatasets.data.hits[selectedIndex - esSearchJobs.data.hits.length]
+        const datasetHit = openSearchDatasets.data.hits[selectedIndex - openSearchJobs.data.hits.length]
         navigate(`/lineage/${encodeNode('DATASET', datasetHit.namespace, datasetHit.name)}`)
       }
     }
   })
 
   const debouncedFetchJobs = useCallback(
     debounce(async (searchTerm) => {
-      fetchEsSearchJobs(searchTerm);
+      fetchOpenSearchJobs(searchTerm);
       setIsDebouncing(false); // Set loading to false after the fetch completes
     }, DEBOUNCE_TIME_MS),
     []
@@ -131,7 +131,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
 
   const debouncedFetchDatasets = useCallback(
     debounce(async (searchTerm) => {
-      fetchEsSearchDatasets(searchTerm);
+      fetchOpenSearchDatasets(searchTerm);
       setIsDebouncing(false); // Set loading to false after the fetch completes
     }, DEBOUNCE_TIME_MS),
     []
@@ -145,9 +145,9 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
 
   useEffect(() => {
     setSelectedIndex(null)
-  }, [esSearchJobs.data.hits, esSearchDatasets.data.hits])
+  }, [openSearchJobs.data.hits, openSearchDatasets.data.hits])
 
-  if (esSearchJobs.data.hits.length === 0 && esSearchDatasets.data.hits.length === 0 && !isDebouncing) {
+  if (openSearchJobs.data.hits.length === 0 && openSearchDatasets.data.hits.length === 0 && !isDebouncing) {
     return (
       <Box my={4}>
         <MqEmpty title={'No Hits'} body={'Keep typing or try a more precise search.'} />
@@ -157,7 +157,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
 
   return (
     <Box>
-      {esSearchJobs.data.hits.map((hit, index) => {
+      {openSearchJobs.data.hits.map((hit, index) => {
         return (
           <Box
             key={`job-${hit.run_id}`}
@@ -226,7 +226,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
                   Match
                 </MqText>
                 <Box>
-                  {Object.entries(esSearchJobs.data.highlights[index]).map(([key, value]) => {
+                  {Object.entries(openSearchJobs.data.highlights[index]).map(([key, value]) => {
                     return value.map((highlightedString: any, idx: number) => {
                       return (
                         <Box
@@ -279,7 +279,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
           </Box>
         )
       })}
-      {esSearchDatasets.data.hits.map((hit, index) => {
+      {openSearchDatasets.data.hits.map((hit, index) => {
         return (
           <Box
             key={`dataset-${index}-${hit.run_id}`}
@@ -295,7 +295,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
                 backgroundColor: theme.palette.action.hover,
               },
               backgroundColor:
-                selectedIndex === index + esSearchJobs.data.hits.length
+                selectedIndex === index + openSearchJobs.data.hits.length
                   ? theme.palette.action.hover
                   : undefined,
             }}
@@ -326,7 +326,7 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
                   Match
                 </MqText>
                 <Box>
-                  {Object.entries(esSearchDatasets.data.highlights[index]).map(([key, value]) => {
+                  {Object.entries(openSearchDatasets.data.highlights[index]).map(([key, value]) => {
                     return value.map((highlightedString: any, idx: number) => {
                       return (
                         <Box
@@ -393,18 +393,18 @@ const EsSearch: React.FC<StateProps & DispatchProps & Props> = ({
 
 const mapStateToProps = (state: IState) => {
   return {
-    esSearchJobs: state.esSearchJobs,
-    esSearchDatasets: state.esSearchDatasets,
+    openSearchJobs: state.openSearchJobs,
+    openSearchDatasets: state.openSearchDatasets,
   }
 }
 
 const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
   bindActionCreators(
     {
-      fetchEsSearchJobs: fetchEsSearchJobs,
-      fetchEsSearchDatasets: fetchEsSearchDatasets,
+      fetchOpenSearchJobs: fetchOpenSearchJobs,
+      fetchOpenSearchDatasets: fetchOpenSearchDatasets,
     },
     dispatch
   )
 
-export default connect(mapStateToProps, mapDispatchToProps)(EsSearch)
+export default connect(mapStateToProps, mapDispatchToProps)(OpenSearch)

web/src/globals.ts

@@ -6,12 +6,14 @@ declare const __NODE_ENV__: string
 declare const __DEVELOPMENT__: boolean
 
 declare const __API_URL__: string
+declare const __API_BETA_URL__: string
 declare const __REACT_APP_ADVANCED_SEARCH__: boolean
 
 declare const __FEEDBACK_FORM_URL__: string
 declare const __API_DOCS_URL__: string
 
 export const API_URL = __API_URL__
+export const API_BETA_URL = __API_BETA_URL__
 export const FEEDBACK_FORM_URL = __FEEDBACK_FORM_URL__
 export const API_DOCS_URL = __API_DOCS_URL__
 export const REACT_APP_ADVANCED_SEARCH = __REACT_APP_ADVANCED_SEARCH__;

web/src/store/actionCreators/actionTypes.ts

@@ -72,11 +72,11 @@ export const SET_SHOW_FULL_GRAPH = 'SET_SHOW_FULL_GRAPH'
 export const FETCH_SEARCH = 'FETCH_SEARCH'
 export const FETCH_SEARCH_SUCCESS = 'FETCH_SEARCH _SUCCESS'
 
-// esSearch
-export const FETCH_ES_SEARCH_JOBS = 'FETCH_ES_SEARCH_JOBS'
-export const FETCH_ES_SEARCH_JOBS_SUCCESS = 'FETCH_ES_SEARCH_JOBS_SUCCESS'
-export const FETCH_ES_SEARCH_DATASETS = 'FETCH_ES_SEARCH_DATASETS'
-export const FETCH_ES_SEARCH_DATASETS_SUCCESS = 'FETCH_ES_SEARCH_DATASETS_SUCCESS'
+// OpenSearch
+export const FETCH_OPEN_SEARCH_JOBS = 'FETCH_OPEN_SEARCH_JOBS'
+export const FETCH_OPEN_SEARCH_JOBS_SUCCESS = 'FETCH_OPEN_SEARCH_JOBS_SUCCESS'
+export const FETCH_OPEN_SEARCH_DATASETS = 'FETCH_OPEN_SEARCH_DATASETS'
+export const FETCH_OPEN_SEARCH_DATASETS_SUCCESS = 'FETCH_OPEN_SEARCH_DATASETS_SUCCESS'
 
 // facets
 export const FETCH_RUN_FACETS = 'FETCH_RUN_FACETS'