forked from FlowiseAI/Flowise
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added meilisearch retriever component (FlowiseAI#2824)
* added meilisearch retriever and credentials.ts * added semantic ratio * removed a TODO implementatio * meilisearch component implemented with searching and upsert functionality (FlowiseAI#3) meilisearch retriever component created , searching for an existing index and upserting a new or existing index has been implemented , component utilizes langchain and meilisearch vector search Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/3 Reviewed-by: mohamed1999akram <[email protected]> * added CI/CD for ntg branch, added proper dockerfile for flowise-ntg (FlowiseAI#4) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/4 Reviewed-by: mohammad <[email protected]> * modified os version , removed linting errors , removed cypress github actions (FlowiseAI#5) added --no-lock-file flag to pass CICD , made the runner run on debian and not ubuntu , removed code that caused warnings to pass linting Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/5 Reviewed-by: omaryassery <[email protected]> * removed unnecessary QEMU install action (FlowiseAI#6) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/6 Reviewed-by: omaryassery <[email protected]> * removed cypress installation and linting from dockerfile (FlowiseAI#7) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/7 Reviewed-by: isameh <[email protected]> * dockerfile-ntg-modification (FlowiseAI#9) dockerfile-ntg modified to copy all working directory before calling pnpm install Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/9 Reviewed-by: isameh <[email protected]> * resolved comments, reverted CI/CD * add test docker build yml back * moved meilisearch to vector store folder * Update Meilisearch.ts --------- Co-authored-by: Henry <[email protected]> Co-authored-by: Henry Heng <[email protected]>
- Loading branch information
1 parent
282bea5
commit 09c1cf7
Showing
6 changed files
with
311 additions
and
0 deletions.
There are no files selected for viewing
32 changes: 32 additions & 0 deletions
32
packages/components/credentials/MeilisearchApi.credential.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
import { INodeParams, INodeCredential } from '../src/Interface' | ||
|
||
class MeilisearchApi implements INodeCredential { | ||
label: string | ||
name: string | ||
version: number | ||
description: string | ||
inputs: INodeParams[] | ||
|
||
constructor() { | ||
this.label = 'Meilisearch API' | ||
this.name = 'meilisearchApi' | ||
this.version = 1.0 | ||
this.description = | ||
'Refer to <a target="_blank" href="https://meilisearch.com">official guide</a> on how to get an API Key, you need a search API KEY for basic searching functionality, admin API KEY is optional but needed for upsert functionality ' | ||
this.inputs = [ | ||
{ | ||
label: 'Meilisearch Search API Key', | ||
name: 'meilisearchSearchApiKey', | ||
type: 'password' | ||
}, | ||
{ | ||
label: 'Meilisearch Admin API Key', | ||
name: 'meilisearchAdminApiKey', | ||
type: 'password', | ||
optional: true | ||
} | ||
] | ||
} | ||
} | ||
|
||
module.exports = { credClass: MeilisearchApi } |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
174 changes: 174 additions & 0 deletions
174
packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,174 @@ | ||
import { getCredentialData, getCredentialParam } from '../../../src' | ||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' | ||
import { Meilisearch } from 'meilisearch' | ||
import { MeilisearchRetriever } from './core' | ||
import { flatten } from 'lodash' | ||
import { Document } from '@langchain/core/documents' | ||
import { v4 as uuidv4 } from 'uuid' | ||
import { Embeddings } from '@langchain/core/embeddings' | ||
|
||
class MeilisearchRetriever_node implements INode { | ||
label: string | ||
name: string | ||
version: number | ||
description: string | ||
type: string | ||
icon: string | ||
category: string | ||
baseClasses: string[] | ||
inputs: INodeParams[] | ||
credential: INodeParams | ||
badge: string | ||
outputs: INodeOutputsValue[] | ||
author?: string | ||
|
||
constructor() { | ||
this.label = 'Meilisearch' | ||
this.name = 'meilisearch' | ||
this.version = 1.0 | ||
this.type = 'Meilisearch' | ||
this.icon = 'Meilisearch.png' | ||
this.category = 'Vector Stores' | ||
this.badge = 'NEW' | ||
this.description = `Upsert embedded data and perform similarity search upon query using Meilisearch hybrid search functionality` | ||
this.baseClasses = ['BaseRetriever'] | ||
this.credential = { | ||
label: 'Connect Credential', | ||
name: 'credential', | ||
type: 'credential', | ||
credentialNames: ['meilisearchApi'] | ||
} | ||
this.inputs = [ | ||
{ | ||
label: 'Document', | ||
name: 'document', | ||
type: 'Document', | ||
list: true, | ||
optional: true | ||
}, | ||
{ | ||
label: 'Embeddings', | ||
name: 'embeddings', | ||
type: 'Embeddings' | ||
}, | ||
{ | ||
label: 'Host', | ||
name: 'host', | ||
type: 'string', | ||
description: 'This is the URL for the desired Meilisearch instance' | ||
}, | ||
{ | ||
label: 'Index Uid', | ||
name: 'indexUid', | ||
type: 'string', | ||
description: 'UID for the index to answer from' | ||
}, | ||
{ | ||
label: 'Top K', | ||
name: 'K', | ||
type: 'number', | ||
description: 'number of top searches to return as context', | ||
additionalParams: true, | ||
optional: true | ||
}, | ||
{ | ||
label: 'Semantic Ratio', | ||
name: 'semanticRatio', | ||
type: 'number', | ||
description: 'percentage of sematic reasoning in meilisearch hybrid search', | ||
additionalParams: true, | ||
optional: true | ||
} | ||
] | ||
this.outputs = [ | ||
{ | ||
label: 'Meilisearch Retriever', | ||
name: 'MeilisearchRetriever', | ||
description: 'retrieve answers', | ||
baseClasses: this.baseClasses | ||
} | ||
] | ||
this.outputs = [ | ||
{ | ||
label: 'Meilisearch Retriever', | ||
name: 'retriever', | ||
baseClasses: this.baseClasses | ||
} | ||
] | ||
} | ||
//@ts-ignore | ||
vectorStoreMethods = { | ||
async upsert(nodeData: INodeData, options: ICommonObject): Promise<any> { | ||
const credentialData = await getCredentialData(nodeData.credential ?? '', options) | ||
const meilisearchAdminApiKey = getCredentialParam('meilisearchAdminApiKey', credentialData, nodeData) | ||
const docs = nodeData.inputs?.document as Document[] | ||
const host = nodeData.inputs?.host as string | ||
const indexUid = nodeData.inputs?.indexUid as string | ||
const embeddings = nodeData.inputs?.embeddings as Embeddings | ||
let embeddingDimension: number = 384 | ||
const client = new Meilisearch({ | ||
host: host, | ||
apiKey: meilisearchAdminApiKey | ||
}) | ||
const flattenDocs = docs && docs.length ? flatten(docs) : [] | ||
const finalDocs = [] | ||
for (let i = 0; i < flattenDocs.length; i += 1) { | ||
if (flattenDocs[i] && flattenDocs[i].pageContent) { | ||
const uniqueId = uuidv4() | ||
const { pageContent, metadata } = flattenDocs[i] | ||
const docEmbedding = await embeddings.embedQuery(pageContent) | ||
embeddingDimension = docEmbedding.length | ||
const documentForIndexing = { | ||
pageContent, | ||
metadata, | ||
objectID: uniqueId, | ||
_vectors: { | ||
ollama: { | ||
embeddings: docEmbedding, | ||
regenerate: false | ||
} | ||
} | ||
} | ||
finalDocs.push(documentForIndexing) | ||
} | ||
} | ||
let index: any | ||
try { | ||
index = await client.getIndex(indexUid) | ||
} catch (error) { | ||
console.error('Error fetching index:', error) | ||
await client.createIndex(indexUid, { primaryKey: 'objectID' }) | ||
} finally { | ||
index = await client.getIndex(indexUid) | ||
} | ||
|
||
try { | ||
await index.updateSettings({ | ||
embedders: { | ||
ollama: { | ||
source: 'userProvided', | ||
dimensions: embeddingDimension | ||
} | ||
} | ||
}) | ||
await index.addDocuments(finalDocs) | ||
} catch (error) { | ||
console.error('Error occurred while adding documents:', error) | ||
} | ||
return | ||
} | ||
} | ||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> { | ||
const credentialData = await getCredentialData(nodeData.credential ?? '', options) | ||
const meilisearchSearchApiKey = getCredentialParam('meilisearchSearchApiKey', credentialData, nodeData) | ||
const host = nodeData.inputs?.host as string | ||
const indexUid = nodeData.inputs?.indexUid as string | ||
const K = nodeData.inputs?.K as string | ||
const semanticRatio = nodeData.inputs?.semanticRatio as string | ||
const embeddings = nodeData.inputs?.embeddings as Embeddings | ||
|
||
const hybridsearchretriever = new MeilisearchRetriever(host, meilisearchSearchApiKey, indexUid, K, semanticRatio, embeddings) | ||
return hybridsearchretriever | ||
} | ||
} | ||
module.exports = { nodeClass: MeilisearchRetriever_node } |
92 changes: 92 additions & 0 deletions
92
packages/components/nodes/vectorstores/Meilisearch/core.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
import { BaseRetriever, type BaseRetrieverInput } from '@langchain/core/retrievers' | ||
import { Document } from '@langchain/core/documents' | ||
import { Meilisearch } from 'meilisearch' | ||
import { Embeddings } from '@langchain/core/embeddings' | ||
|
||
export interface CustomRetrieverInput extends BaseRetrieverInput {} | ||
|
||
export class MeilisearchRetriever extends BaseRetriever { | ||
lc_namespace = ['langchain', 'retrievers'] | ||
private readonly meilisearchSearchApiKey: any | ||
private readonly host: any | ||
private indexUid: string | ||
private K: string | ||
private semanticRatio: string | ||
private embeddings: Embeddings | ||
constructor( | ||
host: string, | ||
meilisearchSearchApiKey: any, | ||
indexUid: string, | ||
K: string, | ||
semanticRatio: string, | ||
embeddings: Embeddings, | ||
fields?: CustomRetrieverInput | ||
) { | ||
super(fields) | ||
this.meilisearchSearchApiKey = meilisearchSearchApiKey | ||
this.host = host | ||
this.indexUid = indexUid | ||
this.embeddings = embeddings | ||
|
||
if (semanticRatio == '') { | ||
this.semanticRatio = '0.5' | ||
} else { | ||
let semanticRatio_Float = parseFloat(semanticRatio) | ||
if (semanticRatio_Float > 1.0) { | ||
this.semanticRatio = '1.0' | ||
} else if (semanticRatio_Float < 0.0) { | ||
this.semanticRatio = '0.0' | ||
} else { | ||
this.semanticRatio = semanticRatio | ||
} | ||
} | ||
|
||
if (K == '') { | ||
K = '4' | ||
} | ||
this.K = K | ||
} | ||
|
||
async _getRelevantDocuments(query: string): Promise<Document[]> { | ||
// Pass `runManager?.getChild()` when invoking internal runnables to enable tracing | ||
// const additionalDocs = await someOtherRunnable.invoke(params, runManager?.getChild()) | ||
const client = new Meilisearch({ | ||
host: this.host, | ||
apiKey: this.meilisearchSearchApiKey | ||
}) | ||
|
||
const index = await client.index(this.indexUid) | ||
const questionEmbedding = await this.embeddings.embedQuery(query) | ||
// Perform the search | ||
const searchResults = await index.search(query, { | ||
vector: questionEmbedding, | ||
limit: parseInt(this.K), // Optional: Limit the number of results | ||
attributesToRetrieve: ['*'], // Optional: Specify which fields to retrieve | ||
hybrid: { | ||
semanticRatio: parseFloat(this.semanticRatio), | ||
embedder: 'ollama' | ||
} | ||
}) | ||
const hits = searchResults.hits | ||
let documents: Document[] = [ | ||
new Document({ | ||
pageContent: 'mock page', | ||
metadata: {} | ||
}) | ||
] | ||
try { | ||
documents = hits.map( | ||
(hit: any) => | ||
new Document({ | ||
pageContent: hit.pageContent, | ||
metadata: { | ||
objectID: hit.objectID | ||
} | ||
}) | ||
) | ||
} catch (e) { | ||
console.error('Error occurred while adding documents:', e) | ||
} | ||
return documents | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.