Integrate visualQnA backend (#768)

Signed-off-by: Yue, Wenjiao <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

VisualQnA/docker/ui/svelte/src/lib/assets/imageData/images.json

@@ -0,0 +1,10 @@
+[
+	{
+		"src": "/extreme_ironing.jpg",
+		"prompt": "what is unusual about this image?"
+	},
+	{
+		"src": "/waterview.jpg",
+		"prompt": "what are the things I should be cautious about when I visit here?"
+	}
+]

VisualQnA/docker/ui/svelte/src/lib/modules/upload/upload.svelte

@@ -4,57 +4,29 @@
 -->
 
 <script lang="ts">
-	import { Card, Button, Hr, Label, Input } from "flowbite-svelte";
+	import { Hr, Label, Input } from "flowbite-svelte";
 	import UploadImg from "./uploadImg.svelte";
-	import { Tooltip } from "flowbite-svelte";
 
 	import { Range } from "flowbite-svelte";
-	import Help from "$lib/assets/upload/help.svelte";
-	import { EnvelopeSolid } from "flowbite-svelte-icons";
+	import { FilePasteSolid } from "flowbite-svelte-icons";
+	import { stepValueStore } from "$lib/shared/stores/common/Store";
 	let stepValue = 512;
-	let UseShow = false;
+    let imageUrl = '';
+
+	$: stepValueStore.set(stepValue);
 </script>
 
 <div class="flex w-full flex-col gap-3 rounded-xl bg-white p-5">
 	<p>Upload Images</p>
-	<UploadImg />
+	<UploadImg imageUrl={imageUrl}/>
 	<Hr classHr="my-8 w-64">or</Hr>
 	<div class="mb-6">
 		<Label for="input-group-1" class="block mb-2">Import from URL</Label>
-		<Input id="email" type="email" placeholder="[email protected]">
-		  <EnvelopeSolid slot="left" class="w-5 h-5 text-gray-500 dark:text-gray-400" />
+		<Input type="text" placeholder=""  bind:value={imageUrl}>
+		  <FilePasteSolid slot="left" class="w-5 h-5 text-gray-500 dark:text-gray-400" />
 		</Input>
 	  </div>
 	<p>Parameters</p>
 	<Range id="range-steps" min="0" max="1024" bind:value={stepValue} step="1" />
 	<p>Max output tokens: {stepValue}</p>
-	<div class="flex flex-row justify-between pb-4">
-		<button
-			class="text-#eee gap-2 bg-white  p-0 hover:bg-[white] flex flex-row items-center"
-			on:click={() => {
-				UseShow = !UseShow;
-			}}><Help />Help Centre</button
-		>
-		<div class="mt-4 flex space-x-3">
-			<Button color="light" class="dark:text-white">Cancel</Button>
-			<Button>Done</Button>
-		</div>
-	</div>
 </div>
-
-{#if UseShow}
-	<div class="mt-5 flex w-full flex-col gap-3 rounded-xl bg-white p-5">
-		<p>Terms of use</p>
-		<p>
-			By using this service, users are required to agree to the following terms:
-			The service is a research preview intended for non-commercial use only. It
-			only provides limited safety measures and may generate offensive content.
-			It must not be used for any illegal, harmful, violent, racist, or sexual
-			purposes. The service may collect user dialogue data for future research.
-			Please click the "Flag" button if you get any inappropriate answer! We
-			will collect those to keep improving our moderator. For an optimal
-			experience, please use desktop computers for this demo, as mobile devices
-			may compromise its quality.
-		</p>
-	</div>
-{/if}

VisualQnA/docker/ui/svelte/src/lib/modules/upload/uploadImg.svelte

@@ -4,50 +4,70 @@
 -->
 
 <script lang="ts">
+	import { base64ImageStore } from "$lib/shared/stores/common/Store";
 	import { Dropzone } from "flowbite-svelte";
 
 	let value = [];
+	export let imageUrl = "";
+
+	$: if (imageUrl) {
+		uploadImage();
+	}
+
+	const uploadImage = async () => {
+		try {
+			if (imageUrl.startsWith("http://") || imageUrl.startsWith("https://")) {
+				base64ImageStore.set(imageUrl);
+				console.log("Image URL Stored:", imageUrl);
+				return;
+			}
+		} catch (error) {
+			console.error("Error converting image to Base64:", error);
+		}
+	};
+
 	const dropHandle = (event) => {
-		value = [];
 		event.preventDefault();
 		if (event.dataTransfer.items) {
-			[...event.dataTransfer.items].forEach((item, i) => {
+			[...event.dataTransfer.items].forEach((item) => {
 				if (item.kind === "file") {
 					const file = item.getAsFile();
-					value.push(file.name);
-					value = value;
+					if (file) {
+						value = [file.name]; // Allow only one file selection
+						readFileAsBase64(file); // Convert to Base64
+					}
 				}
 			});
 		} else {
-			[...event.dataTransfer.files].forEach((file, i) => {
-				value = file.name;
+			[...event.dataTransfer.files].forEach((file) => {
+				value = [file.name]; // Allow only one file selection
+				readFileAsBase64(file); // Convert to Base64
 			});
 		}
-    console.log('dropHandle', value);
-
 	};
 
 	const handleChange = (event) => {
 		const files = event.target.files;
 		if (files.length > 0) {
-			value.push(files[0].name);
-			value = value;
+			value = [files[0].name]; // Allow only one file selection
+			readFileAsBase64(files[0]); // Convert to Base64
 		}
-    console.log('files', files);
+	};
 
+	const readFileAsBase64 = (file) => {
+		const reader = new FileReader();
+		reader.onload = () => {
+			const base64Data = reader.result; // Get Base64 data
+			base64ImageStore.set(base64Data); // Store the Base64 string in the store
+			imageUrl = URL.createObjectURL(file); // Keep the object URL for preview
+		};
+		reader.readAsDataURL(file); // Read the file as a Data URL
 	};
 
 	const showFiles = (files) => {
 		if (files.length === 1) return files[0];
-		let concat = "";
-		files.map((file) => {
-			concat += file;
-			concat += ",";
-			concat += " ";
-		});
-
-		if (concat.length > 40) concat = concat.slice(0, 40);
-		concat += "...";
+		let concat = files.join(", ");
+		if (concat.length > 40) concat = concat.slice(0, 40) + "...";
 		return concat;
 	};
 </script>
@@ -60,28 +80,33 @@
 	}}
 	on:change={handleChange}
 >
-	<svg
-		aria-hidden="true"
-		class="mb-3 h-10 w-10 text-gray-400"
-		fill="none"
-		stroke="currentColor"
-		viewBox="0 0 24 24"
-		xmlns="http://www.w3.org/2000/svg"
-		><path
-			stroke-linecap="round"
-			stroke-linejoin="round"
-			stroke-width="2"
-			d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"
-		/></svg
-	>
-	{#if value.length === 0}
-		<p class="mb-2 text-sm text-gray-500 dark:text-gray-400">
+	{#if value.length === 0 && !imageUrl}
+		<svg
+			aria-hidden="true"
+			class="mx-auto mb-3 h-10 w-10 text-gray-400"
+			fill="none"
+			stroke="currentColor"
+			viewBox="0 0 24 24"
+			xmlns="http://www.w3.org/2000/svg"
+		>
+			<path
+				stroke-linecap="round"
+				stroke-linejoin="round"
+				stroke-width="2"
+				d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"
+			/>
+		</svg>
+		<p class="mb-2 text-center text-sm text-gray-500 dark:text-gray-400">
 			<span class="font-semibold">Click to upload</span> or drag and drop
 		</p>
-		<p class="text-xs text-gray-500 dark:text-gray-400">
+		<p class="text-center text-xs text-gray-500 dark:text-gray-400">
 			SVG, PNG, JPG
 		</p>
-	{:else}
-		<p>{showFiles(value)}</p>
+	{:else if imageUrl}
+		<img src={imageUrl} alt="Uploaded Image" class="m-2 mx-auto block" />
 	{/if}
 </Dropzone>
+
+{#if value.length !== 0}
+	<p class="bg-white p-2 text-center">{showFiles(value)}</p>
+{/if}

VisualQnA/docker/ui/svelte/src/lib/network/chat/Network.ts

@@ -17,14 +17,32 @@ import { SSE } from "sse.js";
 
 const BACKEND_BASE_URL = env.BACKEND_BASE_URL;
 
-export async function fetchTextStream(query: string, knowledge_base_id: string, isCheckedStore: boolean) {
+export async function fetchTextStream(query: string, stepValueStore: number, base64ImageStore: string) {
 	let payload = {};
 	let url = "";
+	base64ImageStore = base64ImageStore.replace(/^data:[a-zA-Z]+\/[a-zA-Z]+;base64,/, "");
 
 	payload = {
-		messages: query,
-		stream: "True",
+		messages: [
+			{
+				role: "user",
+				content: [
+					{
+						type: "text",
+						text: query,
+					},
+					{
+						type: "image_url",
+						image_url: { url: base64ImageStore },
+					},
+				],
+			},
+		],
+		max_tokens: stepValueStore,
+		stream: true,
 	};
+	console.log("payload", payload);
+
 	url = `${BACKEND_BASE_URL}`;
 
 	return new SSE(url, {

VisualQnA/docker/ui/svelte/src/lib/shared/stores/common/Store.ts

@@ -39,3 +39,7 @@ export const knowledge1 = writable<{
 }>();
 
 export const knowledgeName = writable("");
+
+export const base64ImageStore = writable("");
+
+export const stepValueStore = writable(512);

VisualQnA/docker/ui/svelte/src/routes/+page.svelte

@@ -17,9 +17,11 @@
 <script lang="ts">
 	export let data;
 	import {
+	base64ImageStore,
 		ifStoreMsg,
 		isCheckedStore,
 		knowledge1,
+		stepValueStore,
 	} from "$lib/shared/stores/common/Store";
 	import { onMount } from "svelte";
 	import Header from "$lib/shared/components/header/header.svelte";
@@ -46,13 +48,9 @@
 	let query: string = "";
 	let loading: boolean = false;
 	let scrollToDiv: HTMLDivElement;
-	// ·········
 	let chatMessages: Message[] = data.chatMsg ? data.chatMsg : [];
 	console.log("chatMessages", chatMessages);
 
-	// ··············
-
-	$: knowledge_1 = $knowledge1?.id ? $knowledge1.id : "default";
 
 	onMount(async () => {
 		scrollToDiv = document
@@ -88,12 +86,16 @@
 	const callTextStream = async (query: string) => {
 		const eventSource = await fetchTextStream(
 			query,
-			knowledge_1,
-			$isCheckedStore
+			$stepValueStore,
+			$base64ImageStore
 		);
 
 		eventSource.addEventListener("message", (e: any) => {
+			console.log('e', e);
+
 			let Msg = e.data;
+			console.log('Msg', Msg);
+
 			if (Msg.startsWith("b")) {
 				let trimmedData = Msg.slice(2, -1);