Gpt 4 vision

src/playground/src/components/playground/ChatCard.tsx

@@ -3,21 +3,28 @@ import {
   Body1,
   Button,
   CardFooter,
-  Field,
   Textarea,
   Spinner,
-  shorthands
+  shorthands,
 } from "@fluentui/react-components";
 import { Dispatch, useEffect, useRef, useState } from "react";
-import { Delete24Regular, SendRegular } from "@fluentui/react-icons";
+import {
+  Delete24Regular,
+  SendRegular,
+  Attach24Regular,
+} from "@fluentui/react-icons";
 import { Message } from "./Message";
 import { Response } from "./Response";
 import { useEventDataContext } from "../../providers/EventDataProvider";
 import { Card } from "./Card";
 import { ChatResponseMessageExtended } from "../../pages/playground/Chat.state";
+import {
+  ChatMessageContentItem,
+  ChatMessageImageContentItem,
+} from "@azure/openai";
 
 interface CardProps {
-  onPromptEntered: Dispatch<string>;
+  onPromptEntered: Dispatch<ChatMessageContentItem[]>;
   messageList: ChatResponseMessageExtended[];
   onClear: () => void;
   isLoading: boolean;
@@ -28,12 +35,13 @@ const useStyles = makeStyles({
   dialog: {
     display: "block",
   },
+  buttonContainer: {
+    display: "flex",
+    justifyContent: "flex-end", // Align buttons to the right
+  },
   smallButton: {
-    width: "100%",
-    height: "40%",
-    maxWidth: "none",
-    textAlign: "left",
-    marginBottom: "12px"
+    marginBottom: "12px",
+    ...shorthands.margin("4px"),
   },
   startCard: {
     display: "flex",
@@ -43,7 +51,19 @@ const useStyles = makeStyles({
   },
   chatCard: {
     display: "flex",
-    height: "calc(100vh - 92px)",
+    height: "calc(100vh - 100px)",
+  },
+  wrapper: {
+    display: "flex",
+    flexDirection: "column",
+    justifyContent: "flex-end", // Ensure content sticks to the bottom
+    height: "120px",
+    maxHeight: "120px",
+  },
+  userQuery: {
+    flexGrow: 0, // Do not grow to fill remaining space
+    marginBottom: "10px",
+    overflowY: "auto", // Scroll if content overflows
   },
 });
 
@@ -66,16 +86,14 @@ export const ChatCard = ({
   }, [messageList]);
 
   return (
-    <Card header="Chat session" className={chat.chatCard} >
-
+    <Card header="Chat session" className={chat.chatCard}>
       {isAuthorized && (
         <>
           <div
             id={"chatContainer"}
             style={{ overflowY: "auto" }}
             ref={chatContainerRef}
           >
-
             {messageList.length > 1 ? (
               messageList.map((message, index) => {
                 if (message.role === "system") {
@@ -90,11 +108,12 @@ export const ChatCard = ({
             ) : (
               <Card className={chat.startCard}>
                 <Body1 style={{ textAlign: "center" }}>
-                  {!canChat && (<h2>Select a model</h2>)}
+                  {!canChat && <h2>Select a model</h2>}
                   {canChat && (
                     <>
                       <h2>Start chatting</h2>
-                      Test your assistant by sending queries below. Then adjust your assistant setup to improve the assistant's responses.
+                      Test your assistant by sending queries below. Then adjust
+                      your assistant setup to improve the assistant's responses.
                     </>
                   )}
                 </Body1>
@@ -132,60 +151,158 @@ function ChatInput({
   onClear,
   canChat,
 }: {
-  promptSubmitted: Dispatch<string>;
+  promptSubmitted: Dispatch<ChatMessageContentItem[]>;
   onClear: () => void;
   canChat: boolean;
 }) {
   const [userPrompt, setPrompt] = useState("");
+  const [files, setFiles] = useState<{ name: string; dataUrl: string }[]>([]);
+  const maxFiles = 10;
 
   const chat = useStyles();
+
+  // Handle file selection and convert files to base64 data URLs
+  const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const selectedFiles = Array.from(e.target.files || []);
+
+    if (selectedFiles.length + files.length > maxFiles) {
+      alert(`You can only upload up to ${maxFiles} files.`);
+      return;
+    }
+
+    try {
+      const newFiles = await Promise.all(
+        selectedFiles.map(
+          (file) =>
+            new Promise<{ name: string; dataUrl: string }>(
+              (resolve, reject) => {
+                const reader = new FileReader();
+
+                reader.onload = () => {
+                  const dataUrl = reader.result?.toString();
+                  if (dataUrl) {
+                    resolve({ name: file.name, dataUrl });
+                  } else {
+                    reject(new Error("Failed to read file data"));
+                  }
+                };
+
+                reader.onerror = () => {
+                  console.error(`Error reading file: ${file.name}`);
+                  reject(new Error(`Error reading file: ${file.name}`));
+                };
+
+                reader.readAsDataURL(file);
+              }
+            )
+        )
+      );
+
+      setFiles((prevFiles) => [...prevFiles, ...newFiles]);
+    } catch (error) {
+      console.error("Error encoding files to base64:", error);
+      alert("Error uploading files, please try again.");
+    } finally {
+      // Clear the file input value to allow re-uploading the same file
+      if (fileInputRef.current) {
+        fileInputRef.current.value = "";
+      }
+    }
+  };
+
+  const fileInputRef = useRef<HTMLInputElement | null>(null);
+
+  const triggerFileInput = () => {
+    if (fileInputRef.current) {
+      fileInputRef.current.click();
+    }
+  };
+
+  const handleSend = () => {
+    if (files.length > 0) {
+      const promptImageData: ChatMessageContentItem[] = [
+        { type: "text", text: userPrompt },
+        ...files.map(
+          (file): ChatMessageImageContentItem => ({
+            type: "image_url",
+            imageUrl: { url: file.dataUrl },
+          })
+        ),
+      ];
+      promptSubmitted(promptImageData);
+    } else {
+      promptSubmitted([{ type: "text", text: userPrompt }]);
+    }
+    setPrompt(""); // Clear the prompt
+  };
+
+  const clearAll = () => {
+    setFiles([]); // Clear all uploaded files
+    onClear();
+    setPrompt("");
+  };
+
   return (
-    <CardFooter style={{ height: "10vh" }}>
-      <Field className="user-query" style={{ width: "100%" }}>
-        <Textarea
-          value={userPrompt}
-          placeholder="Type user query here (Shift + Enter for new line)"
-          disabled={!canChat}
-          onChange={(event) => setPrompt(event.target.value)}
-          onKeyDown={(event) => {
-            if (
-              event.key === "Enter" &&
-              !event.shiftKey &&
-              hasPrompt(userPrompt)
-            ) {
-              promptSubmitted(userPrompt);
-              setPrompt("");
-              event.preventDefault();
-            }
-          }}
+    <div className={chat.wrapper}>
+      <Textarea
+        className={chat.wrapper}
+        value={userPrompt}
+        placeholder="Type user query here (Shift + Enter for new line)"
+        disabled={!canChat}
+        onChange={(event) => setPrompt(event.target.value)}
+        onKeyDown={(event) => {
+          if (
+            event.key === "Enter" &&
+            !event.shiftKey &&
+            hasPrompt(userPrompt)
+          ) {
+            handleSend();
+            setPrompt("");
+            event.preventDefault();
+          }
+        }}
+      />
+      <div className={chat.buttonContainer}>
+        {/* Hidden file input */}
+        <input
+          ref={fileInputRef}
+          id="upload-button"
+          type="file"
+          accept="image/png, image/jpeg, image/jpg, image/gif, image/webp"
+          onChange={handleFileChange}
+          style={{ display: "none" }} // This hides the file input
+          multiple // Enable multiple file selection
         />
-      </Field>
-      <div>
+
+        {/* Button to trigger the file input */}
         <Button
           className={chat.smallButton}
-          id={"send-button"}
-          icon={<SendRegular />}
+          id="upload-button"
+          icon={<Attach24Regular />}
           iconPosition="before"
-          appearance="primary"
-          onClick={() => {
-            promptSubmitted(userPrompt);
-            setPrompt("");
-          }}
-          disabled={!canChat || !hasPrompt(userPrompt)}
+          disabled={!canChat}
+          onClick={triggerFileInput}
         >
-          Send
+          {files?.length} of {maxFiles}
         </Button>
         <Button
           className={chat.smallButton}
           id="clear-button"
           disabled={!canChat}
           icon={<Delete24Regular />}
           iconPosition="before"
-          onClick={onClear}
-        >
-          Clear
-        </Button>
+          onClick={clearAll}
+        ></Button>
+        <Button
+          className={chat.smallButton}
+          id={"send-button"}
+          icon={<SendRegular />}
+          iconPosition="before"
+          appearance="primary"
+          onClick={handleSend} // Use handleSend function to check for file or prompt
+          disabled={!canChat || !hasPrompt(userPrompt)} // Disable if no input or file
+        ></Button>
       </div>
-    </CardFooter>
+    </div>
   );
 }

src/playground/src/components/playground/Message.tsx

@@ -13,6 +13,12 @@ const useStyles = makeStyles({
     marginBottom: "20px",
     maxWidth: "80%",
     marginLeft: "auto",
+    flexWrap: "wrap", // Enable wrapping for the images
+  },
+  image: {
+    maxWidth: "20%",
+    height: "auto",
+    ...shorthands.margin("2px") // Add spacing between the images manually
   },
   message: {
     fontSize: "medium",
@@ -26,21 +32,48 @@ const useStyles = makeStyles({
     ...shorthands.borderRadius("2px"),
   },
   icon: {
-    minWidth:"24px",
-    maxWidth:"24px",
-    width:"24px",
-    marginTop:"6px"
+    minWidth: "24px",
+    maxWidth: "24px",
+    width: "24px",
+    marginTop: "6px"
   }
 });
 
 
 export const Message = ({ message }: Props) => {
   const styles = useStyles();
 
+  const parsedPrompt = message.content ? JSON.parse(message.content) : null;
+
+  if (Array.isArray(parsedPrompt)) {
+    // Find the text content if it exists
+    const textContent = parsedPrompt.find(item => item.type === 'text')?.text;
+
+    // Find all image URLs
+    const imageUrls = parsedPrompt
+      .filter(item => item.type === 'image_url')
+      .map(item => item.imageUrl?.url);
+
+    return (
+      <div className={styles.container}>
+        {textContent && <div className={styles.message}>{textContent}</div>}
+        {imageUrls.map((url, index) => (
+          <img
+            key={index}
+            src={url}
+            alt={`Message image ${index + 1}`}
+            className={styles.image}// Restrict image width
+          />
+        ))}
+        <Person32Regular className={styles.icon} />
+      </div>
+    );
+  }
+
   return (
     <div className={styles.container}>
       <div className={styles.message}>{message.content}</div>
-      <Person32Regular className={styles.icon}/>
+      <Person32Regular className={styles.icon} />
     </div>
   );
 };