feat(vision): show images in a gallery, expandable images (#4407)

* feat(vision): show images in a gallery, make images expandable * fix redacted * remove mask
Arize-ai · Aug 27, 2024 · 9e2d67f · 9e2d67f
1 parent d839fc4
commit 9e2d67f
Show file tree

Hide file tree

Showing 2 changed files with 143 additions and 19 deletions.
diff --git a/app/src/pages/trace/SpanDetails.tsx b/app/src/pages/trace/SpanDetails.tsx
@@ -90,6 +90,7 @@ import { EditSpanAnnotationsButton } from "./EditSpanAnnotationsButton";
 import { SpanAside } from "./SpanAside";
 import { SpanCodeDropdown } from "./SpanCodeDropdown";
 import { SpanFeedback } from "./SpanFeedback";
+import { SpanImage } from "./SpanImage";
 import { SpanToDatasetExampleDialog } from "./SpanToDatasetExampleDialog";
 
 /**
@@ -1392,6 +1393,13 @@ function LLMPromptsList({ prompts }: { prompts: string[] }) {
   );
 }
 
+const messageContentListCSS = css`
+  display: flex;
+  flex-direction: row;
+  gap: var(--ac-global-dimension-size-200);
+  flex-wrap: wrap;
+`;
+
 /**
  * A list of message contents. Used for multi-modal models.
  */
@@ -1401,37 +1409,32 @@ function MessageContentsList({
   messageContents: AttributeMessageContent[];
 }) {
   return (
-    <ul
-      css={css`
-        display: flex;
-        flex-direction: column;
-        gap: var(--ac-global-dimension-size-100);
-      `}
-    >
+    <ul css={messageContentListCSS} data-testid="message-content-list">
       {messageContents.map((messageContent, idx) => {
         return (
-          <li key={idx}>
-            <MessageContent messageContentAttribute={messageContent} />
-          </li>
+          <MessageContentListItem
+            key={idx}
+            messageContentAttribute={messageContent}
+          />
         );
       })}
     </ul>
   );
 }
 
-const imageCSS = css`
-  max-width: 100%;
-  max-height: 100%;
-  object-fit: cover;
+/**
+ * Display text content in full width.
+ */
+const messageContentTextListItemCSS = css`
+  flex: 1 1 100%;
 `;
-
 /**
  * Displays multi-modal message content. Typically an image or text.
  * Examples:
  * {"message_content":{"text":"What is in this image?","type":"text"}}
  * {"message_content":{"type":"image","image":{"image":{"url":"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}}}}
  */
-function MessageContent({
+function MessageContentListItem({
   messageContentAttribute,
 }: {
   messageContentAttribute: AttributeMessageContent;
@@ -1442,7 +1445,7 @@ function MessageContent({
   const imageUrl = image?.image?.url;
 
   return (
-    <Flex direction="column">
+    <li css={text ? messageContentTextListItemCSS : null}>
       {text ? (
         <pre
           css={css`
@@ -1454,8 +1457,8 @@ function MessageContent({
           {text}
         </pre>
       ) : null}
-      {imageUrl ? <img src={imageUrl} css={imageCSS} /> : null}
-    </Flex>
+      {imageUrl ? <SpanImage url={imageUrl} /> : null}
+    </li>
   );
 }
 

diff --git a/app/src/pages/trace/SpanImage.tsx b/app/src/pages/trace/SpanImage.tsx
@@ -0,0 +1,121 @@
+import React, { ReactNode, useState } from "react";
+import { css } from "@emotion/react";
+
+import { Button, classNames, Icon, Icons } from "@arizeai/components";
+
+type SpanImageProps = {
+  /**
+   * The url of the image. Can be either be a data URL, a URL or a redacted string
+   */
+
+  url: string;
+};
+
+function isRedactedUrl(url: string) {
+  return url === "__REDACTED__";
+}
+
+const imageContainerCSS = css`
+  position: relative;
+  overflow: hidden;
+  width: 400px;
+  height: 200px;
+  border-radius: var(--ac-global-rounding-small);
+  border: 1px solid var(--ac-global-color-grey-500);
+  background-color: var(--ac-global-color-grey-200);
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  transition:
+    width 0.1s ease-in-out,
+    height 0.1s ease-in-out opacity 0.1s ease-in-out;
+  button {
+    position: absolute;
+    right: var(--ac-global-dimension-size-100);
+    top: var(--ac-global-dimension-size-100);
+    z-index: 1;
+    opacity: 0;
+  }
+  &:hover button {
+    opacity: 1;
+  }
+  img {
+    width: inherit;
+    height: inherit;
+    object-fit: contain;
+  }
+
+  &.is-expanded {
+    width: 100%;
+    height: 100%;
+    img {
+      object-fit: cover;
+    }
+  }
+`;
+/**
+ * Displays an image attribute of a span.
+ */
+export function SpanImage(props: SpanImageProps) {
+  const [isExpanded, setIsExpanded] = useState(false);
+  let content: ReactNode;
+  const isRedacted = isRedactedUrl(props.url);
+  if (isRedacted) {
+    content = <RedactedImageSVG />;
+  } else {
+    content = <img src={props.url} />;
+  }
+  return (
+    <div
+      className={classNames({
+        "is-expanded": isExpanded,
+      })}
+      css={imageContainerCSS}
+    >
+      {!isRedacted && (
+        <Button
+          variant="default"
+          size="compact"
+          onClick={() => setIsExpanded(!isExpanded)}
+          icon={
+            <Icon
+              svg={
+                isExpanded ? <Icons.CollapseOutline /> : <Icons.ExpandOutline />
+              }
+            />
+          }
+          aria-label="Expand / Collapse Image"
+        />
+      )}
+      {content}
+    </div>
+  );
+}
+
+const RedactedImageSVG = () => (
+  <svg
+    width="130"
+    height="130"
+    viewBox="0 0 130 130"
+    fill="none"
+    xmlns="http://www.w3.org/2000/svg"
+  >
+    <rect
+      x="1.5"
+      y="1.5"
+      width="127"
+      height="127"
+      rx="6.5"
+      strokeOpacity="0.8"
+      strokeWidth="3"
+    />
+    <path
+      fillRule="evenodd"
+      clipRule="evenodd"
+      d="M40 27.5H90C96.8917 27.5 102.5 33.1083 102.5 40V90C102.5 96.8917 96.8917 102.5 90 102.5H40C33.1083 102.5 27.5 96.8917 27.5 90V40C27.5 33.1083 33.1083 27.5 40 27.5ZM40 35.8333H90C92.3 35.8333 94.1667 37.7 94.1667 40V74.85L80.8208 63.4667C76.6958 59.9583 70.2417 59.9583 66.1542 63.4417L35.8333 88.7417V40C35.8333 37.7 37.7 35.8333 40 35.8333ZM54.5833 50.4167C54.5833 53.8667 51.7833 56.6667 48.3333 56.6667C44.8833 56.6667 42.0833 53.8667 42.0833 50.4167C42.0833 46.9667 44.8833 44.1667 48.3333 44.1667C51.7833 44.1667 54.5833 46.9667 54.5833 50.4167ZM42.3375 94.1667H90C92.3 94.1667 94.1667 92.3 94.1667 90V85.8083L75.4125 69.8083C74.4083 68.9458 72.55 68.9417 71.525 69.8125L42.3375 94.1667Z"
+      fill="var(--ac-global-color-grey-300)"
+    />
+  </svg>
+);