From 9f77176e20576afb74a650d93c61c9e6ad66675f Mon Sep 17 00:00:00 2001 From: Xiaowei Zhu <33129495+zhu-xiaowei@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:29:32 +0800 Subject: [PATCH] feat: support image, document, and video understanding for Amazon Nova --- README.md | 13 +- README_CN.md | 11 +- react-native/android/app/build.gradle | 4 +- react-native/ios/Podfile.lock | 12 +- .../ios/SwiftChat.xcodeproj/project.pbxproj | 8 +- react-native/ios/SwiftChat/Info.plist | 4 +- .../ios/SwiftChat/PrivacyInfo.xcprivacy | 1 + react-native/package-lock.json | 60 +++-- react-native/package.json | 7 +- react-native/src/assets/play.png | Bin 0 -> 866 bytes react-native/src/chat/ChatScreen.tsx | 31 +-- .../chat/component/CustomAddFileComponent.tsx | 131 ++++++++--- .../src/chat/component/CustomChatFooter.tsx | 9 +- .../component/CustomFileListComponent.tsx | 211 +++++++++++++++--- .../chat/component/CustomMarkdownRenderer.tsx | 4 +- .../chat/component/CustomMessageComponent.tsx | 4 + .../chat/component/CustomSendComponent.tsx | 11 +- .../{ProgressBar.tsx => ImageProgressBar.tsx} | 4 +- .../src/chat/util/BedrockMessageConvertor.ts | 28 ++- react-native/src/chat/util/FileUtils.ts | 82 ++++++- react-native/src/types/Chat.ts | 3 + server/src/main.py | 7 +- server/src/requirements.txt | 4 +- 23 files changed, 512 insertions(+), 137 deletions(-) create mode 100644 react-native/src/assets/play.png rename react-native/src/chat/component/{ProgressBar.tsx => ImageProgressBar.tsx} (98%) diff --git a/README.md b/README.md index a50dd58..e77c89f 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,20 @@ macOS platforms. - Real-time streaming chat with AI - AI image generation with progress -- Multimodal support (camera, photos & documents) +- Multimodal support (images, videos & documents) - Conversation history list view and management - Cross-platform support (Android, iOS, macOS) - Tablet-optimized for iPad and Android tablets - Fast launch and responsive performance - Multiple AI model support and switching +**New Features For Amazon Nova 🎉🎉🎉** + +- Stream conversations with Amazon Nova Micro, Lite and Pro +- Understand images, documents and videos with Nova Lite and Pro +- Record 30-second videos directly on Android and iOS for Nova analysis +- Upload large videos (1080p/4K) beyond 8MB with auto compression + ## Architecture ![](/assets/architecture.png) @@ -85,8 +92,8 @@ can find the **API URL** which looks like: `https://xxx.xxx.awsapprunner.com` or ### Step 3: Download the app and setup with API URL and API Key 1. Download the App - - Android App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.5.0/SwiftChat.apk) - - macOS App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.5.0/SwiftChat.dmg) + - Android App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.apk) + - macOS App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.dmg) - iOS (Currently we do not provide the iOS version, you can build it locally with Xcode) 2. Launch the App, open the drawer menu, and tap **Settings**. diff --git a/README_CN.md b/README_CN.md index d8bfc54..6c921d6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -19,6 +19,13 @@ macOS 等多个平台。 - 快速启动和响应性能 - 支持多种 AI 模型及切换 +**Amazon Nova 新功能 🎉🎉🎉** + +- 支持与 Amazon Nova Micro、Lite 和 Pro 进行流式对话 +- 支持 Nova Lite 和 Pro 对图片、文档及视频内容的理解 +- 支持直接在安卓和 iOS 设备上录制最长 30 秒的视频供 Nova 分析 +- 支持自动压缩上传超过8MB的高清视频（1080p/4K） + ## 架构 ![](/assets/architecture.png) @@ -76,8 +83,8 @@ macOS 等多个平台。 ### 第3步: 下载应用并设置 API URL 和 API Key 1. 下载应用 - - Android 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.5.0/SwiftChat.apk) - - macOS 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.5.0/SwiftChat.dmg) + - Android 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.apk) + - macOS 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.dmg) - iOS (目前不提供 iOS 版本，您可以使用 Xcode 在本地构建) 2. 启动应用，点击左侧菜单按钮，并点击底部的 **Settings**。 diff --git a/react-native/android/app/build.gradle b/react-native/android/app/build.gradle index f621bfa..e82ad7b 100644 --- a/react-native/android/app/build.gradle +++ b/react-native/android/app/build.gradle @@ -79,8 +79,8 @@ android { applicationId "com.aws.swiftchat" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 10 - versionName "1.5.0" + versionCode 11 + versionName "1.6.0" ndk { //noinspection ChromeOsAbiSupport abiFilters 'arm64-v8a' diff --git a/react-native/ios/Podfile.lock b/react-native/ios/Podfile.lock index 09d3bcb..a45225a 100644 --- a/react-native/ios/Podfile.lock +++ b/react-native/ios/Podfile.lock @@ -938,7 +938,7 @@ PODS: - React-Mapbuffer (0.74.1): - glog - React-debug - - react-native-compressor (1.9.0): + - react-native-compressor (1.10.1): - DoubleConversion - glog - hermes-engine @@ -982,7 +982,7 @@ PODS: - Yoga - react-native-get-random-values (1.11.0): - React-Core - - react-native-image-picker (7.1.2): + - react-native-image-picker (7.2.2): - DoubleConversion - glog - hermes-engine @@ -1330,7 +1330,7 @@ PODS: - Yoga - RNShare (10.2.1): - React-Core - - RNSVG (15.4.0): + - RNSVG (15.10.1): - React-Core - SocketRocket (0.7.0) - Yoga (0.0.0) @@ -1588,10 +1588,10 @@ SPEC CHECKSUMS: React-jsitracing: 233d1a798fe0ff33b8e630b8f00f62c4a8115fbc React-logger: 7e7403a2b14c97f847d90763af76b84b152b6fce React-Mapbuffer: 11029dcd47c5c9e057a4092ab9c2a8d10a496a33 - react-native-compressor: 6e5044b1c065421bcc2bfb89579ab30f51803e9a + react-native-compressor: 2ae9013718fb351264fcfcdf232eccbbf3d280a2 react-native-document-picker: c4f197741c327270453aa9840932098e0064fd52 react-native-get-random-values: 21325b2244dfa6b58878f51f9aa42821e7ba3d06 - react-native-image-picker: c3afe5472ef870d98a4b28415fc0b928161ee5f7 + react-native-image-picker: dd85e2530d366acf77745830b053294afed66339 react-native-mmkv: 8c9a677e64a1ac89b0c6cf240feea528318b3074 react-native-safe-area-context: b7daa1a8df36095a032dff095a1ea8963cb48371 React-nativeconfig: b0073a590774e8b35192fead188a36d1dca23dec @@ -1625,7 +1625,7 @@ SPEC CHECKSUMS: RNReanimated: f4ff116e33e0afc3d127f70efe928847c7c66355 RNScreens: 5aeecbb09aa7285379b6e9f3c8a3c859bb16401c RNShare: 0fad69ae2d71de9d1f7b9a43acf876886a6cb99c - RNSVG: cb24fb322de8c1ebf59904e7aca0447bb8dbed5a + RNSVG: 7ff26379b2d1871b8571e6f9bc9630de6baf9bdf SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d Yoga: 348f8b538c3ed4423eb58a8e5730feec50bce372 diff --git a/react-native/ios/SwiftChat.xcodeproj/project.pbxproj b/react-native/ios/SwiftChat.xcodeproj/project.pbxproj index 86211a4..f0cfdae 100644 --- a/react-native/ios/SwiftChat.xcodeproj/project.pbxproj +++ b/react-native/ios/SwiftChat.xcodeproj/project.pbxproj @@ -485,7 +485,7 @@ CODE_SIGN_ENTITLEMENTS = SwiftChat/SwiftChat.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 10; + CURRENT_PROJECT_VERSION = 11; DEVELOPMENT_TEAM = BUA6W9H7T3; ENABLE_BITCODE = NO; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; @@ -497,7 +497,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.5.0; + MARKETING_VERSION = 1.6.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", @@ -526,7 +526,7 @@ CODE_SIGN_ENTITLEMENTS = SwiftChat/SwiftChat.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 10; + CURRENT_PROJECT_VERSION = 11; DEVELOPMENT_TEAM = BUA6W9H7T3; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; INFOPLIST_FILE = SwiftChat/Info.plist; @@ -537,7 +537,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.5.0; + MARKETING_VERSION = 1.6.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", diff --git a/react-native/ios/SwiftChat/Info.plist b/react-native/ios/SwiftChat/Info.plist index 1cc84ed..3670a34 100644 --- a/react-native/ios/SwiftChat/Info.plist +++ b/react-native/ios/SwiftChat/Info.plist @@ -34,7 +34,9 @@ NSCameraUsageDescription - Support take camera and summarize it + Support take photos and summarize it + NSMicrophoneUsageDescription + Support record videos and summarize it NSPhotoLibraryUsageDescription Support choose pictures and summarize it UILaunchStoryboardName diff --git a/react-native/ios/SwiftChat/PrivacyInfo.xcprivacy b/react-native/ios/SwiftChat/PrivacyInfo.xcprivacy index 41b8317..d634def 100644 --- a/react-native/ios/SwiftChat/PrivacyInfo.xcprivacy +++ b/react-native/ios/SwiftChat/PrivacyInfo.xcprivacy @@ -9,6 +9,7 @@ NSPrivacyAccessedAPICategoryFileTimestamp NSPrivacyAccessedAPITypeReasons + 3B52.1 C617.1 diff --git a/react-native/package-lock.json b/react-native/package-lock.json index 9428589..cca17be 100644 --- a/react-native/package-lock.json +++ b/react-native/package-lock.json @@ -15,7 +15,7 @@ "react": "18.2.0", "react-native": "0.74.1", "react-native-code-highlighter": "^1.2.2", - "react-native-compressor": "^1.9.0", + "react-native-compressor": "^1.10.1", "react-native-dialog": "^9.3.0", "react-native-document-picker": "^9.3.1", "react-native-element-dropdown": "^2.12.1", @@ -26,11 +26,12 @@ "react-native-get-random-values": "^1.11.0", "react-native-gifted-chat": "^2.4.0", "react-native-haptic-feedback": "^2.2.0", - "react-native-image-picker": "^7.1.2", + "react-native-image-picker": "^7.2.2", "react-native-image-viewing": "^0.2.2", "react-native-marked": "^6.0.4", "react-native-mmkv": "^2.12.2", "react-native-polyfill-globals": "^3.1.0", + "react-native-progress": "^5.0.1", "react-native-reanimated": "^3.14.0", "react-native-safe-area-context": "^4.10.8", "react-native-screens": "^3.32.0", @@ -14343,9 +14344,9 @@ "integrity": "sha512-5+C0X9mopI0+qxyQHzOPEi5v5rxNBQjxydPPiKMQSlX1RBIcJ8uTcqUPssQ9Mo8p6c1IKIWJUSqCj4jAmD0qVQ==" }, "node_modules/react-native-compressor": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/react-native-compressor/-/react-native-compressor-1.9.0.tgz", - "integrity": "sha512-kJgRBTrvI8/yOJs8GhXfc1bZui/Wl3FNKOkc3K5+EIgihzU/vS+UTLkojGcYwoYXxj4xP7GFwp0SJLsNOOJZNw==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/react-native-compressor/-/react-native-compressor-1.10.1.tgz", + "integrity": "sha512-rbwg4q04VfwnmSxLRv7T5tk2Rq86MpTzDh66Sa6ocN5cImeq7W9jqbFu4TMCq4EkyjoX9hLS5VLNayuLlDPVEQ==", "engines": { "node": ">= 16.0.0" }, @@ -14510,9 +14511,9 @@ } }, "node_modules/react-native-image-picker": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/react-native-image-picker/-/react-native-image-picker-7.1.2.tgz", - "integrity": "sha512-b5y5nP60RIPxlAXlptn2QwlIuZWCUDWa/YPUVjgHc0Ih60mRiOg1PSzf0IjHSLeOZShCpirpvSPGnDExIpTRUg==", + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/react-native-image-picker/-/react-native-image-picker-7.2.2.tgz", + "integrity": "sha512-hcSaD/ohDac1ooDbZyKltfmOEAWrQOrPbusKEfbJQ7EfGveBEIF5wfmZsCjXRc0HNpGdQ71P1x2JZckkqu7IRw==", "peerDependencies": { "react": "*", "react-native": "*" @@ -15352,6 +15353,17 @@ "web-streams-polyfill": "*" } }, + "node_modules/react-native-progress": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/react-native-progress/-/react-native-progress-5.0.1.tgz", + "integrity": "sha512-TYfJ4auAe5vubDma2yfFvt7ktSI+UCfysqJnkdHEcLXqAitRFOozgF/cLgN5VNi/iLdaf3ga1ETi2RF4jVZ/+g==", + "dependencies": { + "prop-types": "^15.7.2" + }, + "peerDependencies": { + "react-native-svg": "*" + } + }, "node_modules/react-native-reanimated": { "version": "3.14.0", "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.14.0.tgz", @@ -15403,9 +15415,9 @@ } }, "node_modules/react-native-svg": { - "version": "15.4.0", - "resolved": "https://registry.npmjs.org/react-native-svg/-/react-native-svg-15.4.0.tgz", - "integrity": "sha512-zkBEbme/Dba4yqreg/oI2P6/6LrLywWY7HhaSwpU7Pb5COpTd2fV6/ShsgZz8GRFFdidUPwWmx01FITUsjhkmw==", + "version": "15.10.1", + "resolved": "https://registry.npmjs.org/react-native-svg/-/react-native-svg-15.10.1.tgz", + "integrity": "sha512-Hqz/doQciVFK/Df2v+wsW96oY5jxlta7rZ31KQYo78dlgvAHEaGr6paEOAMvlIruw7EHNQ0Vc1ZmJPJF2kfIPQ==", "dependencies": { "css-select": "^5.1.0", "css-tree": "^1.1.3", @@ -28858,9 +28870,9 @@ "integrity": "sha512-5+C0X9mopI0+qxyQHzOPEi5v5rxNBQjxydPPiKMQSlX1RBIcJ8uTcqUPssQ9Mo8p6c1IKIWJUSqCj4jAmD0qVQ==" }, "react-native-compressor": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/react-native-compressor/-/react-native-compressor-1.9.0.tgz", - "integrity": "sha512-kJgRBTrvI8/yOJs8GhXfc1bZui/Wl3FNKOkc3K5+EIgihzU/vS+UTLkojGcYwoYXxj4xP7GFwp0SJLsNOOJZNw==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/react-native-compressor/-/react-native-compressor-1.10.1.tgz", + "integrity": "sha512-rbwg4q04VfwnmSxLRv7T5tk2Rq86MpTzDh66Sa6ocN5cImeq7W9jqbFu4TMCq4EkyjoX9hLS5VLNayuLlDPVEQ==", "requires": {} }, "react-native-dialog": { @@ -28980,9 +28992,9 @@ "requires": {} }, "react-native-image-picker": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/react-native-image-picker/-/react-native-image-picker-7.1.2.tgz", - "integrity": "sha512-b5y5nP60RIPxlAXlptn2QwlIuZWCUDWa/YPUVjgHc0Ih60mRiOg1PSzf0IjHSLeOZShCpirpvSPGnDExIpTRUg==", + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/react-native-image-picker/-/react-native-image-picker-7.2.2.tgz", + "integrity": "sha512-hcSaD/ohDac1ooDbZyKltfmOEAWrQOrPbusKEfbJQ7EfGveBEIF5wfmZsCjXRc0HNpGdQ71P1x2JZckkqu7IRw==", "requires": {} }, "react-native-image-viewing": { @@ -29651,6 +29663,14 @@ "integrity": "sha512-6ACmV1SjXvZP2LN6J2yK58yNACKddcvoiKLrSQdISx32IdYStfdmGXrbAfpd+TANrTlIaZ2SLoFXohNwhnqm/w==", "requires": {} }, + "react-native-progress": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/react-native-progress/-/react-native-progress-5.0.1.tgz", + "integrity": "sha512-TYfJ4auAe5vubDma2yfFvt7ktSI+UCfysqJnkdHEcLXqAitRFOozgF/cLgN5VNi/iLdaf3ga1ETi2RF4jVZ/+g==", + "requires": { + "prop-types": "^15.7.2" + } + }, "react-native-reanimated": { "version": "3.14.0", "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.14.0.tgz", @@ -29687,9 +29707,9 @@ "integrity": "sha512-Z2LWGYWH7raM4H6Oauttv1tEhaB43XSWJAN8iS6oaSG9CnyrUBeYFF4QpU1AH5RgNeylXQdN8CtbizCHHt6coQ==" }, "react-native-svg": { - "version": "15.4.0", - "resolved": "https://registry.npmjs.org/react-native-svg/-/react-native-svg-15.4.0.tgz", - "integrity": "sha512-zkBEbme/Dba4yqreg/oI2P6/6LrLywWY7HhaSwpU7Pb5COpTd2fV6/ShsgZz8GRFFdidUPwWmx01FITUsjhkmw==", + "version": "15.10.1", + "resolved": "https://registry.npmjs.org/react-native-svg/-/react-native-svg-15.10.1.tgz", + "integrity": "sha512-Hqz/doQciVFK/Df2v+wsW96oY5jxlta7rZ31KQYo78dlgvAHEaGr6paEOAMvlIruw7EHNQ0Vc1ZmJPJF2kfIPQ==", "requires": { "css-select": "^5.1.0", "css-tree": "^1.1.3", diff --git a/react-native/package.json b/react-native/package.json index 18006bf..0938357 100644 --- a/react-native/package.json +++ b/react-native/package.json @@ -1,7 +1,7 @@ { "name": "swift-chat", "description": "Sample Bedrock Cross-platform App - SwiftChat", - "version": "1.5.0", + "version": "1.6.0", "private": true, "scripts": { "android": "react-native run-android", @@ -19,7 +19,7 @@ "react": "18.2.0", "react-native": "0.74.1", "react-native-code-highlighter": "^1.2.2", - "react-native-compressor": "^1.9.0", + "react-native-compressor": "^1.10.1", "react-native-dialog": "^9.3.0", "react-native-document-picker": "^9.3.1", "react-native-element-dropdown": "^2.12.1", @@ -30,11 +30,12 @@ "react-native-get-random-values": "^1.11.0", "react-native-gifted-chat": "^2.4.0", "react-native-haptic-feedback": "^2.2.0", - "react-native-image-picker": "^7.1.2", + "react-native-image-picker": "^7.2.2", "react-native-image-viewing": "^0.2.2", "react-native-marked": "^6.0.4", "react-native-mmkv": "^2.12.2", "react-native-polyfill-globals": "^3.1.0", + "react-native-progress": "^5.0.1", "react-native-reanimated": "^3.14.0", "react-native-safe-area-context": "^4.10.8", "react-native-screens": "^3.32.0", diff --git a/react-native/src/assets/play.png b/react-native/src/assets/play.png new file mode 100644 index 0000000000000000000000000000000000000000..2339e5bfee97986a851507ecef7c881e6fc671c7 GIT binary patch literal 866 zcmeAS@N?(olHy`uVBq!ia0vp^2_VeD3?#3*wSy#D0(?STfBg6XWF9$kC^l7 z@4tBQ;*A?Owr$(Cd-v}9_wR4sym`%EnBsac^uda<@Bw z?ed|O3+K(M%LwrIw6@UG(vlKj`Saq#llwou-2OFngZ3W=2BsuW7srr{ds{BC79DmF zaJjf`x=Z1iB%_v+|G!foPw>3fx8;|3MW}IaXW*)i?eDy-nu;Hun7X@{+tlzBLqMat zWM0xG?kfcjt=VnYZdiphHynBTqGPvb{(<-dD_ZY!cgRRQcS%v0te|Qv6p$;?vC%h` zkA=xrs8KPX`o+xf*$hvG_C`;;>!4?#FtMERrA+gz<^7EFlN0XLv;5#`POh__P!!Rz zJlDaled@EHwF^WxOwoAH_*Ljy)U|R3ezDJajSOMj>l(cdWG&joWVYeXGocd_I>$G0 zI~=O|B-3HwbW!p_Ld%@`1IIOf)*oP6yyXY0P>IQH`^Ekz`5!D=oO$`pe24RY*+k44 zX5HmJQCQ04B*Cz2yX1;i^@Lou7t5}Mui4LVw5Ktxe97J>OpWSj@1C z`KjMLHWfhy-!Q)g5w2f?gx2*ug(+dk}FuN(LBD%mg z?2sgziG08MYli^FTPdtR0vyU+vhQ#e{K&a>rck1#Q6tJlv3rq8=cGQazyp8ZxBEBz zv-#5EpKwy{e0(|IiS-?}Urg&$%vOD`@wu5UeZ&2<;JeA|xwd_f`?)Du`*-{GuSV&w zt=MJQW-B+HDXls_VP>=4B0cBEZ3>^h>Imq&$j7t3^5FRJG)!TC%b|MhFLOI4)H^n8 zxyZt#GA$t2q_f4#^}(Jk7gdxlOxw6iVEbB@yF44;%G{l>N!RqQ$d+8++cU1NO}#C% z{cWxFf(^=I)+(El^{iJ|+h^H`P5UPJ{n3G4%TMf$lX|i7+XQj*s&(^zJe~RY>wMk` XJZ*owRdo&n6DfnItDnm{r-UW|5h1** literal 0 HcmV?d00001 diff --git a/react-native/src/chat/ChatScreen.tsx b/react-native/src/chat/ChatScreen.tsx index e859d78..023023a 100644 --- a/react-native/src/chat/ChatScreen.tsx +++ b/react-native/src/chat/ChatScreen.tsx @@ -31,7 +31,6 @@ import { ChatMode, ChatStatus, FileInfo, - FileType, IMessageWithToken, Usage, } from '../types/Chat.ts'; @@ -48,9 +47,14 @@ import { HapticFeedbackTypes } from 'react-native-haptic-feedback/src/types.ts'; import { clearCachedNode } from './component/CustomMarkdownRenderer.tsx'; import { isMac } from '../App.tsx'; import { CustomChatFooter } from './component/CustomChatFooter.tsx'; -import { checkFileNumberLimit } from './util/FileUtils.ts'; +import { + checkFileNumberLimit, + getFileTypeSummary, + isAllFileReady, +} from './util/FileUtils.ts'; import HeaderTitle from './component/HeaderTitle.tsx'; import { HeaderOptions } from '@react-navigation/elements/src/types.tsx'; +import Toast from 'react-native-toast-message'; const BOT_ID = 2; @@ -411,18 +415,16 @@ function ChatScreen(): React.JSX.Element { // handle onSend const onSend = useCallback((message: IMessage[] = []) => { const files = selectedFilesRef.current; + if (!isAllFileReady(files)) { + Toast.show({ + type: 'info', + text1: 'please wait for all videos to be ready', + }); + return; + } if (message[0]?.text || files.length > 0) { if (!message[0]?.text) { - const hasImages = files.some(file => file.type === FileType.image); - const hasDocs = files.some(file => file.type === FileType.document); - message[0].text = - files.length === 1 - ? 'Summarize this' - : hasImages && hasDocs - ? 'Summarize these docs and images' - : hasImages - ? 'Summarize these images' - : 'Summarize these docs'; + message[0].text = getFileTypeSummary(files); } if (selectedFilesRef.current.length > 0) { message[0].image = JSON.stringify(selectedFilesRef.current); @@ -493,10 +495,11 @@ function ChatScreen(): React.JSX.Element { selectedFiles.length > 0 && ( { - if (isDelete) { + onFileUpdated={(files, isUpdate) => { + if (isUpdate) { setSelectedFiles(files); } else { + console.log('handleNewFileSelected'); handleNewFileSelected(files); } }} diff --git a/react-native/src/chat/component/CustomAddFileComponent.tsx b/react-native/src/chat/component/CustomAddFileComponent.tsx index 8e00ce2..eda6f74 100644 --- a/react-native/src/chat/component/CustomAddFileComponent.tsx +++ b/react-native/src/chat/component/CustomAddFileComponent.tsx @@ -1,5 +1,5 @@ import { Actions } from 'react-native-gifted-chat'; -import { Image, StyleSheet, Text } from 'react-native'; +import { Image, Platform, StyleSheet, Text } from 'react-native'; import React from 'react'; import { ImagePickerResponse, @@ -10,8 +10,14 @@ import { FileInfo, FileType } from '../../types/Chat.ts'; import { pick, types } from 'react-native-document-picker'; import Toast from 'react-native-toast-message'; import { saveFile } from '../util/FileUtils.ts'; -import { Image as Img } from 'react-native-compressor'; +import { + createVideoThumbnail, + getImageMetaData, + getVideoMetaData, + Image as Img, +} from 'react-native-compressor'; import { isMac } from '../../App.tsx'; +import { getTextModel } from '../../storage/StorageUtils.ts'; interface CustomRenderActionsProps { onFileSelected: (files: FileInfo[]) => void; @@ -33,19 +39,23 @@ export const CustomAddFileComponent: React.FC = ({ }) => { const handleChooseFiles = async () => { try { + const chooseType = [ + types.images, + types.pdf, + types.csv, + types.doc, + types.docx, + types.xls, + types.xlsx, + types.plainText, + 'public.html', + ]; + if (isVideoSupported()) { + chooseType.push(types.video); + } const pickResults = await pick({ allowMultiSelection: true, - type: [ - types.images, - types.pdf, - types.csv, - types.doc, - types.docx, - types.xls, - types.xlsx, - types.plainText, - 'public.html', - ], + type: chooseType, }); const files: FileInfo[] = []; await Promise.all( @@ -53,7 +63,7 @@ export const CustomAddFileComponent: React.FC = ({ if (pickResult.name && pickResult.uri) { const fileName = getFileNameWithoutExtension(pickResult.name); const fileNameArr = pickResult.name.split('.'); - const format = fileNameArr[fileNameArr.length - 1]; + let format = fileNameArr[fileNameArr.length - 1].toLowerCase(); const fileType = getFileType(format); if (fileType === FileType.unSupported) { const msg = 'Selected UnSupported Files format: .' + format; @@ -70,21 +80,48 @@ export const CustomAddFileComponent: React.FC = ({ } let localFileUrl: string | null; if (fileType === FileType.image) { - const compressUri = await Img.compress(decodeURI(pickResult.uri)); - localFileUrl = await saveFile(compressUri, pickResult.name); + pickResult.uri = decodeURI(pickResult.uri); + if (format === 'png' || format === 'jpg' || format === 'jpeg') { + pickResult.uri = await Img.compress(pickResult.uri); + const metaData = await getImageMetaData(pickResult.uri); + format = metaData.extension; + } + localFileUrl = await saveFile(pickResult.uri, pickResult.name); + } else if (fileType === FileType.video) { + localFileUrl = pickResult.uri; } else { localFileUrl = await saveFile( decodeURI(pickResult.uri), pickResult.name ); } + + let thumbnailUrl; + let width = 0; + let height = 0; + if (fileType === FileType.video) { + if (Platform.OS === 'android') { + localFileUrl = await saveFile(pickResult.uri, fileName); + pickResult.uri = localFileUrl!; + } + const thumbnail = await createVideoThumbnail(pickResult.uri); + thumbnailUrl = + (await saveFile(thumbnail.path, fileName + '.jpeg')) ?? ''; + const metaData = await getVideoMetaData(pickResult.uri); + width = metaData.width; + height = metaData.height; + } + if (localFileUrl) { files.push({ fileName: fileName, url: localFileUrl, + videoThumbnailUrl: thumbnailUrl, fileSize: pickResult.size ?? 0, type: fileType, format: format.toLowerCase() === 'jpg' ? 'jpeg' : format, + width: width, + height: height, }); } } @@ -129,7 +166,9 @@ export const CustomAddFileComponent: React.FC = ({ 'Take Camera': () => { launchCamera({ saveToPhotos: false, - mediaType: 'photo', + mediaType: isVideoSupported() ? 'mixed' : 'photo', + videoQuality: 'high', + durationLimit: 30, includeBase64: false, includeExtra: true, presentationStyle: 'fullScreen', @@ -143,9 +182,10 @@ export const CustomAddFileComponent: React.FC = ({ 'Choose From Photos': () => { launchImageLibrary({ selectionLimit: 0, - mediaType: 'photo', + mediaType: isVideoSupported() ? 'mixed' : 'photo', includeBase64: false, includeExtra: true, + assetRepresentationMode: 'current', }).then(async res => { const files = await getFiles(res); if (files.length > 0) { @@ -170,6 +210,7 @@ const showInfo = (msg: string) => { const MAX_FILE_SIZE = 4.5 * 1024 * 1024; export const IMAGE_FORMATS = ['png', 'jpg', 'jpeg', 'gif', 'webp']; +export const VIDEO_FORMATS = ['mp4', 'mov', 'mkv', 'webm']; export const DOCUMENT_FORMATS = [ 'pdf', 'csv', @@ -185,6 +226,8 @@ export const DOCUMENT_FORMATS = [ export const getFileType = (format: string) => { if (isImageFormat(format)) { return FileType.image; + } else if (isVideoFormat(format)) { + return FileType.video; } else if (isDocumentFormat(format)) { return FileType.document; } else { @@ -193,42 +236,68 @@ export const getFileType = (format: string) => { }; export const isImageFormat = (format: string) => { - return IMAGE_FORMATS.includes(format.toLowerCase()); + return IMAGE_FORMATS.includes(format); +}; + +export const isVideoFormat = (format: string) => { + return VIDEO_FORMATS.includes(format); }; export const isDocumentFormat = (format: string) => { - return DOCUMENT_FORMATS.includes(format.toLowerCase()); + return DOCUMENT_FORMATS.includes(format); }; const getFileNameWithoutExtension = (fileName: string) => { return fileName.substring(0, fileName.lastIndexOf('.')).trim(); }; +export const isVideoSupported = (): boolean => { + const textModelId = getTextModel().modelId; + return textModelId.includes('nova-pro') || textModelId.includes('nova-lite'); +}; + const getFiles = async (res: ImagePickerResponse) => { const files: FileInfo[] = []; await Promise.all( - res.assets?.map(async image => { - if (image.fileName && image.uri) { - const fileName = getFileNameWithoutExtension(image.fileName); - const fileNameArr = image.fileName.split('.'); - const format = fileNameArr[fileNameArr.length - 1]; + res.assets?.map(async media => { + if (media.fileName && media.uri) { + const fileName = getFileNameWithoutExtension(media.fileName); + const fileNameArr = media.fileName.split('.'); + let format = fileNameArr[fileNameArr.length - 1].toLowerCase(); const fileType = getFileType(format); if (fileType === FileType.unSupported) { const msg = 'Selected UnSupported Files format: .' + format; showInfo(msg); return; } - const compressUri = await Img.compress(image.uri); - const localFileUrl = await saveFile(compressUri, image.fileName); + if (format === 'png' || format === 'jpg' || format === 'jpeg') { + media.uri = await Img.compress(media.uri); + const metaData = await getImageMetaData(media.uri); + format = metaData.extension; + } + let thumbnailUrl; + if (fileType === FileType.video) { + const thumbnail = await createVideoThumbnail(media.uri); + thumbnailUrl = + (await saveFile(thumbnail.path, fileName + '.jpeg')) ?? ''; + } + let localFileUrl: string | null; + if (fileType !== FileType.video) { + localFileUrl = await saveFile(media.uri, media.fileName); + } else { + localFileUrl = media.uri; + } + if (localFileUrl) { files.push({ fileName: fileName, url: localFileUrl, - fileSize: image.fileSize ?? 0, + videoThumbnailUrl: thumbnailUrl, + fileSize: media.fileSize ?? 0, type: fileType, - format: format.toLowerCase() === 'jpg' ? 'jpeg' : format, - width: image.width, - height: image.height, + format: format === 'jpg' ? 'jpeg' : format, + width: media.width, + height: media.height, }); } } diff --git a/react-native/src/chat/component/CustomChatFooter.tsx b/react-native/src/chat/component/CustomChatFooter.tsx index 2ec069f..11a4486 100644 --- a/react-native/src/chat/component/CustomChatFooter.tsx +++ b/react-native/src/chat/component/CustomChatFooter.tsx @@ -5,20 +5,17 @@ import { CustomFileListComponent } from './CustomFileListComponent.tsx'; interface CustomComposerProps { files: FileInfo[]; - onFileSelected: (files: FileInfo[], isDelete?: boolean) => void; + onFileUpdated: (files: FileInfo[], isUpdate?: boolean) => void; } export const CustomChatFooter: React.FC = ({ files, - onFileSelected, + onFileUpdated, }) => { return ( {files.length > 0 && ( - + )} ); diff --git a/react-native/src/chat/component/CustomFileListComponent.tsx b/react-native/src/chat/component/CustomFileListComponent.tsx index f9f5dd4..1af3259 100644 --- a/react-native/src/chat/component/CustomFileListComponent.tsx +++ b/react-native/src/chat/component/CustomFileListComponent.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { Image, ScrollView, @@ -14,11 +14,14 @@ import { ImageSource } from 'react-native-image-viewing/dist/@types'; import Share from 'react-native-share'; import FileViewer from 'react-native-file-viewer'; import { isMac } from '../../App.tsx'; -import { getFullFileUrl } from '../util/FileUtils.ts'; +import { getFullFileUrl, saveFile } from '../util/FileUtils.ts'; +import { getVideoMetaData, Video } from 'react-native-compressor'; +import * as Progress from 'react-native-progress'; +import Toast from 'react-native-toast-message'; interface CustomFileProps { files: FileInfo[]; - onFileSelected?: (files: FileInfo[], isDelete?: boolean) => void; + onFileUpdated?: (files: FileInfo[], isUpdate?: boolean) => void; mode?: DisplayMode; } @@ -27,6 +30,8 @@ export enum DisplayMode { Display = 'display', } +const MAX_VIDEO_SIZE = 8; + const openInFileViewer = (url: string) => { FileViewer.open(url) .then(() => {}) @@ -35,9 +40,22 @@ const openInFileViewer = (url: string) => { }); }; +const CircularProgress = ({ progress }: { progress: number }) => { + return ( + + + + ); +}; + export const CustomFileListComponent: React.FC = ({ files, - onFileSelected, + onFileUpdated, mode = DisplayMode.Edit, }) => { const [visible, setIsVisible] = useState(false); @@ -45,8 +63,13 @@ export const CustomFileListComponent: React.FC = ({ const [imageUrls, setImageUrls] = useState([]); const scrollViewRef = useRef(null); + const [compressionProgress, setCompressionProgress] = useState(0); + const compressingFiles = useRef(''); + const filesRef = useRef(files); + const isCompressing = useRef(false); useEffect(() => { + filesRef.current = files; if (scrollViewRef.current && mode !== DisplayMode.Display) { setTimeout(() => { scrollViewRef.current?.scrollToEnd({ animated: true }); @@ -54,27 +77,115 @@ export const CustomFileListComponent: React.FC = ({ } }, [files, mode]); + const handleCompression = useCallback(async () => { + for (const file of filesRef.current) { + if ( + !isCompressing.current && + file.type === FileType.video && + !file.videoUrl && + compressingFiles.current !== file.url + ) { + compressingFiles.current = file.url; + try { + isCompressing.current = true; + const uri = await Video.compress( + file.url, + { progressDivider: 1, maxSize: 960 }, + progress => { + setCompressionProgress(progress); + } + ); + const metaData = await getVideoMetaData(uri); + console.log('metaData', metaData); + isCompressing.current = false; + compressingFiles.current = ''; + const currentSize = metaData.size / 1024 / 1024; + if (currentSize < MAX_VIDEO_SIZE) { + // save video to files and update video url + const localFileUrl = await saveFile( + uri, + file.fileName + '.' + metaData.extension + ); + if (localFileUrl) { + const updatedFiles = filesRef.current.map(f => + f.url === file.url + ? { ...f, videoUrl: localFileUrl, format: metaData.extension } + : f + ); + onFileUpdated!(updatedFiles, true); + } + } else { + // remove the video + const newFiles = filesRef.current.filter(f => f.url !== file.url); + onFileUpdated!(newFiles, true); + Toast.show({ + type: 'info', + text1: `Video too large: ${currentSize.toFixed( + 1 + )}MB (max ${MAX_VIDEO_SIZE}MB)`, + }); + } + } catch (error) { + Toast.show({ + type: 'info', + text1: 'Video process failed', + }); + compressingFiles.current = ''; + isCompressing.current = false; + // remove the failed video + const newFiles = filesRef.current.filter(f => f.url !== file.url); + onFileUpdated!(newFiles, true); + } + } + } + }, [onFileUpdated]); + + useEffect(() => { + const checkAndCompressVideos = async () => { + await handleCompression(); + }; + checkAndCompressVideos().then(); + }, [files, handleCompression]); + const renderFileItem = (file: FileInfo, fileIndex: number) => { const isImage = file.type === FileType.image; - const fullFileUrl = getFullFileUrl(file.url); + const isDocument = file.type === FileType.document; + const isVideo = file.type === FileType.video; + const fullFileUrl = + isVideo && !file.videoUrl + ? file.url + : getFullFileUrl(file.videoUrl || file.url); const itemKey = `file-${fileIndex}-${file.url}`; + + const isFileCompressing = compressingFiles.current === file.url; + let ratio = 1; + if (file.width && file.height) { + ratio = file.width / file.height; + ratio = ratio < 1 ? 1 : ratio; + } + const isHideDelete = file.type === FileType.video && !file.videoUrl; return ( - {mode === DisplayMode.Edit && ( + {mode === DisplayMode.Edit && !isHideDelete && ( { const newFiles = files.filter(f => f.url !== file.url); - onFileSelected!(newFiles, true); + onFileUpdated!(newFiles, true); }}> - × + + × + )} @@ -92,7 +203,14 @@ export const CustomFileListComponent: React.FC = ({ } }} onPress={() => { - if (isMac || file.type === FileType.document) { + if (isVideo && isFileCompressing) { + return; + } + if ( + isMac || + file.type === FileType.document || + file.type === FileType.video + ) { openInFileViewer(fullFileUrl); } else { const images = files @@ -106,12 +224,27 @@ export const CustomFileListComponent: React.FC = ({ setIsVisible(true); } }}> - {isImage ? ( - + {isImage || isVideo ? ( + + + {isVideo && !isFileCompressing && ( + + )} + {isVideo && isFileCompressing && ( + + )} + ) : ( @@ -156,10 +289,7 @@ export const CustomFileListComponent: React.FC = ({ {mode === DisplayMode.Edit && ( - + )} ; + return ; } if (uri.endsWith('.svg')) { return ; diff --git a/react-native/src/chat/component/CustomMessageComponent.tsx b/react-native/src/chat/component/CustomMessageComponent.tsx index e35dcfe..84d916a 100644 --- a/react-native/src/chat/component/CustomMessageComponent.tsx +++ b/react-native/src/chat/component/CustomMessageComponent.tsx @@ -255,6 +255,10 @@ const styles = StyleSheet.create({ const customMarkedStyles: MarkedStyles = { table: { marginVertical: 4 }, li: { paddingVertical: 4 }, + h1: { fontSize: 28 }, + h2: { fontSize: 24 }, + h3: { fontSize: 20 }, + h4: { fontSize: 18 }, }; export default CustomMessageComponent; diff --git a/react-native/src/chat/component/CustomSendComponent.tsx b/react-native/src/chat/component/CustomSendComponent.tsx index 5322a11..6d36adb 100644 --- a/react-native/src/chat/component/CustomSendComponent.tsx +++ b/react-native/src/chat/component/CustomSendComponent.tsx @@ -24,7 +24,7 @@ const CustomSendComponent: React.FC = ({ const { text } = props; if ( chatMode !== ChatMode.Text || - !getTextModel().modelId.includes('claude-3') || + !isMultiModalModel() || (text && text!.length > 0) || selectedFiles.length > 0 || chatStatus === ChatStatus.Running @@ -74,6 +74,15 @@ const CustomSendComponent: React.FC = ({ } }; +const isMultiModalModel = (): boolean => { + const textModelId = getTextModel().modelId; + return ( + textModelId.includes('claude-3') || + textModelId.includes('nova-pro') || + textModelId.includes('nova-lite') + ); +}; + const styles = StyleSheet.create({ stopContainer: { marginRight: 15, diff --git a/react-native/src/chat/component/ProgressBar.tsx b/react-native/src/chat/component/ImageProgressBar.tsx similarity index 98% rename from react-native/src/chat/component/ProgressBar.tsx rename to react-native/src/chat/component/ImageProgressBar.tsx index aed404c..4fd86b6 100644 --- a/react-native/src/chat/component/ProgressBar.tsx +++ b/react-native/src/chat/component/ImageProgressBar.tsx @@ -47,7 +47,7 @@ const ProgressCircle = ({ progressAnim }: { progressAnim: Animated.Value }) => { ); }; -const ProgressBar = () => { +const ImageProgressBar = () => { const progressAnim = useRef(new Animated.Value(0)).current; const fadeAnim = useRef(new Animated.Value(0)).current; const [showFinishImage, setShowFinishImage] = useState(false); @@ -127,4 +127,4 @@ const styles = StyleSheet.create({ }, }); -export default ProgressBar; +export default ImageProgressBar; diff --git a/react-native/src/chat/util/BedrockMessageConvertor.ts b/react-native/src/chat/util/BedrockMessageConvertor.ts index dad58eb..49740db 100644 --- a/react-native/src/chat/util/BedrockMessageConvertor.ts +++ b/react-native/src/chat/util/BedrockMessageConvertor.ts @@ -21,7 +21,9 @@ export async function getBedrockMessage( const files = JSON.parse(message.image) as FileInfo[]; for (const file of files) { try { - const fileBytes = await getFileBytes(file.url); + const fileUrl = + file.type === FileType.video ? file.videoUrl! : file.url; + const fileBytes = await getFileBytes(fileUrl); if (file.type === FileType.image) { content.push({ image: { @@ -31,6 +33,15 @@ export async function getBedrockMessage( }, }, }); + } else if (file.type === FileType.video) { + content.push({ + video: { + format: file.format.toLowerCase(), + source: { + bytes: fileBytes, + }, + }, + }); } else if (file.type === FileType.document) { let fileName = file.fileName; if (!isValidFilename(fileName)) { @@ -88,6 +99,15 @@ export interface ImageContent { }; } +export interface VideoContent { + video: { + format: string; + source: { + bytes: string; + }; + }; +} + export interface DocumentContent { document: { format: string; @@ -98,7 +118,11 @@ export interface DocumentContent { }; } -export type MessageContent = TextContent | ImageContent | DocumentContent; +export type MessageContent = + | TextContent + | ImageContent + | VideoContent + | DocumentContent; export type BedrockMessage = { role: string; diff --git a/react-native/src/chat/util/FileUtils.ts b/react-native/src/chat/util/FileUtils.ts index 08c9d2f..159f0b8 100644 --- a/react-native/src/chat/util/FileUtils.ts +++ b/react-native/src/chat/util/FileUtils.ts @@ -2,6 +2,7 @@ import RNFS from 'react-native-fs'; import { Platform } from 'react-native'; import { FileInfo, FileType } from '../../types/Chat.ts'; import Toast from 'react-native-toast-message'; +import { getTextModel } from '../../storage/StorageUtils.ts'; export const saveImageToLocal = async ( base64ImageData: string @@ -82,6 +83,8 @@ export const getFullFileUrl = (url: string) => { const MAX_IMAGES = 20; const MAX_DOCUMENTS = 5; +const MAX_NOVA_FILES = 5; +const MAX_NOVA_VIDEOS = 1; export const checkFileNumberLimit = ( prevFiles: FileInfo[], @@ -101,6 +104,34 @@ export const checkFileNumberLimit = ( let processedNewDocs = newDocs; let showWarning = false; + if (isNova()) { + if (prevFiles.length + newFiles.length > MAX_NOVA_FILES) { + showInfo(`Maximum ${MAX_NOVA_FILES} files allowed`); + } + if (prevFiles.length >= MAX_NOVA_FILES) { + return prevFiles; + } + const existingVideos = prevFiles.filter( + file => file.type === FileType.video + ).length; + const newVideos = newFiles.filter(file => file.type === FileType.video); + + if (existingVideos + newVideos.length > MAX_NOVA_VIDEOS) { + showInfo(`Maximum ${MAX_NOVA_VIDEOS} video allowed`); + } + + const filteredNewFiles = + existingVideos >= MAX_NOVA_VIDEOS + ? newFiles.filter(file => file.type !== FileType.video) + : newFiles.filter( + file => + file.type !== FileType.video || + newVideos.indexOf(file) < MAX_NOVA_VIDEOS - existingVideos + ); + + return [...prevFiles, ...filteredNewFiles].slice(0, MAX_NOVA_FILES); + } + if (totalImages > MAX_IMAGES) { const remainingSlots = Math.max(0, MAX_IMAGES - existingImages.length); processedNewImages = newImages.slice(0, remainingSlots); @@ -115,17 +146,52 @@ export const checkFileNumberLimit = ( if (showWarning) { if (totalImages > MAX_IMAGES) { - Toast.show({ - type: 'info', - text1: `Image limit exceeded, maximum ${MAX_IMAGES} images allowed`, - }); + showInfo(`Image limit exceeded, maximum ${MAX_IMAGES} images allowed`); } if (totalDocs > MAX_DOCUMENTS) { - Toast.show({ - type: 'info', - text1: `Document limit exceeded, maximum ${MAX_DOCUMENTS} files allowed`, - }); + showInfo( + `Document limit exceeded, maximum ${MAX_DOCUMENTS} files allowed` + ); } } return [...prevFiles, ...processedNewImages, ...processedNewDocs]; }; + +const isNova = (): boolean => { + const textModelId = getTextModel().modelId; + return textModelId.includes('nova-pro') || textModelId.includes('nova-lite'); +}; + +export const isAllFileReady = (files: FileInfo[]) => { + const videos = files.filter(file => file.type === FileType.video); + if (videos.length > 0) { + return videos.filter(video => video.videoUrl === undefined).length === 0; + } else { + return true; + } +}; + +const showInfo = (msg: string) => { + Toast.show({ + type: 'info', + text1: msg, + }); +}; + +export const getFileTypeSummary = (files: FileInfo[]) => { + if (files.length === 1) { + return 'Summarize this'; + } + + const imgCount = files.filter(file => file.type === FileType.image).length; + const docCount = files.filter(file => file.type === FileType.document).length; + const videoCount = files.filter(file => file.type === FileType.video).length; + + const types = [ + imgCount && `${imgCount > 1 ? 'images' : 'image'}`, + docCount && `${docCount > 1 ? 'docs' : 'doc'}`, + videoCount && `${videoCount > 1 ? 'videos' : 'video'}`, + ].filter(Boolean); + + return `Summarize these ${types.join(' and ')}`; +}; diff --git a/react-native/src/types/Chat.ts b/react-native/src/types/Chat.ts index ddd7a58..aa48fb6 100644 --- a/react-native/src/types/Chat.ts +++ b/react-native/src/types/Chat.ts @@ -57,12 +57,15 @@ export type UpgradeInfo = { export enum FileType { document = 'document', image = 'image', + video = 'video', unSupported = 'unSupported', } export type FileInfo = { fileName: string; url: string; + videoUrl?: string; + videoThumbnailUrl?: string; fileSize: number; format: string; type: FileType; diff --git a/server/src/main.py b/server/src/main.py index 529b9bd..2af55a1 100644 --- a/server/src/main.py +++ b/server/src/main.py @@ -93,9 +93,12 @@ async def converse(request: ConverseRequest, if 'image' in content: image_bytes = base64.b64decode(content['image']['source']['bytes']) content['image']['source']['bytes'] = image_bytes + if 'video' in content: + video_bytes = base64.b64decode(content['video']['source']['bytes']) + content['video']['source']['bytes'] = video_bytes if 'document' in content: - image_bytes = base64.b64decode(content['document']['source']['bytes']) - content['document']['source']['bytes'] = image_bytes + document_bytes = base64.b64decode(content['document']['source']['bytes']) + content['document']['source']['bytes'] = document_bytes command = { "inferenceConfig": {"maxTokens": max_tokens}, "messages": request.messages, diff --git a/server/src/requirements.txt b/server/src/requirements.txt index 709da89..6e7628d 100644 --- a/server/src/requirements.txt +++ b/server/src/requirements.txt @@ -1,4 +1,4 @@ fastapi~=0.112.1 -boto3==1.34.127 +boto3==1.35.74 pydantic~=2.8.2 -uvicorn~=0.30.6 \ No newline at end of file +uvicorn~=0.30.6