From 94168f20e42fd3eb09b66c5b3017334a42c93096 Mon Sep 17 00:00:00 2001 From: merefield Date: Wed, 19 Jun 2024 20:05:42 +0100 Subject: [PATCH] FEATURE: add paint feature --- config/locales/server.en.yml | 7 +++ config/settings.yml | 3 ++ lib/discourse_chatbot/bots/open_ai_bot_rag.rb | 27 ++++++++++ .../functions/paint_function.rb | 52 +++++++++++++++++++ plugin.rb | 3 +- 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 lib/discourse_chatbot/functions/paint_function.rb diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 736f312..f44527d 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -22,6 +22,7 @@ en: chatbot_open_ai_model: "(UNLESS CUSTOM) The model to be accessed. More on supported models at OpenAI: Model overview" chatbot_support_vision: "(EXPERIMENTAL) Bot 'Vision': support sharing of images with bot so you can ask it to describe them. Images must be uploaded to forum and not hotlinked and in RAG mode bot can only see images in current Post it's responding to. Requires compatible model to be selected. 'false' switches vision off, 'directly' will share images potentially on every call, 'via_function' will only share images when requested by a function which might turn out to be cheaper if main model is older generation" chatbot_open_ai_vision_model: "(EXPERIMENTAL) Open AI Vision model used when 'via_function' or 'directly' options are selected. Used for all interaction if 'directly' is set, overriding all other model settings - caution: this will be expensive!" + chatbot_support_picture_creation: "(EXPERIMENTAL) Bot 'Paint': support creation of images based on text descriptions. Currently powered by DALL-E using the same token Open AI token" chatbot_reply_job_time_delay: 'Number of seconds before reply job is run. This helps prevent rate limits being hit and discourages spamming.' chatbot_include_whispers_in_post_history: "Include content of whispers in Post history bot sees (careful!)" chatbot_max_look_behind: "Maximum number of Posts or Chat Messages bot will consider as prompt for completion, the more the more impressive may be its response, but the more costly the interaction will be." @@ -123,6 +124,12 @@ en: parameters: input: the mathematical expression you need to process and get the answer to. Make sure it is Ruby compatible. error: "'%{parameter}' is an invalid mathematical expression, make sure if you are trying to calculate dates use Ruby Time class" + paint: + description: | + creates a picture based on the provided description + parameters: + description: a description of the picture the user wants to create + error: there was an issue creating the picture. forum_search: description: | Search the local forum for information that may help you answer the question. Especially useful when the forum specialises in the subject matter of the query. diff --git a/config/settings.yml b/config/settings.yml index 24638ac..b6d8970 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -216,6 +216,9 @@ plugins: chatbot_open_ai_vision_model: client: false default: gpt-4o + chatbot_support_picture_creation: + client: false + default: false chatbot_reply_job_time_delay: client: false default: 1 diff --git a/lib/discourse_chatbot/bots/open_ai_bot_rag.rb b/lib/discourse_chatbot/bots/open_ai_bot_rag.rb index 972ed30..c0cb7c5 100644 --- a/lib/discourse_chatbot/bots/open_ai_bot_rag.rb +++ b/lib/discourse_chatbot/bots/open_ai_bot_rag.rb @@ -47,6 +47,7 @@ def merge_functions(opts) google_search_function = ::DiscourseChatbot::GoogleSearchFunction.new stock_data_function = ::DiscourseChatbot::StockDataFunction.new escalate_to_staff_function = ::DiscourseChatbot::EscalateToStaffFunction.new + paint_function = ::DiscourseChatbot::PaintFunction.new forum_search_function = nil user_search_from_user_location_function = nil user_search_from_location_function = nil @@ -77,6 +78,7 @@ def merge_functions(opts) functions << forum_search_function if forum_search_function functions << vision_function if vision_function + functions << paint_function if SiteSetting.chatbot_support_picture_creation functions << user_search_from_location_function if user_search_from_location_function functions << user_search_from_user_location_function if user_search_from_user_location_function @@ -196,6 +198,18 @@ def generate_response(opts) else raise "Unexpected finish reason: #{finish_reason}" end + + # If the response is an image, we don't want to continue the loop of thought + return { + "choices" => [ + { + "message" => { + "content" => "#{@inner_thoughts.last[:content]}" + } + } + ] + } if image_url?(@inner_thoughts.last[:content]) + iteration += 1 end end @@ -296,5 +310,18 @@ def legal_urls?(res, post_ids_found, topic_ids_found) true end + + private + + def image_url?(url) + image_extensions = %w[.jpg .jpeg .png .gif .bmp .tiff .webp] + + uri = URI.parse(url) + path = uri.path + + # Check the file extension + return true if image_extensions.any? { |ext| path.downcase.end_with?(ext) } + false + end end end diff --git a/lib/discourse_chatbot/functions/paint_function.rb b/lib/discourse_chatbot/functions/paint_function.rb new file mode 100644 index 0000000..a188877 --- /dev/null +++ b/lib/discourse_chatbot/functions/paint_function.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require_relative '../function' + +module DiscourseChatbot + class PaintFunction < Function + + def name + 'paint_picture' + end + + def description + I18n.t("chatbot.prompt.function.paint.description") + end + + def parameters + [ + { name: "description", type: String, description: I18n.t("chatbot.prompt.function.paint.parameters.description") } , + ] + end + + def required + ['description'] + end + + def process(args) + begin + super(args) + + description = args[parameters[0][:name]] + + client = OpenAI::Client.new do |f| + f.response :logger, Logger.new($stdout), bodies: true if SiteSetting.chatbot_enable_verbose_console_logging + if SiteSetting.chatbot_enable_verbose_rails_logging != "off" + case SiteSetting.chatbot_verbose_rails_logging_destination_level + when "warn" + f.response :logger, Rails.logger, bodies: true, log_level: :warn + else + f.response :logger, Rails.logger, bodies: true, log_level: :info + end + end + end + + response = client.images.generate(parameters: { prompt: description, model: "dall-e-3", size: "1792x1024", quality: "standard" }) + + response.dig("data", 0, "url") + rescue + I18n.t("chatbot.prompt.function.paint.error") + end + end + end +end diff --git a/plugin.rb b/plugin.rb index b13ee68..aa40446 100644 --- a/plugin.rb +++ b/plugin.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # name: discourse-chatbot # about: a plugin that allows you to have a conversation with a configurable chatbot in Discourse Chat, Topics and Private Messages -# version: 0.9.34 +# version: 0.9.35 # authors: merefield # url: https://github.com/merefield/discourse-chatbot @@ -98,6 +98,7 @@ def progress_debug_message(message) ../lib/discourse_chatbot/functions/web_crawler_function.rb ../lib/discourse_chatbot/functions/wikipedia_function.rb ../lib/discourse_chatbot/functions/vision_function.rb + ../lib/discourse_chatbot/functions/paint_function.rb ../lib/discourse_chatbot/functions/google_search_function.rb ../lib/discourse_chatbot/functions/forum_search_function.rb ../lib/discourse_chatbot/functions/forum_user_distance_from_location_function.rb