From cbae98f6eda078906b65a18b4bd3e32850372dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=94=A6=E5=BF=83?= <41134017+Lhcfl@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:41:12 +0800 Subject: [PATCH] FEATURE: Allows CSV file result to be attached in automated PMs (#318) This commit adds an optional setting that allows to attach query results in CSV format as a file to PMs sent by Data Explorer's automation scripts. meta topic: https://meta.discourse.org/t/turn-data-explorer-query-results-into-csv-to-attach-to-discourse-automated-emails/267529 Co-authored-by: Drenmi --- .../query_controller.rb | 56 ++++++------------ config/locales/client.en.yml | 2 + lib/report_generator.rb | 25 ++++++-- lib/result_format_converter.rb | 59 +++++++++++++++++++ plugin.rb | 5 +- spec/report_generator_spec.rb | 20 +++++++ spec/result_format_converter_spec.rb | 35 +++++++++++ 7 files changed, 157 insertions(+), 45 deletions(-) create mode 100644 lib/result_format_converter.rb create mode 100644 spec/result_format_converter_spec.rb diff --git a/app/controllers/discourse_data_explorer/query_controller.rb b/app/controllers/discourse_data_explorer/query_controller.rb index 660ba750..6b6ecac8 100644 --- a/app/controllers/discourse_data_explorer/query_controller.rb +++ b/app/controllers/discourse_data_explorer/query_controller.rb @@ -179,49 +179,27 @@ def run render json: { success: false, errors: [err_msg] }, status: 422 else - pg_result = result[:pg_result] - cols = pg_result.fields + content_disposition = + "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult" + respond_to do |format| format.json do - if params[:download] - response.headers[ - "Content-Disposition" - ] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.json" - end - json = { - success: true, - errors: [], - duration: (result[:duration_secs].to_f * 1000).round(1), - result_count: pg_result.values.length || 0, - params: query_params, - columns: cols, - default_limit: SiteSetting.data_explorer_query_result_limit, - } - json[:explain] = result[:explain] if opts[:explain] - - if !params[:download] - relations, colrender = DataExplorer.add_extra_data(pg_result) - json[:relations] = relations - json[:colrender] = colrender - end - - json[:rows] = pg_result.values - - render json: json + response.headers["Content-Disposition"] = "#{content_disposition}.json" if params[ + :download + ] + + render json: + ResultFormatConverter.convert( + :json, + result, + query_params:, + download: params[:download], + ) end format.csv do - response.headers[ - "Content-Disposition" - ] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv" - - require "csv" - text = - CSV.generate do |csv| - csv << cols - pg_result.values.each { |row| csv << row } - end - - render plain: text + response.headers["Content-Disposition"] = "#{content_disposition}.csv" + + render plain: ResultFormatConverter.convert(:csv, result) end end end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 1ac237fa..be49ded7 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -116,3 +116,5 @@ en: label: Data Explorer Query parameters skip_empty: label: Skip sending PM if there are no results + attach_csv: + label: Attach the CSV file to the PM diff --git a/lib/report_generator.rb b/lib/report_generator.rb index 2f504f6f..3cf27440 100644 --- a/lib/report_generator.rb +++ b/lib/report_generator.rb @@ -9,13 +9,13 @@ def self.generate(query_id, query_params, recipients, opts = {}) recipients = filter_recipients_by_query_access(recipients, query) params = params_to_hash(query_params) - result = DataExplorer.run_query(query, params)[:pg_result] + result = DataExplorer.run_query(query, params) query.update!(last_run_at: Time.now) - return [] if opts[:skip_empty] && result.values.empty? - table = ResultToMarkdown.convert(result) + return [] if opts[:skip_empty] && result[:pg_result].values.empty? + table = ResultToMarkdown.convert(result[:pg_result]) - build_report_pms(query, table, recipients) + build_report_pms(query, table, recipients, attach_csv: opts[:attach_csv], result:) end private @@ -40,8 +40,20 @@ def self.params_to_hash(query_params) params_hash end - def self.build_report_pms(query, table = "", targets = []) + def self.build_report_pms(query, table = "", targets = [], attach_csv: false, result: nil) pms = [] + upload = + if attach_csv + tmp_filename = + "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv" + tmp = Tempfile.new(tmp_filename) + tmp.write(ResultFormatConverter.convert(:csv, result)) + tmp.rewind + UploadCreator.new(tmp, tmp_filename, type: "csv_export").create_for( + Discourse.system_user.id, + ) + end + targets.each do |target| name = target[0] pm_type = "target_#{target[1]}s" @@ -53,6 +65,9 @@ def self.build_report_pms(query, table = "", targets = []) "Query Name:\n#{query.name}\n\nHere are the results:\n#{table}\n\n" + "View query in Data Explorer\n\n" + "Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})" + if upload + pm["raw"] << "\n\nAppendix: [#{upload.original_filename}|attachment](#{upload.short_url})" + end pms << pm end pms diff --git a/lib/result_format_converter.rb b/lib/result_format_converter.rb new file mode 100644 index 00000000..258f07fb --- /dev/null +++ b/lib/result_format_converter.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true +module ::DiscourseDataExplorer + class ResultFormatConverter + def self.convert(file_type, result, opts = {}) + self.new(result, opts).send("to_#{file_type}") + end + + def initialize(result, opts) + @result = result + @opts = opts + end + + private + + attr_reader :result + attr_reader :opts + + def pg_result + @pg_result ||= @result[:pg_result] + end + + def cols + @cols ||= pg_result.fields + end + + def to_csv + require "csv" + CSV.generate do |csv| + csv << cols + pg_result.values.each { |row| csv << row } + end + end + + def to_json + json = { + success: true, + errors: [], + duration: (result[:duration_secs].to_f * 1000).round(1), + result_count: pg_result.values.length || 0, + params: opts[:query_params], + columns: cols, + default_limit: SiteSetting.data_explorer_query_result_limit, + } + json[:explain] = result[:explain] if opts[:explain] + + if !opts[:download] + relations, colrender = DataExplorer.add_extra_data(pg_result) + json[:relations] = relations + json[:colrender] = colrender + end + + json[:rows] = pg_result.values + + json + end + + #TODO: we can move ResultToMarkdown here + end +end diff --git a/plugin.rb b/plugin.rb index 8b6a725c..d62981d2 100644 --- a/plugin.rb +++ b/plugin.rb @@ -79,6 +79,7 @@ module ::DiscourseDataExplorer require_relative "lib/report_generator" require_relative "lib/result_to_markdown" + require_relative "lib/result_format_converter" reloadable_patch do if defined?(DiscourseAutomation) add_automation_scriptable("recurring_data_explorer_result_pm") do @@ -90,6 +91,7 @@ module ::DiscourseDataExplorer field :query_id, component: :choices, required: true, extra: { content: queries } field :query_params, component: :"key-value", accepts_placeholders: true field :skip_empty, component: :boolean + field :attach_csv, component: :boolean version 1 triggerables [:recurring] @@ -99,6 +101,7 @@ module ::DiscourseDataExplorer query_id = fields.dig("query_id", "value") query_params = fields.dig("query_params", "value") || {} skip_empty = fields.dig("skip_empty", "value") || false + attach_csv = fields.dig("attach_csv", "value") || false unless SiteSetting.data_explorer_enabled Rails.logger.warn "#{DiscourseDataExplorer::PLUGIN_NAME} - plugin must be enabled to run automation #{automation.id}" @@ -111,7 +114,7 @@ module ::DiscourseDataExplorer end DiscourseDataExplorer::ReportGenerator - .generate(query_id, query_params, recipients, { skip_empty: }) + .generate(query_id, query_params, recipients, { skip_empty:, attach_csv: }) .each do |pm| begin utils.send_pm(pm, automation_id: automation.id, prefers_encrypt: false) diff --git a/spec/report_generator_spec.rb b/spec/report_generator_spec.rb index c0e50053..e96848ff 100644 --- a/spec/report_generator_spec.rb +++ b/spec/report_generator_spec.rb @@ -130,5 +130,25 @@ expect(result[1]["target_group_names"]).to eq([group.name]) expect(result[2]["target_emails"]).to eq(["john@doe.com"]) end + + it "works with attached csv file" do + SiteSetting.personal_message_enabled_groups = group.id + DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table") + freeze_time + + result = + described_class.generate(query.id, query_params, [user.username], { attach_csv: true }) + + filename = + "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv" + + expect(result[0]["raw"]).to include( + "Hi #{user.username}, your data explorer report is ready.\n\n" + + "Query Name:\n#{query.name}\n\nHere are the results:\nle table\n\n" + + "View query in Data Explorer\n\n" + + "Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})\n\n" + + "Appendix: [#{filename}|attachment](upload://", + ) + end end end diff --git a/spec/result_format_converter_spec.rb b/spec/result_format_converter_spec.rb new file mode 100644 index 00000000..5ea2f24d --- /dev/null +++ b/spec/result_format_converter_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +describe DiscourseDataExplorer::ResultFormatConverter do + fab!(:user) + fab!(:post) + fab!(:query) { DiscourseDataExplorer::Query.find(-1) } + + let(:query_params) { [{ from_days_ago: 0 }, { duration_days: 15 }] } + let(:query_result) { DiscourseDataExplorer::DataExplorer.run_query(query, query_params) } + + before { SiteSetting.data_explorer_enabled = true } + + describe ".convert" do + context "for csv files" do + it "format results as a csv table with headers and columns" do + result = described_class.convert(:csv, query_result) + + table = <<~CSV + liker_user_id,liked_user_id,count + CSV + + expect(result).to include(table) + end + end + + context "for json files" do + it "format results as a json file" do + result = described_class.convert(:json, query_result, { query_params: }) + + expect(result[:columns]).to contain_exactly("liker_user_id", "liked_user_id", "count") + expect(result[:params]).to eq(query_params) + end + end + end +end