diff --git a/app/controllers/discourse_data_explorer/query_controller.rb b/app/controllers/discourse_data_explorer/query_controller.rb
index 660ba750..6b6ecac8 100644
--- a/app/controllers/discourse_data_explorer/query_controller.rb
+++ b/app/controllers/discourse_data_explorer/query_controller.rb
@@ -179,49 +179,27 @@ def run
render json: { success: false, errors: [err_msg] }, status: 422
else
- pg_result = result[:pg_result]
- cols = pg_result.fields
+ content_disposition =
+ "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"
+
respond_to do |format|
format.json do
- if params[:download]
- response.headers[
- "Content-Disposition"
- ] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.json"
- end
- json = {
- success: true,
- errors: [],
- duration: (result[:duration_secs].to_f * 1000).round(1),
- result_count: pg_result.values.length || 0,
- params: query_params,
- columns: cols,
- default_limit: SiteSetting.data_explorer_query_result_limit,
- }
- json[:explain] = result[:explain] if opts[:explain]
-
- if !params[:download]
- relations, colrender = DataExplorer.add_extra_data(pg_result)
- json[:relations] = relations
- json[:colrender] = colrender
- end
-
- json[:rows] = pg_result.values
-
- render json: json
+ response.headers["Content-Disposition"] = "#{content_disposition}.json" if params[
+ :download
+ ]
+
+ render json:
+ ResultFormatConverter.convert(
+ :json,
+ result,
+ query_params:,
+ download: params[:download],
+ )
end
format.csv do
- response.headers[
- "Content-Disposition"
- ] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
-
- require "csv"
- text =
- CSV.generate do |csv|
- csv << cols
- pg_result.values.each { |row| csv << row }
- end
-
- render plain: text
+ response.headers["Content-Disposition"] = "#{content_disposition}.csv"
+
+ render plain: ResultFormatConverter.convert(:csv, result)
end
end
end
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
index 1ac237fa..be49ded7 100644
--- a/config/locales/client.en.yml
+++ b/config/locales/client.en.yml
@@ -116,3 +116,5 @@ en:
label: Data Explorer Query parameters
skip_empty:
label: Skip sending PM if there are no results
+ attach_csv:
+ label: Attach the CSV file to the PM
diff --git a/lib/report_generator.rb b/lib/report_generator.rb
index 2f504f6f..3cf27440 100644
--- a/lib/report_generator.rb
+++ b/lib/report_generator.rb
@@ -9,13 +9,13 @@ def self.generate(query_id, query_params, recipients, opts = {})
recipients = filter_recipients_by_query_access(recipients, query)
params = params_to_hash(query_params)
- result = DataExplorer.run_query(query, params)[:pg_result]
+ result = DataExplorer.run_query(query, params)
query.update!(last_run_at: Time.now)
- return [] if opts[:skip_empty] && result.values.empty?
- table = ResultToMarkdown.convert(result)
+ return [] if opts[:skip_empty] && result[:pg_result].values.empty?
+ table = ResultToMarkdown.convert(result[:pg_result])
- build_report_pms(query, table, recipients)
+ build_report_pms(query, table, recipients, attach_csv: opts[:attach_csv], result:)
end
private
@@ -40,8 +40,20 @@ def self.params_to_hash(query_params)
params_hash
end
- def self.build_report_pms(query, table = "", targets = [])
+ def self.build_report_pms(query, table = "", targets = [], attach_csv: false, result: nil)
pms = []
+ upload =
+ if attach_csv
+ tmp_filename =
+ "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
+ tmp = Tempfile.new(tmp_filename)
+ tmp.write(ResultFormatConverter.convert(:csv, result))
+ tmp.rewind
+ UploadCreator.new(tmp, tmp_filename, type: "csv_export").create_for(
+ Discourse.system_user.id,
+ )
+ end
+
targets.each do |target|
name = target[0]
pm_type = "target_#{target[1]}s"
@@ -53,6 +65,9 @@ def self.build_report_pms(query, table = "", targets = [])
"Query Name:\n#{query.name}\n\nHere are the results:\n#{table}\n\n" +
"View query in Data Explorer\n\n" +
"Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})"
+ if upload
+ pm["raw"] << "\n\nAppendix: [#{upload.original_filename}|attachment](#{upload.short_url})"
+ end
pms << pm
end
pms
diff --git a/lib/result_format_converter.rb b/lib/result_format_converter.rb
new file mode 100644
index 00000000..258f07fb
--- /dev/null
+++ b/lib/result_format_converter.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+module ::DiscourseDataExplorer
+ class ResultFormatConverter
+ def self.convert(file_type, result, opts = {})
+ self.new(result, opts).send("to_#{file_type}")
+ end
+
+ def initialize(result, opts)
+ @result = result
+ @opts = opts
+ end
+
+ private
+
+ attr_reader :result
+ attr_reader :opts
+
+ def pg_result
+ @pg_result ||= @result[:pg_result]
+ end
+
+ def cols
+ @cols ||= pg_result.fields
+ end
+
+ def to_csv
+ require "csv"
+ CSV.generate do |csv|
+ csv << cols
+ pg_result.values.each { |row| csv << row }
+ end
+ end
+
+ def to_json
+ json = {
+ success: true,
+ errors: [],
+ duration: (result[:duration_secs].to_f * 1000).round(1),
+ result_count: pg_result.values.length || 0,
+ params: opts[:query_params],
+ columns: cols,
+ default_limit: SiteSetting.data_explorer_query_result_limit,
+ }
+ json[:explain] = result[:explain] if opts[:explain]
+
+ if !opts[:download]
+ relations, colrender = DataExplorer.add_extra_data(pg_result)
+ json[:relations] = relations
+ json[:colrender] = colrender
+ end
+
+ json[:rows] = pg_result.values
+
+ json
+ end
+
+ #TODO: we can move ResultToMarkdown here
+ end
+end
diff --git a/plugin.rb b/plugin.rb
index 8b6a725c..d62981d2 100644
--- a/plugin.rb
+++ b/plugin.rb
@@ -79,6 +79,7 @@ module ::DiscourseDataExplorer
require_relative "lib/report_generator"
require_relative "lib/result_to_markdown"
+ require_relative "lib/result_format_converter"
reloadable_patch do
if defined?(DiscourseAutomation)
add_automation_scriptable("recurring_data_explorer_result_pm") do
@@ -90,6 +91,7 @@ module ::DiscourseDataExplorer
field :query_id, component: :choices, required: true, extra: { content: queries }
field :query_params, component: :"key-value", accepts_placeholders: true
field :skip_empty, component: :boolean
+ field :attach_csv, component: :boolean
version 1
triggerables [:recurring]
@@ -99,6 +101,7 @@ module ::DiscourseDataExplorer
query_id = fields.dig("query_id", "value")
query_params = fields.dig("query_params", "value") || {}
skip_empty = fields.dig("skip_empty", "value") || false
+ attach_csv = fields.dig("attach_csv", "value") || false
unless SiteSetting.data_explorer_enabled
Rails.logger.warn "#{DiscourseDataExplorer::PLUGIN_NAME} - plugin must be enabled to run automation #{automation.id}"
@@ -111,7 +114,7 @@ module ::DiscourseDataExplorer
end
DiscourseDataExplorer::ReportGenerator
- .generate(query_id, query_params, recipients, { skip_empty: })
+ .generate(query_id, query_params, recipients, { skip_empty:, attach_csv: })
.each do |pm|
begin
utils.send_pm(pm, automation_id: automation.id, prefers_encrypt: false)
diff --git a/spec/report_generator_spec.rb b/spec/report_generator_spec.rb
index c0e50053..e96848ff 100644
--- a/spec/report_generator_spec.rb
+++ b/spec/report_generator_spec.rb
@@ -130,5 +130,25 @@
expect(result[1]["target_group_names"]).to eq([group.name])
expect(result[2]["target_emails"]).to eq(["john@doe.com"])
end
+
+ it "works with attached csv file" do
+ SiteSetting.personal_message_enabled_groups = group.id
+ DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
+ freeze_time
+
+ result =
+ described_class.generate(query.id, query_params, [user.username], { attach_csv: true })
+
+ filename =
+ "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
+
+ expect(result[0]["raw"]).to include(
+ "Hi #{user.username}, your data explorer report is ready.\n\n" +
+ "Query Name:\n#{query.name}\n\nHere are the results:\nle table\n\n" +
+ "View query in Data Explorer\n\n" +
+ "Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})\n\n" +
+ "Appendix: [#{filename}|attachment](upload://",
+ )
+ end
end
end
diff --git a/spec/result_format_converter_spec.rb b/spec/result_format_converter_spec.rb
new file mode 100644
index 00000000..5ea2f24d
--- /dev/null
+++ b/spec/result_format_converter_spec.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+describe DiscourseDataExplorer::ResultFormatConverter do
+ fab!(:user)
+ fab!(:post)
+ fab!(:query) { DiscourseDataExplorer::Query.find(-1) }
+
+ let(:query_params) { [{ from_days_ago: 0 }, { duration_days: 15 }] }
+ let(:query_result) { DiscourseDataExplorer::DataExplorer.run_query(query, query_params) }
+
+ before { SiteSetting.data_explorer_enabled = true }
+
+ describe ".convert" do
+ context "for csv files" do
+ it "format results as a csv table with headers and columns" do
+ result = described_class.convert(:csv, query_result)
+
+ table = <<~CSV
+ liker_user_id,liked_user_id,count
+ CSV
+
+ expect(result).to include(table)
+ end
+ end
+
+ context "for json files" do
+ it "format results as a json file" do
+ result = described_class.convert(:json, query_result, { query_params: })
+
+ expect(result[:columns]).to contain_exactly("liker_user_id", "liked_user_id", "count")
+ expect(result[:params]).to eq(query_params)
+ end
+ end
+ end
+end