From 6dd438244632019620f78b86af0c496a83d3ba6e Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Mon, 4 Nov 2024 16:19:43 +0200 Subject: [PATCH 01/14] optimize webhooks query --- lib/webhook_urls.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/webhook_urls.rb b/lib/webhook_urls.rb index 482e66a70..e57c802ba 100644 --- a/lib/webhook_urls.rb +++ b/lib/webhook_urls.rb @@ -10,7 +10,7 @@ def for_account_id(account_id, events) event_arel = events.map { |event| Arel::Table.new(:webhook_urls)[:events].matches("%\"#{event}\"%") }.reduce(:or) - if Docuseal.multitenant? + if Docuseal.multitenant? || account_id == 1 rel.where(event_arel) else linked_account_rel = From 4635fdd32bada41c9e9f4adf436f6237fc354848 Mon Sep 17 00:00:00 2001 From: Alex Turchyn Date: Tue, 5 Nov 2024 22:34:14 +0200 Subject: [PATCH 02/14] remove replace_timestamps helper --- ...nd_form_completed_webhook_request_job_spec.rb | 16 ++++++++-------- ...end_form_declined_webhook_request_job_spec.rb | 16 ++++++++-------- ...send_form_started_webhook_request_job_spec.rb | 16 ++++++++-------- .../send_form_viewed_webhook_request_job_spec.rb | 16 ++++++++-------- ...bmission_archived_webhook_request_job_spec.rb | 16 ++++++++-------- ...mission_completed_webhook_request_job_spec.rb | 16 ++++++++-------- ...ubmission_created_webhook_request_job_spec.rb | 16 ++++++++-------- ..._template_created_webhook_request_job_spec.rb | 16 ++++++++-------- ..._template_updated_webhook_request_job_spec.rb | 16 ++++++++-------- spec/rails_helper.rb | 16 ---------------- 10 files changed, 72 insertions(+), 88 deletions(-) diff --git a/spec/jobs/send_form_completed_webhook_request_job_spec.rb b/spec/jobs/send_form_completed_webhook_request_job_spec.rb index ceec4c00c..c4a09da84 100644 --- a/spec/jobs/send_form_completed_webhook_request_job_spec.rb +++ b/spec/jobs/send_form_completed_webhook_request_job_spec.rb @@ -26,11 +26,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.completed', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -43,11 +43,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.completed', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_form_declined_webhook_request_job_spec.rb b/spec/jobs/send_form_declined_webhook_request_job_spec.rb index 88a88906b..c53a19992 100644 --- a/spec/jobs/send_form_declined_webhook_request_job_spec.rb +++ b/spec/jobs/send_form_declined_webhook_request_job_spec.rb @@ -26,11 +26,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.declined', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -43,11 +43,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.declined', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_form_started_webhook_request_job_spec.rb b/spec/jobs/send_form_started_webhook_request_job_spec.rb index 863ce826f..929a21878 100644 --- a/spec/jobs/send_form_started_webhook_request_job_spec.rb +++ b/spec/jobs/send_form_started_webhook_request_job_spec.rb @@ -26,11 +26,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.started', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -43,11 +43,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.started', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_form_viewed_webhook_request_job_spec.rb b/spec/jobs/send_form_viewed_webhook_request_job_spec.rb index 495e62395..c3c7c216d 100644 --- a/spec/jobs/send_form_viewed_webhook_request_job_spec.rb +++ b/spec/jobs/send_form_viewed_webhook_request_job_spec.rb @@ -26,11 +26,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.viewed', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -43,11 +43,11 @@ described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'form.viewed', - 'timestamp' => Time.current, - 'data' => Submitters::SerializeForWebhook.call(submitter.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_submission_archived_webhook_request_job_spec.rb b/spec/jobs/send_submission_archived_webhook_request_job_spec.rb index 468430f74..05947f33f 100644 --- a/spec/jobs/send_submission_archived_webhook_request_job_spec.rb +++ b/spec/jobs/send_submission_archived_webhook_request_job_spec.rb @@ -23,11 +23,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.archived', - 'timestamp' => Time.current, - 'data' => submission.reload.as_json(only: %i[id archived_at]) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(submission.reload.as_json(only: %i[id archived_at]).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -40,11 +40,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.archived', - 'timestamp' => Time.current, - 'data' => submission.reload.as_json(only: %i[id archived_at]) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(submission.reload.as_json(only: %i[id archived_at]).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_submission_completed_webhook_request_job_spec.rb b/spec/jobs/send_submission_completed_webhook_request_job_spec.rb index 1101b12af..2cef72ccc 100644 --- a/spec/jobs/send_submission_completed_webhook_request_job_spec.rb +++ b/spec/jobs/send_submission_completed_webhook_request_job_spec.rb @@ -23,11 +23,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.completed', - 'timestamp' => Time.current, - 'data' => Submissions::SerializeForApi.call(submission.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submissions::SerializeForApi.call(submission.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -40,11 +40,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.completed', - 'timestamp' => Time.current, - 'data' => Submissions::SerializeForApi.call(submission.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submissions::SerializeForApi.call(submission.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_submission_created_webhook_request_job_spec.rb b/spec/jobs/send_submission_created_webhook_request_job_spec.rb index e4004dc74..1b2e02fef 100644 --- a/spec/jobs/send_submission_created_webhook_request_job_spec.rb +++ b/spec/jobs/send_submission_created_webhook_request_job_spec.rb @@ -23,11 +23,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.created', - 'timestamp' => Time.current, - 'data' => Submissions::SerializeForApi.call(submission.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submissions::SerializeForApi.call(submission.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -40,11 +40,11 @@ described_class.new.perform('submission_id' => submission.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'submission.created', - 'timestamp' => Time.current, - 'data' => Submissions::SerializeForApi.call(submission.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Submissions::SerializeForApi.call(submission.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_template_created_webhook_request_job_spec.rb b/spec/jobs/send_template_created_webhook_request_job_spec.rb index 532a1beab..14df0b3e2 100644 --- a/spec/jobs/send_template_created_webhook_request_job_spec.rb +++ b/spec/jobs/send_template_created_webhook_request_job_spec.rb @@ -22,11 +22,11 @@ described_class.new.perform('template_id' => template.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'template.created', - 'timestamp' => Time.current, - 'data' => Templates::SerializeForApi.call(template.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Templates::SerializeForApi.call(template.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -39,11 +39,11 @@ described_class.new.perform('template_id' => template.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'template.created', - 'timestamp' => Time.current, - 'data' => Templates::SerializeForApi.call(template.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Templates::SerializeForApi.call(template.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/jobs/send_template_updated_webhook_request_job_spec.rb b/spec/jobs/send_template_updated_webhook_request_job_spec.rb index 1bef3eb2a..4451b071e 100644 --- a/spec/jobs/send_template_updated_webhook_request_job_spec.rb +++ b/spec/jobs/send_template_updated_webhook_request_job_spec.rb @@ -22,11 +22,11 @@ described_class.new.perform('template_id' => template.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'template.updated', - 'timestamp' => Time.current, - 'data' => Templates::SerializeForApi.call(template.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Templates::SerializeForApi.call(template.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook' @@ -39,11 +39,11 @@ described_class.new.perform('template_id' => template.id, 'webhook_url_id' => webhook_url.id) expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: replace_timestamps({ + body: { 'event_type' => 'template.updated', - 'timestamp' => Time.current, - 'data' => Templates::SerializeForApi.call(template.reload) - }.deep_stringify_keys), + 'timestamp' => /.*/, + 'data' => JSON.parse(Templates::SerializeForApi.call(template.reload).to_json) + }, headers: { 'Content-Type' => 'application/json', 'User-Agent' => 'DocuSeal.com Webhook', diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 94f071be6..86a15a9f8 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -67,19 +67,3 @@ Sidekiq::Testing.inline! if example.metadata[:sidekiq] == :inline end end - -def replace_timestamps(data, replace = /.*/) - timestamp_fields = %w[created_at updated_at completed_at sent_at opened_at timestamp] - - data.each do |key, value| - if timestamp_fields.include?(key) && (value.is_a?(String) || value.is_a?(Time)) - data[key] = replace - elsif value.is_a?(Hash) - replace_timestamps(value) - elsif value.is_a?(Array) - value.each { |item| replace_timestamps(item) if item.is_a?(Hash) } - end - end - - data -end From 8fbb8b8f6e11bb4ae62ad480cbf871bea987b1c8 Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Tue, 5 Nov 2024 22:54:59 +0200 Subject: [PATCH 03/14] do not touch updated_at --- lib/submissions/generate_result_attachments.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/submissions/generate_result_attachments.rb b/lib/submissions/generate_result_attachments.rb index d42fa2f43..3bc6a2905 100644 --- a/lib/submissions/generate_result_attachments.rb +++ b/lib/submissions/generate_result_attachments.rb @@ -68,7 +68,7 @@ def call(submitter) name: item['name']) end - return result_attachments.map { |e| e.tap(&:save!) } if image_pdfs.size < 2 + return ApplicationRecord.no_touching { result_attachments.map { |e| e.tap(&:save!) } } if image_pdfs.size < 2 images_pdf = image_pdfs.each_with_object(HexaPDF::Document.new) do |pdf, doc| @@ -87,7 +87,9 @@ def call(submitter) name: template.name ) - (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) } + ApplicationRecord.no_touching do + (result_attachments + [images_pdf_attachment]).map { |e| e.tap(&:save!) } + end end def generate_pdfs(submitter) From e9034d8da702f17c0e326d61c88534e67f1e9da3 Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Wed, 6 Nov 2024 13:44:55 +0200 Subject: [PATCH 04/14] webhook secret priority --- app/jobs/send_form_completed_webhook_request_job.rb | 4 ++-- app/jobs/send_submission_completed_webhook_request_job.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/jobs/send_form_completed_webhook_request_job.rb b/app/jobs/send_form_completed_webhook_request_job.rb index 30810c328..22a724588 100644 --- a/app/jobs/send_form_completed_webhook_request_job.rb +++ b/app/jobs/send_form_completed_webhook_request_job.rb @@ -28,9 +28,9 @@ def perform(params = {}) timestamp: Time.current, data: Submitters::SerializeForWebhook.call(submitter) }.to_json, - **webhook_url.secret.to_h, 'Content-Type' => 'application/json', - 'User-Agent' => USER_AGENT) + 'User-Agent' => USER_AGENT, + **webhook_url.secret.to_h) rescue Faraday::Error nil end diff --git a/app/jobs/send_submission_completed_webhook_request_job.rb b/app/jobs/send_submission_completed_webhook_request_job.rb index 085e721c1..d6037d0e9 100644 --- a/app/jobs/send_submission_completed_webhook_request_job.rb +++ b/app/jobs/send_submission_completed_webhook_request_job.rb @@ -24,9 +24,9 @@ def perform(params = {}) timestamp: Time.current, data: Submissions::SerializeForApi.call(submission) }.to_json, - **webhook_url.secret.to_h, 'Content-Type' => 'application/json', - 'User-Agent' => USER_AGENT) + 'User-Agent' => USER_AGENT, + **webhook_url.secret.to_h) rescue Faraday::Error nil end From 0b0f0b497e8b6eec1ac4d2b5bd4f99973e4ca713 Mon Sep 17 00:00:00 2001 From: Alex Turchyn Date: Wed, 6 Nov 2024 19:27:37 +0200 Subject: [PATCH 05/14] fix acroform select field --- lib/templates/find_acro_fields.rb | 23 ++++++++++++++++++++--- lib/templates/process_document.rb | 9 +++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/templates/find_acro_fields.rb b/lib/templates/find_acro_fields.rb index b6ae50cea..ac2d5c450 100644 --- a/lib/templates/find_acro_fields.rb +++ b/lib/templates/find_acro_fields.rb @@ -6,6 +6,20 @@ module FindAcroFields FIELD_NAME_REGEXP = /\A(?=.*\p{L})[\p{L}\d\s-]+\z/ SKIP_FIELD_DESCRIPTION = %w[undefined].freeze + SELECT_PLACEHOLDER_REGEXP = /\b( + Select | + Choose | + Wählen | + Auswählen | + Sélectionner| + Choisir | + Seleccionar | + Elegir | + Seleziona | + Scegliere | + Selecionar | + Escolher + )\b/ix module_function @@ -143,8 +157,8 @@ def build_field_properties(field) { **attrs, type: 'select', - options: build_options(field[:Opt]), - default_value: field.field_value + options: build_options(field[:Opt], 'select'), + default_value: field.field_value.to_s.match?(SELECT_PLACEHOLDER_REGEXP) ? nil : field.field_value } elsif field.field_type == :Ch && field.concrete_field_type == :multi_select && field[:Opt].present? { @@ -178,11 +192,14 @@ def build_field_properties(field) def build_options(values, type = nil) is_skip_single_value = type.in?(%w[radio multiple]) && values.uniq.size == 1 - values.map do |option| + values.filter_map do |option| is_option_number = option.is_a?(Symbol) && option.to_s.match?(/\A\d+\z/) + option = option[1] if option.is_a?(Array) && option.size == 2 option = option.encode('utf-8', invalid: :replace, undef: :replace, replace: '') if option.is_a?(String) + next if type == 'select' && option.to_s.match?(SELECT_PLACEHOLDER_REGEXP) + { uuid: SecureRandom.uuid, value: is_option_number || is_skip_single_value ? '' : option diff --git a/lib/templates/process_document.rb b/lib/templates/process_document.rb index b840bd9ae..d7f3d36f3 100644 --- a/lib/templates/process_document.rb +++ b/lib/templates/process_document.rb @@ -122,6 +122,15 @@ def maybe_flatten_form(data, pdf) io = StringIO.new + pdf.acro_form.each_field do |field| + next if field.field_type != :Ch || + field[:Opt].blank? || + %i[combo_box editable_combo_box].exclude?(field.concrete_field_type) || + !field.field_value.to_s.match?(FindAcroFields::SELECT_PLACEHOLDER_REGEXP) + + field[:V] = '' + end + pdf.acro_form.create_appearances(force: true) if pdf.acro_form[:NeedAppearances] pdf.acro_form.flatten From c79fca75f331e404e828643aa72bcced88bacbee Mon Sep 17 00:00:00 2001 From: Pete Matsyburka Date: Thu, 7 Nov 2024 20:58:43 +0200 Subject: [PATCH 06/14] add complete button --- app/javascript/submission_form/area.vue | 1 - app/javascript/submission_form/form.vue | 68 +++++++++++++++++++++++-- app/javascript/submission_form/i18n.js | 13 +++++ 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/app/javascript/submission_form/area.vue b/app/javascript/submission_form/area.vue index c77222e6a..5d08ce752 100644 --- a/app/javascript/submission_form/area.vue +++ b/app/javascript/submission_form/area.vue @@ -39,7 +39,6 @@
+ + + + + +