From 908552d689ff99481042ca11519361a764c4fdd2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 26 Sep 2023 15:16:10 -0700 Subject: [PATCH 01/22] Updates the oai feed --- app/models/oral_history_item.rb | 44 +++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index 0e040c6f..ec5bbb97 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -26,35 +26,31 @@ def self.index_logger def self.client(args) - url = args[:url] || "https://webservices.library.ucla.edu/dldataprovider/oai2_0.do" - OAI::Client.new url, :headers => { "From" => "rob@notch8.com" }, :parser => 'rexml', metadata_prefix: 'mods' + url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" + + OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) end def self.fetch(args) - set = args[:set] || "oralhistory" - response = client(args).list_records(set: set, metadata_prefix: 'mods') + response = client(args).list_records end def self.get(args) - response = client(args).get_record(identifier: args[:identifier], metadata_prefix: 'mods', ) + response = client(args).get_record(identifier: args[:identifier] ) end def self.fetch_first_id - response = self.fetch({progress: false, limit:1}) - response.full&.first&.header&.identifier&.split('/')&.last + response = self.fetch({limit:1}) + response.full&.first&.header&.identifier end def self.import(args) return false if !args[:override] && check_for_tmp_file begin create_import_tmp_file - progress = args[:progress] || true limit = args[:limit] || 20000000 # essentially no limit response = self.fetch(args) - if progress - bar = ProgressBar.new(response.doc.elements['//resumptionToken'].attributes['completeListSize'].to_i) - end total = 0 new_record_ids = [] @@ -78,9 +74,6 @@ def self.import(args) yield(total) if block_given? end - if progress - bar.increment! - end total += 1 break if total >= limit end @@ -116,8 +109,8 @@ def self.process_record(record) if record.header.blank? || record.header.identifier.blank? return false end - - history = OralHistoryItem.find_or_new(record.header.identifier.split('/').last) #Digest::MD5.hexdigest(record.header.identifier).to_i(16)) + record_id = record.header.identifier.gsub('/','-') + history = OralHistoryItem.find_or_new(record_id) #Digest::MD5.hexdigest(record.header.identifier).to_i(16)) history.attributes['id_t'] = history.id if record.header.datestamp history.attributes[:timestamp] = Time.parse(record.header.datestamp) @@ -201,7 +194,7 @@ def self.process_record(record) if child.elements['mods:location/mods:url[@usage="timed log"]'].present? time_log_url = child.elements['mods:location/mods:url[@usage="timed log"]'].text - transcript = self.generate_transcript(time_log_url) + transcript = self.generate_xml_transcript(time_log_url) history.attributes["transcripts_json_t"] << { "transcript_t": transcript, "order_i": order @@ -272,6 +265,7 @@ def self.process_record(record) elsif child.name == 'location' child.elements.each do |f| history.attributes['links_t'] << [f.text, f.attributes['displayLabel']].to_json + order = child.elements['mods:part'].present? ? child.elements['mods:part'].attributes['order'] : 1 if f.attributes['displayLabel'] && has_xml_transcripts == false && history.attributes["transcripts_t"].blank? && @@ -279,6 +273,9 @@ def self.process_record(record) f.text.match(/pdf/i) history.should_process_pdf_transcripts = true pdf_text = f.text + history.attributes["transcripts_json_t"] << { + "order_i": order + }.to_json end end elsif child.name == 'physicalDescription' @@ -367,7 +364,7 @@ def self.find_or_new(id) OralHistoryItem.new(id: id) end - def self.generate_transcript(url) + def self.generate_xml_transcript(url) tmpl = Nokogiri::XSLT(File.read('public/convert.xslt')) resp = Net::HTTP.get(URI(url)) @@ -396,10 +393,19 @@ def peak_job_queued? Delayed::Job.where("handler LIKE ? AND last_error IS ?", "%job_class: ProcessPeakJob%#{self.id}%", nil).present? end + def pdf_transcript_job_queued? + Delayed::Job.where("handler LIKE ? AND last_error IS ?", "%job_class: IndexPdfTranscriptJob%#{self.id}%", nil).present? + end + def should_process_peaks? !has_peaks? && !peak_job_queued? end + def should_process_pdf_transcripts + @should_process_pdf_transcripts ||= false + @should_process_pdf_transcripts && !pdf_transcript_job_queued? + end + def self.create_import_tmp_file FileUtils.touch(Rails.root.join('tmp/importer.tmp')) end @@ -411,4 +417,4 @@ def self.remove_import_tmp_file def self.check_for_tmp_file File.exist?(File.join('tmp/importer.tmp')) end -end +end \ No newline at end of file From 6126ad826ceabecd1ca50860c8696c18bba824e2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 26 Sep 2023 19:41:42 -0700 Subject: [PATCH 02/22] Remove the update code from our forked repo --- app/models/oral_history_item.rb | 45 ++++++++++++--------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index ec5bbb97..d4880759 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -25,25 +25,25 @@ def self.index_logger end - def self.client(args) - url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" + def self.client(args) + url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" - OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) - end + OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) + end - def self.fetch(args) - response = client(args).list_records - end + def self.fetch(args) + response = client(args).list_records + end - def self.get(args) - response = client(args).get_record(identifier: args[:identifier] ) - end + def self.get(args) + response = client(args).get_record(identifier: args[:identifier] ) + end - def self.fetch_first_id - response = self.fetch({limit:1}) - response.full&.first&.header&.identifier - end + def self.fetch_first_id + response = self.fetch({limit:1}) + response.full&.first&.header&.identifier + end def self.import(args) return false if !args[:override] && check_for_tmp_file @@ -194,7 +194,7 @@ def self.process_record(record) if child.elements['mods:location/mods:url[@usage="timed log"]'].present? time_log_url = child.elements['mods:location/mods:url[@usage="timed log"]'].text - transcript = self.generate_xml_transcript(time_log_url) + transcript = self.generate_transcript(time_log_url) history.attributes["transcripts_json_t"] << { "transcript_t": transcript, "order_i": order @@ -265,7 +265,6 @@ def self.process_record(record) elsif child.name == 'location' child.elements.each do |f| history.attributes['links_t'] << [f.text, f.attributes['displayLabel']].to_json - order = child.elements['mods:part'].present? ? child.elements['mods:part'].attributes['order'] : 1 if f.attributes['displayLabel'] && has_xml_transcripts == false && history.attributes["transcripts_t"].blank? && @@ -273,9 +272,6 @@ def self.process_record(record) f.text.match(/pdf/i) history.should_process_pdf_transcripts = true pdf_text = f.text - history.attributes["transcripts_json_t"] << { - "order_i": order - }.to_json end end elsif child.name == 'physicalDescription' @@ -364,7 +360,7 @@ def self.find_or_new(id) OralHistoryItem.new(id: id) end - def self.generate_xml_transcript(url) + def self.generate_transcript(url) tmpl = Nokogiri::XSLT(File.read('public/convert.xslt')) resp = Net::HTTP.get(URI(url)) @@ -393,19 +389,10 @@ def peak_job_queued? Delayed::Job.where("handler LIKE ? AND last_error IS ?", "%job_class: ProcessPeakJob%#{self.id}%", nil).present? end - def pdf_transcript_job_queued? - Delayed::Job.where("handler LIKE ? AND last_error IS ?", "%job_class: IndexPdfTranscriptJob%#{self.id}%", nil).present? - end - def should_process_peaks? !has_peaks? && !peak_job_queued? end - def should_process_pdf_transcripts - @should_process_pdf_transcripts ||= false - @should_process_pdf_transcripts && !pdf_transcript_job_queued? - end - def self.create_import_tmp_file FileUtils.touch(Rails.root.join('tmp/importer.tmp')) end From c5e0e9dfa6c7937da6b394034881b68ca04ea5b1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 27 Sep 2023 23:56:52 -0700 Subject: [PATCH 03/22] Give me control over deleting the delayed_jobs --- app/controllers/admin_controller.rb | 5 +++++ app/views/admin/index.html.erb | 2 +- app/views/delayed_jobs/destroy_all.html.erb | 4 ++++ config/routes.rb | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 app/views/delayed_jobs/destroy_all.html.erb diff --git a/app/controllers/admin_controller.rb b/app/controllers/admin_controller.rb index 606457da..12324cf8 100644 --- a/app/controllers/admin_controller.rb +++ b/app/controllers/admin_controller.rb @@ -12,4 +12,9 @@ def run_import def logs send_file(Rails.root.join('log/indexing.log')) end + + def destroy_all_delayed_jobs + Delayed::Job.destroy_all + redirect_to root_path, notice: 'All Delayed::Jobs have been destroyed.' + end end diff --git a/app/views/admin/index.html.erb b/app/views/admin/index.html.erb index 84726fda..fd39130a 100644 --- a/app/views/admin/index.html.erb +++ b/app/views/admin/index.html.erb @@ -19,7 +19,7 @@ <% end %> <%= link_to "Background Jobs", delayed_job_web_path, class: "btn btn-lg btn-primary" %> <%= link_to "Download Logs", admin_logs_path, class: "btn btn-lg btn-primary" %> - + <%= button_to "Destroy All Delayed Jobs", destroy_all_delayed_jobs_path, method: :delete, data: { confirm: 'Are you sure?' }, class: 'btn btn-lg btn-danger' %>
<%= link_to t('blacklight.header_links.logout'), destroy_user_session_path %>
diff --git a/app/views/delayed_jobs/destroy_all.html.erb b/app/views/delayed_jobs/destroy_all.html.erb new file mode 100644 index 00000000..6624de5e --- /dev/null +++ b/app/views/delayed_jobs/destroy_all.html.erb @@ -0,0 +1,4 @@ + +

Destroy All Delayed Jobs

+

Are you sure you want to destroy all Delayed Jobs?

+<%= button_to 'Destroy All', destroy_all_delayed_jobs_path, method: :delete, data: { confirm: 'Are you sure?' }, class: 'btn btn-danger' %> diff --git a/config/routes.rb b/config/routes.rb index 989330b8..39e5d52e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -25,6 +25,8 @@ post 'admin/run_import', to: 'admin#run_import', as: 'run_import' + delete 'destroy_all_delayed_jobs', to: 'admin#destroy_all_delayed_jobs' + mount Blacklight::Engine => '/' mount BlacklightDynamicSitemap::Engine => '/' From 6b5b122a362a8bf725be1b4946b8bce95ed41242 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 28 Sep 2023 22:58:22 -0700 Subject: [PATCH 04/22] oh lonely operator -- thanks for the save on nil class --- app/models/oral_history_item.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index d4880759..62db79fb 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -227,7 +227,7 @@ def self.process_record(record) history.attributes["series_facet"] = child.elements['mods:titleInfo/mods:title'].text history.attributes["series_t"] = child.elements['mods:titleInfo/mods:title'].text history.attributes["series_sort"] = child.elements['mods:titleInfo/mods:title'].text - history.attributes["abstract_display"] = child.elements['mods:abstract'].text + history.attributes["abstract_display"] = child.elements['mods:abstract']&.text history.attributes["abstract_t"] = [] history.attributes["abstract_t"] << child.elements['mods:abstract'].text elsif child.name == "note" From bae68b6e64826dcaf28ff13e5dd206d65da82ad7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 29 Sep 2023 10:11:04 -0700 Subject: [PATCH 05/22] update the oai feed in all places --- app/models/oral_history_item.rb | 36 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index 62db79fb..279f75f0 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -25,25 +25,25 @@ def self.index_logger end - def self.client(args) - url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" + def self.client(args) + url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" - OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) - end + OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) + end - def self.fetch(args) - response = client(args).list_records - end + def self.fetch(args) + response = client(args).list_records + end - def self.get(args) - response = client(args).get_record(identifier: args[:identifier] ) - end + def self.get(args) + response = client(args).get_record(identifier: args[:identifier] ) + end - def self.fetch_first_id - response = self.fetch({limit:1}) - response.full&.first&.header&.identifier - end + def self.fetch_first_id + response = self.fetch({limit:1}) + response.full&.first&.header&.identifier + end def self.import(args) return false if !args[:override] && check_for_tmp_file @@ -370,11 +370,9 @@ def self.generate_transcript(url) end def self.total_records(args = {}) - url = args[:url] || "https://webservices.library.ucla.edu/dldataprovider/oai2_0.do" - set = args[:set] || "oralhistory" - client = OAI::Client.new url, :headers => { "From" => "rob@notch8.com" }, :parser => 'rexml', metadata_prefix: 'mods' - response = client.list_records(set: set, metadata_prefix: 'mods') - response.doc.elements['//resumptionToken'].attributes['completeListSize'].to_i + url = args[:url] || "https://oh-staff.library.ucla.edu/oai/" + + OAI::Client.new(url, http: Faraday.new {|c| c.options.timeout = 300}) end def has_peaks? From 63e28a66f1d7ab84a45c06a26f16e7f1c1a9da09 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 2 Oct 2023 12:36:41 -0700 Subject: [PATCH 06/22] updates seeds --- db/seeds.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index d39836b2..630061c1 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -10,10 +10,4 @@ u = User.find_or_create_by(email: ENV['ADMIN_EMAIL'] || 'admin@example.com') u.password = ENV['ADMIN_PASSWORD'] || 'password' u.save - - # Import the work with the video - OralHistoryItem.import_single("21198-zz002jxz7z") - - # Import additional works with audio - OralHistoryItem.import_single("21198-zz002bfs89") end From 46fe6ddda7a5f7e6ac01940cb1810b2c429a577a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 5 Oct 2023 20:56:23 -0700 Subject: [PATCH 07/22] updated to add a lonely operator when .text is nil --- app/models/oral_history_item.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index 279f75f0..288b2a80 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -229,7 +229,7 @@ def self.process_record(record) history.attributes["series_sort"] = child.elements['mods:titleInfo/mods:title'].text history.attributes["abstract_display"] = child.elements['mods:abstract']&.text history.attributes["abstract_t"] = [] - history.attributes["abstract_t"] << child.elements['mods:abstract'].text + history.attributes["abstract_t"] << child.elements['mods:abstract']&.text elsif child.name == "note" if child.attributes == {} history.attributes["admin_note_display"] = child.text From d12805baaebd7874afabf1a287faebf5f1e04b5d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 5 Oct 2023 21:58:55 -0700 Subject: [PATCH 08/22] Nominal update to kick off deploy --- .env | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 3d81569c..d8f4bb25 100644 --- a/.env +++ b/.env @@ -1,15 +1,25 @@ +# UCLALibrary - .env setup -- Start DEPLOY_HOOK=CHANGEME DOCKER_PORTS=80 -MAKE_WAVES= +# MAKE_WAVES= NEGATIVE_CAPTCHA_SECRET=64fe54311a8e54b637a1da1ff993b560ff5c742211f645f35b8b9bd8b3d2e4015e95dea8db4dc235df0396ddd94d21d18d0c787bcaa5b579cb5f6f2aac90e601 PASSENGER_APP_ENV=development POSTGRES_DB=oral_history POSTGRES_HOST=postgres POSTGRES_PASSWORD=DatabaseFTW POSTGRES_USER=postgres +# Commented out for development purposes @SoftServ REGISTRY_HOST=index.docker.io/ REGISTRY_URI=uclalibrary SITE_URI=oralhistory-test.library.ucla.edu SOLR_URL=http://solr:8983/solr/blacklight-core TAG=dev TEST_DB=oral_history_test +# UCLALibrary - .env setup -- End + +# SoftServ - .env additions/alterations --- Start +# REGISTRY_HOST=ghcr.io +# REGISTRY_URI=oral-history +# ADMIN_EMAIL=admin@example.com +# ADMIN_PASSWORD=testing123 +# SoftServ - .env additions/alterations --- End \ No newline at end of file From 8bdc3a781233fd7c6c641c37283e2a7efa0f7ffe Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 31 Oct 2023 07:59:36 -0700 Subject: [PATCH 09/22] Add waves env back --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index d8f4bb25..a40b06a4 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ # UCLALibrary - .env setup -- Start DEPLOY_HOOK=CHANGEME DOCKER_PORTS=80 -# MAKE_WAVES= +MAKE_WAVES= NEGATIVE_CAPTCHA_SECRET=64fe54311a8e54b637a1da1ff993b560ff5c742211f645f35b8b9bd8b3d2e4015e95dea8db4dc235df0396ddd94d21d18d0c787bcaa5b579cb5f6f2aac90e601 PASSENGER_APP_ENV=development POSTGRES_DB=oral_history From 3911b03879f83a70e252d6c94956a6255c487bb8 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Tue, 7 Nov 2023 12:53:02 -0800 Subject: [PATCH 10/22] Remove session and series metadata --- app/models/oral_history_item.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index 288b2a80..67effb5b 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -133,6 +133,10 @@ def self.process_record(record) history.attributes['links_t'] = [] set.children.each do |child| next if child.class == REXML::Text + + # Skip series and session level metadata + next if child.name == "relatedItem" && ['constituent', 'series'].include?(child.attributes['type']) + if child.name == "titleInfo" child.elements.each('mods:title') do |title| title_text = title.text.to_s.strip From b1b79e92268739aa8e63f1e665461541ca50e02a Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 10 Jan 2024 11:14:31 -0800 Subject: [PATCH 11/22] Tag it staging --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 77fb2065..65f15016 100644 --- a/.env +++ b/.env @@ -20,7 +20,7 @@ SOLR_ADMIN_PASSWORD=CHANGEME SOLR_HOST=solr SOLR_PORT=8983 SOLR_URL="http://${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}@${SOLR_HOST}:${SOLR_PORT}/solr/blacklight-core" -TAG=dev +TAG=staging TEST_DB=oral_history_test # UCLALibrary - .env setup -- End From abae1a9f0063d023fcc1675f26a0ce78fe2e11a1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 10 Jan 2024 11:23:28 -0800 Subject: [PATCH 12/22] really tag it for staging oin the helm chart values file --- charts/stage-oralhistory-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/stage-oralhistory-values.yaml b/charts/stage-oralhistory-values.yaml index 3f30b1eb..3d2aa0b9 100644 --- a/charts/stage-oralhistory-values.yaml +++ b/charts/stage-oralhistory-values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: uclalibrary/oral-history # changing this tag will cause a deploy via ArgoCD - tag: v0.0.2 + tag: v0.0.3 pullPolicy: Always # Chart documentation: https://github.com/bitnami/charts/tree/main/bitnami/solr From d1df0ab819e869cd01814b0ceb7b3bb65b9e7b65 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 10 Jan 2024 11:35:14 -0800 Subject: [PATCH 13/22] Changed the one in the chart.yaml file for testing a deploy --- charts/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index ab0ad2af..370d00eb 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "0.0.2" description: Chart for Oral History Public-Facing App name: oralhistory -version: 1.0.3 +version: 1.0.4 # The `appVersion` is not a required field whereas `version` is required. If # you’re making changes to a helm chart template file and/or the default values From 7f75c643f933b9fc8c028fbdb391c7ace84c01c8 Mon Sep 17 00:00:00 2001 From: "John H. Robinson, IV" Date: Fri, 12 Jan 2024 12:31:06 -0800 Subject: [PATCH 14/22] Set the values file by target environment If the branch name is: - test - starts with test- - ends in -test - has *-test/* - has */test-* then test If the branch name is: - main then prod Otherwise: stage --- .github/workflows/build-dockerhub.yml | 5 ++++- .github/workflows/get-k8s-environment.sh | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/get-k8s-environment.sh diff --git a/.github/workflows/build-dockerhub.yml b/.github/workflows/build-dockerhub.yml index 864d7e81..0aea76f3 100644 --- a/.github/workflows/build-dockerhub.yml +++ b/.github/workflows/build-dockerhub.yml @@ -20,11 +20,14 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set the K8S Environment + run: echo "K8S_DEPLOY_ENVIRONMENT=$($GITHUB_WORKSPACE/.github/workflows/get-k8s-environment.sh)" >> "$GITHUB_ENV" + - name: Read chart yaml to get version for docker tag id: image-tag uses: KJ002/read-yaml@1.6 with: - file: 'charts/prod-oralhistory-values.yaml' + file: 'charts/${{ env.K8S_DEPLOY_ENVIRONMENT }}-oralhistory-values.yaml' key-path: '["image", "tag"]' - name: Report image tag diff --git a/.github/workflows/get-k8s-environment.sh b/.github/workflows/get-k8s-environment.sh new file mode 100755 index 00000000..b840aa30 --- /dev/null +++ b/.github/workflows/get-k8s-environment.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +case "$GITHUB_REF" in + test | test-* | */test | */test-* | *-test/* | *-test) + echo test ;; + main) + echo prod ;; + *) + echo stage ;; +esac From 0a5d6c59b7d2ba3eaea134707bd634dffd96ac39 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 20 Jan 2024 15:26:54 -0800 Subject: [PATCH 15/22] Fix the single import by fixing the format of the id with gsub --- app/models/oral_history_item.rb | 68 ++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index 134d73c3..d8e308ab 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -93,7 +93,8 @@ def self.import(args) end def self.import_single(id) - record = self.get(identifier: id)&.record + converted_id = id.gsub('-','/') + record = self.get(identifier: converted_id)&.record history = process_record(record) history.index_record if ENV['MAKE_WAVES'] && history.attributes["audio_b"] && history.should_process_peaks? @@ -134,10 +135,7 @@ def self.process_record(record) set.children.each do |child| next if child.class == REXML::Text - # Skip series and session level metadata - next if child.name == "relatedItem" && ['constituent', 'series'].include?(child.attributes['type']) - - if child.name == "titleInfo" + if child.name == "titleInfo" # child.elements.each('mods:title') do |title| title_text = title.text.to_s.strip if(child.attributes["type"] == "alternative") && title_text.size > 0 @@ -154,22 +152,30 @@ def self.process_record(record) end end end - elsif child.name == "typeOfResource" - history.attributes["type_of_resource_display"] = child.text - history.attributes["type_of_resource_t"] ||= [] - history.attributes["type_of_resource_t"] << child.text - history.attributes["type_of_resource_facet"] ||= [] - history.attributes["type_of_resource_facet"] << child.text + + # not in new oai feed remove? + # elsif child.name == "typeOfResource" + # history.attributes["type_of_resource_display"] = child.text + # history.attributes["type_of_resource_t"] ||= [] + # history.attributes["type_of_resource_t"] << child.text + # history.attributes["type_of_resource_facet"] ||= [] + # history.attributes["type_of_resource_facet"] << child.text + + # elsif child.name == "accessCondition" history.attributes["rights_display"] = [child.text] history.attributes["rights_t"] = [] history.attributes["rights_t"] << child.text + + # elsif child.name == 'language' child.elements.each('mods:languageTerm') do |e| history.attributes["language_facet"] = LanguageList::LanguageInfo.find(e.text).try(:name) history.attributes["language_sort"] = LanguageList::LanguageInfo.find(e.text).try(:name) history.attributes["language_t"] = [LanguageList::LanguageInfo.find(e.text).try(:name)] end + + # elsif child.name == "subject" child.elements.each('mods:topic') do |e| history.attributes["subject_topic_facet"] ||= [] @@ -177,13 +183,21 @@ def self.process_record(record) history.attributes["subject_t"] ||= [] history.attributes["subject_t"] << e.text end + + # elsif child.name == "name" + + # + # interviewer if child.elements['mods:role/mods:roleTerm'].text == "interviewer" history.attributes["author_display"] = child.elements['mods:namePart'].text history.attributes["author_t"] ||= [] if !history.attributes["author_t"].include?(child.elements['mods:namePart'].text) history.attributes["author_t"] << child.elements['mods:namePart'].text end + + # + # interviewee elsif child.elements['mods:role/mods:roleTerm'].text == "interviewee" history.attributes["interviewee_display"] = child.elements['mods:namePart'].text history.attributes["interviewee_t"] ||= [] @@ -192,10 +206,11 @@ def self.process_record(record) end history.attributes["interviewee_sort"] = child.elements['mods:namePart'].text end + + # elsif child.name == "relatedItem" && child.attributes['type'] == "constituent" time_log_url = '' order = child.elements['mods:part'].present? ? child.elements['mods:part'].attributes['order'] : 1 - if child.elements['mods:location/mods:url[@usage="timed log"]'].present? time_log_url = child.elements['mods:location/mods:url[@usage="timed log"]'].text transcript = self.generate_xml_transcript(time_log_url) @@ -225,8 +240,10 @@ def self.process_record(record) end history.attributes["peaks_t"] ||= [] child_doc_json = child_document.to_json - history.attributes["peaks_t"] << child_doc_json unless history.attributes["peaks_t"].include? child_doc_json + history.attributes["peaks_t"] << child_doc_json unless history.attributes["peaks_t"].include? child_doc_json history.attributes["children_t"] << child_doc_json + + # elsif child.name == "relatedItem" && child.attributes['type'] == "series" history.attributes["series_facet"] = child.elements['mods:titleInfo/mods:title'].text history.attributes["series_t"] = child.elements['mods:titleInfo/mods:title'].text @@ -234,42 +251,60 @@ def self.process_record(record) history.attributes["abstract_display"] = child.elements['mods:abstract']&.text history.attributes["abstract_t"] = [] history.attributes["abstract_t"] << child.elements['mods:abstract']&.text + + # elsif child.name == "note" if child.attributes == {} history.attributes["admin_note_display"] = child.text history.attributes["admin_note_t"] = [] history.attributes["admin_note_t"] << child.text end + + # if child.attributes['type'].to_s.match('biographical') history.attributes["biographical_display"] = child.text history.attributes["biographical_t"] = [] history.attributes["biographical_t"] << child.text end + + # if child.attributes['type'].to_s.match('personpresent') history.attributes['person_present_display'] = child.text history.attributes['person_present_t'] << child.text end + + # if child.attributes['type'].to_s.match('place') history.attributes['place_display'] = child.text history.attributes['place_t'] << child.text end + + # if child.attributes['type'].to_s.match('supportingdocuments') history.attributes['supporting_documents_display'] = child.text history.attributes['supporting_documents_t'] << child.text end + + # if child.attributes['type'].to_s.match('interviewerhistory') history.attributes['interviewer_history_display'] = child.text history.attributes['interviewer_history_t'] << child.text end + + # if child.attributes['type'].to_s.match('processinterview') history.attributes['process_interview_display'] = child.text history.attributes['process_interview_t'] << child.text end history.attributes["description_t"] << child.text + + # elsif child.name == 'location' child.elements.each do |f| history.attributes['links_t'] << [f.text, f.attributes['displayLabel']].to_json order = child.elements['mods:part'].present? ? child.elements['mods:part'].attributes['order'] : 1 + + # elsif child.name == 'physicalDescription' history.attributes["extent_display"] = child.elements['mods:extent'].text history.attributes['extent_t'] = [] history.attributes['extent_t'] << child.elements['mods:extent'].text + + # elsif child.name == 'abstract' history.attributes['interview_abstract_display'] = child.text history.attributes["interview_abstract_t"] = [] @@ -413,7 +452,8 @@ def self.create_import_tmp_file end def self.remove_import_tmp_file - FileUtils.rm(Rails.root.join('tmp/importer.tmp')) + tmp_file = Rails.root.join('tmp/importer.tmp') + FileUtils.rm(tmp_file) if File.exist?(tmp_file) end def self.check_for_tmp_file From 243f7148edd8d7e56e3e5fc1dd915075665de741 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 20 Jan 2024 15:54:09 -0800 Subject: [PATCH 16/22] Trigger chart push from branch to stage --- .github/workflows/push-helm-chart.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-helm-chart.yml b/.github/workflows/push-helm-chart.yml index 9afb5a6c..971e2ef1 100644 --- a/.github/workflows/push-helm-chart.yml +++ b/.github/workflows/push-helm-chart.yml @@ -4,10 +4,11 @@ name: Push Helm Chart to ChartMuseum on: push: paths: - - 'charts/**' - - '!charts/*-*-values.yaml' - branches: - - main + - 'charts/stage-oralhistory-values.yaml' + # - 'charts/**' + # - '!charts/*-*-values.yaml' + # branches: + # - main jobs: push-to-chart-museum: From fccd49692d49e966492f53741975fcf5b0b14edf Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 20 Jan 2024 19:32:15 -0800 Subject: [PATCH 17/22] Putting it back the way I found it --- .github/workflows/push-helm-chart.yml | 9 ++++----- app/models/oral_history_item.rb | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push-helm-chart.yml b/.github/workflows/push-helm-chart.yml index 971e2ef1..9afb5a6c 100644 --- a/.github/workflows/push-helm-chart.yml +++ b/.github/workflows/push-helm-chart.yml @@ -4,11 +4,10 @@ name: Push Helm Chart to ChartMuseum on: push: paths: - - 'charts/stage-oralhistory-values.yaml' - # - 'charts/**' - # - '!charts/*-*-values.yaml' - # branches: - # - main + - 'charts/**' + - '!charts/*-*-values.yaml' + branches: + - main jobs: push-to-chart-museum: diff --git a/app/models/oral_history_item.rb b/app/models/oral_history_item.rb index d8e308ab..87b76da8 100644 --- a/app/models/oral_history_item.rb +++ b/app/models/oral_history_item.rb @@ -243,7 +243,7 @@ def self.process_record(record) history.attributes["peaks_t"] << child_doc_json unless history.attributes["peaks_t"].include? child_doc_json history.attributes["children_t"] << child_doc_json - # + # elsif child.name == "relatedItem" && child.attributes['type'] == "series" history.attributes["series_facet"] = child.elements['mods:titleInfo/mods:title'].text history.attributes["series_t"] = child.elements['mods:titleInfo/mods:title'].text From b70d45e23a0f238d630774962ccf1452879cb164 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 22 Jan 2024 11:51:01 -0800 Subject: [PATCH 18/22] Changes from feedback with pairing with Stephen --- .github/workflows/build-dockerhub.yml | 5 +---- charts/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-dockerhub.yml b/.github/workflows/build-dockerhub.yml index 0aea76f3..864d7e81 100644 --- a/.github/workflows/build-dockerhub.yml +++ b/.github/workflows/build-dockerhub.yml @@ -20,14 +20,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set the K8S Environment - run: echo "K8S_DEPLOY_ENVIRONMENT=$($GITHUB_WORKSPACE/.github/workflows/get-k8s-environment.sh)" >> "$GITHUB_ENV" - - name: Read chart yaml to get version for docker tag id: image-tag uses: KJ002/read-yaml@1.6 with: - file: 'charts/${{ env.K8S_DEPLOY_ENVIRONMENT }}-oralhistory-values.yaml' + file: 'charts/prod-oralhistory-values.yaml' key-path: '["image", "tag"]' - name: Report image tag diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 370d00eb..e8fa72e9 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "0.0.2" description: Chart for Oral History Public-Facing App name: oralhistory -version: 1.0.4 +version: 1.0.0 # The `appVersion` is not a required field whereas `version` is required. If # you’re making changes to a helm chart template file and/or the default values From d236c31c34fd10690d58012d86ef24f2b72f1311 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 22 Jan 2024 11:52:28 -0800 Subject: [PATCH 19/22] Changes from feedback with pairing with Stephen part 2 --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 65f15016..67eab864 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ # UCLALibrary - .env setup -- Start DEPLOY_HOOK=CHANGEME DOCKER_PORTS=80 -MAKE_WAVES= +MAKE_WAVES=true # Set to false to disable Waveform generation NEGATIVE_CAPTCHA_SECRET=64fe54311a8e54b637a1da1ff993b560ff5c742211f645f35b8b9bd8b3d2e4015e95dea8db4dc235df0396ddd94d21d18d0c787bcaa5b579cb5f6f2aac90e601 SECRET_KEY_BASE=CHANGEME PASSENGER_APP_ENV=development @@ -20,7 +20,7 @@ SOLR_ADMIN_PASSWORD=CHANGEME SOLR_HOST=solr SOLR_PORT=8983 SOLR_URL="http://${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}@${SOLR_HOST}:${SOLR_PORT}/solr/blacklight-core" -TAG=staging +TAG=dev TEST_DB=oral_history_test # UCLALibrary - .env setup -- End From 48fa5873df270da4935c4b55f68f2d7fd36ce0c7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 22 Jan 2024 11:53:24 -0800 Subject: [PATCH 20/22] Changes from feedback with pairing with Stephen part 3 --- .github/workflows/get-k8s-environment.sh | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100755 .github/workflows/get-k8s-environment.sh diff --git a/.github/workflows/get-k8s-environment.sh b/.github/workflows/get-k8s-environment.sh deleted file mode 100755 index b840aa30..00000000 --- a/.github/workflows/get-k8s-environment.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -case "$GITHUB_REF" in - test | test-* | */test | */test-* | *-test/* | *-test) - echo test ;; - main) - echo prod ;; - *) - echo stage ;; -esac From 5a3acdd5b508c892560e3ce18f2775a8fd4ae329 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Mon, 5 Feb 2024 17:02:16 -0800 Subject: [PATCH 21/22] Round 1 test in CI --- .github/workflows/build-dockerhub.yml | 36 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-dockerhub.yml b/.github/workflows/build-dockerhub.yml index 864d7e81..2b5f0428 100644 --- a/.github/workflows/build-dockerhub.yml +++ b/.github/workflows/build-dockerhub.yml @@ -1,13 +1,24 @@ name: Build Oral History Web for Docker Hub on: push: - # branches: - # - main - workflow_dispatch: + branches: + - main + pull_request: + branches: + - main + jobs: build-for-docker-hub: runs-on: ubuntu-latest steps: + - name: Set env + run: >- + echo "TAG=${HEAD_TAG::8}" >> ${GITHUB_ENV}; + echo ${HEAD_TAG::8} + env: + HEAD_TAG: ${{ github.event.pull_request.head.sha || github.sha }} + shell: bash + - name: Check out code uses: actions/checkout@v3 @@ -20,19 +31,20 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Read chart yaml to get version for docker tag - id: image-tag - uses: KJ002/read-yaml@1.6 + - name: Retag latest if merge to main action + id: meta-web + uses: docker/metadata-action@v4.1.1 with: - file: 'charts/prod-oralhistory-values.yaml' - key-path: '["image", "tag"]' - - - name: Report image tag - run: echo "${{ steps.image-tag.outputs.data }}" + images: | + name=uclalibrary/oral-history + tags: | + type=raw,value=latest,enable={{is_default_branch}} - name: Build and push uses: docker/build-push-action@v3 with: context: . push: true - tags: uclalibrary/oral-history:${{ steps.image-tag.outputs.data }} + tags: | + ${{ steps.meta-web.outputs.tags }} + uclalibrary/oral-history:${{ env.TAG }} \ No newline at end of file From 7edf6eefd7677d5beb62a8ec0a6be20de94f0573 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 29 Feb 2024 13:38:06 -0800 Subject: [PATCH 22/22] Setup the test values file to have the githsha for deployment to test versus staging - reveret staging --- charts/stage-oralhistory-values.yaml | 2 +- charts/test-oralhistory-values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/stage-oralhistory-values.yaml b/charts/stage-oralhistory-values.yaml index 3d2aa0b9..3f30b1eb 100644 --- a/charts/stage-oralhistory-values.yaml +++ b/charts/stage-oralhistory-values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: uclalibrary/oral-history # changing this tag will cause a deploy via ArgoCD - tag: v0.0.3 + tag: v0.0.2 pullPolicy: Always # Chart documentation: https://github.com/bitnami/charts/tree/main/bitnami/solr diff --git a/charts/test-oralhistory-values.yaml b/charts/test-oralhistory-values.yaml index 56c46f2e..328bc06b 100644 --- a/charts/test-oralhistory-values.yaml +++ b/charts/test-oralhistory-values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: uclalibrary/oral-history # changing this tag will cause a deploy via ArgoCD - tag: v0.0.2 + tag: 5a3acdd5 pullPolicy: Always # Chart documentation: https://github.com/bitnami/charts/tree/main/bitnami/solr