Merge pull request #703 from sanger/develop

Develop into Master

.github/workflows/append_release.yml

@@ -11,7 +11,7 @@ jobs:
   build-and-append-to-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1

.github/workflows/automated_release.yml

@@ -10,7 +10,7 @@ jobs:
   build-and-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - uses: nelonoel/branch-name@v1.0.1
 

.github/workflows/check_release_version.yml

@@ -10,7 +10,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Get specific changed files
         id: changed-files-specific

.github/workflows/docker_ci.yml

@@ -47,7 +47,7 @@ jobs:
           INTEGRATION_TEST_SEED: '/code/spec/data/integration/seed_for_unified_wh.rb'
           RAILS_ENV: test
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: nelonoel/branch-name@v1.0.1
       - name: Build the Docker image
         run: >-

.github/workflows/ruby_ci.yml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
@@ -39,7 +39,7 @@ jobs:
           MYSQL_ALLOW_EMPTY_PASSWORD: yes
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

.release-version

@@ -1 +1 @@
-1.22.0
+1.23.0

.ruby-version

@@ -1 +1 @@
-3.3.5
+3.3.6

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.3.5
+FROM ruby:3.3.6
 
 RUN gem install bundler
 

Gemfile.lock

@@ -7,25 +7,25 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.8.4)
-      actionview (= 7.0.8.4)
-      activesupport (= 7.0.8.4)
+    actionpack (7.0.8.5)
+      actionview (= 7.0.8.5)
+      activesupport (= 7.0.8.5)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.8.4)
-      activesupport (= 7.0.8.4)
+    actionview (7.0.8.5)
+      activesupport (= 7.0.8.5)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activemodel (7.0.8.4)
-      activesupport (= 7.0.8.4)
-    activerecord (7.0.8.4)
-      activemodel (= 7.0.8.4)
-      activesupport (= 7.0.8.4)
-    activesupport (7.0.8.4)
+    activemodel (7.0.8.5)
+      activesupport (= 7.0.8.5)
+    activerecord (7.0.8.5)
+      activemodel (= 7.0.8.5)
+      activesupport (= 7.0.8.5)
+    activesupport (7.0.8.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -45,14 +45,14 @@ GEM
     diff-lcs (1.5.1)
     docile (1.4.0)
     erubi (1.13.0)
-    factory_bot (6.4.5)
+    factory_bot (6.5.0)
       activesupport (>= 5.0.0)
-    factory_bot_rails (6.4.3)
-      factory_bot (~> 6.4)
+    factory_bot_rails (6.4.4)
+      factory_bot (~> 6.5)
       railties (>= 5.0.0)
     ffi (1.17.0)
     formatador (1.1.0)
-    guard (2.18.1)
+    guard (2.19.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
@@ -73,18 +73,18 @@ GEM
     hashie (5.0.0)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    json (2.7.2)
+    json (2.8.2)
     language_server-protocol (3.17.0.3)
-    listen (3.8.0)
+    listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.22.0)
+    loofah (2.23.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-    lumberjack (1.2.9)
+    lumberjack (1.2.10)
     method_source (1.1.0)
-    mini_portile2 (2.8.7)
-    minitest (5.25.1)
+    mini_portile2 (2.8.8)
+    minitest (5.25.2)
     msgpack (1.7.2)
     multi_json (1.15.0)
     mysql2 (0.5.6)
@@ -96,14 +96,14 @@ GEM
       nenv (~> 0.1)
       shellany (~> 0.0)
     parallel (1.26.3)
-    parser (3.3.5.0)
+    parser (3.3.6.0)
       ast (~> 2.4.1)
       racc
-    pry (0.14.2)
+    pry (0.15.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
     racc (1.8.1)
-    rack (2.2.9)
+    rack (2.2.10)
     rack-test (2.1.0)
       rack (>= 1.3)
     rails-dom-testing (2.2.0)
@@ -113,33 +113,33 @@ GEM
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.0.8.4)
-      actionpack (= 7.0.8.4)
-      activesupport (= 7.0.8.4)
+    railties (7.0.8.5)
+      actionpack (= 7.0.8.5)
+      activesupport (= 7.0.8.5)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.2.1)
     rb-fsevent (0.11.2)
-    rb-inotify (0.10.1)
+    rb-inotify (0.11.1)
       ffi (~> 1.0)
     rbtree (0.4.6)
     regexp_parser (2.9.2)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.1)
+    rspec-core (3.13.2)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.1)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-rails (7.0.1)
+    rspec-rails (7.1.0)
       actionpack (>= 7.0)
       activesupport (>= 7.0)
       railties (>= 7.0)
@@ -148,7 +148,7 @@ GEM
       rspec-mocks (~> 3.13)
       rspec-support (~> 3.13)
     rspec-support (3.13.1)
-    rubocop (1.66.1)
+    rubocop (1.68.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -158,17 +158,17 @@ GEM
       rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.32.3)
+    rubocop-ast (1.36.1)
       parser (>= 3.3.1.0)
-    rubocop-performance (1.22.1)
+    rubocop-performance (1.23.0)
       rubocop (>= 1.48.1, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rails (2.26.2)
+    rubocop-rails (2.27.0)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.52.0, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rspec (3.0.5)
+    rubocop-rspec (3.2.0)
       rubocop (~> 1.61)
     ruby-progressbar (1.13.0)
     sanger_warren (0.4.1)
@@ -191,7 +191,7 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.6.0)
-    zeitwerk (2.6.18)
+    zeitwerk (2.7.1)
 
 PLATFORMS
   ruby

README.md

@@ -4,6 +4,34 @@
 
 A denormalised warehouse for multiple LIMS.
 
+Populating a table in the warehouse is done asynchronously. This project is a Railtie application that facilitate asynchronous database population through a message queue. This application listens to a message queue, consumes the message, converts the message into a database model, and persists in the corresponding table. 
+
+Take the example below.
+
+```json
+{
+    "lims": "traction",
+    "aliquot": {
+        "id_lims": "LIMS123456",
+        "lims_uuid": "550e8400-e29b-41d4-a716-446655440000",
+        "aliquot_type": "DNA",
+        "source_type": "Blood",
+        "source_barcode": "SRC123456",
+        "sample_name": "SampleA",
+        "used_by_type": "Research",
+        "used_by_barcode": "USR123456",
+        "volume": 50,
+        "concentration": 200,
+        "last_updated": "2024-07-09T10:15:30Z",
+        "recorded_at": "2024-07-09T09:00:00Z",
+        "created_at": "2024-07-08T08:00:00Z",
+        "insert_size": 350
+    }
+}
+```
+
+If the message above is consumed by `unified_warehouse`, it will create a new record in `aliquot` table with the values set to the given attributes. If an exception occur, it will dead-letter the message. For message serialization logic, please follow [`lib/payload.rb`](https://github.com/sanger/unified_warehouse/blob/beea89418cc1987c426e7967b7c13ff9218b51fb/lib/payload.rb#L31-L33).
+
 ## Usage (Development)
 
 ### Requirement

db/migrate/20241203115427_add_indexes_to_sample.rb

@@ -0,0 +1,11 @@
+class AddIndexesToSample < ActiveRecord::Migration[7.0]
+
+  def change
+    #id_lims and id_sample_lims should be unique together
+    add_index :sample, [:id_lims, :id_sample_lims], unique: true
+
+    #improve search performance
+    add_index :sample, :id_lims, unique: false
+    add_index :sample, :id_sample_lims, unique: false
+  end
+end

db/migrate/20241203134455_add_required_to_uuid_sample_lims_to_sample.rb

@@ -0,0 +1,10 @@
+class AddRequiredToUuidSampleLimsToSample < ActiveRecord::Migration[7.0]
+  def up
+    # uuid_sample_lims should be required
+    change_column_null(:sample, :uuid_sample_lims, false) 
+  end
+
+  def down
+    change_column_null(:sample, :uuid_sample_lims, true)
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_09_19_134424) do
+ActiveRecord::Schema[7.0].define(version: 2024_12_03_134455) do
   create_table "aliquot", charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t|
     t.string "id_lims", null: false, comment: "The LIMS system that the aliquot was created in"
     t.string "aliquot_uuid", null: false, comment: "The UUID of the aliquot in the LIMS system"
@@ -304,7 +304,7 @@
 
   create_table "sample", primary_key: "id_sample_tmp", id: { type: :integer, unsigned: true }, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t|
     t.string "id_lims", limit: 10, null: false, comment: "LIM system identifier, e.g. CLARITY-GCLP, SEQSCAPE"
-    t.string "uuid_sample_lims", limit: 36, comment: "LIMS-specific sample uuid"
+    t.string "uuid_sample_lims", limit: 36, null: false, comment: "LIMS-specific sample uuid"
     t.string "id_sample_lims", null: false, comment: "LIMS-specific sample identifier"
     t.datetime "last_updated", precision: nil, null: false, comment: "Timestamp of last update"
     t.datetime "recorded_at", precision: nil, null: false, comment: "Timestamp of warehouse update"
@@ -367,7 +367,10 @@
     t.string "dna_source"
     t.string "priority_level", comment: "Priority level eg Medium, High etc"
     t.index ["accession_number"], name: "sample_accession_number_index"
+    t.index ["id_lims", "id_sample_lims"], name: "index_sample_on_id_lims_and_id_sample_lims", unique: true
+    t.index ["id_lims"], name: "index_sample_on_id_lims"
     t.index ["id_sample_lims", "id_lims"], name: "index_sample_on_id_sample_lims_and_id_lims", unique: true
+    t.index ["id_sample_lims"], name: "index_sample_on_id_sample_lims"
     t.index ["name"], name: "sample_name_index"
     t.index ["sanger_sample_id"], name: "index_sample_on_sanger_sample_id"
     t.index ["supplier_name"], name: "index_sample_on_supplier_name"

spec/models/sample_spec.rb

@@ -24,41 +24,8 @@
   end
 
   context 'without uuid' do
-    it_behaves_like 'a sample resource'
-
-    let(:json) do
-      {
-        'id' => '1' * 255.to_i,
-        'name' => 'name',
-        'reference_genome' => 'reference genome',
-        'organism' => 'organism',
-        'consent_withdrawn' => true,
-        'accession_number' => 'accession number',
-        'common_name' => 'common name',
-        'description' => 'description',
-        'taxon_id' => 'taxon id',
-        'father' => 'father',
-        'mother' => 'mother',
-        'replicate' => 'replicate',
-        'ethnicity' => 'ethnicity',
-        'gender' => 'gender',
-        'cohort' => 'cohort',
-        'country_of_origin' => 'country of origin',
-        'geographical_region' => 'geographical region',
-        'updated_at' => '2012-03-11 10:22:42',
-        'created_at' => '2012-03-11 10:22:42',
-        'sanger_sample_id' => 'sanger sample id',
-        'control' => true,
-        'empty_supplier_sample_name' => true,
-        'supplier_name' => 'supplier name',
-        'public_name' => 'public name',
-        'sample_visibility' => 'sample visibility',
-        'strain' => 'strain',
-        'updated_by_manifest' => true,
-        'donor_id' => '11111111-2222-3333-4444-555555555556',
-        'developmental_stage' => 'Larval: Day 5 ZFS:0000037',
-        'control_type' => 'positive'
-      }
+    it 'should raiser an error if uuid is null' do
+      expect { create(:sample, uuid_sample_lims: nil) }.to raise_error ActiveRecord::NotNullViolation
     end
   end