diff --git a/.github/workflows/append_release.yml b/.github/workflows/append_release.yml index 08b4707d..a9f7d25a 100644 --- a/.github/workflows/append_release.yml +++ b/.github/workflows/append_release.yml @@ -11,7 +11,7 @@ jobs: build-and-append-to-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 diff --git a/.github/workflows/automated_release.yml b/.github/workflows/automated_release.yml index 89d02913..db17c0d1 100644 --- a/.github/workflows/automated_release.yml +++ b/.github/workflows/automated_release.yml @@ -10,7 +10,7 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: nelonoel/branch-name@v1.0.1 diff --git a/.github/workflows/check_release_version.yml b/.github/workflows/check_release_version.yml index 3f441c5f..217c379b 100644 --- a/.github/workflows/check_release_version.yml +++ b/.github/workflows/check_release_version.yml @@ -10,7 +10,7 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Get specific changed files id: changed-files-specific diff --git a/.github/workflows/docker_ci.yml b/.github/workflows/docker_ci.yml index 98f0fbbf..9c637be3 100644 --- a/.github/workflows/docker_ci.yml +++ b/.github/workflows/docker_ci.yml @@ -47,7 +47,7 @@ jobs: INTEGRATION_TEST_SEED: '/code/spec/data/integration/seed_for_unified_wh.rb' RAILS_ENV: test steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: nelonoel/branch-name@v1.0.1 - name: Build the Docker image run: >- diff --git a/.github/workflows/ruby_ci.yml b/.github/workflows/ruby_ci.yml index 165da224..789afe4b 100644 --- a/.github/workflows/ruby_ci.yml +++ b/.github/workflows/ruby_ci.yml @@ -9,7 +9,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -39,7 +39,7 @@ jobs: MYSQL_ALLOW_EMPTY_PASSWORD: yes steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/.release-version b/.release-version index 57807d6d..a6c2798a 100644 --- a/.release-version +++ b/.release-version @@ -1 +1 @@ -1.22.0 +1.23.0 diff --git a/.ruby-version b/.ruby-version index fa7adc7a..9c25013d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.3.5 +3.3.6 diff --git a/Dockerfile b/Dockerfile index f5464a13..4d6a1e19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:3.3.5 +FROM ruby:3.3.6 RUN gem install bundler diff --git a/Gemfile.lock b/Gemfile.lock index 9b60545b..53325a82 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,25 +7,25 @@ GIT GEM remote: https://rubygems.org/ specs: - actionpack (7.0.8.4) - actionview (= 7.0.8.4) - activesupport (= 7.0.8.4) + actionpack (7.0.8.5) + actionview (= 7.0.8.5) + activesupport (= 7.0.8.5) rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actionview (7.0.8.4) - activesupport (= 7.0.8.4) + actionview (7.0.8.5) + activesupport (= 7.0.8.5) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activemodel (7.0.8.4) - activesupport (= 7.0.8.4) - activerecord (7.0.8.4) - activemodel (= 7.0.8.4) - activesupport (= 7.0.8.4) - activesupport (7.0.8.4) + activemodel (7.0.8.5) + activesupport (= 7.0.8.5) + activerecord (7.0.8.5) + activemodel (= 7.0.8.5) + activesupport (= 7.0.8.5) + activesupport (7.0.8.5) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -45,14 +45,14 @@ GEM diff-lcs (1.5.1) docile (1.4.0) erubi (1.13.0) - factory_bot (6.4.5) + factory_bot (6.5.0) activesupport (>= 5.0.0) - factory_bot_rails (6.4.3) - factory_bot (~> 6.4) + factory_bot_rails (6.4.4) + factory_bot (~> 6.5) railties (>= 5.0.0) ffi (1.17.0) formatador (1.1.0) - guard (2.18.1) + guard (2.19.0) formatador (>= 0.2.4) listen (>= 2.7, < 4.0) lumberjack (>= 1.0.12, < 2.0) @@ -73,18 +73,18 @@ GEM hashie (5.0.0) i18n (1.14.6) concurrent-ruby (~> 1.0) - json (2.7.2) + json (2.8.2) language_server-protocol (3.17.0.3) - listen (3.8.0) + listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.22.0) + loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) - lumberjack (1.2.9) + lumberjack (1.2.10) method_source (1.1.0) - mini_portile2 (2.8.7) - minitest (5.25.1) + mini_portile2 (2.8.8) + minitest (5.25.2) msgpack (1.7.2) multi_json (1.15.0) mysql2 (0.5.6) @@ -96,14 +96,14 @@ GEM nenv (~> 0.1) shellany (~> 0.0) parallel (1.26.3) - parser (3.3.5.0) + parser (3.3.6.0) ast (~> 2.4.1) racc - pry (0.14.2) + pry (0.15.0) coderay (~> 1.1) method_source (~> 1.0) racc (1.8.1) - rack (2.2.9) + rack (2.2.10) rack-test (2.1.0) rack (>= 1.3) rails-dom-testing (2.2.0) @@ -113,9 +113,9 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.0.8.4) - actionpack (= 7.0.8.4) - activesupport (= 7.0.8.4) + railties (7.0.8.5) + actionpack (= 7.0.8.5) + activesupport (= 7.0.8.5) method_source rake (>= 12.2) thor (~> 1.0) @@ -123,7 +123,7 @@ GEM rainbow (3.1.1) rake (13.2.1) rb-fsevent (0.11.2) - rb-inotify (0.10.1) + rb-inotify (0.11.1) ffi (~> 1.0) rbtree (0.4.6) regexp_parser (2.9.2) @@ -131,15 +131,15 @@ GEM rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.1) + rspec-core (3.13.2) rspec-support (~> 3.13.0) rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.1) + rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-rails (7.0.1) + rspec-rails (7.1.0) actionpack (>= 7.0) activesupport (>= 7.0) railties (>= 7.0) @@ -148,7 +148,7 @@ GEM rspec-mocks (~> 3.13) rspec-support (~> 3.13) rspec-support (3.13.1) - rubocop (1.66.1) + rubocop (1.68.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -158,17 +158,17 @@ GEM rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.3) + rubocop-ast (1.36.1) parser (>= 3.3.1.0) - rubocop-performance (1.22.1) + rubocop-performance (1.23.0) rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rails (2.26.2) + rubocop-rails (2.27.0) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rspec (3.0.5) + rubocop-rspec (3.2.0) rubocop (~> 1.61) ruby-progressbar (1.13.0) sanger_warren (0.4.1) @@ -191,7 +191,7 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.6.0) - zeitwerk (2.6.18) + zeitwerk (2.7.1) PLATFORMS ruby diff --git a/README.md b/README.md index 83377b3f..0eb676c5 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,34 @@ A denormalised warehouse for multiple LIMS. +Populating a table in the warehouse is done asynchronously. This project is a Railtie application that facilitate asynchronous database population through a message queue. This application listens to a message queue, consumes the message, converts the message into a database model, and persists in the corresponding table. + +Take the example below. + +```json +{ + "lims": "traction", + "aliquot": { + "id_lims": "LIMS123456", + "lims_uuid": "550e8400-e29b-41d4-a716-446655440000", + "aliquot_type": "DNA", + "source_type": "Blood", + "source_barcode": "SRC123456", + "sample_name": "SampleA", + "used_by_type": "Research", + "used_by_barcode": "USR123456", + "volume": 50, + "concentration": 200, + "last_updated": "2024-07-09T10:15:30Z", + "recorded_at": "2024-07-09T09:00:00Z", + "created_at": "2024-07-08T08:00:00Z", + "insert_size": 350 + } +} +``` + +If the message above is consumed by `unified_warehouse`, it will create a new record in `aliquot` table with the values set to the given attributes. If an exception occur, it will dead-letter the message. For message serialization logic, please follow [`lib/payload.rb`](https://github.com/sanger/unified_warehouse/blob/beea89418cc1987c426e7967b7c13ff9218b51fb/lib/payload.rb#L31-L33). + ## Usage (Development) ### Requirement diff --git a/db/migrate/20241203115427_add_indexes_to_sample.rb b/db/migrate/20241203115427_add_indexes_to_sample.rb new file mode 100644 index 00000000..d26b769c --- /dev/null +++ b/db/migrate/20241203115427_add_indexes_to_sample.rb @@ -0,0 +1,11 @@ +class AddIndexesToSample < ActiveRecord::Migration[7.0] + + def change + #id_lims and id_sample_lims should be unique together + add_index :sample, [:id_lims, :id_sample_lims], unique: true + + #improve search performance + add_index :sample, :id_lims, unique: false + add_index :sample, :id_sample_lims, unique: false + end +end diff --git a/db/migrate/20241203134455_add_required_to_uuid_sample_lims_to_sample.rb b/db/migrate/20241203134455_add_required_to_uuid_sample_lims_to_sample.rb new file mode 100644 index 00000000..aba76bd4 --- /dev/null +++ b/db/migrate/20241203134455_add_required_to_uuid_sample_lims_to_sample.rb @@ -0,0 +1,10 @@ +class AddRequiredToUuidSampleLimsToSample < ActiveRecord::Migration[7.0] + def up + # uuid_sample_lims should be required + change_column_null(:sample, :uuid_sample_lims, false) + end + + def down + change_column_null(:sample, :uuid_sample_lims, true) + end +end diff --git a/db/schema.rb b/db/schema.rb index c1190ee9..1aaedb85 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_09_19_134424) do +ActiveRecord::Schema[7.0].define(version: 2024_12_03_134455) do create_table "aliquot", charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| t.string "id_lims", null: false, comment: "The LIMS system that the aliquot was created in" t.string "aliquot_uuid", null: false, comment: "The UUID of the aliquot in the LIMS system" @@ -304,7 +304,7 @@ create_table "sample", primary_key: "id_sample_tmp", id: { type: :integer, unsigned: true }, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| t.string "id_lims", limit: 10, null: false, comment: "LIM system identifier, e.g. CLARITY-GCLP, SEQSCAPE" - t.string "uuid_sample_lims", limit: 36, comment: "LIMS-specific sample uuid" + t.string "uuid_sample_lims", limit: 36, null: false, comment: "LIMS-specific sample uuid" t.string "id_sample_lims", null: false, comment: "LIMS-specific sample identifier" t.datetime "last_updated", precision: nil, null: false, comment: "Timestamp of last update" t.datetime "recorded_at", precision: nil, null: false, comment: "Timestamp of warehouse update" @@ -367,7 +367,10 @@ t.string "dna_source" t.string "priority_level", comment: "Priority level eg Medium, High etc" t.index ["accession_number"], name: "sample_accession_number_index" + t.index ["id_lims", "id_sample_lims"], name: "index_sample_on_id_lims_and_id_sample_lims", unique: true + t.index ["id_lims"], name: "index_sample_on_id_lims" t.index ["id_sample_lims", "id_lims"], name: "index_sample_on_id_sample_lims_and_id_lims", unique: true + t.index ["id_sample_lims"], name: "index_sample_on_id_sample_lims" t.index ["name"], name: "sample_name_index" t.index ["sanger_sample_id"], name: "index_sample_on_sanger_sample_id" t.index ["supplier_name"], name: "index_sample_on_supplier_name" diff --git a/spec/models/sample_spec.rb b/spec/models/sample_spec.rb index 33c66c9e..656701de 100644 --- a/spec/models/sample_spec.rb +++ b/spec/models/sample_spec.rb @@ -24,41 +24,8 @@ end context 'without uuid' do - it_behaves_like 'a sample resource' - - let(:json) do - { - 'id' => '1' * 255.to_i, - 'name' => 'name', - 'reference_genome' => 'reference genome', - 'organism' => 'organism', - 'consent_withdrawn' => true, - 'accession_number' => 'accession number', - 'common_name' => 'common name', - 'description' => 'description', - 'taxon_id' => 'taxon id', - 'father' => 'father', - 'mother' => 'mother', - 'replicate' => 'replicate', - 'ethnicity' => 'ethnicity', - 'gender' => 'gender', - 'cohort' => 'cohort', - 'country_of_origin' => 'country of origin', - 'geographical_region' => 'geographical region', - 'updated_at' => '2012-03-11 10:22:42', - 'created_at' => '2012-03-11 10:22:42', - 'sanger_sample_id' => 'sanger sample id', - 'control' => true, - 'empty_supplier_sample_name' => true, - 'supplier_name' => 'supplier name', - 'public_name' => 'public name', - 'sample_visibility' => 'sample visibility', - 'strain' => 'strain', - 'updated_by_manifest' => true, - 'donor_id' => '11111111-2222-3333-4444-555555555556', - 'developmental_stage' => 'Larval: Day 5 ZFS:0000037', - 'control_type' => 'positive' - } + it 'should raiser an error if uuid is null' do + expect { create(:sample, uuid_sample_lims: nil) }.to raise_error ActiveRecord::NotNullViolation end end