From 576478cdcb2ab67331338197c3c181f7bcea529e Mon Sep 17 00:00:00 2001 From: Zeke Gabrielse Date: Mon, 11 Nov 2024 16:20:54 -0600 Subject: [PATCH] add base for oci routes --- .../release_engines/oci/blobs_controller.rb | 33 +++++++++++++++ .../oci/manifests_controller.rb | 35 ++++++++++++++++ app/models/release_artifact.rb | 5 +-- app/workers/process_oci_image_worker.rb | 19 +++++++-- config/routes.rb | 41 ++++++++++++++++++- spec/factories/release_artifact.rb | 4 +- spec/workers/process_oci_image_worker_spec.rb | 24 +++++++++-- 7 files changed, 146 insertions(+), 15 deletions(-) create mode 100644 app/controllers/api/v1/release_engines/oci/blobs_controller.rb create mode 100644 app/controllers/api/v1/release_engines/oci/manifests_controller.rb diff --git a/app/controllers/api/v1/release_engines/oci/blobs_controller.rb b/app/controllers/api/v1/release_engines/oci/blobs_controller.rb new file mode 100644 index 0000000000..2ab70650f7 --- /dev/null +++ b/app/controllers/api/v1/release_engines/oci/blobs_controller.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Api::V1::ReleaseEngines + class Oci::BlobsController < Api::V1::BaseController + before_action :scope_to_current_account! + before_action :require_active_subscription! + before_action :authenticate_with_token + before_action :set_artifact + + def show + authorize! artifact + + redirect_to vanity_v1_account_release_artifact_url(artifact.account, artifact, filename: artifact.filename, host: request.host), + status: :see_other + end + + private + + attr_reader :artifact + + def set_artifact + scoped_artifacts = authorized_scope(current_account.release_artifacts.blobs) + .for_package(params[:namespace]) + + # see: https://github.com/opencontainers/image-spec/blob/main/descriptor.md#digests + algorithm, encoded = params[:digest].split(':', 2) + + Current.resource = @artifact = scoped_artifacts.find_by!( + filename: "blobs/#{algorithm}/#{encoded}", + ) + end + end +end diff --git a/app/controllers/api/v1/release_engines/oci/manifests_controller.rb b/app/controllers/api/v1/release_engines/oci/manifests_controller.rb new file mode 100644 index 0000000000..7540a28d7d --- /dev/null +++ b/app/controllers/api/v1/release_engines/oci/manifests_controller.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Api::V1::ReleaseEngines + class Oci::ManifestsController < Api::V1::BaseController + before_action :scope_to_current_account! + before_action :require_active_subscription! + before_action :authenticate_with_token + before_action :set_artifact + + def show + authorize! artifact, + to: :show? + + manifest = artifact.manifest + + # for etag support + return unless + stale?(manifest, cache_control: { max_age: 1.day, private: true }) + + render body: manifest.content + end + + private + + attr_reader :artifact + + def set_artifact + Current.resource = @artifact = authorized_scope(current_account.release_artifacts.tarballs) + .for_package(params[:namespace]) + .for_release(params[:reference]) + .order_by_version + .first! + end + end +end diff --git a/app/models/release_artifact.rb b/app/models/release_artifact.rb index 5d7c46c38e..ae6bf624e6 100644 --- a/app/models/release_artifact.rb +++ b/app/models/release_artifact.rb @@ -452,15 +452,14 @@ class ReleaseArtifact < ApplicationRecord scope :gems, -> { for_filetype(:gem) } scope :tarballs, -> { for_filetypes(:tgz, :tar, :'tar.gz') } + scope :blobs, -> { where("release_artifacts.filename LIKE 'blobs/%'") } # prefixes are fast delegate :version, :semver, :channel, :tag, :tag?, :licensed?, :open?, :closed?, allow_nil: true, to: :release - def key_for(path) = "artifacts/#{account_id}/#{release_id}/#{path}" - def key = key_for(filename) - + def key = "artifacts/#{account_id}/#{release_id}/#{filename}" def presigner = Aws::S3::Presigner.new(client:) def client diff --git a/app/workers/process_oci_image_worker.rb b/app/workers/process_oci_image_worker.rb index 71a8fc19e6..339b0d8c47 100644 --- a/app/workers/process_oci_image_worker.rb +++ b/app/workers/process_oci_image_worker.rb @@ -57,18 +57,29 @@ def perform(artifact_id) content:, ) in %r{^blobs/sha256/} if entry.file? - key = artifact.key_for(entry.name) + blob = ReleaseArtifact.create!( + account_id: artifact.account_id, + environment_id: artifact.environment_id, + release_id: artifact.release_id, + filename: entry.name, + filesize: entry.size, + ) # skip if already uploaded next if - client.head_object(bucket: artifact.bucket, key:).successful? rescue false + client.head_object(bucket: blob.bucket, key: blob.key).successful? rescue false # upload blob in chunks - client.put_object(bucket: artifact.bucket, key:) do |writer| - while (chunk = entry.read(16 * 1024)) # write in chunks + obj = client.put_object(bucket: blob.bucket, key: blob.key) do |writer| + while chunk = entry.read(16 * 1024) # write in chunks writer.write(chunk) end end + + blob.update!( + etag: obj.etag.delete('"'), + status: 'UPLOADED', + ) else end end diff --git a/config/routes.rb b/config/routes.rb index 153a8b4e5c..62a122e434 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -66,7 +66,7 @@ product_id: /[^\/]*/ , package_id: /[^\/]*/ , release_id: /[^\/]*/ , - id: /.*/ + id: /.*/, } end end @@ -96,7 +96,7 @@ scope module: :npm, constraints: MimeTypeConstraint.new(:json, raise_on_no_match: true), defaults: { format: :json } do get ':package', to: 'package_metadata#show', as: :npm_package_metadata, constraints: { # see: https://docs.npmjs.com/cli/v9/configuring-npm/package-json#name - package: %r{(?:@([a-z0-9][a-z0-9-]*[a-z0-9])(/|%2F))?([a-z0-9][a-z0-9._-]*[a-z0-9])} + package: /(?:@([a-z0-9][a-z0-9-]*[a-z0-9])(\/|%2F))?([a-z0-9][a-z0-9._-]*[a-z0-9])/, } end @@ -106,6 +106,26 @@ end end + concern :oci do + # NOTE(ezekg) /v2 namespace is handled outside of this because docker wants it to always be at the root + scope module: :oci, defaults: { format: :binary } do + # see: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests + match ':namespace/manifests/:reference', via: %i[head get], to: 'manifests#show', as: :oci_manifest, constraints: { + namespace: /[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*/, + reference: /[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}/, + } + + # see: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-blobs + get ':namespace/blobs/:digest', to: 'blobs#show', as: :oci_blob, constraints: { + namespace: /[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*/, + digest: /[^\/]*/, + } + + # ignore other requests entirely for now e.g. GET /v2/:namespace/referrers/:digest + match ':namespace/*wildcard', via: :all, to: -> env { [405, {}, []] } + end + end + concern :v1 do get :ping, to: 'health#general_ping' @@ -485,6 +505,9 @@ scope :npm do concerns :npm end + scope :oci do + concerns :oci + end end end @@ -632,6 +655,20 @@ concerns :npm end end + + scope module: 'api/v1/release_engines', constraints: { subdomain: 'oci.pkg' } do + # NOTE(ezekg) /v2 namespace is handled here because docker wants it at the root + scope :v2 do + case + when Keygen.multiplayer? + scope ':account_id', as: :account do + concerns :oci + end + when Keygen.singleplayer? + concerns :oci + end + end + end end %w[500 503].each do |code| diff --git a/spec/factories/release_artifact.rb b/spec/factories/release_artifact.rb index 133dfcec43..8c51cd7418 100644 --- a/spec/factories/release_artifact.rb +++ b/spec/factories/release_artifact.rb @@ -55,7 +55,7 @@ trait :npm_package do release { build(:release, :npm, account:, environment:) } filename { "#{release.name.underscore.parameterize}-#{release.version}.tgz" } - filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 25.megabytes.to_i) } + filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 32.megabytes.to_i) } filetype { build(:filetype, key: 'tgz', account:) } platform { nil } arch { nil } @@ -64,7 +64,7 @@ trait :oci_image do release { build(:release, :oci, account:, environment:) } filename { "#{release.name.underscore.parameterize}.tar" } - filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 1.gigabyte.to_i) } + filesize { Faker::Number.between(from: 1.megabyte.to_i, to: 512.megabytes.to_i) } filetype { build(:filetype, key: 'tar', account:) } platform { nil } arch { nil } diff --git a/spec/workers/process_oci_image_worker_spec.rb b/spec/workers/process_oci_image_worker_spec.rb index de9b30a157..a672ba6dd8 100644 --- a/spec/workers/process_oci_image_worker_spec.rb +++ b/spec/workers/process_oci_image_worker_spec.rb @@ -51,6 +51,7 @@ context 'when artifact is processing' do let(:artifact) { create(:artifact, :oci_image, :processing, account:) } + let(:release) { artifact.release } it 'should store manifest' do expect { subject.perform_async(artifact.id) }.to change { artifact.reload.manifest } @@ -64,12 +65,27 @@ Aws.config[:s3][:stub_responses][:head_object] = [Aws::S3::Errors::NotFound] end + it 'should store blobs' do + expect { subject.perform_async(artifact.id) }.to change { release.reload.artifacts } + + expect(release.artifacts).to satisfy { |artifacts| + artifacts in [ + *, + ReleaseArtifact(filename: 'blobs/sha256/33735bd63cf84d7e388d9f6d297d348c523c044410f553bd878c6d7829612735', status: 'UPLOADED'), + ReleaseArtifact(filename: 'blobs/sha256/43c4264eed91be63b206e17d93e75256a6097070ce643c5e8f0379998b44f170', status: 'UPLOADED'), + ReleaseArtifact(filename: 'blobs/sha256/91ef0af61f39ece4d6710e465df5ed6ca12112358344fd51ae6a3b886634148b', status: 'UPLOADED'), + ReleaseArtifact(filename: 'blobs/sha256/beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d', status: 'UPLOADED'), + * + ] + } + end + it 'should upload blobs' do expect { subject.perform_async(artifact.id) }.to upload( - { key: artifact.key_for('blobs/sha256/33735bd63cf84d7e388d9f6d297d348c523c044410f553bd878c6d7829612735') }, - { key: artifact.key_for('blobs/sha256/43c4264eed91be63b206e17d93e75256a6097070ce643c5e8f0379998b44f170') }, - { key: artifact.key_for('blobs/sha256/91ef0af61f39ece4d6710e465df5ed6ca12112358344fd51ae6a3b886634148b') }, - { key: artifact.key_for('blobs/sha256/beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d') }, + { key: %r{blobs/sha256/33735bd63cf84d7e388d9f6d297d348c523c044410f553bd878c6d7829612735} }, + { key: %r{blobs/sha256/43c4264eed91be63b206e17d93e75256a6097070ce643c5e8f0379998b44f170} }, + { key: %r{blobs/sha256/91ef0af61f39ece4d6710e465df5ed6ca12112358344fd51ae6a3b886634148b} }, + { key: %r{blobs/sha256/beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d} }, ) end end