diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 24a223ed..c18aa082 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -99,6 +99,7 @@ mod m0000790_alter_sbom_alter_document_id; mod m0000800_alter_product_version_range_scheme; mod m0000810_fix_get_purl; mod m0000820_create_conversation; +mod m0000830_perf_indexes; pub struct Migrator; @@ -205,6 +206,7 @@ impl MigratorTrait for Migrator { Box::new(m0000800_alter_product_version_range_scheme::Migration), Box::new(m0000810_fix_get_purl::Migration), Box::new(m0000820_create_conversation::Migration), + Box::new(m0000830_perf_indexes::Migration), ] } } diff --git a/migration/src/m0000830_perf_indexes.rs b/migration/src/m0000830_perf_indexes.rs new file mode 100644 index 00000000..cb28232b --- /dev/null +++ b/migration/src/m0000830_perf_indexes.rs @@ -0,0 +1,116 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +#[allow(deprecated)] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_index( + Index::create() + .table(VersionedPurl::Table) + .name(Indexes::VersionedPurlBasePurlIdIDX.to_string()) + .col(VersionedPurl::BasePurlId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(PurlStatus::Table) + .name(Indexes::PurlStatusBasePurlIdIDX.to_string()) + .col(PurlStatus::BasePurlId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(SbomPackagePurlRef::Table) + .name(Indexes::SbomPackagePurlRefSbomIdIDX.to_string()) + .col(SbomPackagePurlRef::SbomId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(SbomPackagePurlRef::Table) + .name(Indexes::SbomPackagePurlRefNodeIdIDX.to_string()) + .col(SbomPackagePurlRef::NodeId) + .to_owned(), + ) + .await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index( + Index::drop() + .if_exists() + .table(SbomPackagePurlRef::Table) + .name(Indexes::SbomPackagePurlRefNodeIdIDX.to_string()) + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .if_exists() + .table(SbomPackagePurlRef::Table) + .name(Indexes::SbomPackagePurlRefSbomIdIDX.to_string()) + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .if_exists() + .table(PurlStatus::Table) + .name(Indexes::PurlStatusBasePurlIdIDX.to_string()) + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .if_exists() + .table(VersionedPurl::Table) + .name(Indexes::VersionedPurlBasePurlIdIDX.to_string()) + .to_owned(), + ) + .await?; + Ok(()) + } +} + +#[allow(clippy::enum_variant_names)] +#[derive(DeriveIden)] +enum Indexes { + VersionedPurlBasePurlIdIDX, + PurlStatusBasePurlIdIDX, + SbomPackagePurlRefSbomIdIDX, + SbomPackagePurlRefNodeIdIDX, +} + +#[derive(DeriveIden)] +enum VersionedPurl { + Table, + BasePurlId, +} + +#[derive(DeriveIden)] +enum PurlStatus { + Table, + BasePurlId, +} + +#[derive(DeriveIden)] +enum SbomPackagePurlRef { + Table, + SbomId, + NodeId, +} diff --git a/modules/fundamental/src/ai/service/tools/sbom_info.rs b/modules/fundamental/src/ai/service/tools/sbom_info.rs index a32e7562..7ba01f14 100644 --- a/modules/fundamental/src/ai/service/tools/sbom_info.rs +++ b/modules/fundamental/src/ai/service/tools/sbom_info.rs @@ -67,7 +67,7 @@ For example, input "quarkus" instead of "quarkus 3.2.11". Err(_) => None, Ok(id) => { log::info!("Fetching SBOM details by Id: {}", id); - service.fetch_sbom_details(id, &self.db).await? + service.fetch_sbom_details(id, vec![], &self.db).await? } }; @@ -76,7 +76,9 @@ For example, input "quarkus" instead of "quarkus 3.2.11". Err(_) => None, Ok(id) => { log::info!("Fetching SBOM details by UUID: {}", id); - service.fetch_sbom_details(Id::Uuid(id), &self.db).await? + service + .fetch_sbom_details(Id::Uuid(id), vec![], &self.db) + .await? } }; } @@ -100,7 +102,7 @@ For example, input "quarkus" instead of "quarkus 3.2.11". 0 => None, 1 => { service - .fetch_sbom_details(Id::Uuid(results.items[0].head.id), &self.db) + .fetch_sbom_details(Id::Uuid(results.items[0].head.id), vec![], &self.db) .await? } _ => { diff --git a/modules/fundamental/src/purl/service/test.rs b/modules/fundamental/src/purl/service/test.rs index 32e435b2..00ad6f6e 100644 --- a/modules/fundamental/src/purl/service/test.rs +++ b/modules/fundamental/src/purl/service/test.rs @@ -734,7 +734,7 @@ async fn gc_purls(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> { let sbom_service = SbomService::new(ctx.db.clone()); let sbom = sbom_service - .fetch_sbom_details(id, &ctx.db) + .fetch_sbom_details(id, vec![], &ctx.db) .await? .expect("fetch_sbom"); assert_eq!( diff --git a/modules/fundamental/src/sbom/endpoints/mod.rs b/modules/fundamental/src/sbom/endpoints/mod.rs index bf704230..86359299 100644 --- a/modules/fundamental/src/sbom/endpoints/mod.rs +++ b/modules/fundamental/src/sbom/endpoints/mod.rs @@ -263,7 +263,11 @@ pub async fn get_sbom_advisories( _: Require, ) -> actix_web::Result { let id = Id::from_str(&id).map_err(Error::IdKey)?; - match fetcher.fetch_sbom_details(id, db.as_ref()).await? { + let statuses: Vec = vec!["affected".to_string()]; + match fetcher + .fetch_sbom_details(id, statuses, db.as_ref()) + .await? + { Some(v) => Ok(HttpResponse::Ok().json(v.advisories)), None => Ok(HttpResponse::NotFound().finish()), } diff --git a/modules/fundamental/src/sbom/model/details.rs b/modules/fundamental/src/sbom/model/details.rs index fcaa8086..bb60cf0d 100644 --- a/modules/fundamental/src/sbom/model/details.rs +++ b/modules/fundamental/src/sbom/model/details.rs @@ -47,8 +47,9 @@ impl SbomDetails { (sbom, node): (sbom::Model, Option), service: &SbomService, tx: &C, + statuses: Vec, ) -> Result, Error> { - let mut relevant_advisory_info = sbom + let mut query = sbom .find_related(sbom_package::Entity) .join(JoinType::Join, sbom_package::Relation::Node.def()) .join(JoinType::LeftJoin, sbom_package::Relation::Purl.def()) @@ -60,6 +61,20 @@ impl SbomDetails { JoinType::LeftJoin, qualified_purl::Relation::VersionedPurl.def(), ) + .join(JoinType::LeftJoin, versioned_purl::Relation::BasePurl.def()) + .join(JoinType::Join, base_purl::Relation::PurlStatus.def()) + .join(JoinType::Join, purl_status::Relation::Status.def()); + + if !statuses.is_empty() { + query = query + .filter(Expr::col((status::Entity, status::Column::Slug)).is_in(statuses.clone())); + } + + let mut relevant_advisory_info = query + .join( + JoinType::LeftJoin, + purl_status::Relation::VersionRange.def(), + ) .filter(SimpleExpr::FunctionCall( Func::cust(VersionMatches) .arg(Expr::col(( @@ -68,13 +83,6 @@ impl SbomDetails { ))) .arg(Expr::col((version_range::Entity, Asterisk))), )) - .join(JoinType::LeftJoin, versioned_purl::Relation::BasePurl.def()) - .join(JoinType::Join, base_purl::Relation::PurlStatus.def()) - .join(JoinType::Join, purl_status::Relation::Status.def()) - .join( - JoinType::LeftJoin, - purl_status::Relation::VersionRange.def(), - ) .join(JoinType::LeftJoin, purl_status::Relation::ContextCpe.def()) .join(JoinType::Join, purl_status::Relation::Advisory.def()) .join(JoinType::Join, purl_status::Relation::Vulnerability.def()) @@ -149,7 +157,7 @@ impl SbomDetails { JOIN "version_range" ON "product_version_range"."version_range_id" = "version_range"."id" AND version_matches("product_version"."version", "version_range".*) -- now find matching purls in these statuses - JOIN base_purl ON "product_status"."package" LIKE CONCAT("base_purl"."namespace", '/', "base_purl"."name") OR "product_status"."package" = "base_purl"."name" + JOIN base_purl ON product_status.package = base_purl.name OR product_status.package LIKE CONCAT(base_purl.namespace, '/', base_purl.name) JOIN "versioned_purl" ON "versioned_purl"."base_purl_id" = "base_purl"."id" JOIN "qualified_purl" ON "qualified_purl"."versioned_purl_id" = "versioned_purl"."id" join sbom_package_purl_ref ON sbom_package_purl_ref.qualified_purl_id = qualified_purl.id AND sbom_package_purl_ref.sbom_id = sbom.sbom_id @@ -162,13 +170,14 @@ impl SbomDetails { JOIN "vulnerability" ON "product_status"."vulnerability_id" = "vulnerability"."id" WHERE "sbom"."sbom_id" = $1 + AND ($2::text[] = ARRAY[]::text[] OR "status"."slug" = ANY($2::text[])) "#; let result: Vec = tx .query_all(Statement::from_sql_and_values( DbBackend::Postgres, product_advisory_info, - [sbom.sbom_id.into()], + [sbom.sbom_id.into(), statuses.into()], )) .await?; diff --git a/modules/fundamental/src/sbom/service/sbom.rs b/modules/fundamental/src/sbom/service/sbom.rs index 8d18774b..493cc9bd 100644 --- a/modules/fundamental/src/sbom/service/sbom.rs +++ b/modules/fundamental/src/sbom/service/sbom.rs @@ -61,11 +61,12 @@ impl SbomService { pub async fn fetch_sbom_details( &self, id: Id, + statuses: Vec, connection: &C, ) -> Result, Error> { Ok(match self.fetch_sbom(id, connection).await? { - Some(row) => SbomDetails::from_entity(row, self, connection).await?, + Some(row) => SbomDetails::from_entity(row, self, connection, statuses).await?, None => None, }) } diff --git a/modules/fundamental/src/sbom/service/test.rs b/modules/fundamental/src/sbom/service/test.rs index 850997c1..24637604 100644 --- a/modules/fundamental/src/sbom/service/test.rs +++ b/modules/fundamental/src/sbom/service/test.rs @@ -22,7 +22,9 @@ async fn sbom_details_status(ctx: &TrustifyContext) -> Result<(), anyhow::Error> let id_3_2_12 = results[3].id.clone(); - let details = service.fetch_sbom_details(id_3_2_12, &ctx.db).await?; + let details = service + .fetch_sbom_details(id_3_2_12, vec![], &ctx.db) + .await?; assert!(details.is_some()); @@ -31,7 +33,7 @@ async fn sbom_details_status(ctx: &TrustifyContext) -> Result<(), anyhow::Error> log::debug!("{details:#?}"); let details = service - .fetch_sbom_details(Id::Uuid(details.summary.head.id), &ctx.db) + .fetch_sbom_details(Id::Uuid(details.summary.head.id), vec![], &ctx.db) .await?; assert!(details.is_some()); diff --git a/modules/fundamental/src/vulnerability/service/test.rs b/modules/fundamental/src/vulnerability/service/test.rs index ca8bb5ec..8b26a469 100644 --- a/modules/fundamental/src/vulnerability/service/test.rs +++ b/modules/fundamental/src/vulnerability/service/test.rs @@ -133,7 +133,9 @@ async fn commons_compress(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let sat_id = ingest_results[1].id.clone(); - let sat_sbom = sbom_service.fetch_sbom_details(sat_id, &ctx.db).await?; + let sat_sbom = sbom_service + .fetch_sbom_details(sat_id, vec![], &ctx.db) + .await?; assert!(sat_sbom.is_some()); let sat_sbom = sat_sbom.unwrap(); @@ -155,7 +157,9 @@ async fn commons_compress(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let quarkus_id = ingest_results[3].id.clone(); - let quarkus_sbom = sbom_service.fetch_sbom_details(quarkus_id, &ctx.db).await?; + let quarkus_sbom = sbom_service + .fetch_sbom_details(quarkus_id, vec![], &ctx.db) + .await?; assert!(quarkus_sbom.is_some()); @@ -224,7 +228,9 @@ async fn product_statuses(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let quarkus_id = ingest_results[1].id.clone(); - let quarkus_sbom = sbom_service.fetch_sbom_details(quarkus_id, &ctx.db).await?; + let quarkus_sbom = sbom_service + .fetch_sbom_details(quarkus_id, vec![], &ctx.db) + .await?; assert!(quarkus_sbom.is_some()); diff --git a/modules/fundamental/tests/sbom/reingest.rs b/modules/fundamental/tests/sbom/reingest.rs index 64d51740..399c0e4d 100644 --- a/modules/fundamental/tests/sbom/reingest.rs +++ b/modules/fundamental/tests/sbom/reingest.rs @@ -55,13 +55,13 @@ async fn quarkus(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_ne!(result1.id, result2.id); let mut sbom1 = sbom - .fetch_sbom_details(result1.id, &ctx.db) + .fetch_sbom_details(result1.id, vec![], &ctx.db) .await? .expect("v1 must be found"); log::info!("SBOM1: {sbom1:?}"); let mut sbom2 = sbom - .fetch_sbom_details(result2.id, &ctx.db) + .fetch_sbom_details(result2.id, vec![], &ctx.db) .await? .expect("v2 must be found"); log::info!("SBOM2: {sbom2:?}"); @@ -129,13 +129,13 @@ async fn nhc(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_ne!(result1.id, result2.id); let mut sbom1 = sbom - .fetch_sbom_details(result1.id, &ctx.db) + .fetch_sbom_details(result1.id, vec![], &ctx.db) .await? .expect("v1 must be found"); log::info!("SBOM1: {sbom1:?}"); let mut sbom2 = sbom - .fetch_sbom_details(result2.id, &ctx.db) + .fetch_sbom_details(result2.id, vec![], &ctx.db) .await? .expect("v2 must be found"); log::info!("SBOM2: {sbom2:?}"); @@ -182,13 +182,13 @@ async fn nhc_same(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_eq!(result1.id, result2.id); let mut sbom1 = sbom - .fetch_sbom_details(result1.id, &ctx.db) + .fetch_sbom_details(result1.id, vec![], &ctx.db) .await? .expect("v1 must be found"); log::info!("SBOM1: {sbom1:?}"); let mut sbom2 = sbom - .fetch_sbom_details(result2.id, &ctx.db) + .fetch_sbom_details(result2.id, vec![], &ctx.db) .await? .expect("v2 must be found"); log::info!("SBOM2: {sbom2:?}"); @@ -249,13 +249,13 @@ async fn nhc_same_content(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_ne!(result1.id, result2.id); let mut sbom1 = sbom - .fetch_sbom_details(result1.id, &ctx.db) + .fetch_sbom_details(result1.id, vec![], &ctx.db) .await? .expect("v1 must be found"); log::info!("SBOM1: {sbom1:?}"); let mut sbom2 = sbom - .fetch_sbom_details(result2.id, &ctx.db) + .fetch_sbom_details(result2.id, vec![], &ctx.db) .await? .expect("v2 must be found"); log::info!("SBOM2: {sbom2:?}"); @@ -306,13 +306,13 @@ async fn syft_rerun(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { assert_ne!(result1.id, result2.id); let mut sbom1 = sbom - .fetch_sbom_details(result1.id, &ctx.db) + .fetch_sbom_details(result1.id, vec![], &ctx.db) .await? .expect("v1 must be found"); log::info!("SBOM1: {sbom1:?}"); let mut sbom2 = sbom - .fetch_sbom_details(result2.id, &ctx.db) + .fetch_sbom_details(result2.id, vec![], &ctx.db) .await? .expect("v2 must be found"); log::info!("SBOM2: {sbom2:?}"); diff --git a/modules/graphql/src/sbomstatus.rs b/modules/graphql/src/sbomstatus.rs index 23707e77..7df0cd73 100644 --- a/modules/graphql/src/sbomstatus.rs +++ b/modules/graphql/src/sbomstatus.rs @@ -30,7 +30,7 @@ impl SbomStatusQuery { let sbom_service = SbomService::new(db.deref().clone()); let sbom_details: Option = sbom_service - .fetch_sbom_details(Id::Uuid(id), db.as_ref()) + .fetch_sbom_details(Id::Uuid(id), vec![], db.as_ref()) .await .unwrap_or_default();