From f203d8ebda71be20e476d7a82db213e47c03ac46 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 9 Mar 2022 18:09:34 -0500 Subject: [PATCH] tests/fixture: Add concept of "owner" Prep for chunk splitting - this is a bit like the dpkg/rpm database. --- lib/Cargo.toml | 1 + lib/src/fixture.rs | 107 +++++++++++++++++++++++++++++++++++++++++++ lib/tests/it/main.rs | 3 ++ 3 files changed, 111 insertions(+) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e5fba266..067b6779 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -33,6 +33,7 @@ openat-ext = "0.2.0" openssl = "0.10.33" ostree = { features = ["v2021_5", "cap-std-apis"], version = "0.13.5" } pin-project = "1.0" +regex = "1.5.4" serde = { features = ["derive"], version = "1.0.125" } serde_json = "1.0.64" structopt = "0.3.21" diff --git a/lib/src/fixture.rs b/lib/src/fixture.rs index f7127646..26bdee10 100644 --- a/lib/src/fixture.rs +++ b/lib/src/fixture.rs @@ -2,6 +2,7 @@ #![allow(missing_docs)] +use crate::objectsource::{ObjectMeta, ObjectSourceMeta}; use crate::prelude::*; use crate::{gio, glib}; use anyhow::{anyhow, Context, Result}; @@ -10,13 +11,16 @@ use cap_std::fs::Dir; use cap_std_ext::prelude::CapStdExtCommandExt; use chrono::TimeZone; use fn_error_context::context; +use once_cell::sync::Lazy; use ostree::cap_std; +use regex::Regex; use sh_inline::bash_in; use std::borrow::Cow; use std::convert::{TryFrom, TryInto}; use std::io::Write; use std::ops::Add; use std::process::Stdio; +use std::rc::Rc; use std::sync::Arc; const OSTREE_GPG_HOME: &[u8] = include_bytes!("fixtures/ostree-gpg-test-home.tar.gz"); @@ -119,6 +123,21 @@ impl FileDef { } } +/// This is like a package database, mapping our test fixture files to package names +static OWNERS: Lazy> = Lazy::new(|| { + [ + ("usr/lib/modules/.*/initramfs", "initramfs"), + ("usr/lib/modules", "kernel"), + ("usr/bin/(ba)?sh", "bash"), + ("usr/bin/hardlink.*", "testlink"), + ("usr/etc/someconfig.conf", "someconfig"), + ("usr/etc/polkit.conf", "a-polkit-config"), + ] + .iter() + .map(|(k, v)| (Regex::new(k).unwrap(), *v)) + .collect() +}); + static CONTENTS_V0: &str = indoc::indoc! { r##" r usr/lib/modules/5.10.18-200.x86_64/vmlinuz this-is-a-kernel r usr/lib/modules/5.10.18-200.x86_64/initramfs this-is-an-initramfs @@ -239,6 +258,81 @@ fn relative_path_components(p: &Utf8Path) -> impl Iterator .filter(|p| matches!(p, Utf8Component::Normal(_))) } +/// Walk over the whole filesystem, and generate mappings from content object checksums +/// to the package that owns them. +/// +/// In the future, we could compute this much more efficiently by walking that +/// instead. But this design is currently oriented towards accepting a single ostree +/// commit as input. +fn build_mapping_recurse( + path: &mut Utf8PathBuf, + dir: &gio::File, + ret: &mut ObjectMeta, +) -> Result<()> { + use std::collections::btree_map::Entry; + let cancellable = gio::NONE_CANCELLABLE; + let e = dir.enumerate_children( + "standard::name,standard::type", + gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, + cancellable, + )?; + for child in e { + let childi = child?; + let name: Utf8PathBuf = childi.name().try_into()?; + let child = dir.child(&name); + path.push(&name); + match childi.file_type() { + gio::FileType::Regular | gio::FileType::SymbolicLink => { + let child = child.downcast::().unwrap(); + + let owner = OWNERS + .iter() + .find_map(|(r, owner)| { + if r.is_match(path.as_str()) { + Some(Rc::from(*owner)) + } else { + None + } + }) + .ok_or_else(|| anyhow!("Unowned path {}", path))?; + + if !ret.set.contains(&*owner) { + ret.set.insert(ObjectSourceMeta { + identifier: Rc::clone(&owner), + name: Rc::clone(&owner), + srcid: Rc::clone(&owner), + change_time_offset: u32::MAX, + }); + } + + let checksum = child.checksum().unwrap().to_string(); + match ret.map.entry(checksum) { + Entry::Vacant(v) => { + v.insert(owner); + } + Entry::Occupied(v) => { + let prev_owner = v.get(); + if **prev_owner != *owner { + anyhow::bail!( + "Duplicate object ownership {} ({} and {})", + path.as_str(), + prev_owner, + owner + ); + } + } + } + } + gio::FileType::Directory => { + build_mapping_recurse(path, &child, ret)?; + } + o => anyhow::bail!("Unhandled file type: {}", o), + } + path.pop(); + } + Ok(()) +} + #[derive(Debug)] pub struct Fixture { // Just holds a reference @@ -485,6 +579,19 @@ impl Fixture { Ok(()) } + /// Gather object metadata for the current commit. + pub fn get_object_meta(&self) -> Result { + let cancellable = gio::NONE_CANCELLABLE; + + // Load our base commit + let root = self.srcrepo.read_commit(self.testref(), cancellable)?.0; + + let mut ret = ObjectMeta::default(); + build_mapping_recurse(&mut Utf8PathBuf::from("/"), &root, &mut ret)?; + + Ok(ret) + } + #[context("Exporting tar")] pub fn export_tar(&self) -> Result<&'static Utf8Path> { let cancellable = gio::NONE_CANCELLABLE; diff --git a/lib/tests/it/main.rs b/lib/tests/it/main.rs index aae84879..c21aecb9 100644 --- a/lib/tests/it/main.rs +++ b/lib/tests/it/main.rs @@ -202,6 +202,9 @@ fn test_tar_export_structure() -> Result<()> { use tar::EntryType::{Directory, Regular}; let mut fixture = Fixture::new_v1()?; + // Just test that we can retrieve ownership for all objects + let _objmeta = fixture.get_object_meta()?; + let src_tar = fixture.export_tar()?; let src_tar = std::io::BufReader::new(fixture.dir.open(src_tar)?); let mut src_tar = tar::Archive::new(src_tar);