From 5eee3e7a20e19d57ceda50e7ef2705d697bad355 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 20 Nov 2024 17:33:45 -0800 Subject: [PATCH] Add support for Debian distros This adds Debian Bullseye and Bookworm support. The only real change needed here was to update the distro config to allow us to enable the `bullseye-backports` repo so the tests can install a more recent version of go (and as such pass the test). Signed-off-by: Brian Goff --- cmd/frontend/main.go | 2 ++ frontend/deb/distro/container.go | 5 +++- frontend/deb/distro/distro.go | 5 ++++ frontend/deb/distro/install.go | 11 ++++++-- frontend/deb/distro/worker.go | 1 + frontend/debian/bookworm.go | 25 ++++++++++++++++++ frontend/debian/bullseye.go | 44 ++++++++++++++++++++++++++++++++ frontend/debian/common.go | 32 +++++++++++++++++++++++ helpers.go | 14 ++++++++++ spec.go | 12 --------- test/debian_test.go | 21 +++++++++++++++ website/docs/targets.md | 8 ++++++ 12 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 frontend/debian/bookworm.go create mode 100644 frontend/debian/bullseye.go create mode 100644 frontend/debian/common.go create mode 100644 test/debian_test.go diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index 36d3e65e3..6940ea153 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -6,6 +6,7 @@ import ( "github.com/Azure/dalec/frontend" "github.com/Azure/dalec/frontend/azlinux" + "github.com/Azure/dalec/frontend/debian" "github.com/Azure/dalec/frontend/debug" "github.com/Azure/dalec/frontend/ubuntu" "github.com/Azure/dalec/frontend/windows" @@ -36,6 +37,7 @@ func main() { frontend.WithBuiltinHandler(azlinux.AzLinux3TargetKey, azlinux.NewAzlinux3Handler()), frontend.WithBuiltinHandler(windows.DefaultTargetKey, windows.Handle), ubuntu.Handlers, + debian.Handlers, frontend.WithTargetForwardingHandler, )); err != nil { bklog.L.WithError(err).Fatal("error running frontend") diff --git a/frontend/deb/distro/container.go b/frontend/deb/distro/container.go index 20a6a08c2..d3f74ee98 100644 --- a/frontend/deb/distro/container.go +++ b/frontend/deb/distro/container.go @@ -23,7 +23,10 @@ func (c *Config) BuildContainer(worker llb.State, sOpt dalec.SourceOpts, client opts = append(opts, dalec.ProgressGroup("Build Container Image")) - withRepos, err := c.RepoMounts(spec.GetInstallRepos(targetKey), sOpt, opts...) + repos := dalec.GetExtraRepos(c.ExtraRepos, "install") + repos = append(repos, spec.GetInstallRepos(targetKey)...) + + withRepos, err := c.RepoMounts(repos, sOpt, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/deb/distro/distro.go b/frontend/deb/distro/distro.go index 9e41efe01..42ab0be30 100644 --- a/frontend/deb/distro/distro.go +++ b/frontend/deb/distro/distro.go @@ -32,6 +32,11 @@ type Config struct { RepoPlatformConfig *dalec.RepoPlatformConfig DefaultOutputImage string + + // ExtraRepos is used by distributions that want to enable extra repositories + // that are not inthe base worker config. + // A prime example of this is adding Debian backports on debian distrubutions. + ExtraRepos []dalec.PackageRepositoryConfig } func (cfg *Config) BuildImageConfig(ctx context.Context, resolver llb.ImageMetaResolver, spec *dalec.Spec, platform *ocispecs.Platform, targetKey string) (*dalec.DockerImageSpec, error) { diff --git a/frontend/deb/distro/install.go b/frontend/deb/distro/install.go index 08f370ef7..4d1811c51 100644 --- a/frontend/deb/distro/install.go +++ b/frontend/deb/distro/install.go @@ -144,7 +144,10 @@ func (d *Config) InstallBuildDeps(sOpt dalec.SourceOpts, spec *dalec.Spec, targe return in, errors.Wrap(err, "error creating intermediate package for installing build dependencies") } - customRepos, err := d.RepoMounts(spec.GetBuildRepos(targetKey), sOpt, opts...) + repos := dalec.GetExtraRepos(d.ExtraRepos, "build") + repos = append(repos, spec.GetBuildRepos(targetKey)...) + + customRepos, err := d.RepoMounts(repos, sOpt, opts...) if err != nil { return in, err } @@ -153,6 +156,7 @@ func (d *Config) InstallBuildDeps(sOpt dalec.SourceOpts, spec *dalec.Spec, targe dalec.WithConstraints(opts...), customRepos, InstallLocalPkg(pkg, opts...), + dalec.WithMountedAptCache(d.AptCachePrefix), ).Root(), nil }) } @@ -166,7 +170,10 @@ func (d *Config) InstallTestDeps(sOpt dalec.SourceOpts, targetKey string, spec * return func(in llb.State) llb.State { return in.Async(func(ctx context.Context, in llb.State, c *llb.Constraints) (llb.State, error) { - withRepos, err := d.RepoMounts(spec.GetTestRepos(targetKey), sOpt, opts...) + repos := dalec.GetExtraRepos(d.ExtraRepos, "test") + repos = append(repos, spec.GetTestRepos(targetKey)...) + + withRepos, err := d.RepoMounts(repos, sOpt, opts...) if err != nil { return in, err } diff --git a/frontend/deb/distro/worker.go b/frontend/deb/distro/worker.go index 8bdf19cc8..b532e4cf7 100644 --- a/frontend/deb/distro/worker.go +++ b/frontend/deb/distro/worker.go @@ -79,6 +79,7 @@ func (cfg *Config) Worker(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (ll Run( dalec.WithConstraints(opts...), AptInstall(cfg.BuilderPackages, opts...), + dalec.WithMountedAptCache(cfg.AptCachePrefix), ). // This file prevents installation of things like docs in ubuntu // containers We don't want to exclude this because tests want to diff --git a/frontend/debian/bookworm.go b/frontend/debian/bookworm.go new file mode 100644 index 000000000..8eeac6e98 --- /dev/null +++ b/frontend/debian/bookworm.go @@ -0,0 +1,25 @@ +package debian + +import ( + "github.com/Azure/dalec/frontend/deb/distro" +) + +const ( + BookwormDefaultTargetKey = "bookworm" + BookwormAptCachePrefix = "bookworm" + BookwormWorkerContextName = "dalec-bookworm-worker" + + bookwormRef = "mcr.microsoft.com/mirror/docker/library/debian:bookworm" + bookwormVersionID = "debian12" +) + +var ( + BookwormConfig = &distro.Config{ + ImageRef: bookwormRef, + AptCachePrefix: BookwormAptCachePrefix, + VersionID: bookwormVersionID, + ContextRef: BookwormWorkerContextName, + DefaultOutputImage: bookwormRef, + BuilderPackages: basePackages, + } +) diff --git a/frontend/debian/bullseye.go b/frontend/debian/bullseye.go new file mode 100644 index 000000000..276f0d16d --- /dev/null +++ b/frontend/debian/bullseye.go @@ -0,0 +1,44 @@ +package debian + +import ( + "github.com/Azure/dalec" + "github.com/Azure/dalec/frontend/deb/distro" +) + +const ( + BullseyeDefaultTargetKey = "bullseye" + BullseyeAptCachePrefix = "bullseye" + BullseyeWorkerContextName = "dalec-bullseye-worker" + + bullseyeRef = "mcr.microsoft.com/mirror/docker/library/debian:bullseye" + bullseyeVersionID = "debian11" +) + +var ( + BullseyeConfig = &distro.Config{ + ImageRef: bullseyeRef, + AptCachePrefix: BullseyeAptCachePrefix, + VersionID: bullseyeVersionID, + ContextRef: BullseyeWorkerContextName, + DefaultOutputImage: bullseyeRef, + BuilderPackages: basePackages, + + // Ubuntu typically has backports repos already in it but Debian does not. + // Without this the go modules test will fail since there is no viable + // version of go except with the backports repository added. + ExtraRepos: []dalec.PackageRepositoryConfig{ + { + Envs: []string{"build", "test", "install"}, + Config: map[string]dalec.Source{ + "backports.list": { + Inline: &dalec.SourceInline{ + File: &dalec.SourceInlineFile{ + Contents: "deb http://deb.debian.org/debian bullseye-backports main", + }, + }, + }, + }, + }, + }, + } +) diff --git a/frontend/debian/common.go b/frontend/debian/common.go new file mode 100644 index 000000000..f54b2d11a --- /dev/null +++ b/frontend/debian/common.go @@ -0,0 +1,32 @@ +package debian + +import ( + "context" + + "github.com/Azure/dalec/frontend" + gwclient "github.com/moby/buildkit/frontend/gateway/client" +) + +var ( + basePackages = []string{ + "aptitude", + "dpkg-dev", + "devscripts", + "equivs", + "fakeroot", + "dh-make", + "build-essential", + "dh-apparmor", + "dh-make", + "dh-exec", + } + + targets = map[string]gwclient.BuildFunc{ + BookwormDefaultTargetKey: BookwormConfig.Handle, + BullseyeDefaultTargetKey: BullseyeConfig.Handle, + } +) + +func Handlers(ctx context.Context, client gwclient.Client, m *frontend.BuildMux) error { + return frontend.LoadBuiltinTargets(targets)(ctx, client, m) +} diff --git a/helpers.go b/helpers.go index cf6f1ea2b..12f9fe11b 100644 --- a/helpers.go +++ b/helpers.go @@ -605,3 +605,17 @@ func BaseImageConfig(platform *ocispecs.Platform) *DockerImageSpec { return img } + +func (p *PackageDependencies) GetExtraRepos(env string) []PackageRepositoryConfig { + return GetExtraRepos(p.ExtraRepos, env) +} + +func GetExtraRepos(repos []PackageRepositoryConfig, env string) []PackageRepositoryConfig { + var out []PackageRepositoryConfig + for _, repo := range repos { + if slices.Contains(repo.Envs, env) { + out = append(repos, repo) + } + } + return out +} diff --git a/spec.go b/spec.go index a788c19cf..c26a0c34f 100644 --- a/spec.go +++ b/spec.go @@ -5,7 +5,6 @@ import ( "fmt" "io/fs" "regexp" - "slices" "strings" "time" @@ -359,17 +358,6 @@ type PackageDependencies struct { ExtraRepos []PackageRepositoryConfig `yaml:"extra_repos,omitempty" json:"extra_repos,omitempty"` } -func (p *PackageDependencies) GetExtraRepos(env string) []PackageRepositoryConfig { - var repos []PackageRepositoryConfig - for _, repo := range p.ExtraRepos { - if slices.Contains(repo.Envs, env) { - repos = append(repos, repo) - } - } - - return repos -} - // PackageRepositoryConfig type PackageRepositoryConfig struct { // Keys are the list of keys that need to be imported to use the configured diff --git a/test/debian_test.go b/test/debian_test.go new file mode 100644 index 000000000..5815976d9 --- /dev/null +++ b/test/debian_test.go @@ -0,0 +1,21 @@ +package test + +import ( + "testing" + + "github.com/Azure/dalec/frontend/debian" +) + +func TestBookworm(t *testing.T) { + t.Parallel() + + ctx := startTestSpan(baseCtx, t) + testLinuxDistro(ctx, t, debLinuxTestConfigFor(debian.BookwormDefaultTargetKey, debian.BookwormConfig)) +} + +func TestBullseye(t *testing.T) { + t.Parallel() + + ctx := startTestSpan(baseCtx, t) + testLinuxDistro(ctx, t, debLinuxTestConfigFor(debian.BullseyeDefaultTargetKey, debian.BullseyeConfig, withPackageOverride("golang", "golang-1.19"))) +} diff --git a/website/docs/targets.md b/website/docs/targets.md index bf0b63eee..a2cf1d37f 100644 --- a/website/docs/targets.md +++ b/website/docs/targets.md @@ -24,6 +24,14 @@ bionic/deb (default) Builds a deb package. bionic/dsc Builds a Debian source package. bionic/testing/container Builds a container image for testing purposes only. bionic/worker Builds the worker image. +bookworm/deb (default) Builds a deb package. +bookworm/dsc Builds a Debian source package. +bookworm/testing/container Builds a container image for testing purposes only. +bookworm/worker Builds the worker image. +bullseye/deb (default) Builds a deb package. +bullseye/dsc Builds a Debian source package. +bullseye/testing/container Builds a container image for testing purposes only. +bullseye/worker Builds the worker image. debug/gomods Outputs all the gomodule dependencies for the spec debug/resolve Outputs the resolved dalec spec file with build args applied. debug/sources Outputs all sources from a dalec spec file.