From dce4d67fff3266c8ba98efcbdbc67cbf4d10f74f Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 4 Oct 2024 13:03:50 -0700 Subject: [PATCH 1/2] deb: Use custom install script instead of dh-exec dh-exec has some limitations that we can't really get around. The immediate need is it can't copy a directory to a new name in the target location. As such we need more flexibility in how we set things up, so this switches to bash so we can have the power that we need. Signed-off-by: Brian Goff --- frontend/deb/debroot.go | 19 +++++++------- .../deb/templates/debian_install_header.sh | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 frontend/deb/templates/debian_install_header.sh diff --git a/frontend/deb/debroot.go b/frontend/deb/debroot.go index ceb7494e9..a18fb8aa4 100644 --- a/frontend/deb/debroot.go +++ b/frontend/deb/debroot.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - _ "embed" "fmt" "io" "path/filepath" @@ -13,6 +12,8 @@ import ( "strings" "sync" + _ "embed" + "github.com/Azure/dalec" "github.com/Azure/dalec/frontend/pkg/bkfs" "github.com/moby/buildkit/client/llb" @@ -25,6 +26,9 @@ const customSystemdPostinstFile = "custom_systemd_postinst.sh.partial" //go:embed templates/patch-header.txt var patchHeader []byte +//go:embed templates/debian_install_header.sh +var debianInstall []byte + // This creates a directory in the debian root directory for each patch, and copies the patch files into it. // The format for each patch dir matches what would normaly be under `debian/patches`, just that this is a separate dir for every source we are patching // This is purely for documenting in the source package how patches are applied in a more readable way than the big merged patch file. @@ -276,13 +280,12 @@ func createInstallScripts(worker llb.State, spec *dalec.Spec, dir string) []llb. return nil } - states := make([]llb.State, 0, len(spec.Artifacts.Binaries)+len(spec.Artifacts.Manpages)) + states := make([]llb.State, 1) base := llb.Scratch().File(llb.Mkdir(dir, 0o755, llb.WithParents(true))) installBuf := bytes.NewBuffer(nil) writeInstallHeader := sync.OnceFunc(func() { - fmt.Fprintln(installBuf, "#!/usr/bin/dh-exec") - fmt.Fprintln(installBuf) + fmt.Fprintln(installBuf, string(debianInstall)) }) writeInstall := func(src, dir, name string) { @@ -290,12 +293,10 @@ func createInstallScripts(worker llb.State, spec *dalec.Spec, dir string) []llb. // first time it is called. writeInstallHeader() - if filepath.Base(src) != name { - fmt.Fprintln(installBuf, src, "=>", filepath.Join(dir, name)) - return - } + name = strings.TrimSuffix(name, "*") + dest := filepath.Join("debian", spec.Name, dir, name) + fmt.Fprintln(installBuf, "do_install", filepath.Dir(dest), dest, src) - fmt.Fprintln(installBuf, src, dir+"/") } if len(spec.Artifacts.Binaries) > 0 { diff --git a/frontend/deb/templates/debian_install_header.sh b/frontend/deb/templates/debian_install_header.sh new file mode 100644 index 000000000..518d496e5 --- /dev/null +++ b/frontend/deb/templates/debian_install_header.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eux + +do_install() { + local parent="${1}" + shift + + local dest="${1}" + shift + + mkdir -p "${parent}" + + local files=($@) + + # When the number of files passed in is more than 1, then dest *must* refer + # to a directory + if test ${#files[@]} -gt 1; then + mkdir -p "${dest}" + fi + + for src in ${files[@]}; do + cp --reflink=auto -a "${src}" "${dest}" + done +} From ce3202be9e560657257cb78797c83dabdb783c7f Mon Sep 17 00:00:00 2001 From: Jose Blanquicet Date: Thu, 3 Oct 2024 13:55:31 +0200 Subject: [PATCH 2/2] Add support for headers artifacts Signed-off-by: Jose Blanquicet --- artifacts.go | 7 ++++ docs/spec.schema.json | 7 ++++ frontend/deb/debroot.go | 9 ++++ frontend/rpm/template.go | 15 +++++++ frontend/rpm/template_test.go | 41 +++++++++++++++++++ test/azlinux_test.go | 77 +++++++++++++++++++++++++++-------- website/docs/artifacts.md | 29 +++++++++++++ 7 files changed, 168 insertions(+), 17 deletions(-) diff --git a/artifacts.go b/artifacts.go index b0c8039cf..f19b8677a 100644 --- a/artifacts.go +++ b/artifacts.go @@ -37,6 +37,10 @@ type Artifacts struct { // the [ImageConfig]. Links []ArtifactSymlinkConfig `yaml:"links,omitempty" json:"links,omitempty"` + // Headers is a list of header files and/or folders to be installed. + // On linux this would typically be installed to /usr/include/. + Headers map[string]ArtifactConfig `yaml:"headers,omitempty" json:"headers,omitempty"` + // TODO: other types of artifacts (libexec, etc) } @@ -135,5 +139,8 @@ func (a *Artifacts) IsEmpty() bool { if len(a.Links) > 0 { return false } + if len(a.Headers) > 0 { + return false + } return true } diff --git a/docs/spec.schema.json b/docs/spec.schema.json index 4fa9bb53b..6604bb613 100644 --- a/docs/spec.schema.json +++ b/docs/spec.schema.json @@ -132,6 +132,13 @@ }, "type": "array", "description": "Links is the list of symlinks to be installed with the package\nLinks should only be used if the *package* should contain the link.\nFor making a container compatible with another image, use [PostInstall] in\nthe [ImageConfig]." + }, + "headers": { + "additionalProperties": { + "$ref": "#/$defs/ArtifactConfig" + }, + "type": "object", + "description": "Headers is a list of header files and/or folders to be installed.\nOn linux this would typically be installed to /usr/include/." } }, "additionalProperties": false, diff --git a/frontend/deb/debroot.go b/frontend/deb/debroot.go index a18fb8aa4..483998862 100644 --- a/frontend/deb/debroot.go +++ b/frontend/deb/debroot.go @@ -388,6 +388,15 @@ func createInstallScripts(worker llb.State, spec *dalec.Spec, dir string) []llb. } } + if len(spec.Artifacts.Headers) > 0 { + sorted := dalec.SortMapKeys(spec.Artifacts.Headers) + for _, key := range sorted { + cfg := spec.Artifacts.Headers[key] + resolved := cfg.ResolveName(key) + writeInstall(key, filepath.Join("/usr/include", cfg.SubPath), resolved) + } + } + if units := spec.Artifacts.Systemd.GetUnits(); len(units) > 0 { // deb-systemd will look for service files in DEBIAN/[.]. // To handle this we'll create symlinks to the actual unit files in the source. diff --git a/frontend/rpm/template.go b/frontend/rpm/template.go index ea930fbfa..a446308d5 100644 --- a/frontend/rpm/template.go +++ b/frontend/rpm/template.go @@ -546,6 +546,12 @@ func (w *specWrapper) Install() fmt.Stringer { fmt.Fprintln(b, "ln -sf", l.Source, "%{buildroot}/"+l.Dest) } + headersKeys := dalec.SortMapKeys(w.Spec.Artifacts.Headers) + for _, h := range headersKeys { + cfg := w.Spec.Artifacts.Headers[h] + copyArtifact(`%{buildroot}/%{_includedir}`, h, &cfg) + } + b.WriteString("\n") return b } @@ -673,6 +679,15 @@ func (w *specWrapper) Files() fmt.Stringer { fmt.Fprintln(b, l.Dest) } + if len(w.Spec.Artifacts.Headers) > 0 { + headersKeys := dalec.SortMapKeys(w.Spec.Artifacts.Headers) + for _, h := range headersKeys { + hf := w.Spec.Artifacts.Headers[h] + path := filepath.Join(`%{_includedir}`, hf.SubPath, hf.ResolveName(h)) + fmt.Fprintln(b, path) + } + } + b.WriteString("\n") return b } diff --git a/frontend/rpm/template_test.go b/frontend/rpm/template_test.go index 106ef65f6..db76a2105 100644 --- a/frontend/rpm/template_test.go +++ b/frontend/rpm/template_test.go @@ -369,6 +369,47 @@ fi want := `%files %license %{_licensedir}/test-pkg/licenses/LICENSE.md +` + assert.Equal(t, want, got) + }) + + t.Run("test headers templating using defaults", func(t *testing.T) { + t.Parallel() + w := &specWrapper{Spec: &dalec.Spec{ + Name: "test-pkg", + Artifacts: dalec.Artifacts{ + Headers: map[string]dalec.ArtifactConfig{ + "test-headers": {}, + }, + }, + }} + + got := w.Files().String() + want := `%files +%{_includedir}/test-headers + +` + assert.Equal(t, want, got) + }) + + t.Run("test headers templating using ArtifactConfig", func(t *testing.T) { + t.Parallel() + w := &specWrapper{Spec: &dalec.Spec{ + Name: "test-pkg", + Artifacts: dalec.Artifacts{ + Headers: map[string]dalec.ArtifactConfig{ + "test-headers": { + Name: "sub-module-headers", + SubPath: "sub-module", + }, + }, + }, + }} + + got := w.Files().String() + want := `%files +%{_includedir}/sub-module/sub-module-headers + ` assert.Equal(t, want, got) }) diff --git a/test/azlinux_test.go b/test/azlinux_test.go index 5b98bb2c0..f2fb20db1 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -508,7 +508,6 @@ echo "$BAR" > bar.txt "src": { Inline: &dalec.SourceInline{ Dir: &dalec.SourceInlineDir{ - Files: map[string]*dalec.SourceInlineFile{ "simple.service": { Contents: ` @@ -523,7 +522,8 @@ Restart=always [Install] WantedBy=multi-user.target -`}, +`, + }, }, }, }, @@ -544,7 +544,7 @@ WantedBy=multi-user.target Files: map[string]dalec.FileCheckOutput{ filepath.Join(testConfig.SystemdDir.Units, "system/simple.service"): { CheckOutput: dalec.CheckOutput{Contains: []string{"ExecStart=/usr/bin/service"}}, - Permissions: 0644, + Permissions: 0o644, }, // symlinked file in multi-user.target.wants should point to simple.service. filepath.Join(testConfig.SystemdDir.Targets, "multi-user.target.wants/simple.service"): { @@ -572,7 +572,7 @@ WantedBy=multi-user.target Files: map[string]dalec.FileCheckOutput{ filepath.Join(testConfig.SystemdDir.Units, "system/simple.service"): { CheckOutput: dalec.CheckOutput{Contains: []string{"ExecStart=/usr/bin/service"}}, - Permissions: 0644, + Permissions: 0o644, }, filepath.Join(testConfig.SystemdDir.Targets, "multi-user.target.wants/simple.service"): { NotExist: true, @@ -601,7 +601,7 @@ WantedBy=multi-user.target Files: map[string]dalec.FileCheckOutput{ filepath.Join(testConfig.SystemdDir.Units, "system/phony.service"): { CheckOutput: dalec.CheckOutput{Contains: []string{"ExecStart=/usr/bin/service"}}, - Permissions: 0644, + Permissions: 0o644, }, filepath.Join(testConfig.SystemdDir.Targets, "multi-user.target.wants/phony.service"): { NotExist: true, @@ -631,7 +631,6 @@ WantedBy=multi-user.target "src": { Inline: &dalec.SourceInline{ Dir: &dalec.SourceInlineDir{ - Files: map[string]*dalec.SourceInlineFile{ "foo.service": { Contents: ` @@ -647,7 +646,8 @@ Restart=always [Install] WantedBy=multi-user.target -`}, +`, + }, "foo.socket": { Contents: ` @@ -703,7 +703,7 @@ Environment="FOO_ARGS=--some-foo-args" Files: map[string]dalec.FileCheckOutput{ filepath.Join(testConfig.SystemdDir.Units, "system/foo.service"): { CheckOutput: dalec.CheckOutput{Contains: []string{"ExecStart=/usr/bin/foo"}}, - Permissions: 0644, + Permissions: 0o644, }, filepath.Join(testConfig.SystemdDir.Targets, "multi-user.target.wants/foo.service"): { NotExist: true, @@ -713,11 +713,11 @@ Environment="FOO_ARGS=--some-foo-args" }, filepath.Join(testConfig.SystemdDir.Units, "system/foo.service.d/foo.conf"): { CheckOutput: dalec.CheckOutput{Contains: []string{"Environment"}}, - Permissions: 0644, + Permissions: 0o644, }, filepath.Join(testConfig.SystemdDir.Units, "system/foo.socket.d/env.conf"): { CheckOutput: dalec.CheckOutput{Contains: []string{"Environment"}}, - Permissions: 0644, + Permissions: 0o644, }, }, }, @@ -745,7 +745,6 @@ Environment="FOO_ARGS=--some-foo-args" "src": { Inline: &dalec.SourceInline{ Dir: &dalec.SourceInlineDir{ - Files: map[string]*dalec.SourceInlineFile{ "foo.conf": { Contents: ` @@ -773,7 +772,7 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot Files: map[string]dalec.FileCheckOutput{ filepath.Join(testConfig.SystemdDir.Units, "system/foo.service.d/foo.conf"): { CheckOutput: dalec.CheckOutput{Contains: []string{"Environment"}}, - Permissions: 0644, + Permissions: 0o644, }, }, }, @@ -1059,7 +1058,7 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) }) - t.Run("docs and licenses are handled correctly", func(t *testing.T) { + t.Run("docs and headers and licenses are handled correctly", func(t *testing.T) { t.Parallel() spec := &dalec.Spec{ Name: "test-docs-handled", @@ -1103,6 +1102,30 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }, }, }, + "src5": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "header.h": { + Contents: "message=hello", + Permissions: 0o644, + }, + }, + }, + }, + }, + "src6": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "header.h": { + Contents: "message=hello", + Permissions: 0o644, + }, + }, + }, + }, + }, }, Artifacts: dalec.Artifacts{ Docs: map[string]dalec.ArtifactConfig{ @@ -1117,15 +1140,37 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot SubPath: "license-subpath", }, }, + Headers: map[string]dalec.ArtifactConfig{ + // Files with and without ArtifactConfig + "src1": { + Name: "renamed-src1", + SubPath: "header-subpath-src1", + }, + "src2": {}, + // Directories with and without ArtifactConfig + "src5": { + Name: "renamed-src5", + SubPath: "header-subpath-src5", + }, + "src6": {}, + }, }, Tests: []*dalec.TestSpec{ { - Name: "Doc files should be created in correct place", + Name: "Doc and lib and header files should be created in correct place", Files: map[string]dalec.FileCheckOutput{ "/usr/share/doc/test-docs-handled/src1": {}, "/usr/share/doc/test-docs-handled/subpath/src2": {}, filepath.Join(testConfig.LicenseDir, "test-docs-handled/src3"): {}, filepath.Join(testConfig.LicenseDir, "test-docs-handled/license-subpath/src4"): {}, + "/usr/include/header-subpath-src1/renamed-src1": {}, + "/usr/include/src2": {}, + "/usr/include/header-subpath-src5/renamed-src5": { + IsDir: true, + }, + "/usr/include/src6": { + IsDir: true, + }, }, }, }, @@ -1290,7 +1335,7 @@ func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetCo } func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) { - var pkgName = "dalec-test-package" + pkgName := "dalec-test-package" getTestPackageSpec := func(version string) *dalec.Spec { depSpec := &dalec.Spec{ @@ -1411,14 +1456,12 @@ func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) sr := newSolveRequest(withSpec(ctx, t, spec), withBuildContext(ctx, t, cfg.Worker.ContextName, worker), withBuildTarget(cfg.Target.Container)) res := solveT(ctx, t, gwc, sr) _, err := res.SingleRef() - if err != nil { t.Fatal(err) } }) }) } - } func testLinuxLibArtirfacts(ctx context.Context, t *testing.T, cfg testLinuxConfig) { diff --git a/website/docs/artifacts.md b/website/docs/artifacts.md index f20b5b1fc..a7869a1b7 100644 --- a/website/docs/artifacts.md +++ b/website/docs/artifacts.md @@ -232,3 +232,32 @@ artifacts: - source: /usr/lib/golang/go dest: /usr/bin/go ``` + +### Headers + +Headers are header to be included with the package. On Linux these typically go +under `/usr/include/`. + +Headers are a mapping of file path to [artifact configuration](#artifact-configuration). +The file path is the path to a file or directory that must be available after +the build section has finished. This path is relative to the working directory +of the build phase *before* any directory changes are made. + +```yaml +artifacts: + headers: + src/my_header.h: +``` + +or for a directory: + +```yaml +artifacts: + headers: + src/my_headers/: +``` + +Note that headers are not installed within a subdirectory of `/usr/include/` +with the name of the package. They are installed directly into `/usr/include/`. +For instance, for the above examples, the headers would be installed to +`/usr/include/my_header.h` and `/usr/include/my_headers/` respectively.