diff --git a/api/credentials/builtin/init.go b/api/credentials/builtin/init.go index 7b7cc5062d..d06d239ae2 100644 --- a/api/credentials/builtin/init.go +++ b/api/credentials/builtin/init.go @@ -2,4 +2,5 @@ package builtin import ( _ "ocm.software/ocm/api/credentials/builtin/github" + _ "ocm.software/ocm/api/tech/git/identity" ) diff --git a/api/oci/cpi/support/base.go b/api/oci/cpi/support/base.go index 60cbf0a79c..5d3568bf33 100644 --- a/api/oci/cpi/support/base.go +++ b/api/oci/cpi/support/base.go @@ -31,18 +31,18 @@ func (a *artifactBase) IsReadOnly() bool { } func (a *artifactBase) IsIndex() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsIndex() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsIndex() } func (a *artifactBase) IsManifest() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsManifest() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsManifest() } func (a *artifactBase) IsValid() bool { - d := a.state.GetState().(*artdesc.Artifact) - return d.IsValid() + d, ok := a.state.GetState().(*artdesc.Artifact) + return ok && d.IsValid() } func (a *artifactBase) blob() (cpi.BlobAccess, error) { diff --git a/api/oci/internal/uniform.go b/api/oci/internal/uniform.go index d35d102afb..32e3b0ef89 100644 --- a/api/oci/internal/uniform.go +++ b/api/oci/internal/uniform.go @@ -31,7 +31,7 @@ type UniformRepositorySpec struct { // Host is the hostname of an oci ref. Host string `json:"host,omitempty"` // Info is the file path used to host ctf component versions - Info string `json:"filePath,omitempty"` + Info string `json:"info,omitempty"` // CreateIfMissing indicates whether a file based or dynamic repo should be created if it does not exist CreateIfMissing bool `json:"createIfMissing,omitempty"` diff --git a/api/ocm/elements/artifactaccess/gitaccess/options.go b/api/ocm/elements/artifactaccess/gitaccess/options.go new file mode 100644 index 0000000000..5b569f1f0f --- /dev/null +++ b/api/ocm/elements/artifactaccess/gitaccess/options.go @@ -0,0 +1,58 @@ +package gitaccess + +import ( + "github.com/mandelsoft/goutils/optionutils" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + URL string + Ref string + Commit string +} + +var _ Option = (*Options)(nil) + +func (o *Options) ApplyTo(opts *Options) { + if o.URL != "" { + opts.URL = o.URL + } +} + +func (o *Options) Apply(opts ...Option) { + optionutils.ApplyOptions(o, opts...) +} + +// ////////////////////////////////////////////////////////////////////////////// +// Local options + +type url string + +func (h url) ApplyTo(opts *Options) { + opts.URL = string(h) +} + +func WithURL(h string) Option { + return url(h) +} + +type ref string + +func (h ref) ApplyTo(opts *Options) { + opts.Ref = string(h) +} + +func WithRef(h string) Option { + return ref(h) +} + +type commitSpec string + +func (h commitSpec) ApplyTo(opts *Options) { + opts.Commit = string(h) +} + +func WithCommit(c string) Option { + return commitSpec(c) +} diff --git a/api/ocm/elements/artifactaccess/gitaccess/resource.go b/api/ocm/elements/artifactaccess/gitaccess/resource.go new file mode 100644 index 0000000000..24ade26b33 --- /dev/null +++ b/api/ocm/elements/artifactaccess/gitaccess/resource.go @@ -0,0 +1,25 @@ +package gitaccess + +import ( + "github.com/mandelsoft/goutils/optionutils" + + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/compdesc" + "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/elements/artifactaccess/genericaccess" + access "ocm.software/ocm/api/ocm/extensions/accessmethods/git" + resourcetypes "ocm.software/ocm/api/ocm/extensions/artifacttypes" +) + +const TYPE = resourcetypes.DIRECTORY_TREE + +func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, opts ...Option) cpi.ArtifactAccess[M] { + eff := optionutils.EvalOptions(opts...) + if meta.GetType() == "" { + meta.SetType(TYPE) + } + + spec := access.New(eff.URL, access.WithRef(eff.Ref), access.WithCommit(eff.Commit)) + // is global access, must work, otherwise there is an error in the lib. + return genericaccess.MustAccess(ctx, meta, spec) +} diff --git a/api/ocm/extensions/accessmethods/git/README.md b/api/ocm/extensions/accessmethods/git/README.md new file mode 100644 index 0000000000..53adeffbd9 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/README.md @@ -0,0 +1,93 @@ + +# Access Method `git` - Git Commit Access + +## Synopsis + +```yaml +type: git/v1 +``` + +Provided blobs use the following media type for: `application/x-tgz` + +The artifact content is provided as gnu-zipped tar archive + +### Description + +This method implements the access of the content of a git commit stored in a +git repository. + +Supported specification version is `v1alpha1` + +### Specification Versions + +#### Version `v1alpha1` + +The type specific specification fields are: + +- **`repository`** *string* + + Repository URL with or without scheme. + +- **`ref`** (optional) *string* + + Original ref used to get the commit from + +- **`commit`** *string* + + The sha/id of the git commit + +### Go Bindings + +The go binding can be found [here](method.go) + +#### Example + +```go +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "io" + + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/cpi" + me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" +) + +func main() { + ctx := ocm.New() + accessSpec := me.New( + "https://github.com/octocat/Hello-World.git", + me.WithRef("refs/heads/master"), + ) + method, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + if err != nil { + panic(err) + } + content, err := method.GetContent() + if err != nil { + panic(err) + } + unzippedContent, err := gzip.NewReader(bytes.NewReader(content)) + + r := tar.NewReader(unzippedContent) + + file, err := r.Next() + if err != nil { + panic(err) + } + + if file.Name != "README.md" { + panic("Expected README.md") + } + + data, err := io.ReadAll(r) + if err != nil { + panic(err) + } + fmt.Println(string(data)) +} +``` diff --git a/api/ocm/extensions/accessmethods/git/cli.go b/api/ocm/extensions/accessmethods/git/cli.go new file mode 100644 index 0000000000..fc0dce7d90 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/cli.go @@ -0,0 +1,43 @@ +package git + +import ( + "ocm.software/ocm/api/ocm/extensions/accessmethods/options" + "ocm.software/ocm/api/utils/cobrautils/flagsets" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + Type, AddConfig, + options.RepositoryOption, + options.ReferenceOption, + options.CommitOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository", "repo", "repoUrl", "repoURL") + flagsets.AddFieldByOptionP(opts, options.CommitOption, config, "commit") + flagsets.AddFieldByOptionP(opts, options.ReferenceOption, config, "ref") + return nil +} + +var usage = ` +This method implements the access of the content of a git commit stored in a +Git repository. +` + +var formatV1 = ` +The type specific specification fields are: + +- **repoUrl** *string* + + Repository URL with or without scheme. + +- **ref** (optional) *string* + + Original ref used to get the commit from + +- **commit** *string* + + The sha/id of the git commit +` diff --git a/api/ocm/extensions/accessmethods/git/method.go b/api/ocm/extensions/accessmethods/git/method.go new file mode 100644 index 0000000000..b43b96841e --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/method.go @@ -0,0 +1,140 @@ +package git + +import ( + "fmt" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/mandelsoft/goutils/errors" + giturls "github.com/whilp/git-urls" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm/cpi/accspeccpi" + "ocm.software/ocm/api/ocm/internal" + "ocm.software/ocm/api/tech/git/identity" + "ocm.software/ocm/api/utils/blobaccess/blobaccess" + gitblob "ocm.software/ocm/api/utils/blobaccess/git" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/runtime" +) + +const ( + Type = "git" + TypeV1Alpha1 = Type + runtime.VersionSeparator + "v1alpha1" +) + +func init() { + // If we remove the default registration, also the docs are gone. + // so we leave the default registration in. + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](Type, accspeccpi.WithDescription(usage))) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1Alpha1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) +} + +// AccessSpec describes the access for a GitHub registry. +type AccessSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // Repository is the repository URL + Repository string `json:"repository"` + + // Ref defines the hash of the commit + Ref string `json:"ref,omitempty"` + + // Commit defines the hash of the commit in string format to checkout from the Ref + Commit string `json:"commit,omitempty"` +} + +// AccessSpecOptions defines a set of options which can be applied to the access spec. +type AccessSpecOptions func(s *AccessSpec) + +func WithCommit(commit string) AccessSpecOptions { + return func(s *AccessSpec) { + s.Commit = commit + } +} + +func WithRef(ref string) AccessSpecOptions { + return func(s *AccessSpec) { + s.Ref = ref + } +} + +// New creates a new git registry access spec version v1. +func New(url string, opts ...AccessSpecOptions) *AccessSpec { + s := &AccessSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + Repository: url, + } + for _, o := range opts { + o(s) + } + return s +} + +func (a *AccessSpec) Describe(internal.Context) string { + return fmt.Sprintf("git commit %s[%s]", a.Repository, a.Ref) +} + +func (*AccessSpec) IsLocal(internal.Context) bool { + return false +} + +func (a *AccessSpec) GlobalAccessSpec(accspeccpi.Context) accspeccpi.AccessSpec { + return a +} + +func (*AccessSpec) GetType() string { + return Type +} + +func (a *AccessSpec) AccessMethod(cva internal.ComponentVersionAccess) (internal.AccessMethod, error) { + _, err := giturls.Parse(a.Repository) + if err != nil { + return nil, errors.ErrInvalidWrap(err, "repository repoURL", a.Repository) + } + if err := plumbing.ReferenceName(a.Ref).Validate(); err != nil { + return nil, errors.ErrInvalidWrap(err, "commit hash", a.Ref) + } + creds, _, err := getCreds(a.Repository, cva.GetContext().CredentialsContext()) + if err != nil { + return nil, fmt.Errorf("failed to get credentials for repository %s: %w", a.Repository, err) + } + + octx := cva.GetContext() + + opts := []gitblob.Option{ + gitblob.WithLoggingContext(octx), + gitblob.WithCredentialContext(octx), + gitblob.WithURL(a.Repository), + gitblob.WithRef(a.Ref), + gitblob.WithCommit(a.Commit), + gitblob.WithCachingFileSystem(vfsattr.Get(octx)), + } + if creds != nil { + opts = append(opts, gitblob.WithCredentials(creds)) + } + + factory := func() (blobaccess.BlobAccess, error) { + return gitblob.BlobAccess(opts...) + } + + return accspeccpi.AccessMethodForImplementation(accspeccpi.NewDefaultMethodImpl( + cva, + a, + "", + mime.MIME_TGZ, + factory, + ), nil) +} + +func getCreds(repoURL string, cctx credentials.Context) (credentials.Credentials, credentials.ConsumerIdentity, error) { + id, err := identity.GetConsumerId(repoURL) + if err != nil { + return nil, nil, err + } + creds, err := credentials.CredentialsForConsumer(cctx.CredentialsContext(), id, identity.IdentityMatcher) + if creds == nil || err != nil { + return nil, id, err + } + return creds, id, nil +} diff --git a/api/ocm/extensions/accessmethods/git/method_test.go b/api/ocm/extensions/accessmethods/git/method_test.go new file mode 100644 index 0000000000..3a3ac663a4 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/method_test.go @@ -0,0 +1,167 @@ +package git_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "embed" + "fmt" + "io" + "os" + "time" + + _ "embed" + + "github.com/go-git/go-git/v5/plumbing" + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/cpi" + me "ocm.software/ocm/api/ocm/extensions/accessmethods/git" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("Method based on Filesystem", func() { + var ( + ctx ocm.Context + expectedBlobContent []byte + accessSpec *me.AccessSpec + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + var repoDir string + + BeforeEach(func() { + repoDir = GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo := Must(git.PlainInit(repoDir, false)) + + repoBase := filepath.Join("testdata", "repo") + repoTestData := Must(testData.ReadDir(repoBase)) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file := Must(testData.Open(path)) + + fileInRepo := Must(os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0o600, + )) + + Must(io.Copy(fileInRepo, file)) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt := Must(repo.Worktree()) + Expect(wt.AddGlob("*")).To(Succeed()) + commit := Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + })) + + path := filepath.Join("testdata", "repo", "file_in_repo") + + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), + me.WithRef(plumbing.Master.String()), + me.WithCommit(commit.String()), + ) + + expectedBlobContent = Must(testData.ReadFile(path)) + }) + + It("downloads artifacts with full ref", func() { + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("file_in_repo")) + Expect(file.Size).To(Equal(int64(len(expectedBlobContent)))) + + data := Must(io.ReadAll(r)) + Expect(data).To(Equal(expectedBlobContent)) + }) + + It("downloads artifacts without commit because the url reference is enough", func() { + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), me.WithRef(plumbing.Master.String())) + + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("file_in_repo")) + Expect(file.Size).To(Equal(int64(len(expectedBlobContent)))) + + data := Must(io.ReadAll(r)) + Expect(data).To(Equal(expectedBlobContent)) + }) + + It("cannot download artifacts ref without a reference", func() { + accessSpec = me.New(fmt.Sprintf("file://%s", repoDir), me.WithCommit(accessSpec.Commit)) + + _, err := accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid reference name")) + }) +}) + +var _ = Describe("Method based on Real Repository", func() { + host := "github.com:443" + reachable := PingTCPServer(host, time.Second) == nil + var url string + BeforeEach(func() { + if !reachable { + Skip(fmt.Sprintf("no connection to %s, skipping test connection to remote", url)) + } + // This repo is a public repo owned by the Github Kraken Bot, so its as good of a public available + // example as any. + url = fmt.Sprintf("https://%s/octocat/Hello-World.git", host) + }) + + It("can download remote artifacts", func() { + ctx := ocm.New() + accessSpec := me.New(url, me.WithRef(plumbing.Master.String())) + + m := Must(accessSpec.AccessMethod(&cpi.DummyComponentVersionAccess{Context: ctx})) + content := Must(m.Get()) + unzippedContent := Must(gzip.NewReader(bytes.NewReader(content))) + + r := tar.NewReader(unzippedContent) + + file := Must(r.Next()) + Expect(file.Name).To(Equal("README")) + }) +}) diff --git a/api/ocm/extensions/accessmethods/git/suite_test.go b/api/ocm/extensions/accessmethods/git/suite_test.go new file mode 100644 index 0000000000..59ce0ec1fc --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/suite_test.go @@ -0,0 +1,13 @@ +package git + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Github Test Suite") +} diff --git a/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo b/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/ocm/extensions/accessmethods/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/api/ocm/extensions/accessmethods/init.go b/api/ocm/extensions/accessmethods/init.go index 5f4094d3d5..08ce51bf32 100644 --- a/api/ocm/extensions/accessmethods/init.go +++ b/api/ocm/extensions/accessmethods/init.go @@ -1,6 +1,7 @@ package accessmethods import ( + _ "ocm.software/ocm/api/ocm/extensions/accessmethods/git" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/github" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" _ "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" diff --git a/api/tech/git/auth.go b/api/tech/git/auth.go new file mode 100644 index 0000000000..276d12a962 --- /dev/null +++ b/api/tech/git/auth.go @@ -0,0 +1,46 @@ +package git + +import ( + "errors" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + gssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/tech/git/identity" +) + +var ErrNoValidGitCredentials = errors.New("no valid credentials found for git authentication") + +type AuthMethod = transport.AuthMethod + +// AuthFromCredentials creates a git authentication method from the given credentials. +// If no valid credentials are found, ErrNoValidGitCredentials is returned. +// However, one can still perform anonymous operations with the git client if the repo allows it. +func AuthFromCredentials(creds credentials.Credentials) (AuthMethod, error) { + if creds == nil { + return nil, ErrNoValidGitCredentials + } + + if creds.ExistsProperty(identity.ATTR_PRIVATE_KEY) { + return gssh.NewPublicKeysFromFile( + creds.GetProperty(identity.ATTR_USERNAME), + creds.GetProperty(identity.ATTR_PRIVATE_KEY), + creds.GetProperty(identity.ATTR_PASSWORD), + ) + } + + if creds.ExistsProperty(identity.ATTR_TOKEN) { + return &http.TokenAuth{Token: creds.GetProperty(identity.ATTR_TOKEN)}, nil + } + + if creds.ExistsProperty(identity.ATTR_USERNAME) { + return &http.BasicAuth{ + Username: creds.GetProperty(identity.ATTR_USERNAME), + Password: creds.GetProperty(identity.ATTR_PASSWORD), + }, nil + } + + return nil, ErrNoValidGitCredentials +} diff --git a/api/tech/git/fs.go b/api/tech/git/fs.go new file mode 100644 index 0000000000..bfbb4c5386 --- /dev/null +++ b/api/tech/git/fs.go @@ -0,0 +1,177 @@ +package git + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + + "github.com/go-git/go-billy/v5" + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" + "github.com/mandelsoft/vfs/pkg/vfs" +) + +func VFSBillyFS(fsToWrap vfs.FileSystem) (billy.Filesystem, error) { + if fsToWrap == nil { + fsToWrap = vfs.New(memoryfs.New()) + } + fi, err := fsToWrap.Stat(".") + if err != nil || !fi.IsDir() { + return nil, fmt.Errorf("invalid vfs for billy conversion: %w", err) + } + + return &fs{ + FileSystem: fsToWrap, + }, nil +} + +type fs struct { + vfs.FileSystem +} + +var _ billy.Filesystem = &fs{} + +// file is a wrapper around a vfs.File that implements billy.File. +// it uses a mutex to lock the file, so it can be used concurrently from the same process, but +// not across processes (like a flock). +type file struct { + vfs.File + lockMu sync.Mutex +} + +var _ billy.File = &file{} + +func (f *file) Lock() error { + f.lockMu.Lock() + return nil +} + +func (f *file) Unlock() error { + f.lockMu.Unlock() + return nil +} + +var _ billy.File = &file{} + +func (f *fs) Create(filename string) (billy.File, error) { + vfsFile, err := f.FileSystem.Create(filename) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile) +} + +// vfsToBillyFileInfo converts a vfs.File to a billy.File +func (f *fs) vfsToBillyFileInfo(vf vfs.File) (billy.File, error) { + return &file{ + File: vf, + }, nil +} + +func (f *fs) Open(filename string) (billy.File, error) { + vfsFile, err := f.FileSystem.Open(filename) + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile) +} + +func (f *fs) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { + if flag&os.O_CREATE != 0 { + if err := f.FileSystem.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return nil, err + } + } + vfsFile, err := f.FileSystem.OpenFile(filename, flag, perm) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile) +} + +func (f *fs) Stat(filename string) (os.FileInfo, error) { + fi, err := f.FileSystem.Stat(filename) + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + return fi, err +} + +func (f *fs) Rename(oldpath, newpath string) error { + dir := filepath.Dir(newpath) + if dir != "." { + if err := f.FileSystem.MkdirAll(dir, 0o755); err != nil { + return err + } + } + return f.FileSystem.Rename(oldpath, newpath) +} + +func (f *fs) Join(elem ...string) string { + return filepath.Join(elem...) +} + +func (f *fs) TempFile(dir, prefix string) (billy.File, error) { + vfsFile, err := vfs.TempFile(f.FileSystem, dir, prefix) + if err != nil { + return nil, err + } + return f.vfsToBillyFileInfo(vfsFile) +} + +func (f *fs) ReadDir(path string) ([]os.FileInfo, error) { + return vfs.ReadDir(f.FileSystem, path) +} + +func (f *fs) Lstat(filename string) (os.FileInfo, error) { + fi, err := f.FileSystem.Lstat(filename) + if err != nil { + if errors.Is(err, syscall.ENOENT) { + return nil, os.ErrNotExist + } + } + return fi, err +} + +func (f *fs) Chroot(path string) (billy.Filesystem, error) { + fi, err := f.FileSystem.Stat(path) + if os.IsNotExist(err) { + if err = f.FileSystem.MkdirAll(path, 0o755); err != nil { + return nil, err + } + fi, err = f.FileSystem.Stat(path) + } + + if err != nil { + return nil, err + } else if !fi.IsDir() { + return nil, fmt.Errorf("path %s is not a directory", path) + } + + chfs, err := projectionfs.New(f.FileSystem, path) + if err != nil { + return nil, err + } + + return &fs{ + FileSystem: chfs, + }, nil +} + +func (f *fs) Root() string { + if root := projectionfs.Root(f.FileSystem); root != "" { + return root + } + if canonicalRoot, err := vfs.Canonical(f.FileSystem, "/", true); err == nil { + return canonicalRoot + } + return "/" +} + +var _ billy.Filesystem = &fs{} diff --git a/api/tech/git/identity/identity.go b/api/tech/git/identity/identity.go new file mode 100644 index 0000000000..20ae3fe257 --- /dev/null +++ b/api/tech/git/identity/identity.go @@ -0,0 +1,131 @@ +package identity + +import ( + "net" + + giturls "github.com/whilp/git-urls" + + "ocm.software/ocm/api/credentials/cpi" + "ocm.software/ocm/api/credentials/identity/hostpath" + "ocm.software/ocm/api/utils/listformat" +) + +const CONSUMER_TYPE = "Git" + +var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) + +func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { + return identityMatcher(pattern, cur, id) +} + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_TOKEN, "HTTP token authentication", + ATTR_PRIVATE_KEY, "Private Key authentication certificate", + }) + cpi.RegisterStandardIdentity(CONSUMER_TYPE, identityMatcher, + `Git credential matcher + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +const ( + ID_HOSTNAME = hostpath.ID_HOSTNAME + ID_PATHPREFIX = hostpath.ID_PATHPREFIX + ID_PORT = hostpath.ID_PORT + ID_SCHEME = hostpath.ID_SCHEME +) + +const ( + ATTR_TOKEN = cpi.ATTR_TOKEN + ATTR_USERNAME = cpi.ATTR_USERNAME + ATTR_PASSWORD = cpi.ATTR_PASSWORD + ATTR_PRIVATE_KEY = cpi.ATTR_PRIVATE_KEY +) + +func GetConsumerId(repoURL string) (cpi.ConsumerIdentity, error) { + host := "" + port := "" + defaultPort := "" + scheme := "" + path := "" + + if repoURL != "" { + u, err := giturls.Parse(repoURL) + if err == nil { + host = u.Host + } else { + return nil, err + } + + scheme = u.Scheme + switch scheme { + case "http": + defaultPort = "80" + case "https": + defaultPort = "443" + case "git": + defaultPort = "9418" + case "ssh": + defaultPort = "22" + case "file": + host = "localhost" + path = u.Path + } + } + + if h, p, err := net.SplitHostPort(host); err == nil { + host, port = h, p + } + + id := cpi.ConsumerIdentity{ + cpi.ID_TYPE: CONSUMER_TYPE, + ID_HOSTNAME: host, + } + + if port != "" { + id[ID_PORT] = port + } else if defaultPort != "" { + id[ID_PORT] = defaultPort + } + + if path != "" { + id[ID_PATHPREFIX] = path + } + + id[ID_SCHEME] = scheme + + return id, nil +} + +func TokenCredentials(token string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_TOKEN: token, + } +} + +func BasicAuthCredentials(username, password string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_USERNAME: username, + ATTR_PASSWORD: password, + } +} + +func PrivateKeyCredentials(username, privateKey string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_USERNAME: username, + ATTR_PRIVATE_KEY: privateKey, + } +} + +func GetCredentials(ctx cpi.ContextProvider, repoURL string) (cpi.Credentials, error) { + id, err := GetConsumerId(repoURL) + if err != nil { + return nil, err + } + return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id, IdentityMatcher) +} diff --git a/api/tech/git/identity/identity_test.go b/api/tech/git/identity/identity_test.go new file mode 100644 index 0000000000..13a95f4e2c --- /dev/null +++ b/api/tech/git/identity/identity_test.go @@ -0,0 +1,132 @@ +package identity_test + +import ( + "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "ocm.software/ocm/api/tech/git/identity" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext" + "ocm.software/ocm/api/oci" + common "ocm.software/ocm/api/utils/misc" +) + +var _ = Describe("consumer id handling", func() { + repo := "https://github.com/torvalds/linux.git" + + Context("id determination", func() { + It("handles https repos", func() { + id := testutils.Must(GetConsumerId(repo)) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "443", + "hostname", "github.com", + "scheme", "https", + ))) + }) + + It("handles http repos", func() { + id := testutils.Must(GetConsumerId("http://github.com/torvalds/linux.git")) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "80", + "hostname", "github.com", + "scheme", "http", + ))) + }) + + It("handles ssh standard format repos", func() { + id := testutils.Must(GetConsumerId("ssh://github.com/torvalds/linux.git")) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "22", + "hostname", "github.com", + "scheme", "ssh", + ))) + }) + + It("handles ssh git @ format repos", func() { + id := testutils.Must(GetConsumerId("git@github.com:torvalds/linux.git")) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "22", + "hostname", "github.com", + "scheme", "ssh", + ))) + }) + + It("handles git format repos", func() { + id := testutils.Must(GetConsumerId("git://github.com/torvalds/linux.git")) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "port", "9418", + "hostname", "github.com", + "scheme", "git", + ))) + }) + + It("handles file format repos", func() { + id := testutils.Must(GetConsumerId("file:///path/to/linux/repo")) + Expect(id).To(Equal(credentials.NewConsumerIdentity(CONSUMER_TYPE, + "scheme", "file", + "hostname", "localhost", + "pathprefix", "/path/to/linux/repo", + ))) + }) + }) + + Context("query credentials", func() { + var ctx oci.Context + var credctx credentials.Context + + BeforeEach(func() { + ctx = oci.New(datacontext.MODE_EXTENDED) + credctx = ctx.CredentialsContext() + }) + + It("Basic Auth", func() { + user, pass := "linus", "torvalds" + id := testutils.Must(GetConsumerId(repo)) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_USERNAME, user, + ATTR_PASSWORD, pass, + ), + ) + + creds := testutils.Must(GetCredentials(ctx, repo)) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_USERNAME: user, + ATTR_PASSWORD: pass, + })) + }) + + It("Token Auth", func() { + token := "mytoken" + id := testutils.Must(GetConsumerId(repo)) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_TOKEN, token, + ), + ) + + creds := testutils.Must(GetCredentials(ctx, repo)) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_TOKEN: token, + })) + }) + + It("Public Key Auth", func() { + user, key := "linus", "path/to/my/id_rsa" + id := testutils.Must(GetConsumerId(repo)) + credctx.SetCredentialsForConsumer(id, + credentials.CredentialsFromList( + ATTR_USERNAME, user, + ATTR_PRIVATE_KEY, key, + ), + ) + + creds := testutils.Must(GetCredentials(ctx, repo)) + Expect(creds).To(BeEquivalentTo(common.Properties{ + ATTR_USERNAME: user, + ATTR_PRIVATE_KEY: key, + })) + }) + }) +}) diff --git a/api/tech/git/identity/suite_test.go b/api/tech/git/identity/suite_test.go new file mode 100644 index 0000000000..d79c7330b1 --- /dev/null +++ b/api/tech/git/identity/suite_test.go @@ -0,0 +1,13 @@ +package identity_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Git Identity Suite") +} diff --git a/api/tech/git/logging.go b/api/tech/git/logging.go new file mode 100644 index 0000000000..1fce027755 --- /dev/null +++ b/api/tech/git/logging.go @@ -0,0 +1,5 @@ +package git + +import "ocm.software/ocm/api/utils/logging" + +var REALM = logging.DefineSubRealm("git repository", "git") diff --git a/api/tech/git/resolver.go b/api/tech/git/resolver.go new file mode 100644 index 0000000000..ef8098ee0b --- /dev/null +++ b/api/tech/git/resolver.go @@ -0,0 +1,212 @@ +package git + +import ( + "context" + "errors" + "fmt" + "net/http" + "sync" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/plumbing/transport" + gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/filesystem" + mlog "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/memoryfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/utils/logging" +) + +const LogAttrProtocol = "protocol" + +func init() { + // override the logging realm for http based git clients + gitclient.InstallProtocol("http", githttp.NewClient(&http.Client{ + Transport: logging.NewRoundTripper(http.DefaultTransport, logging.DynamicLogger(REALM, mlog.NewAttribute(LogAttrProtocol, "http"))), + })) + gitclient.InstallProtocol("https", githttp.NewClient(&http.Client{ + Transport: logging.NewRoundTripper(http.DefaultTransport, logging.DynamicLogger(REALM, mlog.NewAttribute(LogAttrProtocol, "https"))), + })) + // TODO Determine how we ideally log for ssh+git protocol +} + +var DefaultWorktreeBranch = plumbing.NewBranchReferenceName("ocm") + +type client struct { + opts ClientOptions + + // vfs tracks the current filesystem where the repo will be stored (at the root) + vfs vfs.FileSystem + + // repo is a reference to the git repository if it is already open + repo *git.Repository + repoMu sync.Mutex +} + +// Client is a heavy abstraction over the go git Client that opinionates the remote as git.DefaultRemoteName +// as well as access to it via high level functions that are usually required for operation within OCM CTFs that are stored +// within Git. It is not general-purpose. +type Client interface { + // Repository returns the git repository for the client initialized in the Filesystem given to Setup. + // If Setup is not called before Repository, it will an in-memory filesystem. + // Repository will attempt to initially clone the repository if it does not exist. + // If the repository is already open or cloned in the filesystem, it will attempt to open & return the existing repository. + // If the remote repository does not exist, a new repository will be created with a dummy commit and the remote + // configured to the given URL. At that point it is up to the remote to accept an initial push to the repository or not with the + // given AuthMethod. + Repository(ctx context.Context) (*git.Repository, error) + + // Setup will override the current filesystem with the given filesystem. This will be the filesystem where the repository will be stored. + // There can be only one filesystem per client. + // If the filesystem contains a repository already, it can be consumed by a subsequent call to Repository. + Setup(context.Context, vfs.FileSystem) error +} + +type ClientOptions struct { + // URL is the URL of the git repository to clone or open. + URL string + // Ref is the reference to the repository to clone or open. + // If empty, it will default to plumbing.HEAD of the remote repository. + // If the remote does not exist, it will attempt to push to the remote with DefaultWorktreeBranch on Client.Update. + // To point to a remote branch, use refs/heads/. + // To point to a tag, use refs/tags/. + Ref string + // Commit is the commit hash to checkout after cloning the repository. + // If empty, it will default to the plumbing.HEAD of the Ref. + Commit string + // AuthMethod is the authentication method to use for the repository. + AuthMethod AuthMethod +} + +var _ Client = &client{} + +func NewClient(opts ClientOptions) (Client, error) { + pref := plumbing.HEAD + if opts.Ref != "" { + pref = plumbing.ReferenceName(opts.Ref) + if err := pref.Validate(); err != nil { + return nil, fmt.Errorf("invalid reference %q: %w", opts.Ref, err) + } + } + + return &client{ + vfs: memoryfs.New(), + opts: opts, + }, nil +} + +func (c *client) Repository(ctx context.Context) (*git.Repository, error) { + c.repoMu.Lock() + defer c.repoMu.Unlock() + if c.repo != nil { + return c.repo, nil + } + + billyFS, err := VFSBillyFS(c.vfs) + if err != nil { + return nil, err + } + + strg, err := GetStorage(billyFS) + if err != nil { + return nil, err + } + + depth := 0 + if c.opts.Commit == "" { + depth = 1 // if we have no dedicated commit we can checkout HEAD, and thus a shallow clone is ok + } + + newRepo := false + repo, err := git.Open(strg, billyFS) + if errors.Is(err, git.ErrRepositoryNotExists) { + repo, err = git.CloneContext(ctx, strg, billyFS, &git.CloneOptions{ + Auth: c.opts.AuthMethod, + URL: c.opts.URL, + RemoteName: git.DefaultRemoteName, + ReferenceName: plumbing.ReferenceName(c.opts.Ref), + SingleBranch: true, + Depth: depth, + ShallowSubmodules: depth == 1, + Tags: git.AllTags, + }) + newRepo = true + } + if errors.Is(err, transport.ErrEmptyRemoteRepository) { + repo, err = git.Open(strg, billyFS) + if err != nil { + return nil, fmt.Errorf("failed to open repository based on URL %q after it was determined to be an empty clone: %w", c.opts.URL, err) + } + } + + if err != nil { + return nil, err + } + if newRepo { + if err := c.newRepository(ctx, repo); err != nil { + return nil, err + } + } + + c.repo = repo + + return repo, nil +} + +func (c *client) newRepository(ctx context.Context, repo *git.Repository) error { + if err := repo.FetchContext(ctx, &git.FetchOptions{ + Auth: c.opts.AuthMethod, + RemoteName: git.DefaultRemoteName, + Depth: 0, + Tags: git.AllTags, + Force: false, + }); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + worktree, err := repo.Worktree() + if err != nil { + return err + } + + var hash plumbing.Hash + if c.opts.Commit != "" { + hash = plumbing.NewHash(c.opts.Commit) + } + + if err := worktree.Checkout(&git.CheckoutOptions{ + Hash: hash, + Branch: DefaultWorktreeBranch, + Create: true, + Keep: true, + }); err != nil { + return err + } + + return nil +} + +func GetStorage(base billy.Filesystem) (storage.Storer, error) { + dotGit, err := base.Chroot(git.GitDirName) + if err != nil { + return nil, err + } + + return filesystem.NewStorage( + dotGit, + cache.NewObjectLRUDefault(), + ), nil +} + +func (c *client) Setup(ctx context.Context, system vfs.FileSystem) error { + c.vfs = system + if _, err := c.Repository(ctx); err != nil { + return fmt.Errorf("failed to setup repository %q: %w", c.opts.URL, err) + } + return nil +} diff --git a/api/tech/git/resolver_test.go b/api/tech/git/resolver_test.go new file mode 100644 index 0000000000..d128d07808 --- /dev/null +++ b/api/tech/git/resolver_test.go @@ -0,0 +1,112 @@ +package git_test + +import ( + "embed" + "fmt" + "io" + "os" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + . "github.com/mandelsoft/goutils/testutils" + "github.com/mandelsoft/vfs/pkg/cwdfs" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + self "ocm.software/ocm/api/tech/git" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("standard tests with local file repo", func() { + var ( + ctx ocm.Context + expectedBlobContent []byte + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := cwdfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + var repoDir string + var repoURL string + var ref string + var commit string + + BeforeEach(func() { + repoDir = GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo := Must(git.PlainInit(repoDir, false)) + + repoBase := filepath.Join("testdata", "repo") + repoTestData := Must(testData.ReadDir(repoBase)) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file := Must(testData.Open(path)) + + fileInRepo := Must(os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0o600, + )) + + Must(io.Copy(fileInRepo, file)) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt := Must(repo.Worktree()) + Expect(wt.AddGlob("*")).To(Succeed()) + commit = Must(wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + })).String() + + path := filepath.Join("testdata", "repo", "file_in_repo") + repoURL = fmt.Sprintf("file://%s", repoDir) + ref = plumbing.Master.String() + + expectedBlobContent = Must(testData.ReadFile(path)) + }) + + It("Resolver client can setup repository", func(ctx SpecContext) { + client := Must(self.NewClient(self.ClientOptions{ + URL: repoURL, + Ref: ref, + Commit: commit, + })) + + tempVFS, err := projectionfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + + Expect(client.Setup(ctx, tempVFS)).To(Succeed()) + + repo := Must(client.Repository(ctx)) + Expect(repo).ToNot(BeNil()) + + file := Must(tempVFS.Stat("file_in_repo")) + Expect(file.Size()).To(Equal(int64(len(expectedBlobContent)))) + + }) +}) diff --git a/api/tech/git/suite_test.go b/api/tech/git/suite_test.go new file mode 100644 index 0000000000..73e11ee635 --- /dev/null +++ b/api/tech/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM Git Tech Test Suite") +} diff --git a/api/tech/git/testdata/repo/file_in_repo b/api/tech/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/tech/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/api/tech/oras/fetcher.go b/api/tech/oras/fetcher.go index a12df685fc..95b54a7cea 100644 --- a/api/tech/oras/fetcher.go +++ b/api/tech/oras/fetcher.go @@ -65,6 +65,7 @@ func (c *OrasFetcher) resolveDescriptor(ctx context.Context, desc ociv1.Descript if err != nil { return ociv1.Descriptor{}, fmt.Errorf("failed to resolve descriptor %q: %w", desc.Digest.String(), err) } + return desc, nil } @@ -79,10 +80,9 @@ func (c *OrasFetcher) resolveDescriptor(ctx context.Context, desc ociv1.Descript return ociv1.Descriptor{}, fmt.Errorf("failed to resolve manifest %q: %w", desc.Digest.String(), err) } - desc = mdesc - } else { - desc = bdesc + + return mdesc, nil } - return desc, err + return bdesc, err } diff --git a/api/utils/blobaccess/git/access.go b/api/utils/blobaccess/git/access.go new file mode 100644 index 0000000000..d9aba5b631 --- /dev/null +++ b/api/utils/blobaccess/git/access.go @@ -0,0 +1,136 @@ +package git + +import ( + "context" + + gogit "github.com/go-git/go-git/v5" + "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/finalizer" + "github.com/mandelsoft/goutils/optionutils" + "github.com/mandelsoft/vfs/pkg/projectionfs" + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/opencontainers/go-digest" + + "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/utils/blobaccess/bpi" + "ocm.software/ocm/api/utils/blobaccess/file" + "ocm.software/ocm/api/utils/iotools" + "ocm.software/ocm/api/utils/mime" + "ocm.software/ocm/api/utils/tarutils" +) + +// BlobAccess clones the repository into a temporary filesystem, packs it into a tar.gz file, +// and returns a BlobAccess object for the tar.gz file. +func BlobAccess(opt ...Option) (_ bpi.BlobAccess, rerr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagation(&rerr) + + options := optionutils.EvalOptions(opt...) + if options.URL == "" { + return nil, errors.New("no URL specified") + } + log := options.Logger("RepoUrl", options.URL) + + if err := options.ConfigureAuthMethod(); err != nil { + return nil, err + } + + c, err := git.NewClient(options.ClientOptions) + if err != nil { + return nil, err + } + + tmpFS, cleanup, err := options.CachingFilesystem() + if err != nil { + return nil, err + } else if cleanup != nil { + finalize.With(cleanup) + } + + // store the repo in a temporary filesystem subfolder, so the tgz can go in the root without issues. + if err := tmpFS.MkdirAll("repository", 0o700); err != nil { + return nil, err + } + + repositoryFS, err := projectionfs.New(tmpFS, "repository") + if err != nil { + return nil, err + } + finalize.With(func() error { + return tmpFS.RemoveAll("repository") + }) + + // redirect the client to the temporary filesystem for storage of the repo, otherwise it would use memory + if err := c.Setup(context.Background(), repositoryFS); err != nil { + return nil, err + } + + // get the repository, triggering a clone if not present into the filesystem provided by setup + if _, err := c.Repository(context.Background()); err != nil { + return nil, err + } + + filteredRepositoryFS := &filteredVFS{ + FileSystem: repositoryFS, + filter: func(s string) bool { + return s != gogit.GitDirName + }, + } + + // pack all downloaded files into a tar.gz file + fs := tmpFS + tgz, err := vfs.TempFile(fs, "", "git-*.tar.gz") + if err != nil { + return nil, err + } + + dw := iotools.NewDigestWriterWith(digest.SHA256, tgz) + finalize.Close(dw) + + if err := tarutils.TgzFs(filteredRepositoryFS, dw); err != nil { + return nil, err + } + + log.Debug("created", "file", tgz.Name()) + + return file.BlobAccessForTemporaryFilePath( + mime.MIME_TGZ, + tgz.Name(), + file.WithFileSystem(fs), + file.WithDigest(dw.Digest()), + file.WithSize(dw.Size()), + ), nil +} + +type filteredVFS struct { + vfs.FileSystem + filter func(string) bool +} + +func (f *filteredVFS) Open(name string) (vfs.File, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Open(name) +} + +func (f *filteredVFS) OpenFile(name string, flags int, perm vfs.FileMode) (vfs.File, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.OpenFile(name, flags, perm) +} + +func (f *filteredVFS) Stat(name string) (vfs.FileInfo, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Stat(name) +} + +func (f *filteredVFS) Lstat(name string) (vfs.FileInfo, error) { + if !f.filter(name) { + return nil, vfs.SkipDir + } + return f.FileSystem.Lstat(name) +} diff --git a/api/utils/blobaccess/git/access_test.go b/api/utils/blobaccess/git/access_test.go new file mode 100644 index 0000000000..a35495a7aa --- /dev/null +++ b/api/utils/blobaccess/git/access_test.go @@ -0,0 +1,153 @@ +package git_test + +import ( + "embed" + _ "embed" + "fmt" + "io" + "os" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/mandelsoft/filepath/pkg/filepath" + . "github.com/mandelsoft/goutils/testutils" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + gitblob "ocm.software/ocm/api/utils/blobaccess/git" + "ocm.software/ocm/api/utils/tarutils" +) + +//go:embed testdata/repo +var testData embed.FS + +var _ = Describe("git Blob Access", func() { + var ( + ctx ocm.Context + url string + ) + + ctx = ocm.New() + + BeforeEach(func() { + tempVFS, err := projectionfs.New(osfs.New(), GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + tmpcache.Set(ctx, &tmpcache.Attribute{Path: ".", Filesystem: tempVFS}) + vfsattr.Set(ctx, tempVFS) + }) + + Context("git filesystem repository", func() { + BeforeEach(func() { + repoDir := GinkgoT().TempDir() + filepath.PathSeparatorString + "repo" + + repo, err := git.PlainInit(repoDir, false) + Expect(err).ToNot(HaveOccurred()) + + repoBase := filepath.Join("testdata", "repo") + repoTestData, err := testData.ReadDir(repoBase) + Expect(err).ToNot(HaveOccurred()) + + for _, entry := range repoTestData { + path := filepath.Join(repoBase, entry.Name()) + repoPath := filepath.Join(repoDir, entry.Name()) + + file, err := testData.Open(path) + Expect(err).ToNot(HaveOccurred()) + + fileInRepo, err := os.OpenFile( + repoPath, + os.O_CREATE|os.O_RDWR|os.O_TRUNC, + 0o600, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = io.Copy(fileInRepo, file) + Expect(err).ToNot(HaveOccurred()) + + Expect(fileInRepo.Close()).To(Succeed()) + Expect(file.Close()).To(Succeed()) + } + + wt, err := repo.Worktree() + Expect(err).ToNot(HaveOccurred()) + Expect(wt.AddGlob("*")).To(Succeed()) + _, err = wt.Commit("OCM Test Commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "OCM Test", + Email: "dummy@ocm.software", + When: time.Now(), + }, + }) + Expect(err).ToNot(HaveOccurred()) + + url = fmt.Sprintf("file://%s", repoDir) + }) + + It("blobaccess for simple repository", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingContext(ctx), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("file_in_repo")) + }) + + }) + + Context("git http repository", func() { + host := "github.com:443" + reachable := PingTCPServer(host, time.Second) == nil + BeforeEach(func() { + if !reachable { + Skip(fmt.Sprintf("no connection to %s, skipping test connection to remote", url)) + } + // This repo is a public repo owned by the Github Kraken Bot, so its as good of a public available + // example as any. + url = fmt.Sprintf("https://%s/octocat/Hello-World.git", host) + }) + + It("hello world from github without explicit branch", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingContext(ctx), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README")) + }) + + It("hello world from github with master branch", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingContext(ctx), + gitblob.WithRef(plumbing.Master.String()), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README")) + }) + + It("hello world from github with custom ref", func() { + b := Must(gitblob.BlobAccess( + gitblob.WithURL(url), + gitblob.WithLoggingContext(ctx), + gitblob.WithCachingContext(ctx), + gitblob.WithRef("refs/heads/test"), + )) + defer Close(b) + files := Must(tarutils.ListArchiveContentFromReader(Must(b.Reader()))) + Expect(files).To(ConsistOf("README", "CONTRIBUTING.md")) + }) + }) +}) diff --git a/api/utils/blobaccess/git/digest.go b/api/utils/blobaccess/git/digest.go new file mode 100644 index 0000000000..4d61d15310 --- /dev/null +++ b/api/utils/blobaccess/git/digest.go @@ -0,0 +1,7 @@ +package git + +import "github.com/opencontainers/go-digest" + +type Digest interface { + digest.Digest +} diff --git a/api/utils/blobaccess/git/options.go b/api/utils/blobaccess/git/options.go new file mode 100644 index 0000000000..44ebd7cd14 --- /dev/null +++ b/api/utils/blobaccess/git/options.go @@ -0,0 +1,194 @@ +package git + +import ( + "github.com/mandelsoft/goutils/optionutils" + "github.com/mandelsoft/logging" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/projectionfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/datacontext" + "ocm.software/ocm/api/datacontext/attrs/tmpcache" + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/tech/git" + "ocm.software/ocm/api/tech/git/identity" + ocmlog "ocm.software/ocm/api/utils/logging" + "ocm.software/ocm/api/utils/stdopts" +) + +type Option = optionutils.Option[*Options] + +type Options struct { + git.ClientOptions + + stdopts.StandardContexts + stdopts.PathFileSystem +} + +func (o *Options) Logger(keyValuePairs ...interface{}) logging.Logger { + return ocmlog.LogContext(o.LoggingContext.Value, o.CredentialContext.Value).Logger(git.REALM).WithValues(keyValuePairs...) +} + +func (o *Options) Cache() *tmpcache.Attribute { + if o.CachingPath.Value != "" { + return tmpcache.New(o.CachingPath.Value, o.CachingFileSystem.Value) + } + if o.CachingContext.Value != nil { + return tmpcache.Get(o.CachingContext.Value) + } + return tmpcache.Get(o.CredentialContext.Value) +} + +func (o *Options) ApplyTo(opts *Options) { + if opts == nil { + return + } + if o.CredentialContext.Value != nil { + opts.CredentialContext = o.CredentialContext + } + if o.Credentials.Value != nil { + opts.Credentials = o.Credentials + } + if o.LoggingContext.Value != nil { + opts.LoggingContext = o.LoggingContext + } + if o.CachingFileSystem.Value != nil { + opts.CachingFileSystem = o.CachingFileSystem + } + if o.URL != "" { + opts.URL = o.URL + } + if o.Ref != "" { + opts.Ref = o.Ref + } + if o.Commit != "" { + opts.Commit = o.Commit + } +} + +func (o *Options) ConfigureAuthMethod() error { + if o.ClientOptions.AuthMethod != nil { + return nil + } + + var err error + + if o.Credentials.Value != nil { + if o.ClientOptions.AuthMethod, err = git.AuthFromCredentials(o.Credentials.Value); err != nil { + return err + } + } + + if o.CredentialContext.Value == nil { + return nil + } + + creds, err := identity.GetCredentials(o.CredentialContext.Value, o.URL) + if err != nil { + return err + } + + if creds != nil { + if o.ClientOptions.AuthMethod, err = git.AuthFromCredentials(creds); err != nil { + return err + } + } + + return nil +} + +func (o *Options) CachingFilesystem() (vfs.FileSystem, func() error, error) { + if o.PathFileSystem.Value != nil { + return o.PathFileSystem.Value, nil, nil + } + if o.CachingFileSystem.Value != nil { + return o.CachingFileSystem.Value, nil, nil + } + + if o.CachingContext.Value != nil { + if fs := vfsattr.Get(o.CachingContext.Value); fs != nil { + return fs, nil, nil + } + + if fromtmp := tmpcache.Get(o.CachingContext.Value); fromtmp != nil { + fs, err := projectionfs.New(fromtmp.Filesystem, fromtmp.Path) + if err != nil { + return nil, nil, err + } + return fs, nil, nil + } + } + tmpfs, err := osfs.NewTempFileSystem() + return tmpfs, func() error { return vfs.Cleanup(tmpfs) }, err +} + +func option[S any, T any](v T) optionutils.Option[*Options] { + return optionutils.WithGenericOption[S, *Options](v) +} + +func WithCredentialContext(ctx credentials.ContextProvider) Option { + return option[stdopts.CredentialContextOptionBag](ctx) +} + +func WithLoggingContext(ctx logging.ContextProvider) Option { + return option[stdopts.LoggingContextOptionBag](ctx) +} + +func WithCachingContext(ctx datacontext.Context) Option { + return option[stdopts.CachingContextOptionBag](ctx) +} + +func WithCachingFileSystem(fs vfs.FileSystem) Option { + return option[stdopts.CachingFileSystemOptionBag](fs) +} + +func WithPathFileSystem(fs vfs.FileSystem) Option { + return option[stdopts.PathFileSystemOptionBag](fs) +} + +func WithCredentials(c credentials.Credentials) Option { + return option[stdopts.CredentialsOptionBag](c) +} + +//////////////////////////////////////////////////////////////////////////////// + +type URLOptionBag interface { + SetURL(v string) +} + +func (o *Options) SetURL(v string) { + o.URL = v +} + +func WithURL(url string) Option { + return option[URLOptionBag](url) +} + +//////////////////////////////////////////////////////////////////////////////// + +type RefOptionBag interface { + SetRef(v string) +} + +func (o *Options) SetRef(v string) { + o.Ref = v +} + +func WithRef(ref string) Option { + return option[RefOptionBag](ref) +} + +//////////////////////////////////////////////////////////////////////////////// + +type CommitOptionBag interface { + SetCommit(v string) +} + +func (o *Options) SetCommit(v string) { + o.Commit = v +} + +func WithCommit(ref string) Option { + return option[CommitOptionBag](ref) +} diff --git a/api/utils/blobaccess/git/suite_test.go b/api/utils/blobaccess/git/suite_test.go new file mode 100644 index 0000000000..1e348d8652 --- /dev/null +++ b/api/utils/blobaccess/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "git Blob Access Test Suite") +} diff --git a/api/utils/blobaccess/git/testdata/repo/file_in_repo b/api/utils/blobaccess/git/testdata/repo/file_in_repo new file mode 100644 index 0000000000..5eced95754 --- /dev/null +++ b/api/utils/blobaccess/git/testdata/repo/file_in_repo @@ -0,0 +1 @@ +Foobar \ No newline at end of file diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go new file mode 100644 index 0000000000..4d812d5c8f --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/cli.go @@ -0,0 +1,20 @@ +package git + +import ( + "ocm.software/ocm/api/utils/cobrautils/flagsets" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/options" +) + +func ConfigHandler() flagsets.ConfigOptionTypeSetHandler { + return flagsets.NewConfigOptionTypeSetHandler( + TYPE, AddConfig, + options.RepositoryOption, + options.VersionOption, + ) +} + +func AddConfig(opts flagsets.ConfigOptions, config flagsets.Config) error { + flagsets.AddFieldByOptionP(opts, options.RepositoryOption, config, "repository") + flagsets.AddFieldByOptionP(opts, options.VersionOption, config, "ref") + return nil +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go new file mode 100644 index 0000000000..b664f261eb --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/input_test.go @@ -0,0 +1,125 @@ +package git_test + +import ( + "fmt" + "os" + + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/datacontext/attrs/vfsattr" + "ocm.software/ocm/api/ocm" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + "ocm.software/ocm/api/utils/accessio" + . "ocm.software/ocm/cmds/ocm/testhelper" +) + +const ( + ARCH = "test.ctf" + CONSTRUCTOR = "component-constructor.yaml" + VERSION = "v1" +) + +var _ = Describe("Test Environment", func() { + var env *TestEnv + + BeforeEach(func() { + env = NewTestEnv(TestData()) + }) + + AfterEach(func() { + Expect(env.Cleanup()).To(Succeed()) + }) + + It("add git repo described by access type specification", func() { + constructor := fmt.Sprintf(`--- +name: test.de/x +version: %s +provider: + name: ocm +resources: +- name: hello-world + type: git + version: 0.0.1 + access: + type: git + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" + ref: refs/heads/master + repository: https://github.com/octocat/Hello-World.git +`, VERSION) + Expect( + env.WriteFile(CONSTRUCTOR, []byte(constructor), os.ModePerm), + ).To(Succeed()) + + Expect(env.Execute( + "add", + "cv", + "--create", + "--file", + ARCH, + "--force", + "--type", + "directory", + CONSTRUCTOR, + )).To(Succeed()) + + ctx := ocm.New() + vfsattr.Set(ctx, env.FileSystem()) + r := Must(ctf.Open(ctx, ctf.ACC_READONLY, ARCH, 0o400, accessio.FormatDirectory, accessio.PathFileSystem(env.FileSystem()))) + DeferCleanup(r.Close) + + c := Must(r.LookupComponent("test.de/x")) + DeferCleanup(c.Close) + cv := Must(c.LookupVersion(VERSION)) + DeferCleanup(cv.Close) + cd := cv.GetDescriptor() + Expect(len(cd.Resources)).To(Equal(1)) + }) + + It("add git repo described by cli options through blob access via input described in file", func() { + + constructor := fmt.Sprintf(`--- +name: test.de/x +version: %s +provider: + name: ocm +resources: +- name: hello-world + type: git + version: 0.0.1 + input: + type: git + ref: refs/heads/master + commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" + repository: https://github.com/octocat/Hello-World.git +`, VERSION) + Expect( + env.WriteFile(CONSTRUCTOR, []byte(constructor), os.ModePerm), + ).To(Succeed()) + + Expect(env.Execute( + "add", + "cv", + "--file", + ARCH, + "--create", + "--force", + "--type", + "directory", + CONSTRUCTOR, + )).To(Succeed()) + + ctx := ocm.New() + vfsattr.Set(ctx, env.FileSystem()) + r := Must(ctf.Open(ctx, ctf.ACC_READONLY, ARCH, 0o400, accessio.FormatDirectory, accessio.PathFileSystem(env.FileSystem()))) + DeferCleanup(r.Close) + + c := Must(r.LookupComponent("test.de/x")) + DeferCleanup(c.Close) + cv := Must(c.LookupVersion(VERSION)) + DeferCleanup(cv.Close) + cd := cv.GetDescriptor() + Expect(len(cd.Resources)).To(Equal(1)) + }) +}) diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go new file mode 100644 index 0000000000..60b1178cc1 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/spec.go @@ -0,0 +1,77 @@ +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" + giturls "github.com/whilp/git-urls" + "k8s.io/apimachinery/pkg/util/validation/field" + + "ocm.software/ocm/api/utils/blobaccess" + "ocm.software/ocm/api/utils/blobaccess/git" + "ocm.software/ocm/api/utils/runtime" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs" +) + +type Spec struct { + inputs.InputSpecBase `json:",inline"` + + // Repository is the Git Repository URL + Repository string `json:"repository"` + + // Ref is the Git Ref to check out. + // If empty, the default HEAD (remotes/origin/HEAD) of the remote is used. + Ref string `json:"ref,omitempty"` + + // Commit is the Git Commit to check out. + // If empty, the default HEAD of the Ref is used. + Commit string `json:"commit,omitempty"` +} + +var _ inputs.InputSpec = (*Spec)(nil) + +func New(repository, ref, commit string) *Spec { + return &Spec{ + InputSpecBase: inputs.InputSpecBase{ + ObjectVersionedType: runtime.ObjectVersionedType{ + Type: TYPE, + }, + }, + Repository: repository, + Ref: ref, + Commit: commit, + } +} + +func (s *Spec) Validate(fldPath *field.Path, _ inputs.Context, _ string) field.ErrorList { + var allErrs field.ErrorList + + if path := fldPath.Child("repository"); s.Repository == "" { + allErrs = append(allErrs, field.Invalid(path, s.Repository, "no repository")) + } else { + if _, err := giturls.Parse(s.Repository); err != nil { + allErrs = append(allErrs, field.Invalid(path, s.Repository, err.Error())) + } + } + + if ref := fldPath.Child("ref"); s.Ref != "" { + if err := plumbing.ReferenceName(s.Ref).Validate(); err != nil { + allErrs = append(allErrs, field.Invalid(ref, s.Ref, "invalid ref")) + } + } + + return allErrs +} + +func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (blobaccess.BlobAccess, string, error) { + blob, err := git.BlobAccess( + git.WithURL(s.Repository), + git.WithRef(s.Ref), + git.WithCommit(s.Commit), + git.WithCredentialContext(ctx), + git.WithLoggingContext(ctx), + git.WithCachingContext(ctx), + ) + if err != nil { + return nil, "", err + } + return blob, "", nil +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go new file mode 100644 index 0000000000..b2afdea537 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/suite_test.go @@ -0,0 +1,13 @@ +package git_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Input Type git") +} diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml new file mode 100644 index 0000000000..1ebad10285 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/testdata/resources1.yaml @@ -0,0 +1,6 @@ +name: hello-world +type: git +input: + type: git + repository: https://github.com/octocat/Hello-World.git + version: refs/heads/master \ No newline at end of file diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go new file mode 100644 index 0000000000..f4040197ae --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/git/type.go @@ -0,0 +1,33 @@ +package git + +import ( + "ocm.software/ocm/api/oci/annotations" + "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs" +) + +const TYPE = "git" + +func init() { + inputs.DefaultInputTypeScheme.Register(inputs.NewInputType(TYPE, &Spec{}, usage, ConfigHandler())) +} + +const usage = ` +The repository type allows accessing an arbitrary git repository +using the manifest annotation ` + annotations.COMPVERS_ANNOTATION + `. +The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + +This blob type specification supports the following fields: +- **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + +- **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + +- **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. +` diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go index a03de2fceb..a73a166973 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/init.go @@ -6,6 +6,7 @@ import ( _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/docker" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/dockermulti" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/file" + _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/git" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/helm" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/maven" _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types/npm" diff --git a/docs/reference/ocm_add_resource-configuration.md b/docs/reference/ocm_add_resource-configuration.md index 2e541891bd..2d84437b48 100644 --- a/docs/reference/ocm_add_resource-configuration.md +++ b/docs/reference/ocm_add_resource-configuration.md @@ -327,6 +327,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -607,6 +630,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1alpha1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_resources.md b/docs/reference/ocm_add_resources.md index 99d746c1ad..f350c6d481 100644 --- a/docs/reference/ocm_add_resources.md +++ b/docs/reference/ocm_add_resources.md @@ -339,6 +339,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -619,6 +642,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1alpha1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_source-configuration.md b/docs/reference/ocm_add_source-configuration.md index 05153898b8..ecec431c58 100644 --- a/docs/reference/ocm_add_source-configuration.md +++ b/docs/reference/ocm_add_source-configuration.md @@ -327,6 +327,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -607,6 +630,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1alpha1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_add_sources.md b/docs/reference/ocm_add_sources.md index 54f7d17b70..4434cecd57 100644 --- a/docs/reference/ocm_add_sources.md +++ b/docs/reference/ocm_add_sources.md @@ -337,6 +337,29 @@ with the field type in the input field: Options used to configure fields: --inputCompress, --inputPath, --mediaType +- Input type git + + The repository type allows accessing an arbitrary git repository + using the manifest annotation software.ocm/component-version. + The ref can be used to further specify the branch or tag to checkout, otherwise the remote HEAD is used. + + This blob type specification supports the following fields: + - **repository** *string* + + This REQUIRED property describes the URL of the git repository to access. All git URL formats are supported. + + - **ref** *string* + + This OPTIONAL property can be used to specify the remote branch or tag to checkout (commonly called ref). + If not set, the default HEAD (remotes/origin/HEAD) of the remote is used. + + - **commit** *string* + + This OPTIONAL property can be used to specify the commit hash to checkout. + If not set, the default HEAD of the ref is used. + + Options used to configure fields: --inputRepository, --inputVersion + - Input type helm The path must denote an helm chart archive or directory @@ -617,6 +640,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1alpha1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index 71e3340e28..d0ba951423 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -110,6 +110,19 @@ The following credential consumer types are used/supported: - key: secret key use to access the credential server + - Git: Git credential matcher + + It matches the Git consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Git evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - token: HTTP token authentication + - privateKey: Private Key authentication certificate + + - Github: GitHub credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index 55ed158abc..4ef43b318d 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -36,6 +36,19 @@ Matchers exist for the following usage contexts or consumer types: - key: secret key use to access the credential server + - Git: Git credential matcher + + It matches the Git consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Git evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - token: HTTP token authentication + - privateKey: Private Key authentication certificate + + - Github: GitHub credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_logging.md b/docs/reference/ocm_logging.md index 9a1c45800b..c73dc6cc3e 100644 --- a/docs/reference/ocm_logging.md +++ b/docs/reference/ocm_logging.md @@ -27,6 +27,7 @@ The following *realms* are used by the command line tool: - ocm/credentials/dockerconfig: docker config handling as credential repository - ocm/credentials/vault: HashiCorp Vault Access - ocm/downloader: Downloaders + - ocm/git: git repository - ocm/maven: Maven repository - ocm/npm: NPM registry - ocm/oci/docker: Docker repository handling diff --git a/docs/reference/ocm_ocm-accessmethods.md b/docs/reference/ocm_ocm-accessmethods.md index 63d1ad8b44..ba86b327a5 100644 --- a/docs/reference/ocm_ocm-accessmethods.md +++ b/docs/reference/ocm_ocm-accessmethods.md @@ -15,6 +15,30 @@ The access method specification can be put below the access field. If always requires the field type describing the kind and version shown below. +- Access type git + + This method implements the access of the content of a git commit stored in a + Git repository. + + The following versions are supported: + - Version v1alpha1 + + The type specific specification fields are: + + - **repoUrl** *string* + + Repository URL with or without scheme. + + - **ref** (optional) *string* + + Original ref used to get the commit from + + - **commit** *string* + + The sha/id of the git commit + + Options used to configure fields: --accessRepository, --commit, --reference + - Access type gitHub This method implements the access of the content of a git commit stored in a diff --git a/flake.nix b/flake.nix index befdba47c8..ce7f2ffed1 100644 --- a/flake.nix +++ b/flake.nix @@ -35,7 +35,7 @@ state = if (self ? rev) then "clean" else "dirty"; # This vendorHash represents a derivative of all go.mod dependencies and needs to be adjusted with every change - vendorHash = "sha256-D5uEj20XEPSVXggzZsLiBM8KifkxapX9ISUQbDsgPmk="; + vendorHash = "sha256-V+Uhb45+8Wa+kruXL/FpgBCAzUFfcls95FIC5PcB9zY="; src = ./.; diff --git a/go.mod b/go.mod index fbd7e49fce..7ee0a20f49 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( github.com/fluxcd/pkg/ssa v0.43.0 github.com/gertd/go-pluralize v0.2.1 github.com/ghodss/yaml v1.0.0 + github.com/go-git/go-billy/v5 v5.6.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/go-logr/logr v1.4.2 github.com/go-openapi/strfmt v0.23.0 github.com/go-openapi/swag v0.23.0 @@ -68,6 +70,7 @@ require ( github.com/texttheater/golang-levenshtein v1.0.1 github.com/tonglil/buflogr v1.1.1 github.com/ulikunitz/xz v0.5.12 + github.com/whilp/git-urls v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 @@ -177,6 +180,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/orderedmap v1.7.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect @@ -187,6 +191,7 @@ require ( github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-errors/errors v1.5.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect @@ -237,12 +242,14 @@ require ( github.com/huandu/xstrings v1.5.0 // indirect github.com/in-toto/in-toto-golang v0.9.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/letsencrypt/boulder v0.0.0-20241010192615-6692160cedfa // indirect @@ -279,6 +286,7 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -293,11 +301,13 @@ require ( github.com/sassoftware/relic v7.2.1+incompatible // indirect github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect github.com/segmentio/ksuid v1.0.4 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sigstore/fulcio v1.6.5 // indirect github.com/sigstore/protobuf-specs v0.3.2 // indirect github.com/sigstore/timestamp-authority v1.2.3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect @@ -317,6 +327,7 @@ require ( github.com/vbatts/tar-split v0.11.6 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/go-gitlab v0.112.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xlab/treeprint v1.2.0 // indirect @@ -352,6 +363,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiserver v0.32.0 // indirect k8s.io/component-base v0.32.0 // indirect diff --git a/go.sum b/go.sum index 0dd5a350bd..d59e16a4e7 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,7 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.9 h1:2zJy5KA+l0loz1HzEGqyNnjd3fyZA31ZBCGKacp6lLg= @@ -159,6 +160,8 @@ github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6q github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA= github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -365,12 +368,16 @@ github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elliotchance/orderedmap v1.7.0 h1:FirjcM/NbcyudJhaIF9MG/RjIh5XHm2xb1SFquZ8k0g= github.com/elliotchance/orderedmap v1.7.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/proto v1.12.1 h1:6n/Z2pZAnBwuhU66Gs8160B8rrrYKo7h2F2sCOnNceE= github.com/emicklei/proto v1.12.1/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -409,10 +416,20 @@ github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlL github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8= +github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -628,6 +645,8 @@ github.com/in-toto/in-toto-golang v0.9.0/go.mod h1:xsBVrVsHNsB61++S6Dy2vWosKhuA3 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E= github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc= @@ -657,6 +676,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -826,6 +847,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= @@ -895,8 +918,8 @@ github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbm github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= @@ -926,8 +949,11 @@ github.com/sigstore/timestamp-authority v1.2.3/go.mod h1:q2tJKJzP34hLIbVu3Y1A9bB github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY= @@ -1006,10 +1032,14 @@ github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23env github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= github.com/weppos/publicsuffix-go v0.40.3-0.20240815124645-a8ed110559c9 h1:4pH9wXOWQdW8kVMJ8P/kxbuxJKR+iNvDeC8zEVLy7eM= github.com/weppos/publicsuffix-go v0.40.3-0.20240815124645-a8ed110559c9/go.mod h1:o4XOb/pL91sSlesP+I2Xcp38P4/emRvDF6N6xUWvwzg= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/go-gitlab v0.112.0 h1:6Z0cqEooCvBMfBIHw+CgO4AKGRV8na/9781xOb0+DKw= github.com/xanzy/go-gitlab v0.112.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1103,6 +1133,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -1190,6 +1221,7 @@ golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1299,6 +1331,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -1317,6 +1350,8 @@ gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=