From 0c66570648b3852a6801a084a5cbfd4dc5315d31 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Thu, 4 Jan 2024 13:54:19 +0100 Subject: [PATCH] Extended Tour Documentation (#614) * tour doc * complete tour02 * credential tour03 * tour04/01 * tour04/02-03 * tour04/04 * tour05 * tour06 * fix install requirements * incorporate review --- Makefile | 2 +- .../lib/tour/01-getting-started/README.md | 264 ++++- .../lib/tour/01-getting-started/example.go | 72 +- examples/lib/tour/01-getting-started/main.go | 2 +- .../01-basic-componentversion-creation.go | 97 +- .../02-composition-version.go | 15 +- .../README.md | 302 +++++- .../02-composing-a-component-version/main.go | 2 +- .../01-using-credentials.go | 30 +- .../02-basic-credential-management.go | 106 +- .../03-credential-repositories.go | 44 +- .../03-working-with-credentials/README.md | 465 ++++++++- .../03-working-with-credentials/common.go | 6 +- .../tour/03-working-with-credentials/main.go | 2 +- .../01-basic-config-management.go | 42 +- .../02-handle-arbitrary-config.go | 38 +- .../03-using-ocm-config.go | 87 +- .../04-write-config-type.go | 100 +- .../05-write-config-consumer.go | 60 +- .../lib/tour/04-working-with-config/README.md | 922 +++++++++++++++++- .../lib/tour/04-working-with-config/main.go | 7 +- .../tour/04-working-with-config/settings.yaml | 4 + .../README.md | 148 ++- .../common.go | 4 +- .../example.go | 35 +- .../main.go | 2 +- .../01-basic-signing.go | 23 +- .../02-using-context-settings.go | 95 +- .../06-signing-component-versions/README.md | 327 ++++++- .../06-signing-component-versions/common.go | 4 +- .../06-signing-component-versions/main.go | 15 +- .../settings.yaml | 4 + examples/lib/tour/README.md | 3 + examples/lib/tour/doc.go | 7 + .../tour/docsrc/01-getting-started/README.md | 200 ++++ .../README.md | 249 +++++ .../03-working-with-credentials/README.md | 363 +++++++ .../docsrc/04-working-with-config/README.md | 603 ++++++++++++ .../04-working-with-config/settings.yaml | 4 + .../README.md | 128 +++ .../06-signing-component-versions/README.md | 220 +++++ .../settings.yaml | 4 + examples/lib/tour/docsrc/README.md | 15 + go.mod | 2 +- hack/Makefile | 7 +- hack/install-requirements.sh | 1 + pkg/contexts/credentials/config/type.go | 3 + pkg/contexts/credentials/interface.go | 9 + pkg/contexts/ocm/attrs/signingattr/config.go | 30 +- pkg/runtime/utils.go | 20 + pkg/signing/signutils/utils.go | 2 - 51 files changed, 4972 insertions(+), 224 deletions(-) create mode 100644 examples/lib/tour/04-working-with-config/settings.yaml create mode 100644 examples/lib/tour/06-signing-component-versions/settings.yaml create mode 100644 examples/lib/tour/doc.go create mode 100644 examples/lib/tour/docsrc/01-getting-started/README.md create mode 100644 examples/lib/tour/docsrc/02-composing-a-component-version/README.md create mode 100644 examples/lib/tour/docsrc/03-working-with-credentials/README.md create mode 100644 examples/lib/tour/docsrc/04-working-with-config/README.md create mode 100644 examples/lib/tour/docsrc/04-working-with-config/settings.yaml create mode 100644 examples/lib/tour/docsrc/05-transporting-component-versions/README.md create mode 100644 examples/lib/tour/docsrc/06-signing-component-versions/README.md create mode 100644 examples/lib/tour/docsrc/06-signing-component-versions/settings.yaml create mode 100644 examples/lib/tour/docsrc/README.md diff --git a/Makefile b/Makefile index c946d97875..e675297335 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ test: .PHONY: generate generate: - @$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminst/... + @$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminst/... $(REPO_ROOT)/examples/... .PHONY: generate-deepcopy generate-deepcopy: controller-gen diff --git a/examples/lib/tour/01-getting-started/README.md b/examples/lib/tour/01-getting-started/README.md index b9623c4739..528b705156 100644 --- a/examples/lib/tour/01-getting-started/README.md +++ b/examples/lib/tour/01-getting-started/README.md @@ -1,13 +1,271 @@ + + + # Basic Usage of OCM Repositories This [tour](example.go) illustrates the basic usage of the API to access component versions in an OCM repository. -You can just call the main program with some config file argument -with the following content: +## Running the example + +You can call the main program with a config file argument +(`--config `), where the config file has the following content: ```yaml component: github.com/mandelsoft/examples/cred1 repository: ghcr.io/mandelsoft/ocm version: 0.1.0 -``` \ No newline at end of file +``` + +## Walkthrough + +The basic entry point for using the OCM library is always +an [OCM Context object](../../contexts.md). It bundles all +configuration settings and type registrations, like +access methods, repository types, etc, and +configuration settings, like credentials, +which should be used when working with the OCM +ecosystem. + +Therefore, the first step is always to get access to such +a context object. Our example uses the default context +provided by the library, which covers the complete +type registration contained in the executable. + +It can be accessed by a function of the `pkg/contexts/ocm` package. + +```go + ctx := ocm.DefaultContext() +``` + +The context acts as the central entry +point to get access to OCM elements. +First, we get a repository, to look for +component versions. We use the OCM +repository hosted on `ghcr.io`, which is providing the standard OCM +components. + +For every storage technology used to store +OCM components, there is a serializable +descriptor object, the *repository specification*. +It describes the information required to access +the repository and can be used to store the serialized +form as part of other resources, for example +Kubernetes resources or configuration settings. +The available repository implementations can be found +under `.../pkg/contexts/ocm/repositories`. + +```go + spec := ocireg.NewRepositorySpec("ghcr.io/open-component-model/ocm") +``` + +The context can now be used to map the descriptor +into a repository object, which then provides access +to the OCM elements stored in this repository. + +```go + repo, err := ctx.RepositoryForSpec(spec) + if err != nil { + return errors.Wrapf(err, "cannot setup repository") + } +``` + +To release potentially allocated temporary resources, many objects +must be closed, if they are not used anymore. +This is typically done by a `defer` statement placed after a +successful object retrieval. + +```go + defer repo.Close() +``` + +Now we look for the versions of the component +available in this repository. + +```go + versions, err := c.ListVersions() + if err != nil { + return errors.Wrapf(err, "cannot query version names") + } +``` + +OCM version names must follow the *SemVer* rules. +Therefore, we can simply order the versions and print them. + +```go + err = semverutils.SortVersions(versions) + if err != nil { + return errors.Wrapf(err, "cannot sort versions") + } + fmt.Printf("versions for component ocm.software/ocmcli: %s\n", strings.Join(versions, ", ")) +``` + +Now, we have a look at the latest version. It is +the last one in the list. + +```go + cv, err := c.LookupVersion(versions[len(versions)-1]) + if err != nil { + return errors.Wrapf(err, "cannot get latest version") + } + defer cv.Close() +``` + + + +The component version object provides access +to the component descriptor + +```go + cd := cv.GetDescriptor() + fmt.Printf("resources of the latest version:\n") + fmt.Printf(" version: %s\n", cv.GetVersion()) + fmt.Printf(" provider: %s\n", cd.Provider.Name) +``` + +and the resources described by the component version. + +```go + for i, r := range cv.GetResources() { + fmt.Printf(" %2d: name: %s\n", i+1, r.Meta().GetName()) + fmt.Printf(" extra identity: %s\n", r.Meta().GetExtraIdentity()) + fmt.Printf(" resource type: %s\n", r.Meta().GetType()) + acc, err := r.Access() + if err != nil { + fmt.Printf(" access: error: %s\n", err) + } else { + fmt.Printf(" access: %s\n", acc.Describe(ctx)) + } + } +``` + +This results in the following output (the shown version might +differ, because the code always describes the latest version): + +``` +resources of the latest version: + version: 0.6.0 + provider: ocm.software + 1: name: ocmcli + extra identity: "architecture"="amd64","os"="linux" + resource type: executable + access: Local blob sha256:6672528b57fd77cefa4c5a3395431b6a5aa14dc3ddad3ffe52343a7a518c2cd3[] + 2: name: ocmcli + extra identity: "architecture"="arm64","os"="linux" + resource type: executable + access: Local blob sha256:9088cb8bbef1593b905d6bd3af6652165ff82cebd0d86540a7be9637324d036b[] + 3: name: ocmcli-image + extra identity: + resource type: ociImage + access: OCI artifact ghcr.io/open-component-model/ocm/ocm.software/ocmcli/ocmcli-image:0.6.0 +``` + +Resources have some metadata, like their identity and a resource type. +And, most importantly, they describe how the content of the resource +(as blob) can be accessed. +This is done by an *access specification*, again a serializable descriptor, +like the repository specification. + +The component version used here contains the executables for the OCM CLI +for various platforms. The next step is to +get the executable for the actual environment. +The identity of a resource described by a component version +consists of a set of properties. The property `name` is mandatory. But there may be more identity attributes +finally stored as ``extraIdentity` in the component descriptor. + +A convention is to use dedicated identity properties to indicate the +operating system and the architecture for executables. + +```go + id := metav1.NewIdentity("ocmcli", + extraid.ExecutableOperatingSystem, runtime.GOOS, + extraid.ExecutableArchitecture, runtime.GOARCH, + ) + + res, err := cv.GetResource(id) + if err != nil { + return errors.Wrapf(err, "resource %s", id) + } +``` + +Now we want to retrieve the executable. The library provides two +basic ways to do this. + +First, there is the direct way to gain access to the blob by using +the basic model operations to get a reader for the resource blob. +Therefore, in a first step we get the access method for the resource + +```go + var m ocm.AccessMethod + m, err = res.AccessMethod() + if err != nil { + return errors.Wrapf(err, "cannot get access method") + } + defer m.Close() +``` + +The method needs to be closed, because the method +object may cache the technical blob representation +generated by accessing the underlying access technology. +(for example, accessing an OCI image requires a sequence of +backend requests for the manifest, the layers, etc, which will +then be packaged into a tar archive returned as blob). +This caching may not be required, if the backend directly +returns a blob. + +Now, we get access to the reader providing the blob content. +The blob features a mime type, which can be used to understand +the format of the blob. Here, we have a plain octet stream. + +```go + fmt.Printf(" found blob with mime type %s\n", m.MimeType()) + reader, err = m.Reader() +``` + +Because this code sequence is a common operation, there is a +utility function handling this sequence. A shorter way to get +a resource reader is as follows: + +```go + reader, err = utils.GetResourceReader(res) +``` + +Before we download the content we check the error and prepare +closing the reader, again + +```go + if err != nil { + return errors.Wrapf(err, "cannot get resource reader") + } + defer reader.Close() +``` + +Now, we just read the content and copy it to the intended +output file. + +```go + file, err := os.OpenFile("/tmp/ocmcli", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0766) + if err != nil { + return errors.Wrapf(err, "cannot open output file") + } + defer file.Close() + + n, err := io.Copy(file, reader) + if err != nil { + return errors.Wrapf(err, "write executable") + } + fmt.Printf("%d bytes written\n", n) +``` + +Another way to download a resource is to use registered *downloaders*. +`download.DownloadResource` is used to download resources with specific handlers for +selected resource and mime type combinations. +The executable downloader is registered by default and automatically +sets the `X` flag for the written file. + +```go + _, err = download.DownloadResource(ctx, res, "/tmp/ocmcli", download.WithPrinter(common.NewPrinter(os.Stdout))) + if err != nil { + return errors.Wrapf(err, "download failed") + } +``` diff --git a/examples/lib/tour/01-getting-started/example.go b/examples/lib/tour/01-getting-started/example.go index 5762d96cd0..cf46535078 100644 --- a/examples/lib/tour/01-getting-started/example.go +++ b/examples/lib/tour/01-getting-started/example.go @@ -30,7 +30,9 @@ func GettingStarted() error { // configuration settings, like credentials, // which should be used when working with the OCM // ecosystem. + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // The context acts as the central entry // point to get access to OCM elements. @@ -48,19 +50,28 @@ func GettingStarted() error { // Kubernetes resources. // The available repository implementations can be found // under .../pkg/contexts/ocm/repositories. + // --- begin repository spec --- spec := ocireg.NewRepositorySpec("ghcr.io/open-component-model/ocm") + // --- end repository spec --- // And the context can now be used to map the descriptor // into a repository object, which then provides access // to the OCM elements stored in this repository. + // --- begin repository --- repo, err := ctx.RepositoryForSpec(spec) if err != nil { return errors.Wrapf(err, "cannot setup repository") } + // --- end repository --- + // to release potentially allocated temporary resources, // many objects must be closed, if they should not be used - // anymore, to release potentially allocated temporary resources. + // anymore. + // This is typically done by a `defer` statement placed after a + // successful object retrieval. + // --- begin close --- defer repo.Close() + // --- end close --- // Now, we look up the OCM CLI component. // All kinds of repositories, regardless of their type @@ -73,32 +84,49 @@ func GettingStarted() error { // Now we look for the versions of the component // available in this repository. + // --- begin versions --- versions, err := c.ListVersions() if err != nil { return errors.Wrapf(err, "cannot query version names") } + // --- end versions --- - // OCM version names must follow the semver rules. + // OCM version names must follow the SemVer rules. + // Therefore, we can simply order the versions and print them. + // --- begin semver --- err = semverutils.SortVersions(versions) if err != nil { return errors.Wrapf(err, "cannot sort versions") } fmt.Printf("versions for component ocm.software/ocmcli: %s\n", strings.Join(versions, ", ")) + // --- end semver --- - // Now, we have a look at the latest version + // Now, we have a look at the latest version. it is + // the last one in the list. + // --- begin lookup version --- cv, err := c.LookupVersion(versions[len(versions)-1]) if err != nil { return errors.Wrapf(err, "cannot get latest version") } defer cv.Close() + // --- end lookup version --- - // Have a look at the component descriptor + fmt.Printf("--- begin version ---\n") + // The component version object provides access + // to the component descriptor. + // --- begin component descriptor --- cd := cv.GetDescriptor() fmt.Printf("resources of the latest version:\n") fmt.Printf(" version: %s\n", cv.GetVersion()) fmt.Printf(" provider: %s\n", cd.Provider.Name) + // --- end component descriptor --- // and list all the included resources. + // Resources have some metadata, like the resource identity and a resource type. + // And they describe how the content of the resource (as blob) can be accessed. + // This is done by an *access specification*, again a serializable descriptor, + // like the repository specification. + // --- begin resources --- for i, r := range cv.GetResources() { fmt.Printf(" %2d: name: %s\n", i+1, r.Meta().GetName()) fmt.Printf(" extra identity: %s\n", r.Meta().GetExtraIdentity()) @@ -110,11 +138,17 @@ func GettingStarted() error { fmt.Printf(" access: %s\n", acc.Describe(ctx)) } } + // --- end resources --- + fmt.Printf("--- end version ---\n") // Get the executable for the actual environment. // The identity of a resource described by a component version - // consists of a set of properties. The property name must - // always be given. + // consists of a set of properties. The property `name` is mandatory. + // But there may be more identity attributes + // finally stored as ``extraIdentity` in the component descriptor. + // A convention is to use dedicated identity properties to indicate the + // operating system and the architecture for executables. + // --- begin find executable --- id := metav1.NewIdentity("ocmcli", extraid.ExecutableOperatingSystem, runtime.GOOS, extraid.ExecutableArchitecture, runtime.GOARCH, @@ -124,6 +158,7 @@ func GettingStarted() error { if err != nil { return errors.Wrapf(err, "resource %s", id) } + // --- end find executable --- // download to /tmp/ocmcli using basic model // operations. @@ -136,35 +171,47 @@ func GettingStarted() error { // for the resource blob. // First, get the access method for the resource. // Second, request a reader for the blob. + // --- begin getting access --- var m ocm.AccessMethod m, err = res.AccessMethod() if err != nil { return errors.Wrapf(err, "cannot get access method") } + // --- end getting access --- + // the method needs to be closed, because the method // object may cache the technical blob representation - // generated accessing the underlying access technology. + // generated by accessing the underlying access technology. // (for example, accessing an OCI image requires a sequence of - // backend accesses for the manifest, the layers, etc which will + // backend requests for the manifest, the layers, etc, which will // then be packaged into a tar archive returned as blob). // This caching may not be required, if the backend directly // returns a blob. + // --- begin closing access --- defer m.Close() + // --- end closing access --- // the method now also provides information abount the returned // blob format in form of a mime type. + // --- begin getting reader --- fmt.Printf(" found blob with mime type %s\n", m.MimeType()) reader, err = m.Reader() + // --- end getting reader --- } else { // because this is a common operation, there is a - // utility function handling this sequence. + // utility function handling this code sequence. + // --- begin utility function --- reader, err = utils.GetResourceReader(res) + // --- end utility function --- } + // --- begin closing reader --- if err != nil { return errors.Wrapf(err, "cannot get resource reader") } defer reader.Close() + // --- end closing reader --- + // --- begin copy --- file, err := os.OpenFile("/tmp/ocmcli", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0766) if err != nil { return errors.Wrapf(err, "cannot open output file") @@ -176,16 +223,19 @@ func GettingStarted() error { return errors.Wrapf(err, "write executable") } fmt.Printf("%d bytes written\n", n) + // --- end copy --- // alternatively, a registered downloader for executables can be used. - // Download is used to download resources with specific handlers for the + // download.DownloadResource is used to download resources with specific handlers for the // selected resource and mime type combinations. // The executable downloader is registered by default and automatically - // sets the X flag. + // sets the `X `flag for the written file. + // --- begin download --- _, err = download.DownloadResource(ctx, res, "/tmp/ocmcli", download.WithPrinter(common.NewPrinter(os.Stdout))) if err != nil { return errors.Wrapf(err, "download failed") } + // --- end download --- return nil } diff --git a/examples/lib/tour/01-getting-started/main.go b/examples/lib/tour/01-getting-started/main.go index bd222058de..0b551a99c9 100644 --- a/examples/lib/tour/01-getting-started/main.go +++ b/examples/lib/tour/01-getting-started/main.go @@ -10,7 +10,7 @@ import ( ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" func main() { err := GettingStarted() diff --git a/examples/lib/tour/02-composing-a-component-version/01-basic-componentversion-creation.go b/examples/lib/tour/02-composing-a-component-version/01-basic-componentversion-creation.go index 08785c7097..64d46a7eb2 100644 --- a/examples/lib/tour/02-composing-a-component-version/01-basic-componentversion-creation.go +++ b/examples/lib/tour/02-composing-a-component-version/01-basic-componentversion-creation.go @@ -28,6 +28,9 @@ import ( // This can be called on any component version, regardless of // its origin. func setupVersion(cv ocm.ComponentVersionAccess) error { + // The provider is a structure with a name and some labels. + // We just set the name here by directly setting the `Name` attribute. + // --- begin setup provider --- provider := &compdesc.Provider{ Name: "acme.org", } @@ -36,6 +39,7 @@ func setupVersion(cv ocm.ComponentVersionAccess) error { if err != nil { return errors.Wrapf(err, "cannot set provider") } + // --- end setup provider --- //////////////////////////////////////////////////////////////////////////// // Ok, a component version not describing any resources @@ -48,17 +52,21 @@ func setupVersion(cv ocm.ComponentVersionAccess) error { // A resources has some metadata, like an identity // and a type. // The identity is just a set of string properties, - // with at least containing the name property. + // at least containing the `name` property. // additional identity properties can be added via // options. + // The type represents the logical meaning of the + // resource, here an `ociImage`. + // --- begin setup resource meta --- meta, err := elements.ResourceMeta("image", resourcetypes.OCI_IMAGE) if err != nil { // without metadata options, there will be never be an error, // bit to be complete, we just handle the error case, here. return errors.Wrapf(err, "invalid resource meta") } + // --- end setup resource meta --- - // And most important it requires content. + // And most importantly, it requires content. // Content can be already present in some external // repository. As long, as there is an access type // for this kind of repository, we can just refer to it. @@ -66,31 +74,37 @@ func setupVersion(cv ocm.ComponentVersionAccess) error { // OCM ecosystem. // Supported access types can be found under // .../pkg/contexts/ocm/accessmethods. + // --- begin setup image access --- acc := ociartifact.New("ghcr.io/open-component-model/ocm/ocm.software/toi/installers/helminstaller/helminstaller:0.4.0") + // --- end setup image access --- // Once we have both, the metadata and the content specification, - // we can now add the resource. + // we can now add the resource to our component version. // The SetResource methods will replace an existing resource with the same // identity, or add the resource, if no such resource exists in the component // version. + // --- begin setup resource --- err = cv.SetResource(meta, acc) if err != nil { return errors.Wrapf(err, "cannot add access to ocmcli-image)") } + // --- end setup resource --- //////////////////////////////////////////////////////////////////////////// // Now, we will add a second resource, some unspecific yaml data. // Therefore, we use the generic YAML resource type. // In practice, you should always use a resource type describing // the real meaning of the content, for example something like - // `kubernetesManifest`, This enables tools working with specific content + // `kubernetesManifest`. This enables tools working with specific content // to understand the resource set of a component version. fmt.Printf(" setting blob resource 'descriptor'...\n") + // --- begin setup second meta --- meta, err = elements.ResourceMeta("descriptor", resourcetypes.OCM_YAML) if err != nil { return errors.Wrapf(err, "invalid resource meta") } + // --- end setup second meta --- basic := true yamldata := ` @@ -102,17 +116,32 @@ data: some very important data required to understand this component // Besides referring to external resources, another possibility // to add content is to directly provide the content blob. The // used abstraction here is blobaccess.BlobAccess. - // Any blob content provided by an implementation of this - // interface can be added as resource. - // There are various access implementations for blobs - // taken from the local host, for example, from the filesystem, - // or from other repositories (for example by mapping - // an access type specification into a blob access). + // + // Any blob content, which can be provided by an implementation of this + // interface, can be added as resource to a component version. + // The library provides various access implementations for blobs + // taken from the local host or from other repositories. + // For example, this could be some file system content. + // To describe blobs taken from external repositories + // an access type specification can be mapped to a blob access. + // Hereby, blobs are stored along with the component descriptor + // instead of storing a reference to content in an external repository. + // // The most simple form is to directly provide a byte sequence, // for example some YAML data. // A blob always must provide a mime type, describing the // technical format of the blob's byte sequence. + // This is different + // from the resource type. A logical resource, like a *Helm chart* can be + // represented + // in different technical formats, for example a Helm chart archive + // or as OCI image archive. While the type described the + // logical content, the meaning of the resource, its mime type + // described the technical blob format used to represent + // the resource as byte sequence. + // --- begin string blob access --- blob := blobaccess.ForString(mime.MIME_YAML, yamldata) + // --- end string blob access --- // when storing the blob, it is possible to provide some // optional additional information: @@ -121,39 +150,53 @@ data: some very important data required to understand this component // (for example the image repository of an OCI image stored // as local blob) // - an additional access type, which provides an alternative - // global technology specific access to the same content. - // we don't use it, here. + // global technology specific access to the same content + // (we don't use it, here). + // --- begin setup by blob access --- err = cv.SetResourceBlob(meta, blob, "", nil) if err != nil { return errors.Wrapf(err, "cannot add yaml document") } + // --- end setup by blob access --- + + // Resources added by blobs will be stored along with the component + // version metadata in the same repository, no external + // repository is required. } else { // The above blob example describes the basic operations, // which can be used to compose any kind of resource // from any kind of source. - // For selected use cases there are convenience helpers, + // For selected use cases there are convenience helpers available, // which can be used to compose a resource access object. // This is basically the same interface returned by GetResource // functions on the component version from the last example. // Such objects can directly be used to add/modify a resource in a // component version. - // The above case could be written as follows, also: + // + // The above case could also be written as follows: + // --- begin setup by access --- res := textblob.ResourceAccess(cv.GetContext(), meta, yamldata, textblob.WithimeType(mime.MIME_YAML)) err = cv.SetResourceAccess(res) if err != nil { return errors.Wrapf(err, "cannot add yaml document") } + // --- end setup by access --- + + // The resource access is an abstraction of external access via access + // methods or direct blob access objects and additionally + // contain all the required resource metadata. } // There are even more complex blob sources, for example - // for helm charts stored in the filesystem, or even for images + // for Helm charts stored in the file system, or even for images // generated by docker builds. - // Here, we just compose a multi-platform image built with buildx + // Here, we just compose a multi-platform image built with `buildx` // from these sources (components/ocmcli) featuring two flavors. // (you have to execute `make image.multi` in components/ocmcli // before executing this example. fmt.Printf(" setting blob resource 'ocmcli'...\n") + // --- begin setup by docker --- meta, err = elements.ResourceMeta("ocmcli", resourcetypes.OCI_IMAGE) if err != nil { return errors.Wrapf(err, "invalid resource meta") @@ -171,6 +214,7 @@ data: some very important data required to understand this component if err != nil { return errors.Wrapf(err, "cannot add ocmcli") } + // --- end setup by docker --- return err } @@ -184,11 +228,14 @@ func addVersion(repo ocm.Repository, name, version string) error { // now we compose a new component version, first we create // a new version backed by this repository. + // The result is a memory based representation, which is not yet persisted. + // --- begin new version --- cv, err := repo.NewComponentVersion(name, version) if err != nil { return errors.Wrapf(err, "cannot create new version") } defer cv.Close() + // --- end new version --- err = setupVersion(cv) if err != nil { @@ -282,24 +329,28 @@ func listVersions(repo ocm.Repository, list ...string) error { func ComposingAComponentVersionA() error { // yes, we need an OCM context, again + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // To compose and store a new component version - // we finally need some OCM repository to - // store the component, The most simple - // external repository could be the filesystem. - // OCM defines a distribution format, the - // Common Transport Format (CTF) for this, + // we need some OCM repository to + // store the component. The most simple + // external repository could be the file system. + // For this purpose OCM defines a distribution format, the + // Common Transport Format (CTF), // which is an extension of the OCI distribution // specification. - // There are three flavours, Directory, Tar or TGZ. + // There are three flavors, Directory, Tar or TGZ. // The implementation provides a regular OCM repository - // interface, like used in the previous example. + // interface, like the one used in the previous example. + // --- begin create ctf --- repo, err := ctfocm.Open(ctx, ctfocm.ACC_WRITABLE|ctfocm.ACC_CREATE, "/tmp/example02.ctf", 0o0744, ctfocm.FormatDirectory) if err != nil { return errors.Wrapf(err, "cannot create transport repository") } defer repo.Close() + // --- end create ctf --- // now we create a first component version in this repository. err = addVersion(repo, "acme.org/example02", "v0.1.0") diff --git a/examples/lib/tour/02-composing-a-component-version/02-composition-version.go b/examples/lib/tour/02-composing-a-component-version/02-composition-version.go index b26e609891..1820e5b68b 100644 --- a/examples/lib/tour/02-composing-a-component-version/02-composition-version.go +++ b/examples/lib/tour/02-composing-a-component-version/02-composition-version.go @@ -14,18 +14,23 @@ import ( func ComposingAComponentVersionB() error { // yes, we need an OCM context, again + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // now we compose a component version without a repository. - // later we add this to a new repository. - + // later, we add this to a new repository. + // --- begin new version --- cv := composition.NewComponentVersion(ctx, "acme.org/example2", "v0.1.0") + // --- end new version --- // just use the same component version setup from variant A + // --- begin setup version --- err := setupVersion(cv) if err != nil { return errors.Wrapf(err, "version composition") } + // --- end setup version --- // even on this internal component version, the API is the same fmt.Printf("*** composition version ***\n") @@ -36,13 +41,17 @@ func ComposingAComponentVersionB() error { // Here, we are using an internal composition repository. // It has no storage backend and can be used to internally compose // a set of component versions, which can then be transferred - // to any other repository (see example 4) + // to any other repository (see tour 5) + // --- begin create composition repository --- repo := composition.NewRepository(ctx) + // --- end create composition repository --- + // --- begin add version --- err = repo.AddComponentVersion(cv) if err != nil { return errors.Wrapf(err, "cannot add version") } + // --- end add version --- var list []string diff --git a/examples/lib/tour/02-composing-a-component-version/README.md b/examples/lib/tour/02-composing-a-component-version/README.md index 6a7ac375cf..74953c7385 100644 --- a/examples/lib/tour/02-composing-a-component-version/README.md +++ b/examples/lib/tour/02-composing-a-component-version/README.md @@ -1,10 +1,306 @@ + + + # Composing a Component Version -This tor illustrates the basic usage of the API to +This tour illustrates the basic usage of the API to create/compose component versions. It covers two basic scenarios: -- [`basic`](01-basic-componentversion-creation.go) Create a component version stored in the filesystem +- [`basic`](01-basic-componentversion-creation.go) Create a component version stored in the file system - [`compose`](02-composition-version.go) Create a component version stored in memory using a non-persistent composition version. -You can just call the main program with the scenario as argument. +## Running the example + +You can call the main program with the scenario as argument. Configuration is not required. + +## Walkthrough + +### Basic Component Version Creation + +The first variant just creates a new component version +in an OCM repository. To avoid the requirement for +credentials a file system based repository is created, using +the *Common Transport Format* (CTF). + +As usual, we start with getting access to an OCM context +object: + +```go + ctx := ocm.DefaultContext() +``` + +To compose and store a new component version +we need some OCM repository to +store the component. The most simple +external repository could be the file system. +For this purpose OCM defines a distribution format, the +*Common Transport Format* (CTF), +which is an extension of the OCI distribution +specification. +There are three flavors, *Directory*, *Tar* or *TGZ*. +The implementation provides a regular OCM repository +interface, like the one used in the previous example. + +```go + repo, err := ctfocm.Open(ctx, ctfocm.ACC_WRITABLE|ctfocm.ACC_CREATE, "/tmp/example02.ctf", 0o0744, ctfocm.FormatDirectory) + if err != nil { + return errors.Wrapf(err, "cannot create transport repository") + } + defer repo.Close() +``` + +Once we have a repository we can compose a new version. +First, we create a new version backed by this repository. +The result is a memory based representation, which is not yet persisted. + +```go + cv, err := repo.NewComponentVersion(name, version) + if err != nil { + return errors.Wrapf(err, "cannot create new version") + } + defer cv.Close() +``` + +Now, we can configure the component version. It only exists in memory +so far, but is already connected to the repository. + +The setup of the component version is put into a +separate method (`setupVersion`), so it can be reused for the second variant. + +First, we configure the component version provider. + +```go + provider := &compdesc.Provider{ + Name: "acme.org", + } + fmt.Printf(" setting provider...\n") + err := cv.SetProvider(provider) + if err != nil { + return errors.Wrapf(err, "cannot set provider") + } +``` + +The provider is a structure with a name and some labels. +We just set the name here by directly setting the `Name` attribute. + +Now, we fill the component version with content. +First, we add some resource already located in +an external registry. We use an OCI image here. +A resources has some metadata, like an identity +and a type. +The identity is just a set of string properties, +at least containing the `name` property. +Additional identity properties can be added via +options. +The type represents the logical meaning of the +resource, here an `ociImage`. + +```go + meta, err := elements.ResourceMeta("image", resourcetypes.OCI_IMAGE) + if err != nil { + // without metadata options, there will be never be an error, + // bit to be complete, we just handle the error case, here. + return errors.Wrapf(err, "invalid resource meta") + } +``` + +In this example, we just use the `name` property +without any extra identity. + +And most importantly, a resource requires content. +Content can already be present in some external +repository. As long, as there is an access type +for this kind of repository, we can just refer to it. +Here, we just use an image provided by the +OCM ecosystem. +Supported access types can be found under +.../pkg/contexts/ocm/accessmethods. + +```go + acc := ociartifact.New("ghcr.io/open-component-model/ocm/ocm.software/toi/installers/helminstaller/helminstaller:0.4.0") +``` + +Once we have both, the metadata and the content specification, +we can now add the resource to our component version. +The `SetResource` methods will replace an existing resource with the same +identity, or add the resource, if no such resource exists in the component +version. + +```go + err = cv.SetResource(meta, acc) + if err != nil { + return errors.Wrapf(err, "cannot add access to ocmcli-image)") + } +``` + +Now, we will add a second resource, some unspecific yaml data. +Therefore, we use the generic YAML resource type. +In practice, you should always use a resource type describing +the real meaning of the content, for example something like +`kubernetesManifest`. This enables tools working with specific content +to understand the resource set of a component version. + +```go + meta, err = elements.ResourceMeta("descriptor", resourcetypes.OCM_YAML) + if err != nil { + return errors.Wrapf(err, "invalid resource meta") + } +``` + +Besides referring to external resources, another possibility +to add content is to directly provide the content blob. The +used abstraction here is `blobaccess.BlobAccess`. + +Any blob content, which can be provided by an implementation of this +interface, can be added as resource to a component version. +The library provides various access implementations for blobs +taken from the local host or from other repositories. +For example, this could be some file system content. +To describe blobs taken from external repositories +an access type specification can be mapped to a blob access. +Hereby, blobs are stored along with the component descriptor +instead of storing a reference to content in an external repository. + +The most simple form is to directly provide a byte sequence, +for example some YAML data. +A blob always must provide a mime type, describing the +technical format of the blob's byte sequence. This is different +from the resource type. A logical resource, like a *Helm chart* can be +represented in different technical formats, for example a Helm chart +archive or as OCI image archive. While the type described the +logical content, the meaning of the resource, its mime type +described the technical blob format used to represent +the resource as byte sequence. + +```go + blob := blobaccess.ForString(mime.MIME_YAML, yamldata) +``` + +When storing the blob, it is possible to provide some +optional additional information: +- a name of the resource described by the blob, which could + be used to do a later upload into an external repository + (for example the image repository of an OCI image stored + as local blob) +- an additional access type, which provides an alternative + global technology specific access to the same content + (we don't use it, here). + +```go + err = cv.SetResourceBlob(meta, blob, "", nil) + if err != nil { + return errors.Wrapf(err, "cannot add yaml document") + } +``` + +Resources added by blobs will be stored along with the component +version metadata in the same repository, no external +repository is required. + +The above blob example describes the basic operations, +which can be used to compose any kind of resource +from any kind of source. +For selected use cases there are convenience helpers available, +which can be used to compose a resource access object. +This is basically the same interface returned by `GetResource` +functions on the component version from the last example. +Such objects can directly be used to add/modify a resource in a +component version. + +The above case could also be written as follows: + +```go + res := textblob.ResourceAccess(cv.GetContext(), meta, yamldata, + textblob.WithimeType(mime.MIME_YAML)) + err = cv.SetResourceAccess(res) + if err != nil { + return errors.Wrapf(err, "cannot add yaml document") + } +``` + +The resource access is an abstraction of external access via access +methods or direct blob access objects and additionally +contain all the required resource metadata. + +There are even more complex blob sources, for example +for Helm charts stored in the file system, or even for images +generated by docker builds. +Here, we just compose a multi-platform image built with `buildx` +from these sources (components/ocmcli) featuring two flavors. +(you have to execute `make image.multi` in components/ocmcli +before executing this example. + +```go + meta, err = elements.ResourceMeta("ocmcli", resourcetypes.OCI_IMAGE) + if err != nil { + return errors.Wrapf(err, "invalid resource meta") + } + res := dockermultiblob.ResourceAccess(cv.GetContext(), meta, + dockermultiblob.WithPrinter(common.StdoutPrinter), + dockermultiblob.WithHint("ocm.software/ocmci"), + dockermultiblob.WithVersion(current_version), + dockermultiblob.WithVariants( + fmt.Sprintf("ocmcli-image:%s-linux-amd64", current_version), + fmt.Sprintf("ocmcli-image:%s-linux-arm64", current_version), + ), + ) + err = cv.SetResourceAccess(res) + if err != nil { + return errors.Wrapf(err, "cannot add ocmcli") + } +``` + +### Composition Environment + +The second variant just creates a new component version +in a memory based composition environment, no persistence is +required. Like all component versions, such component versions +can be added to any repository later. + +As usual, we start with getting access to an OCM context +object: + +```go + ctx := ocm.DefaultContext() +``` + +Now, we can create a new component version in the composition +environment. This does not require a repository or component object. + +```go + cv := composition.NewComponentVersion(ctx, "acme.org/example2", "v0.1.0") +``` + +To configure the component version, we can just reuse the coding +from the example above, the component version interface is just the same. +We just call the `setupVersion` function for the created component version access. + +```go + err := setupVersion(cv) + if err != nil { + return errors.Wrapf(err, "version composition") + } +``` + +The resulting component version can be added to any OCM repository, +like the one from the previous example. +Here, we use another feature of the composition environment. It also provides +complete memory based OCM repositories. +It has no storage backend and can be used to internally compose +a set of component versions, which can then be transferred +to any other repository (see [tour 05](../05-transporting-component-versions/README.md)) + +```go + repo := composition.NewRepository(ctx) +``` + +This repository object behaves like any other OCM repository object. We can just +add the new component version. + +```go + err = repo.AddComponentVersion(cv) + if err != nil { + return errors.Wrapf(err, "cannot add version") + } +``` \ No newline at end of file diff --git a/examples/lib/tour/02-composing-a-component-version/main.go b/examples/lib/tour/02-composing-a-component-version/main.go index b5b1864b66..074c55e4a0 100644 --- a/examples/lib/tour/02-composing-a-component-version/main.go +++ b/examples/lib/tour/02-composing-a-component-version/main.go @@ -11,7 +11,7 @@ import ( ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" var current_version string diff --git a/examples/lib/tour/03-working-with-credentials/01-using-credentials.go b/examples/lib/tour/03-working-with-credentials/01-using-credentials.go index 0dbdc70284..387aa9e493 100644 --- a/examples/lib/tour/03-working-with-credentials/01-using-credentials.go +++ b/examples/lib/tour/03-working-with-credentials/01-using-credentials.go @@ -14,16 +14,19 @@ import ( func UsingCredentialsA(cfg *helper.Config) error { // yes, we need an OCM context, again + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- - // So far, we just use memory or filesystem based + // So far, we just used memory or file system based // OCM repositories to create component versions. // If we want to store something in a remotely accessible - // repository typically some credentials are required. + // repository typically some credentials are required + // for write access. // - // The OCM library uses a generic abstraction for credentials- + // The OCM library uses a generic abstraction for credentials. // It is just set of properties. To offer various credential sources - // There is an interface credentials.Credentials provides, + // there is an interface credentials.Credentials provided, // whose implementations provide access to those properties. // A simple property based implementation is credentials.DirectCredentials. // @@ -32,12 +35,15 @@ func UsingCredentialsA(cfg *helper.Config) error { // The example config file provides such credentials // for an OCI registry. + // --- begin new credentials --- creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) + // --- end new credentials --- - // now we can use the OCI repository access creation from - // example, but we pass the credentials as additional parameter. - // To give you the chance to specify your own registry the URL + // now we can use the OCI repository access creation from the first tour, + // but we pass the credentials as additional parameter. + // To give you the chance to specify your own registry, the URL // is taken from the config file. + // --- begin repository access --- spec := ocireg.NewRepositorySpec(cfg.Repository, nil) repo, err := ctx.RepositoryForSpec(spec, creds) @@ -45,10 +51,12 @@ func UsingCredentialsA(cfg *helper.Config) error { return err } defer repo.Close() + // --- end repository access --- // if registry name and credentials are fine, we should be able // now to add a new component version using the coding - // from the previous example. + // from the previous example, but now we use a public repository, instead + // of a memory or file system based one. // now we create a component version in this repository. err = addVersion(repo, "acme.org/example03", "v0.1.0") @@ -56,13 +64,15 @@ func UsingCredentialsA(cfg *helper.Config) error { return err } - // list the versions as known from example 1 + // In contrast to our first tour we cannot list components, here. // OCI registries do not support component listers, therefore we - // just list the actually added version. + // just look up the actually added version to verify the result. + // --- begin lookup --- cv, err := repo.LookupComponentVersion("acme.org/example03", "v0.1.0") if err != nil { return errors.Wrapf(err, "added version not found") } defer cv.Close() return errors.Wrapf(describeVersion(cv), "describe failed") + // --- end lookup --- } diff --git a/examples/lib/tour/03-working-with-credentials/02-basic-credential-management.go b/examples/lib/tour/03-working-with-credentials/02-basic-credential-management.go index 4cd8527215..fe9b6104e4 100644 --- a/examples/lib/tour/03-working-with-credentials/02-basic-credential-management.go +++ b/examples/lib/tour/03-working-with-credentials/02-basic-credential-management.go @@ -19,28 +19,32 @@ import ( ) func UsingCredentialsB(cfg *helper.Config, create bool) error { + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- - // Passing credentials directly at the respository + // Passing credentials directly at the repository // is fine, as long only the component version // will be accessed. But as soon as described // resource content will be read, the required // credentials and credential types are dependent - // on the concrete conmponent version, because + // on the concrete component version, because // it might contain any kind of access method // referring to any kind of resource repository // type. // // To solve this problem of passing any set // of credentials the OCM context object is - // used to store credentials. This handled + // used to store credentials. This is handled // by a sub context, the Credentials context. + // --- begin cred context --- credctx := ctx.CredentialsContext() + // --- end cred context --- // The credentials context brings together - // provider of credentials, for example a - // vault or a local docker/config.json + // providers of credentials, for example a + // Vault or a local Docker config.json // and credential consumers like GitHub or // OCI registries. // It must be able to distinguish various kinds @@ -51,88 +55,117 @@ func UsingCredentialsB(cfg *helper.Config, create bool) error { // consumer type specific set of properties // describing the concrete instance of such // a consumer, for example an OCI artifact in - // an OCI registry s identified by a host and + // an OCI registry is identified by a host and // a repository path. // // A credential provider like a vault just provides - // named credential set and typically does not + // named credential sets and typically does not // know anything about the use case for these sets. - // The task of the credential context is now to + // The task of the credential context is to // provide credentials for a dedicated consumer. // Therefore, it maintains a configurable // mapping of credential sources (credentials in // a credential repository) and a dedicated consumer. // - // This mapping defines a usecase, also based on + // This mapping defines a use case, also based on // a property set and dedicated credentials. // If credentials are required for a dedicated // consumer, it matches the defined mappings and // returned the best matching entry. // - // Matching? Lets take GitHub OCI registry as an + // Matching? Let's take the GitHub OCI registry as an // example. There are different owners for - // different repository path (the GitHub org/user). - // Therfore, different credentials needs to be provided + // different repository paths (the GitHub org/user). + // Therefore, different credentials need to be provided // for different repository paths. - // For example credentials for ghcr.io/acme can be used + // For example, credentials for ghcr.io/acme can be used // for a repository ghcr.io/acme/ocm/myimage. // To start with the credentials context we just // provide an explicit mapping for our use case. + // first, we create our credentials object as before. + // --- begin new credentials --- + creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) + // --- end new credentials --- + + // Then we determine the consumer id for our use case. + // The repository implementation provides a function + // for this task. It provides the most general property + // set for an OCI based OCM repository. + // --- begin consumer id --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "invalid consumer") } - creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) + // --- end consumer id --- + // the used functions above are just convenience wrappers - // arround the core type ConsumerId, which might be provided - // which might be for dedicated repository technologies. + // around the core type ConsumerId, which might be provided + // for dedicated repository/consumer technologies. // everything can be done directly with the core interface. + // --- begin set credentials --- credctx.SetCredentialsForConsumer(id, creds) + // --- end set credentials --- // now the context is prepared to provide credentials // for any usage of our OCI registry, regardless // of its type. - // lets test, whether it could provide credentials + // let's test, whether it could provide credentials // for storing our component version. // first we get the repository object for our OCM repository. + // --- begin get repository --- spec := ocireg.NewRepositorySpec(cfg.Repository, nil) repo, err := ctx.RepositoryForSpec(spec, creds) if err != nil { return err } defer repo.Close() + // --- end get repository --- - // a credential consumer may provide might provide consumer id information + // second, we determine the consumer id for our intended repository access. + // a credential consumer may provide consumer id information // for a dedicated sub user context. - // This is supported by the OCM repo implementation for OCI registres. + // This is supported by the OCM repo implementation for OCI registries. // The usage context is here the component name. - id = credentials.GetProvidedConsumerId(repo, credentials.StringUsageContext("acme.org/example3")) + + // --- begin get access id --- + id = credentials.GetProvidedConsumerId(repo, credentials.StringUsageContext("acme.org/example03")) if id == nil { return fmt.Errorf("repository does not support consumer id queries") } fmt.Printf("usage context: %s\n", id) + // --- end get access id --- + + // third, we ask the credential context for appropriate credentials. + // the basic context method `credctx.GetCredentialsForConsumer` returns + // a credentials source interface able to provide credentials + // for a changing credentials source. Here, we use a convenience + // function, which directly provides a credentials interface for the + // actually valid credentials. + // an error is only provided if something went wrong while determining + // the credentials. Delivering NO credentials is a valid result. + // the returned interface then offers access to the credential properties. + // via various methods. - // the returned credentials are provided via an interface, which might change its - // content, if the underlying credential source changes. + // --- begin get credentials --- creds, err = credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) if err != nil { return errors.Wrapf(err, "no credentials") } - // an error is only provided if something went wrong while determining - // the credentials. Delivering NO credentials is a valid result. if creds == nil { return fmt.Errorf("no credentials found") } fmt.Printf("credentials: %s\n", obfuscate(creds.Properties())) + // --- end get credentials --- // Now we can continue with our basic component version composition // from the last example, or we just display the content. + // --- begin add version --- if create { // now we create a component version in this repository. err = addVersion(repo, "acme.org/example03", "v0.1.0") @@ -140,10 +173,12 @@ func UsingCredentialsB(cfg *helper.Config, create bool) error { return err } } + // --- end add version --- // list the versions as known from example 1 // OCI registries do not support component listers, therefore we - // just list the actually added version. + // just get and describe the actually added version. + // --- begin show version --- cv, err := repo.LookupComponentVersion("acme.org/example03", "v0.1.0") if err != nil { return errors.Wrapf(err, "added version not found") @@ -154,12 +189,14 @@ func UsingCredentialsB(cfg *helper.Config, create bool) error { if err != nil { return errors.Wrapf(err, "describe failed") } + // --- end show version --- - // as you have seen in the resource list, out image artifact has been + // as you have seen in the resource list, our image artifact has been // uploaded to the OCI registry and the access method has be changed // to ociArtifact. // It is not longer a local blob. + // --- begin examine cli --- res, err := cv.GetResourcesByName("ocmcli") if err != nil { return errors.Wrapf(err, "accessing ocmcli resource") @@ -174,20 +211,30 @@ func UsingCredentialsB(cfg *helper.Config, create bool) error { defer meth.Close() fmt.Printf("accessing oci image now with %s\n", meth.AccessSpec().Describe(ctx)) + // --- end examine cli --- - // this resource access points effectively to the same repository. + // this resource access effectively points to the ame OCI registry, + // but a completely different repository. // If you are using ghcr.io, this freshly created repo is private, // therefore, you need credentials for accessing the content. - // Because the credentials context now knows the required credentials, - // the access method as credential consumer can access the blob. + // An access method also acts as credential consumer, which + // tries to get required credentials from the credential context. + // Optionally, an access method can act as provider for a consumer id, so that + // it is possible to query the used consumer id from the method object. + // --- begin image credentials --- id = credentials.GetProvidedConsumerId(meth, credentials.StringUsageContext("acme.org/example3")) if id == nil { fmt.Printf("no consumer id info for access method\n") } else { fmt.Printf("usage context: %s\n", id) } + // --- end image credentials --- + + // Because the credentials context now knows the required credentials, + // the access method as credential consumer can access the blob. + // --- begin image access --- writer, err := os.OpenFile("/tmp/example3", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { return errors.Wrapf(err, "cannot write output file") @@ -204,5 +251,6 @@ func UsingCredentialsB(cfg *helper.Config, create bool) error { return errors.Wrapf(err, "cannot copy content") } fmt.Printf("blob has %d bytes\n", n) + // --- end image access --- return nil } diff --git a/examples/lib/tour/03-working-with-credentials/03-credential-repositories.go b/examples/lib/tour/03-working-with-credentials/03-credential-repositories.go index 26a1c3a66e..ec1fb57b97 100644 --- a/examples/lib/tour/03-working-with-credentials/03-credential-repositories.go +++ b/examples/lib/tour/03-working-with-credentials/03-credential-repositories.go @@ -17,8 +17,10 @@ import ( ) func UsingCredentialsRepositories(cfg *helper.Config) error { + // --- begin context --- ctx := ocm.DefaultContext() credctx := ctx.CredentialsContext() + // --- end context --- // The OCM toolset embraces multiple storage // backend technologies, for OCM meta data as well @@ -29,50 +31,68 @@ func UsingCredentialsRepositories(cfg *helper.Config) error { // // The credential management provides so-called // credential repositories. Such a repository - // is able to provide any number of names + // is able to provide any number of named // credential sets. This way any special // credential store can be connected to the - // OCM credential management jsu by providing + // OCM credential management just by providing // an own implementation for the repository interface. // One such case is the docker config json, a config // file used by docker login to store - // credentials for dedicatd OCI regsitries. + // credentials for dedicated OCI registries. + + // --- begin docker config --- dspec := dockerconfig.NewRepositorySpec("~/.docker/config.json") + // --- end docker config --- // There are general credential stores, like a HashiCorp Vault // or type-specific ones, like the docker config json // used to configure credentials for the docker client. // (working with OCI registries). // Those specialized repository implementation are not only able to - // provide credential sets, they also know about the usage context. - // Such repository implementations are able to provide credential - // mappings for consumer ids, also. + // provide credential sets, they also know about the usage context + // of the provided credentials + // Therefore, such repository implementations are also able to provide + // credential mappings for consumer ids. This is supported by the credential + // repository API provided by this library. // The docker config is such a case, so we can instruct the // repository to automatically propagate appropriate the consumer id - // mappings. + // mappings. This feature is typically enabled by a dedicated specfication + // option. + + // --- begin propagation --- dspec = dspec.WithConsumerPropagation(true) + // --- end propagation --- - // now we can jsut add the repository for this specification to - // the credential context. + // now we can just add the repository for this specification to + // the credential context by getting the repository object for our + // specification. + // --- begin add repo --- _, err := credctx.RepositoryForSpec(dspec) if err != nil { return errors.Wrapf(err, "invalid credential repository") } + // --- end add repo --- + // we are not interested in the repository object, so we just ignore // the result. // so, if you have done the appropriate docker login for your // OCI registry, it should be possible now to get the credentials // for the configured repository. + + // We first query the consumer id for the repository, again. + // --- begin get consumer id --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "invalid consumer") } + // --- end get consumer id --- - // the returned credentials are provided via an interface, which might change its - // content, if the underlying credential source changes. + // and then get the credentials from the credentials context + // like in the previous example. + // --- begin get credentials --- creds, err := credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) if err != nil { return errors.Wrapf(err, "no credentials") @@ -83,6 +103,6 @@ func UsingCredentialsRepositories(cfg *helper.Config) error { return fmt.Errorf("no credentials found") } fmt.Printf("credentials: %s\n", obfuscate(creds.Properties())) - + // --- end get credentials --- return nil } diff --git a/examples/lib/tour/03-working-with-credentials/README.md b/examples/lib/tour/03-working-with-credentials/README.md index 56407a1b75..cfeb572121 100644 --- a/examples/lib/tour/03-working-with-credentials/README.md +++ b/examples/lib/tour/03-working-with-credentials/README.md @@ -1,21 +1,474 @@ + + + # Working with Credentials This tour illustrates the basic handling of credentials using the OCM library. The library provides an extensible framework to bring together credential providers -and credential cosunmers in a technology-agnostic way. +and credential consunmers in a technology-agnostic way. It covers four basic scenarios: - [`basic`](01-using-credentials.go) Writing to a repository with directly specified credentials. -- [`generic`](02-basic-credential-management.go) Using credentials via the credential management. -- [`read`](02-basic-credential-management.go) Read the previously component version using the credential management. +- [`context`](02-basic-credential-management.go) Using credentials via the credential management to publish a component version. +- [`read`](02-basic-credential-management.go) Read the previously created component version using the credential management. - [`credrepo`](03-credential-repositories.go) Providing credentials via credential repositories. -You can just call the main program with some config file option (`--config `) and the name of the scenario. -The config file should have the following content: +## Running the example + +You can call the main program with a config file option (`--config `) and the name of the scenario. +The config file should have content similar to: ```yaml repository: ghcr.io/mandelsoft/ocm username: password: -``` \ No newline at end of file +``` + +Set your favorite OCI registry and don't forget to add the repository prefix for your OCM repository hosted in this registry. + +## Walkthrough + +### Writing to a repository with directly specified credentials. + +As usual, we start with getting access to an OCM context +object. + +```go + ctx := ocm.DefaultContext() +``` + +So far, we just used memory or file system based +OCM repositories to create component versions. +If we want to store something in a remotely accessible +repository typically some credentials are required +for write access. + +The OCM library uses a generic abstraction for credentials. +It is just set of properties. To offer various credential sources +there is an interface `credentials.Credentials` provided, +whose implementations provide access to those properties. +A simple property based implementation is `credentials.DirectCredentials. + + +The most simple use case is to provide the credentials +directly for the repository access creation. +The example config file provides such credentials +for an OCI registry. + +```go + creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) +``` + +Now, we can use the OCI repository access creation from the [first tour](../01-getting-started/README.md#walkthrough), +but we pass the credentials as additional parameter. +To give you the chance to specify your own registry, the URL +is taken from the config file. + +```go + spec := ocireg.NewRepositorySpec(cfg.Repository, nil) + + repo, err := ctx.RepositoryForSpec(spec, creds) + if err != nil { + return err + } + defer repo.Close() +``` + +If registry name and credentials are fine, we should be able +now to add a new component version to this repository using the coding +from the previous examples, but now we use a public repository, instead +of a memory or file system based one. This coding is in function `addVersion` +in `common.go` (It is shared by the other examples, also). + +```go + cv, err := repo.NewComponentVersion(name, version) + if err != nil { + return errors.Wrapf(err, "cannot create new version") + } + defer cv.Close() + + err = setupVersion(cv) + if err != nil { + return errors.Wrapf(err, "cannot setup new version") + } + + // finally, wee add the new version to the repository. + fmt.Printf("adding component version\n") + err = repo.AddComponentVersion(cv) + if err != nil { + return errors.Wrapf(err, "cannot save version") + } +``` + +In contrast to our [first tour](../01-getting-started/README.md#walkthrough) +we cannot list components, here. +OCI registries do not support component listers, therefore we +just look up the actually added version to verify the result. + +```go + cv, err := repo.LookupComponentVersion("acme.org/example03", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "added version not found") + } + defer cv.Close() + return errors.Wrapf(describeVersion(cv), "describe failed") +``` + +The coding for `describeVersion` is similar to the one shown in the [first tour](../01-getting-started/README.md#describe-version). + +### Using the Credential Management + +Passing credentials directly at the repository +is fine, as long only the component version +will be accessed. But as soon as described +resource content will be read, the required +credentials and credential types are dependent +on the concrete component version, because +it might contain any kind of access method +referring to any kind of resource repository +type. + +To solve this problem of passing any set +of credentials the OCM context object is +used to store credentials. This is handled +by a sub context, the *Credentials context*. + +As usual, we start with the default OCM context. + +```go + ctx := ocm.DefaultContext() +``` + +It is now used to gain access to the appropriate +credential context. + + +```go + credctx := ctx.CredentialsContext() +``` + +The credentials context brings together +providers of credentials, for example a +Vault or a local Docker config.json +and credential consumers like GitHub or +OCI registries. +It must be able to distinguish various kinds +of consumers. This is done by identifying +a dedicated consumer with a set of properties +called `credentials.ConsumerId`. It consists +at least of a consumer type property and a +consumer type specific set of properties +describing the concrete instance of such +a consumer, for example an OCI artifact in +an OCI registry is identified by a host and +a repository path. + +A credential provider like a vault just provides +named credential sets and typically does not +know anything about the use case for these sets. +The task of the credential context is to +provide credentials for a dedicated consumer. +Therefore, it maintains a configurable +mapping of credential sources (credentials in +a credential repository) and a dedicated consumer. + +This mapping defines a use case, also based on +a property set and dedicated credentials. +If credentials are required for a dedicated +consumer, it matches the defined mappings and +returned the best matching entry. + +Matching? Let's take the GitHub OCI registry as an +example. There are different owners for +different repository paths (the GitHub org/user). +Therefore, different credentials need to be provided +for different repository paths. +For example, credentials for `ghcr.io/acme` can be used +for a repository `ghcr.io/acme/ocm/myimage`. + +To start with the credentials context we just +provide an explicit mapping for our use case. + +First, we create our credentials object as before. + +```go + creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) +``` + +Then we determine the consumer id for our use case. +The repository implementation provides a function +for this task. It provides the most general property +set (no repository path) for an OCI based OCM repository. + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } +``` + +The used functions above are just convenience wrappers +around the core type ConsumerId, which might be provided +for dedicated repository/consumer technologies. +Everything can be done directly with the core interface +and property name constants provided by the dedicted technologies. + +Once we have the id we can finally set the credentials for this +id. + +```go + credctx.SetCredentialsForConsumer(id, creds) +``` + +Now, the context is prepared to provide credentials +for any usage of our OCI registry +Let's test, whether it could provide credentials +for storing our component version. + +First, we get the repository object for our OCM repository. + +```go + spec := ocireg.NewRepositorySpec(cfg.Repository, nil) + repo, err := ctx.RepositoryForSpec(spec, creds) + if err != nil { + return err + } + defer repo.Close() +``` + +Second, we determine the consumer id for our intended repository acccess. +A credential consumer may provide consumer id information +for a dedicated sub user context. +This is supported by the OCM repo implementation for OCI registries. +The usage context is here the component name. + +```go + id = credentials.GetProvidedConsumerId(repo, credentials.StringUsageContext("acme.org/example03")) + if id == nil { + return fmt.Errorf("repository does not support consumer id queries") + } + fmt.Printf("usage context: %s\n", id) +``` + +Third, we ask the credential context for appropriate credentials. +The basic context method `credctx.GetCredentialsForConsumer` returns +a credentials source interface able to provide credentials +for a changing credentials source. Here, we use a convenience +function, which directly provides a credentials interface for the +actually valid credentials. +An error is only provided if something went wrong while determining +the credentials. Delivering NO credentials is a valid result. +The returned interface then offers access to the credential properties. +via various methods. + +```go + creds, err = credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) + if err != nil { + return errors.Wrapf(err, "no credentials") + } + if creds == nil { + return fmt.Errorf("no credentials found") + } + fmt.Printf("credentials: %s\n", obfuscate(creds.Properties())) +``` + +Now, we can continue with our basic component version composition +from the last example, or we just display the content. + +The following code snipped shows the code for the `context` variant +creating a new version, the `read` variant just omits the version creation. +The rest of the example is identical. + +```go + if create { + // now we create a component version in this repository. + err = addVersion(repo, "acme.org/example03", "v0.1.0") + if err != nil { + return err + } + } +``` + +Let's verify the created content and list the versions as known from tour 1. +OCI registries do not support component listers, therefore we +just get and describe the actually added version. + +```go + cv, err := repo.LookupComponentVersion("acme.org/example03", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "added version not found") + } + defer cv.Close() + + err = describeVersion(cv) + if err != nil { + return errors.Wrapf(err, "describe failed") + } +``` + +As we can see in the resource list, our image artifact has been +uploaded to the OCI registry as OCI artifact and the access method has be changed +to `ociArtifact`. It is not longer a local blob. + +```go + res, err := cv.GetResourcesByName("ocmcli") + if err != nil { + return errors.Wrapf(err, "accessing ocmcli resource") + } + if len(res) != 1 { + return fmt.Errorf("oops, there are %d entries for ocmcli", len(res)) + } + meth, err := res[0].AccessMethod() + if err != nil { + return errors.Wrapf(err, "cannot get access method") + } + defer meth.Close() + + fmt.Printf("accessing oci image now with %s\n", meth.AccessSpec().Describe(ctx)) +``` + +This resource access effectively points to the same OCI registry, +but a completely different repository. +If you are using *ghcr.io*, this freshly created repo is private, +therefore, we need credentials for accessing the content. +An access method also acts as credential consumer, which +tries to get required credentials from the credential context. +Optionally, an access method can act as provider for a consumer id, so that +it is possible to query the used consumer id from the method object. + +```go + id = credentials.GetProvidedConsumerId(meth, credentials.StringUsageContext("acme.org/example3")) + if id == nil { + fmt.Printf("no consumer id info for access method\n") + } else { + fmt.Printf("usage context: %s\n", id) + } +``` + +Because the credentials context now knows the required credentials, +the access method as credential consumer can access the blob. + +```go + writer, err := os.OpenFile("/tmp/example3", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return errors.Wrapf(err, "cannot write output file") + } + defer writer.Name() + + reader, err := meth.Reader() + if err != nil { + return errors.Wrapf(err, "cannot get reader") + } + defer reader.Close() + n, err := io.Copy(writer, reader) + if err != nil { + return errors.Wrapf(err, "cannot copy content") + } + fmt.Printf("blob has %d bytes\n", n) +``` + +### Providing credentials via credential repositories + +The OCM toolset embraces multiple storage +backend technologies, for OCM metadata as well +as for artifacts described by a component version. +All those technologies typically have their own +way to configure credentials for command line +tools or servers. + +The credential management provides so-called +credential repositories. Such a repository +is able to provide any number of named +credential sets. This way any special +credential store can be connected to the +OCM credential management just by providing +an own implementation for the repository interface. + +One such case is the docker config json, a config +file used by docker login to store +credentials for dedicated OCI registries. + +We start again by providing access to the +OCM context and the connected credential context. + + +```go + ctx := ocm.DefaultContext() + credctx := ctx.CredentialsContext() +``` + +In package `.../contexts/credentials/repositories` you can find +packages for predefined implementations for some standard credential repositories, +for example `dockerconfig`. + +```go + dspec := dockerconfig.NewRepositorySpec("~/.docker/config.json") +``` + +There are general credential stores, like a HashiCorp Vault +or type-specific ones, like the docker config json +used to configure credentials for the docker client. +(working with OCI registries). +Those specialized repository implementations are not only able to +provide credential sets, they also know about the usage context +of the provided credentials. +Therefore, such repository implementations are also able to provide +credential mappings for consumer ids. This is supported by the credential +repository API provided by this library. + +The docker config is such a case, so we can instruct the +repository to automatically propagate appropriate the consumer id +mappings. This feature is typically enabled by a dedicated specfication +option. + +```go + dspec = dspec.WithConsumerPropagation(true) +``` + +Implementations for more generic credential repositories can also use this +feature, if the repository allows adding arbitrary metadata. This is for +example used by the `vault` implementation. It uses dedicated attributes +to allow the user to configure intended consumer id properties. + +Now, we can just add the repository for this specification to +the credential context by getting the repository object for our +specification. + +```go + _, err := credctx.RepositoryForSpec(dspec) + if err != nil { + return errors.Wrapf(err, "invalid credential repository") + } +``` + +We are not interested in the repository object, so we just ignore +the result. + +So, if you have done the appropriate docker login for your +OCI registry, it should be possible now to get the credentials +for the configured repository. + +We first query the consumer id for the repository, again. + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } +``` + +and then get the credentials from the credentials context like in the previous example. + +```go + creds, err := credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) + if err != nil { + return errors.Wrapf(err, "no credentials") + } + // an error is only provided if something went wrong while determining + // the credentials. Delivering NO credentials is a valid result. + if creds == nil { + return fmt.Errorf("no credentials found") + } + fmt.Printf("credentials: %s\n", obfuscate(creds.Properties())) +``` diff --git a/examples/lib/tour/03-working-with-credentials/common.go b/examples/lib/tour/03-working-with-credentials/common.go index 4846450c8c..503fa71a83 100644 --- a/examples/lib/tour/03-working-with-credentials/common.go +++ b/examples/lib/tour/03-working-with-credentials/common.go @@ -105,7 +105,7 @@ data: some very important data required to understand this component // Any blob content provided by an implementation of this // interface can be added as resource. // There are various access implementations for blobs - // taken from the local host, for example, from the filesystem, + // taken from the local host, for example, from the file system, // or from other repositories (for example by mapping // an access type specification into a blob access). // The most simple form is to directly provide a byte sequence, @@ -147,7 +147,7 @@ data: some very important data required to understand this component } // There are even more complex blob sources, for example - // for helm charts stored in the filesystem, or even for images + // for Helm charts stored in the file system, or even for images // generated by docker builds. // Here, we just compose a multi-platform image built with buildx // from these sources (components/ocmcli) featuring two flavors. @@ -184,6 +184,7 @@ func addVersion(repo ocm.Repository, name, version string) error { // now we compose a new component version, first we create // a new version backed by this repository. + // --- begin create version --- cv, err := repo.NewComponentVersion(name, version) if err != nil { return errors.Wrapf(err, "cannot create new version") @@ -201,6 +202,7 @@ func addVersion(repo ocm.Repository, name, version string) error { if err != nil { return errors.Wrapf(err, "cannot save version") } + // --- end create version --- return nil } diff --git a/examples/lib/tour/03-working-with-credentials/main.go b/examples/lib/tour/03-working-with-credentials/main.go index 7abd06c7e0..a705fa1167 100644 --- a/examples/lib/tour/03-working-with-credentials/main.go +++ b/examples/lib/tour/03-working-with-credentials/main.go @@ -13,7 +13,7 @@ import ( ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" var current_version string diff --git a/examples/lib/tour/04-working-with-config/01-basic-config-management.go b/examples/lib/tour/04-working-with-config/01-basic-config-management.go index bea3a74d7c..6e8d66533e 100644 --- a/examples/lib/tour/04-working-with-config/01-basic-config-management.go +++ b/examples/lib/tour/04-working-with-config/01-basic-config-management.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/go-test/deep" + "github.com/open-component-model/ocm/examples/lib/helper" "github.com/open-component-model/ocm/pkg/contexts/config" "github.com/open-component-model/ocm/pkg/contexts/credentials" @@ -20,19 +21,29 @@ import ( func BasicConfigurationHandling(cfg *helper.Config) error { // configuration is handled by the configuration context. + // --- begin default context --- ctx := config.DefaultContext() + // --- end default context --- // the configuration context handles configuration objects. // a configuration object is any object implementing - // the config.Config interface. + // the config.Config interface. The task of a config object + // is to apply configuration to some target object. // one such object is the configuration object for // credentials. + // It finally applies settings to a credential context. + // --- begin cred config --- creds := credcfg.New() - - // here we can configure credential settings: - // credential repositories and consumer is mappings. + // --- end cred config --- + + // here, we can configure credential settings: + // credential repositories and consumer id mappings. + // We do this by setting the credentials provided + // by our config file for the consumer id used + // by our configured OCI registry. + // --- begin configure creds --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "invalid consumer") @@ -41,21 +52,25 @@ func BasicConfigurationHandling(cfg *helper.Config) error { id, directcreds.NewRepositorySpec(cfg.GetCredentials().Properties()), ) + // --- end configure creds --- - // credential objects are typically serializable and deserializable. + // configuration objects are typically serializable and deserializable. + // --- begin marshal --- spec, err := json.MarshalIndent(creds, " ", " ") if err != nil { return errors.Wrapf(err, "marshal credential config") } fmt.Printf("this a a credential configuration object:\n%s\n", string(spec)) + // --- end marshal --- - // like all the other maifest based description this format always includes + // like all the other manifest based descriptions this format always includes // a type field, which can be used to deserialize a specification into // the appropriate object. - // This can ebe done by the config context. It accepts YAML or JSON. + // This can be done by the config context. It accepts YAML or JSON. + // --- begin unmarshal --- o, err := ctx.GetConfigForData(spec, nil) if err != nil { return errors.Wrapf(err, "deserialize config") @@ -65,13 +80,16 @@ func BasicConfigurationHandling(cfg *helper.Config) error { fmt.Printf("diff:\n%v\n", diff) return fmt.Errorf("invalid des/erialization") } + // --- end unmarshal --- // regardless what variant is used (direct object or descriptor) // the config object can be added to a config context. + // --- begin apply config --- err = ctx.ApplyConfig(creds, "explicit cred setting") if err != nil { return errors.Wrapf(err, "cannot apply config") } + // --- end apply config --- // Every config object implements the // ApplyTo(ctx config.Context, target interface{}) error method. @@ -85,13 +103,14 @@ func BasicConfigurationHandling(cfg *helper.Config) error { // an object, which wants to be configured calls the config // context to apply pending configs. // The config context manages a queue of config objects - // and applys them to an object to be configured. + // and applies them to an object to be configured. - // If ask he credential context now for credentials, + // If the credential context is asked now for credentials, // it asks the config context for pending config objects - // and apply them. - // Theregore, we now should the configured creentials, here. + // and applies them. + // Therefore, we now should be able to get the configured credentials. + // --- begin get credentials --- credctx := credentials.DefaultContext() found, err := credentials.CredentialsForConsumer(credctx, id) @@ -112,5 +131,6 @@ func BasicConfigurationHandling(cfg *helper.Config) error { if found.GetProperty(credentials.ATTR_PASSWORD) != cfg.Password { return fmt.Errorf("password mismatch") } + // --- end get credentials --- return nil } diff --git a/examples/lib/tour/04-working-with-config/02-handle-arbitrary-config.go b/examples/lib/tour/04-working-with-config/02-handle-arbitrary-config.go index e130d31f5b..c3c7864f7e 100644 --- a/examples/lib/tour/04-working-with-config/02-handle-arbitrary-config.go +++ b/examples/lib/tour/04-working-with-config/02-handle-arbitrary-config.go @@ -38,37 +38,62 @@ func HandleArbitraryConfiguration(cfg *helper.Config) error { // The configuration management provides a configuration object // for it own. + // --- begin config config --- generic := configcfg.New() + // --- end config config --- - // the generic config holds a list of config objects, + // the generic config holds a list of any other config objects, // or their specification formats. - // Additionally, it is possible to configure names sets + // Additionally, it is possible to configure named sets // of configurations, which can later be enabled - // on-demand at the config context. + // on-demand by their name at the config context. // we recycle our credential config from the last example. + // --- begin sub config --- creds, err := credConfig(cfg) if err != nil { return err } + // --- end sub config --- + + // now, we can add this credential config object to + // our generic config list. + // --- begin add config --- err = generic.AddConfig(creds) if err != nil { return errors.Wrapf(err, "adding config") } + // --- end add config --- - // credential objects are typically serializable and deserializable. + // as we have seen in the previous example, config objects are typically + // serializable and deserializable. // this also holds for the generic config object of the config context. + // --- begin serialized --- spec, err := json.MarshalIndent(generic, " ", " ") if err != nil { return errors.Wrapf(err, "marshal credential config") } + fmt.Printf("this a a generic configuration object:\n%s\n", string(spec)) + // --- end serialized --- // the result is a config object hosting a list (with 1 entry) // of other config object specifications. - fmt.Printf("this a a generic configuration object:\n%s\n", string(spec)) - // the generic config object can be added to a config context, again. + // The generic config object can be added to a config context, again, like + // any other config object. If it is asked to configure a configuration + // context it uses the methods of the configuration context to apply the + // contained list of config objects (and the named set of config lists). + // Therefore, all config objects applied to a configuration context are + // asked to configure the configuration context itself when queued to the + // list of applied configuration objects. + + // If we now ask the default credential context (which uses the default + // configuration context to configure itself) for credentials for our OCI registry, + // the credential mapping provided by the config object added to the generic one, + // will be found. + + // --- begin query --- ctx := config.DefaultContext() err = ctx.ApplyConfig(creds, "generic setting") if err != nil { @@ -87,5 +112,6 @@ func HandleArbitraryConfiguration(cfg *helper.Config) error { } fmt.Printf("consumer id: %s\n", id) fmt.Printf("credentials: %s\n", obfuscate(found)) + // --- end query --- return nil } diff --git a/examples/lib/tour/04-working-with-config/03-using-ocm-config.go b/examples/lib/tour/04-working-with-config/03-using-ocm-config.go index f6e4872ee1..fe1b58e955 100644 --- a/examples/lib/tour/04-working-with-config/03-using-ocm-config.go +++ b/examples/lib/tour/04-working-with-config/03-using-ocm-config.go @@ -27,37 +27,47 @@ func HandleOCMConfig(cfg *helper.Config) error { // library function, which can be used to configure an OCM // context and all related other contexts with a single call // based on a central configuration file (~/.ocmconfig) + + // --- begin central config --- ctx := ocm.DefaultContext() _, err := utils.Configure(ctx, "") if err != nil { return errors.Wrapf(err, "configuration") } + // --- end central config --- - // It is typically such a generic configuration specification, + // This file typically contains the serialization of such a generic + // configuration specification (or any other serialized configuration object), // enriched with specialized config specifications for - // credentials, default repositories signing keys and any + // credentials, default repositories, signing keys and any // other configuration specification. + // // Most important are here the credentials. - // Because OCM embraces lots of storage technologies - // for artifact storage as well as storing OCM meta data, - // tzere are typically multiple technology specific ways + // Because OCM embraces lots of storage technologies for artifact + // storage as well as storing OCM component version metadata, + // there are typically multiple technology specific ways // to configure credentials for command line tools. - // Using the credentials settings shown in the previous examples, - // it ius possible to specify credentials for all - // required purposes, but the configuration mangement provides + // Using the credentials settings shown in the previous tour, + // it is possible to specify credentials for all + // required purposes, and the configuration management provides // an extensible way to embed native technology specific ways // to provide credentials just by adding an appropriate type - // of config objects, which reads the specialized stoarge and - // feeds it into the credential context. + // of credential repository, which reads the specialized storage and + // feeds it into the credential context. Those specifications + // can be added via the credential configuration object to + // the central configuration. // - // One such config object type is the docker config type. It - // reads a dockerconfig.json file and fed in the credentials. - // because it is sed for a dedicated purpose (credentials for + // One such repository type is the Docker config type. It + // reads a `dockerconfig.json` file and feeds in the credentials. + // Because it is used for a dedicated purpose (credentials for // OCI registries), it not only can feed the credentials, but // also their mapping to consumer ids. - // create the specification for a new credential repository of - // type dockerconfig. + // We first create the specification for a new credential repository of + // type `dockerconfig` describing the default location + // of the standard Docker config file. + + // --- begin docker config --- credspec := dockerconfig.NewRepositorySpec("~/.docker/config.json", true) // add this repository specification to a credential configuration. @@ -66,13 +76,18 @@ func HandleOCMConfig(cfg *helper.Config) error { if err != nil { return errors.Wrapf(err, "invalid credential config") } + // --- end docker config --- - // By adding the default location for the standard docker config - // file, all credentials provided by the docker login + // By adding the default location for the standard Docker config + // file, all credentials provided by the `docker login` command // are available in the OCM toolset, also. // A typical minimal .ocmconfig file can be composed as follows. + // We add this config object to an empty generic configuration object + // and print the serialized form. The result can be used as + // default initial OCM configuration file. + // --- begin default config --- ocmcfg := configcfg.New() err = ocmcfg.AddConfig(ccfg) @@ -84,19 +99,23 @@ func HandleOCMConfig(cfg *helper.Config) error { // the result is a typical minimal ocm configuration file // just providing the credentials configured with // doicker login. - fmt.Printf("this a typical ocm config file:\n%s\n", string(spec)) + fmt.Printf("this a typical ocm config file:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(spec)) + // --- end default config --- // Besides from a file, such a config can be provided as data, also, // taken from any other source, for example from a Kubernetes secret + // --- begin by data --- err = utils.ConfigureByData(ctx, spec, "from data") if err != nil { return errors.Wrapf(err, "configuration") } + // --- end by data --- // If you have provided your OCI credentials with - // docker login, they should now be available. + // `docker login`, they should now be available. + // --- begin query --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "invalid consumer") @@ -107,5 +126,35 @@ func HandleOCMConfig(cfg *helper.Config) error { } fmt.Printf("consumer id: %s\n", id) fmt.Printf("credentials: %s\n", obfuscate(found)) + // --- end query --- + + // the configuration library function does not only read the + // ocm config file, it also applies [*spiff*](github.com/mandelsoft/spiff) + // processing to the provided YAML/JSON content. *Spiff* is an + // in-domain yaml-based templating engine. Therefore, you can use + // any spiff dynaml expression to define values or even complete + // sub structures. + + // --- begin spiff --- + ocmcfg = configcfg.New() + ccfg = credcfg.New() + cspec := credentials.CredentialsSpecFromList("clientCert", `(( read("~/ocm/keys/myClientCert.pem") ))`) + id = credentials.NewConsumerIdentity("ApplicationServer.acme.org", "hostname", "app.acme.org") + ccfg.AddConsumer(id, cspec) + ocmcfg.AddConfig(ccfg) + // --- end spiff --- + + spec, err = yaml.Marshal(ocmcfg) + if err != nil { + return errors.Wrapf(err, "marshal ocm config") + } + fmt.Printf("this a typical ocm config file using spiff file operations:\n--- begin spiffocmconfig ---\n%s--- end spiffocmconfig ---\n", string(spec)) + + // this config object is not directly usable, because the cert value is not + // a valid certificate. We use it here just to generate the serialized form. + // if this is used with the above library functions, the finally generated + // config object will contain the read file content, which is hopefully a + // valid certificate. + return nil } diff --git a/examples/lib/tour/04-working-with-config/04-write-config-type.go b/examples/lib/tour/04-working-with-config/04-write-config-type.go index 931dab7e75..7e7404e21a 100644 --- a/examples/lib/tour/04-working-with-config/04-write-config-type.go +++ b/examples/lib/tour/04-working-with-config/04-write-config-type.go @@ -5,6 +5,7 @@ package main import ( + "encoding/json" "fmt" "github.com/open-component-model/ocm/examples/lib/helper" @@ -13,6 +14,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials" ociidentity "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/runtime" "sigs.k8s.io/yaml" @@ -21,19 +23,28 @@ import ( // TYPE is the name of our new configuration object type. // To be globally unique, it should always end with a // DNS domain owned by the provider of the new type. +// --- begin type name --- const TYPE = "example.config.acme.org" -// ExampleConfigSpec is a new type of config specification +// --- end type name --- + +// ExampleConfigSpec is the new Go type for the config specification // covering our example configuration. +// It just encapsulates our simple configuration structure +// used to configure the examples of our tour. +// --- begin config type --- type ExampleConfigSpec struct { // ObjectVersionedType is the base type providing the type feature - // form config specifications. + // for (config) specifications. runtime.ObjectVersionedType `json:",inline"` // Config is our example config representation. helper.Config `json:",inline"` } +// --- end config type --- + // NewConfig provides a config object for out helper configuration. +// --- begin constructor --- func NewConfig(cfg *helper.Config) cpi.Config { return &ExampleConfigSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(TYPE), @@ -41,13 +52,49 @@ func NewConfig(cfg *helper.Config) cpi.Config { } } +// --- end constructor --- + +// additional setters can be used to configure the configuration object. +// Here, programmatic objects (like an ocm.RepositorySpec) are +// converted to a form storable in the configuration object. +// --- begin setters --- + +// SetTargetRepository takes a repository specification +// and adds its serialized form to the config object. +func (c *ExampleConfigSpec) SetTargetRepository(target ocm.RepositorySpec) error { + data, err := json.Marshal(target) + if err != nil { + return err + } + c.Target = data + return nil +} + +// SetTargetRepositoryData sets the target repository specification +// from a byte sequence. +func (c *ExampleConfigSpec) SetTargetRepositoryData(data []byte) error { + err := runtime.CheckSpecification(data) + if err != nil { + return err + } + c.Target = data + return nil +} + +// --- end setters --- + +// --- begin config interface --- + // RepositoryTarget consumes a repository name. type RepositoryTarget interface { SetRepository(r string) } +// --- end config interface --- + // ApplyTo is used to apply the provided configuration settings // to a dedicated object, which wants to be configured. +// --- begin method apply ---. func (c *ExampleConfigSpec) ApplyTo(_ cpi.Context, tgt interface{}) error { switch t := tgt.(type) { @@ -80,6 +127,20 @@ func (c *ExampleConfigSpec) ApplyTo(_ cpi.Context, tgt interface{}) error { return nil } +// --- end method apply --- + +// to enable automatic deserialization of our new config type, +// we have to tell the configuration management about our +// new type. This is done by a registration function, +// which gets called with a dedicated type object for +// the new config type. +// a type object describes the config type, its type name, how +// it is serialized and deserialized and some description. +// we use a standard type object, here, instead of implementing +// an own one. It is parameterized by the Go pointer type for +// our specification object. + +// --- begin init ---. func init() { // register the new config type, so that is can be used // by the config management to deserialize appropriately @@ -87,26 +148,37 @@ func init() { cpi.RegisterConfigType(cpi.NewConfigType[*ExampleConfigSpec](TYPE, "this ia config object type based on the example config data.")) } +// --- end init ---. + func WriteConfigType(cfg *helper.Config) error { - // after preparing aout new special config type + // after preparing a new special config type // we can feed it into the config management. + // because of the registration the config management + // now knows about this new type. + // A usual, we gain access to our required + // contexts. + // --- begin default context --- credctx := credentials.DefaultContext() // the credential context is based on a config context // used to configure it. ctx := credctx.ConfigContext() + // --- end default context --- - // create our new config based on the actual settings - // and apply it to the config context. + // to setup our environment we create our new config based on the actual + // settings and apply it to the config context. + // --- begin apply --- examplecfg := NewConfig(cfg) ctx.ApplyConfig(examplecfg, "special acme config") + // --- end apply --- + // If you omit the above call, no credentials // will be found later. - // _, _ = ctx, examplecfg // now we should be prepared to get the credentials + // --- begin query credentials --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "cannot get consumer id") @@ -120,12 +192,14 @@ func WriteConfigType(cfg *helper.Config) error { return errors.Wrapf(err, "credentials") } fmt.Printf("credentials: %s\n", obfuscate(creds)) + // --- end query credentials --- // Because of the new credential type, such a specification can // now be added to the ocm config, also. // So, we could use our special tour config file content // directly as part of the ocm config. + // --- begin in ocmconfig --- ocmcfg := configcfg.New() err = ocmcfg.AddConfig(examplecfg) @@ -136,11 +210,14 @@ func WriteConfigType(cfg *helper.Config) error { // the result is a minimal ocm configuration file // just providing our new example configuration. - fmt.Printf("this a typical ocm config file:\n%s\n", string(spec)) + fmt.Printf("this a typical ocm config file:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(spec)) + // --- end in ocmconfig --- // above, we added a new kind of target, the RepositoryTarget interface. // Just by providing an implementation for this interface, we can // configure such an object using the config management. + + // --- begin apply interface --- target := &SimpleRepositoryTarget{} _, err = ctx.ApplyTo(0, target) @@ -148,6 +225,7 @@ func WriteConfigType(cfg *helper.Config) error { return errors.Wrapf(err, "applying to new target") } fmt.Printf("repository for target: %s\n", target.repository) + // --- end apply interface --- // This way any specialized configuration object can be added // by a user of the OCM library. It can be used to configure @@ -158,10 +236,14 @@ func WriteConfigType(cfg *helper.Config) error { // to be configured and which autoconfigure themselves when // used. Our simple repository target is just an example // for some kind of ad-hoc configuration. - // This is shown in the next example. + // a complete scenario is shown in the next example. return nil } +// --- begin demo target --- + +// SimpleRepositoryTarget is demo target object +// just implementing our new configuration interface. type SimpleRepositoryTarget struct { repository string } @@ -171,3 +253,5 @@ var _ RepositoryTarget = (*SimpleRepositoryTarget)(nil) func (t *SimpleRepositoryTarget) SetRepository(repo string) { t.repository = repo } + +// --- end demo target --- diff --git a/examples/lib/tour/04-working-with-config/05-write-config-consumer.go b/examples/lib/tour/04-working-with-config/05-write-config-consumer.go index af95a8d29f..8bfe5958cd 100644 --- a/examples/lib/tour/04-working-with-config/05-write-config-consumer.go +++ b/examples/lib/tour/04-working-with-config/05-write-config-consumer.go @@ -24,18 +24,22 @@ import ( // able to provide an OCI repository reference. // It has a setter and a getter (the setter is // provided by our ad-hoc SimpleRepositoryTarget). +// --- begin type --- type RepositoryProvider struct { lock sync.Mutex - // updater is a utility, which ia able to - // configure an object basesd a a managed configuration + // cpi.Updater is a utility, which is able to + // configure an object based on a managed configuration // watermark. It remembers which config objects from the - // config queue are already applies, and replays + // config queue are already applied, and replays // the config objects applied to the config context // after the last update. updater cpi.Updater SimpleRepositoryTarget } +// --- end type --- + +// --- begin constructor --- func NewRepositoryProvider(ctx cpi.ContextProvider) *RepositoryProvider { p := &RepositoryProvider{} // To do its work, the updater needs a connection to @@ -45,22 +49,31 @@ func NewRepositoryProvider(ctx cpi.ContextProvider) *RepositoryProvider { return p } +// --- end constructor --- + +// the magic now happens in the methods provided +// by our configurable object. +// the first step for methods of configurable objects +// dependent on potential configuration is always +// to update itself using the embedded updater. +// +// Please note, the config management reverses the +// request direction. Applying a config object to +// the config context does not configure dependent objects, +// it just manages a config queue, which is used by potential +// configuration targets to configure themselves. +// The actual configuration action is always initiated +// by the object, which wants to be configured. +// The reason for this is to avoid references from the +// management to managed objects. This would prohibit +// the garbage collection of all configurable objects. + // GetRepository returns a repository ref. +// --- begin method --- func (p *RepositoryProvider) GetRepository() (string, error) { p.lock.Lock() defer p.lock.Unlock() - // the first step for methods of configurable objects - // dependent on potential configuration is always - // to update itself using the embedded updater. - // Please remember, the config management reverses the - // request direction. Applying a config object to - // the config context does not configure dependent objects, - // it just manages a config queue, which is used by potential - // configuration targets to configure themselves. - // The reason for this is to avoid references from the - // management to managed objects. This would prohibit - // the garbage collection of all configurable objects. err := p.updater.Update() if err != nil { return "", err @@ -70,15 +83,22 @@ func (p *RepositoryProvider) GetRepository() (string, error) { return p.repository, nil } +// --- end method --- + func WriteConfigTargets(cfg *helper.Config) error { + // --- begin default context --- credctx := credentials.DefaultContext() + // --- end default context --- // after defining or repository provider type // we can now use it. + // --- begin object --- prov := NewRepositoryProvider(credctx) + // --- end object --- // If we ask now for a repository we will get the empty // answer. + // --- begin initial query --- repo, err := prov.GetRepository() if err != nil { errors.Wrapf(err, "get repo") @@ -86,17 +106,21 @@ func WriteConfigTargets(cfg *helper.Config) error { if repo != "" { return fmt.Errorf("Oops, found repository %q", repo) } + // --- end initial query --- // Now, we apply our config from the last example. + // --- begin apply config --- ctx := credctx.ConfigContext() examplecfg := NewConfig(cfg) err = ctx.ApplyConfig(examplecfg, "special acme config") if err != nil { errors.Wrapf(err, "apply config") } + // --- end apply config --- - // asking for a repository now will return the configured - // ref. + // without any further action, asking for a repository now will return the + // configured ref. + // --- begin query --- repo, err = prov.GetRepository() if err != nil { errors.Wrapf(err, "get repo") @@ -105,10 +129,12 @@ func WriteConfigTargets(cfg *helper.Config) error { return fmt.Errorf("no repository provided") } fmt.Printf("using repository: %s\n", repo) + // --- end query --- // now, we should also be prepared to get the credentials, // our config object configures the provider as well as // the credential context. + // --- begin credentials --- id, err := oci.GetConsumerIdForRef(repo) if err != nil { return errors.Wrapf(err, "cannot get consumer id") @@ -120,6 +146,6 @@ func WriteConfigTargets(cfg *helper.Config) error { return errors.Wrapf(err, "credentials") } fmt.Printf("credentials: %s\n", obfuscate(creds)) - + // --- end credentials --- return nil } diff --git a/examples/lib/tour/04-working-with-config/README.md b/examples/lib/tour/04-working-with-config/README.md index 48397d7179..2b156dc19a 100644 --- a/examples/lib/tour/04-working-with-config/README.md +++ b/examples/lib/tour/04-working-with-config/README.md @@ -1,3 +1,6 @@ + + + # Working with Configurations This tour illustrates the basic configuration management @@ -6,18 +9,933 @@ an extensible framework to bring together configuration settings and configurable objects. It covers five basic scenarios: -- [`basic`](01-basic-config-management.go) Basic configuration management illustarting the configuration of credentials. +- [`basic`](01-basic-config-management.go) Basic configuration management illustrating the configuration of credentials. - [`generic`](02-handle-arbitrary-config.go) Handling of arbitrary configuration. - [`ocm`](03-using-ocm-config.go) Central configuration - [`provide`](04-write-config-type.go) Providing new config object types - [`consume`](05-write-config-consumer.go) Preparing objects to be configured by the config management +## Running the example -You can just call the main program with some config file option (`--config `) and the name of the scenario. +You can call the main program with a config file option (`--config `) and the name of the scenario. The config file should have the following content: ```yaml repository: ghcr.io/mandelsoft/ocm username: password: +``` + +Set your favorite OCI registry and don't forget to add the repository prefix for your OCM repository hosted in this registry. + +## Walkthrough + +### Basic Configuration Management + +Similar to the other context areas, Configuration is handled by the configuration contexts. +Therefore, for the example, we just get the default configuration context. + +```go + ctx := config.DefaultContext() +``` + +The configuration context handles configuration objects. +A configuration object is any object implementing +the `config.Config` interface. The task of a config object +is to apply configuration to some target object. + +One such object is the configuration object for +credentials provided by the credentials context. +It finally applies settings to a credential context. + +```go + creds := credcfg.New() +``` + +Here, we can configure credential settings: +credential repositories and consumer id mappings. +We do this by setting the credentials provided +by our config file for the consumer id used +by our configured OCI registry. + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } + creds.AddConsumer( + id, + directcreds.NewRepositorySpec(cfg.GetCredentials().Properties()), + ) +``` + +(Credential) Configuration objects are typically serializable and deserializable. + +```go + spec, err := json.MarshalIndent(creds, " ", " ") + if err != nil { + return errors.Wrapf(err, "marshal credential config") + } + + fmt.Printf("this a a credential configuration object:\n%s\n", string(spec)) +``` + +Like all the other manifest based descriptions this format always includes +a type field, which can be used to deserialize a specification into +the appropriate object. +This can be done by the config context. It accepts YAML or JSON. + +```go + o, err := ctx.GetConfigForData(spec, nil) + if err != nil { + return errors.Wrapf(err, "deserialize config") + } + + if diff := deep.Equal(o, creds); len(diff) != 0 { + fmt.Printf("diff:\n%v\n", diff) + return fmt.Errorf("invalid des/erialization") + } +``` + +Regardless what variant is used (direct specification object or descriptor) +the config object can be added to a config context. + +```go + err = ctx.ApplyConfig(creds, "explicit cred setting") + if err != nil { + return errors.Wrapf(err, "cannot apply config") + } +``` + +Every config object implements the +`ApplyTo(ctx config.Context, target interface{}) error` method. +It takes an object, which wants to be configured. +The config object then decides, whether it provides +settings for the given object and calls the appropriate +methods on this object (after a type cast). + +Here is the code snippet from the apply method of the credential +config object ([.../pkg/contexts/credentials/config/type.go](../../../../../pkg/contexts/credentials/config/type.go)): + +```go + +func (a *Config) ApplyTo(ctx cfgcpi.Context, target interface{}) error { + list := errors.ErrListf("applying config") + t, ok := target.(cpi.Context) + if !ok { + return cfgcpi.ErrNoContext(ConfigType) + } + for _, e := range a.Consumers { + t.SetCredentialsForConsumer(e.Identity, CredentialsChain(e.Credentials...)) + } + ... +``` + +This way the config mechanism reverts the configuration +request, it does not actively configure something, instead +an object, which wants to be configured calls the config +context to apply pending configs. +To do this the config context manages a queue of config objects +and applies them to an object to be configured. + +If the credential context is asked now for credentials, +it asks the config context for pending config objects +and applies them. +Therefore, we now should be able to get the configured credentials. + +```go + credctx := credentials.DefaultContext() + + found, err := credentials.CredentialsForConsumer(credctx, id) + if err != nil { + return errors.Wrapf(err, "cannot get credentials") + } + // an error is only provided if something went wrong while determining + // the credentials. Delivering NO credentials is a valid result. + if found == nil { + return fmt.Errorf("no credentials found") + } + fmt.Printf("consumer id: %s\n", id) + fmt.Printf("credentials: %s\n", obfuscate(found)) + + if found.GetProperty(credentials.ATTR_USERNAME) != cfg.Username { + return fmt.Errorf("password mismatch") + } + if found.GetProperty(credentials.ATTR_PASSWORD) != cfg.Password { + return fmt.Errorf("password mismatch") + } +``` + +### Handling of Arbitrary Configuration + +The config management not only manages configuration objects for any +other configurable object, it also provides a configuration object of +its own. The task of the object is to handle other configuration objects +to be applied to a configuration object. + +```go + generic := configcfg.New() +``` + +The generic config object holds a list of any other config objects, +or their specification formats. +Additionally, it is possible to configure named sets +of configurations, which can later be enabled +on-demand by their name at the config context. + +We recycle our credential config from the last example to get +a config object to be added to our generic config object. + +```go + creds, err := credConfig(cfg) + if err != nil { + return err + } +``` + +Now, we can add this credential config object to +our generic config list. + +```go + err = generic.AddConfig(creds) + if err != nil { + return errors.Wrapf(err, "adding config") + } +``` + +As we have seen in our previous example, config objects are typically +serializable and deserializable. This also holds for the generic config +object of the config context. + +```go + spec, err := json.MarshalIndent(generic, " ", " ") + if err != nil { + return errors.Wrapf(err, "marshal credential config") + } + + fmt.Printf("this a a generic configuration object:\n%s\n", string(spec)) +``` + +The result is a config object hosting a list (with 1 entry) +of other config object specifications. + +The generic config object can be added to a config context, again, like +any other config object. If it is asked to configure a configuration +context it uses the methods of the configuration context to apply the +contained list of config objects (and the named set of config lists). +Therefore, all config objects applied to a configuration context are +asked to configure the configuration context itself when queued to the +list of applied configuration objects. + +If we now ask the default credential context (which uses the default +configuration context to configure itself) for credentials for our OCI registry, +the credential mapping provided by the config object added to the generic one, +will be found. + +```go + ctx := config.DefaultContext() + err = ctx.ApplyConfig(creds, "generic setting") + if err != nil { + return errors.Wrapf(err, "cannot apply config") + } + credctx := credentials.DefaultContext() + + // query now works, also. + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } + found, err := credentials.CredentialsForConsumer(credctx, id) + if err != nil { + return errors.Wrapf(err, "cannot get credentials") + } + fmt.Printf("consumer id: %s\n", id) + fmt.Printf("credentials: %s\n", obfuscate(found)) +``` + +The very same mechanism is used to provide central configuration in a +configuration file for the OCM ecosystem, as will be shown in the next example. + +### Central Configuration + +Although the configuration of an OCM context can +be done by a sequence of explicit calls according to the mechanisms +shown in the examples before, a simple convenience +library function is provided, which can be used to configure an OCM +context and all related other contexts with a single call +based on a central configuration file (`~/.ocmconfig`) + +```go + ctx := ocm.DefaultContext() + _, err := utils.Configure(ctx, "") + if err != nil { + return errors.Wrapf(err, "configuration") + } +``` + +This file typically contains the serialization of such a generic +configuration specification (or any other serialized configuration object), +enriched with specialized config specifications for +credentials, default repositories, signing keys and any +other configuration specification. + +#### Standard Configuration File + +Most important are here the credentials. +Because OCM embraces lots of storage technologies for artifact +storage as well as storing OCM component version metadata, +there are typically multiple technology specific ways +to configure credentials for command line tools. +Using the credentials settings shown in the previous tour, +it is possible to specify credentials for all +required purposes, and the configuration management provides +an extensible way to embed native technology specific ways +to provide credentials just by adding an appropriate type +of credential repository, which reads the specialized storage and +feeds it into the credential context. Those specifications +can be added via the credential configuration object to +the central configuration. + +One such repository type is the Docker config type. It +reads a `dockerconfig.json` file and feeds in the credentials. +Because it is used for a dedicated purpose (credentials for +OCI registries), it not only can feed the credentials, but +also their mapping to consumer ids. + +We first create the specification for a new credential repository of +type `dockerconfig` describing the default location +of the standard Docker config file. + +```go + credspec := dockerconfig.NewRepositorySpec("~/.docker/config.json", true) + + // add this repository specification to a credential configuration. + ccfg := credcfg.New() + err = ccfg.AddRepository(credspec) + if err != nil { + return errors.Wrapf(err, "invalid credential config") + } +``` + +By adding the default location for the standard Docker config +file, all credentials provided by the `docker login` command +are available in the OCM toolset, also. + +A typical minimal .ocmconfig file can be composed as follows. +We add this config object to an empty generic configuration object +and print the serialized form. The result can be used as +default initial OCM configuration file. + +```go + ocmcfg := configcfg.New() + err = ocmcfg.AddConfig(ccfg) + + spec, err := yaml.Marshal(ocmcfg) + if err != nil { + return errors.Wrapf(err, "marshal ocm config") + } + + // the result is a typical minimal ocm configuration file + // just providing the credentials configured with + // doicker login. + fmt.Printf("this a typical ocm config file:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(spec)) +``` + +The result should look similar to (but with reordered fields): +```yaml +type: generic.config.ocm.software +configurations: + - type: credentials.config.ocm.software + repositories: + - repository: + type: DockerConfig + dockerConfigFile: ~/.docker/config.json + propagateConsumerIdentity: true +``` + +Because of the ordered map keys the actual output looks a little bit confusing: + +```yaml +configurations: +- repositories: + - repository: + dockerConfigFile: ~/.docker/config.json + propagateConsumerIdentity: true + type: DockerConfig + type: credentials.config.ocm.software +type: generic.config.ocm.software +``` + +Besides from a file, such a config can be provided as data, also, +taken from any other source, for example from a Kubernetes secret. + +```go + err = utils.ConfigureByData(ctx, spec, "from data") + if err != nil { + return errors.Wrapf(err, "configuration") + } +``` + +If you have provided your OCI credentials with +`docker login`, they should now be available. + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } + found, err := credentials.CredentialsForConsumer(ctx, id) + if err != nil { + return errors.Wrapf(err, "cannot get credentials") + } + fmt.Printf("consumer id: %s\n", id) + fmt.Printf("credentials: %s\n", obfuscate(found)) +``` + +#### Templating + +The configuration library function does not only read the +ocm config file, it also applies [*spiff*](github.com/mandelsoft/spiff) +processing to the provided YAML/JSON content. *Spiff* is an +in-domain yaml-based templating engine. Therefore, you can use +any spiff dynaml expression to define values or even complete +sub structures. + +```go + ocmcfg = configcfg.New() + ccfg = credcfg.New() + cspec := credentials.CredentialsSpecFromList("clientCert", `(( read("~/ocm/keys/myClientCert.pem") ))`) + id = credentials.NewConsumerIdentity("ApplicationServer.acme.org", "hostname", "app.acme.org") + ccfg.AddConsumer(id, cspec) + ocmcfg.AddConfig(ccfg) +``` + +This config object is not directly usable, because the cert value is not +a valid certificate. We use it here just to generate the serialized form. + +```yaml +configurations: +- consumers: + - credentials: + - credentialsName: Credentials + properties: + clientCert: (( read("~/ocm/keys/myClientCert.pem") )) + type: Credentials + identity: + hostname: app.acme.org + type: ApplicationServer.acme.org + type: credentials.config.ocm.software +type: generic.config.ocm.software +``` + +If this is used with the above library functions, the finally generated +config object will contain the read file content, which is hopefully a +valid certificate. + +### Providing new config object types + +So far, we just used existing config types to configure existing objects. +But the configuration management is highly extensible, and it is quite +simple to provide new config types, which can be used to configure +any new or existing object, which is prepared to consume configuration. + +The next [chapter](#preparing-objects-to-be-configured-by-the-config-management) will show how to prepare an +object to be automatically configurable by +the configuration management. Here, we focus on the implementation of +new config object types. Therefore, we want to configure the +credential context by a new configuration object. + +#### The Configuration Object Type + +Typically, every kind of configuration object lives in its own package, +which always have the same layout. + +A configuration object has a *type*, the configuration type. Therefore, +the package declares a constant `TYPE`. + +It is the name of our new configuration object type. +To be globally unique, it should always end with a +DNS domain owned by the provider of the new type. + +```go +const TYPE = "example.config.acme.org" + +``` + +Next, we need a Go type. `ExampleConfigSpec` is the new Go type for the +config specification covering our example configuration. +It just encapsulates our simple configuration structure +used to configure the examples of our tour. + +```go +type ExampleConfigSpec struct { + // ObjectVersionedType is the base type providing the type feature + // for (config) specifications. + runtime.ObjectVersionedType `json:",inline"` + // Config is our example config representation. + helper.Config `json:",inline"` +} + +``` + +Every config type structure must contain a field (and the appropriate methods) +for storing the config type name. This is done by embedding the +type `runtime.ObjectVersionedType` from the `runtime` package. This package +contains everything to work with specification objects and +serialization/deserialization. + +As second field we just embed the config structure used to read the tour +config. This way any kind of configuration information can be mapped +to the configuration management. + +A config type typically provide a constructor for a config object of +this type: + +```go +func NewConfig(cfg *helper.Config) cpi.Config { + return &ExampleConfigSpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(TYPE), + Config: *cfg, + } +} + +``` + +Additional setters can be used to configure the configuration object. +Here, programmatic objects (like an `ocm.RepositorySpec`) are +converted to a form storable in the configuration object. + +```go + +// SetTargetRepository takes a repository specification +// and adds its serialized form to the config object. +func (c *ExampleConfigSpec) SetTargetRepository(target ocm.RepositorySpec) error { + data, err := json.Marshal(target) + if err != nil { + return err + } + c.Target = data + return nil +} + +// SetTargetRepositoryData sets the target repository specification +// from a byte sequence. +func (c *ExampleConfigSpec) SetTargetRepositoryData(data []byte) error { + err := runtime.CheckSpecification(data) + if err != nil { + return err + } + c.Target = data + return nil +} + +``` + +The utility function `runtime.CheckSpecification` can be used to +check a byte sequence to be a valid specification. +It just checks for a valid YAML document featuring a non-empty +`type` field: + +```go + +// CheckSpecification checks a byte sequence to describe a +// valid minimum specification object. +func CheckSpecification(data []byte) error { + var obj ObjectTypedObject + + err := DefaultYAMLEncoding.Unmarshal(data, &obj) + if err != nil { + return errors.ErrInvalidWrap(err, "repository specification", string(data)) + } + if obj.GetType() == "" { + return errors.ErrInvalidWrap(fmt.Errorf("non-empty type field required"), "repository specification", string(data)) + } + return nil +} + +``` + +The most important method to implement is `ApplyTo(_ cpi.Context, tgt interface{}) error`, +which must be implemented by all configuration objects. +Its task is to apply the described configuration settings to a dedicated +object. + +```go +func (c *ExampleConfigSpec) ApplyTo(_ cpi.Context, tgt interface{}) error { + + switch t := tgt.(type) { + // if the target is a credentials context + // configure the credentials to be used for the + // described OCI repository. + case credentials.Context: + // determine the consumer id for our target repository- + id, err := oci.GetConsumerIdForRef(c.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } + // create the credentials. + creds := c.GetCredentials() + + // configure the targeted credential context with + // the provided credentials (see previous examples). + t.SetCredentialsForConsumer(id, creds) + + // if the target consumes an OCI repository, propagate + // the provided OCI repository ref. + case RepositoryTarget: + t.SetRepository(c.Repository) + + // all other targets are ignored, we don't have + // something to set at these objects. + default: + return cpi.ErrNoContext(TYPE) + } + return nil +} + +``` + +Therefore, it decides, whether it is able to handle a dedicated type of target +object and how to configure it. This way a configuration object +may apply is settings or even parts of its setting to any kind of target object. + +Our configuration object supports two kinds of target objects: +if the target is a credentials context +it configures the credentials to be used for the +described OCI repository similar to our [credential management example](../03-working-with-credentials/README.md#using-the-credential-management). + +But we want to accept more types of target objects. Therefore, we +introduce an own interface declaring the methods required for applying +some configuration settings. + +```go + +// RepositoryTarget consumes a repository name. +type RepositoryTarget interface { + SetRepository(r string) +} + +``` + +By checking the target object against this interface, we are able +to configure any kind of object, as long as it provides the necessary +configuration methods. + +Now, we are nearly prepared to use our new configuration, there is just one step +missing. To enable the automatic recognition of our new type (for example +in the ocm config file), we have to tell the configuration management +about the new type. This is done by an `init()` function in our config package. + +Here, we call a registration function, +which gets called with a dedicated type object for the new config type. +A *type object* describes the config type, its type name, how +it is serialized and deserialized and some description. +We use a standard type object, here, instead of implementing +an own one. It is parameterized by the Go pointer type (`*ExampleConfigSpec`) for +our specification object. + +```go +func init() { + // register the new config type, so that is can be used + // by the config management to deserialize appropriately + // typed specifications. + cpi.RegisterConfigType(cpi.NewConfigType[*ExampleConfigSpec](TYPE, "this ia config object type based on the example config data.")) +} + +``` + +#### Using our new Config Object + +After preparing a new special config type +we can feed it into the config management. +Because of the registration the config management +now knows about this new type. + +A usual, we gain access to our required contexts. + +```go + credctx := credentials.DefaultContext() + + // the credential context is based on a config context + // used to configure it. + ctx := credctx.ConfigContext() +``` + +To setup our environment we create our new config based on the actual settings +and apply it to the config context. + +```go + examplecfg := NewConfig(cfg) + ctx.ApplyConfig(examplecfg, "special acme config") +``` + +Now, we should be prepared to get the credentials +the usual way. + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "cannot get consumer id") + } + fmt.Printf("usage context: %s\n", id) + + // the returned credentials are provided via an interface, which might change its + // content, if the underlying credential source changes. + creds, err := credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) + if err != nil { + return errors.Wrapf(err, "credentials") + } + fmt.Printf("credentials: %s\n", obfuscate(creds)) +``` + +#### Using in the OCM Configuration + +Because of the new credential type, such a specification can +now be added to the ocm config, also. +So, we could use our special tour config file content +directly as part of the ocm config. + +```go + ocmcfg := configcfg.New() + err = ocmcfg.AddConfig(examplecfg) + + spec, err := yaml.Marshal(ocmcfg) + if err != nil { + return errors.Wrapf(err, "marshal ocm config") + } + + // the result is a minimal ocm configuration file + // just providing our new example configuration. + fmt.Printf("this a typical ocm config file:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(spec)) +``` + +The resulting config file looks as follows: + +```yaml +configurations: +- component: github.com/mandelsoft/examples/cred1 + password: ghp_xyz + type: example.config.acme.org + username: mandelsoft + version: 0.1.0 +type: generic.config.ocm.software +``` + +#### Applying to our Configuration Interface + +Above, we added a new kind of target, the `RepositoryTarget` interface. +By providing an implementation for this interface, we can +configure such an object using the config management. +We just provide a simple implementation for this interface, just storing the configured +repository specification. + +```go + +// SimpleRepositoryTarget is demo target object +// just implementing our new configuration interface. +type SimpleRepositoryTarget struct { + repository string +} + +var _ RepositoryTarget = (*SimpleRepositoryTarget)(nil) + +func (t *SimpleRepositoryTarget) SetRepository(repo string) { + t.repository = repo +} + +``` + +The context management now is able to apply our config to such an object. + +```go + target := &SimpleRepositoryTarget{} + + _, err = ctx.ApplyTo(0, target) + if err != nil { + return errors.Wrapf(err, "applying to new target") + } + fmt.Printf("repository for target: %s\n", target.repository) +``` + +This way any specialized configuration object can be added +by a user of the OCM library. It can be used to configure +existing objects or even new object types, even in combination. + +What is still required is a way +to implement new config targets, objects, which wants +to be configured and which autoconfigure themselves when +used. Our simple repository target is just an example +for some kind of ad-hoc configuration. +A complete scenario is shown in the next example. + +### Preparing Objects to be Configured by the Config Management + +We already have our new acme.org config object type, +and a target interface which must be implemented by a target +object to be configurable. The last example showed how +such an object can be configured in an ad-hoc manner +by directly requesting it to be configured by the config +management. + +Now, we want to provide an object, which configures +itself when used. +Therefore, we introduce a Go type `RepositoryProvider`, +which should be an object, which is +able to provide an OCI repository reference. +It has a setter and a getter (the setter is +provided by our ad-hoc `SimpleRepositoryTarget`). + +To be able to configure itself, the object must know about +the config context it should use to configure itself. + +Therefore, our type contains an additional field `updater`. +Its type `cpi.Updater` is a utility provided by the configuration +management, which holds a reference to a configuration context +and is able to +configure an object based on a managed configuration +watermark. It remembers which config objects from the +config queue are already applied, and replays +the config objects applied to the config context +after the last update. + +Finally, a mutex field is contained, which is used to +synchronize updates later. + +```go +type RepositoryProvider struct { + lock sync.Mutex + // cpi.Updater is a utility, which is able to + // configure an object based on a managed configuration + // watermark. It remembers which config objects from the + // config queue are already applied, and replays + // the config objects applied to the config context + // after the last update. + updater cpi.Updater + SimpleRepositoryTarget +} + +``` + +For this type a constructor is provided, which initializes +the `updater` field with the desired configuration context. + +```go +func NewRepositoryProvider(ctx cpi.ContextProvider) *RepositoryProvider { + p := &RepositoryProvider{} + // To do its work, the updater needs a connection to + // the config context to use and the object, which should be + // configured. + p.updater = cpi.NewUpdater(ctx.ConfigContext(), p) + return p +} + +``` + +The magic now happens in the methods provided +by our configurable object. +The first step for methods of configurable objects +dependent on potential configuration is always +to update itself using the embedded updater. + +Please note, the config management reverses the +request direction. Applying a config object to +the config context does not configure dependent objects, +it just manages a config queue, which is used by potential +configuration targets to configure themselves. +The actual configuration action is always initiated +by the object, which want to be configured. +The reason for this is to avoid references from the +management to managed objects. This would prohibit +the garbage collection of all configurable objects +as long as the configuration context exists. + +```go +func (p *RepositoryProvider) GetRepository() (string, error) { + p.lock.Lock() + defer p.lock.Unlock() + + err := p.updater.Update() + if err != nil { + return "", err + } + // now, we can do our regular function, aka + // providing a repository ref. + return p.repository, nil +} + +``` + +After defining our repository provider type we can now start to use it +together with the configuration management and out configuration object. + +As usual, we first determine out context to use. + +```go + credctx := credentials.DefaultContext() +``` + +New, we create our provide configurable object by binding it +to the config context. + +```go + prov := NewRepositoryProvider(credctx) +``` + +If we ask now for a repository we will get the empty +answer, because nothing is configured, yet. + +```go + repo, err := prov.GetRepository() + if err != nil { + errors.Wrapf(err, "get repo") + } + if repo != "" { + return fmt.Errorf("Oops, found repository %q", repo) + } +``` + +Now, we apply our config from the last example. Therefore, we create and initialize +the config object with our program settings and apply it to the config +context. + +```go + ctx := credctx.ConfigContext() + examplecfg := NewConfig(cfg) + err = ctx.ApplyConfig(examplecfg, "special acme config") + if err != nil { + errors.Wrapf(err, "apply config") + } +``` + +Without any further action, asking for a repository now will return the +configured ref. The configurable object automatically catches the +new configuration from the config context. + +```go + repo, err = prov.GetRepository() + if err != nil { + errors.Wrapf(err, "get repo") + } + if repo == "" { + return fmt.Errorf("no repository provided") + } + fmt.Printf("using repository: %s\n", repo) +``` + +Now, we should also be prepared to get the credentials, +our config object configures the provider as well as +the credential context. + +```go + id, err := oci.GetConsumerIdForRef(repo) + if err != nil { + return errors.Wrapf(err, "cannot get consumer id") + } + fmt.Printf("usage context: %s\n", id) + + creds, err := credentials.CredentialsForConsumer(credctx, id, ociidentity.IdentityMatcher) + if err != nil { + return errors.Wrapf(err, "credentials") + } + fmt.Printf("credentials: %s\n", obfuscate(creds)) ``` \ No newline at end of file diff --git a/examples/lib/tour/04-working-with-config/main.go b/examples/lib/tour/04-working-with-config/main.go index e25482bcd5..ca59b4738b 100644 --- a/examples/lib/tour/04-working-with-config/main.go +++ b/examples/lib/tour/04-working-with-config/main.go @@ -13,14 +13,17 @@ import ( ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" var current_version string func init() { data, err := os.ReadFile("VERSION") if err != nil { - panic("VERSION not found") + data, err = os.ReadFile("../../../../../VERSION") + if err != nil { + panic("VERSION not found") + } } current_version = strings.TrimSpace(string(data)) } diff --git a/examples/lib/tour/04-working-with-config/settings.yaml b/examples/lib/tour/04-working-with-config/settings.yaml new file mode 100644 index 0000000000..513e95095a --- /dev/null +++ b/examples/lib/tour/04-working-with-config/settings.yaml @@ -0,0 +1,4 @@ +username: mandelsoft +password: ghp_xyz +component: github.com/mandelsoft/examples/cred1 +version: 0.1.0 \ No newline at end of file diff --git a/examples/lib/tour/05-transporting-component-versions/README.md b/examples/lib/tour/05-transporting-component-versions/README.md index a5b494a501..89da4f5c59 100644 --- a/examples/lib/tour/05-transporting-component-versions/README.md +++ b/examples/lib/tour/05-transporting-component-versions/README.md @@ -1,10 +1,14 @@ + + + # Transporting Component Versions This [tour](example.go) illustrates the basic support for transporting content from one environment into another. +## Running the example -You can just call the main program with some config file option (`--config `). +You can call the main program with a config file option (`--config `). The config file should have the following content: ```yaml @@ -31,8 +35,142 @@ targetRepository: ocmConfig: ``` -The actual version of the example just works with the filesystem +The actual version of the example just works with the file system target, because it is not possible to specify credentials for the -target repository in this simple config file. But, if you specific an [OCM config file](../04-working-with-config/README.md) you can -add more credential settings to make target repositories possible -requiring credentials. \ No newline at end of file +target repository in this simple config file. But, if you specify an [OCM config file](../04-working-with-config/README.md#standard-configuration-file) you can +add more predefined credential settings to make it possible to use +target repositories requiring credentials. The credentials are +automatically taken from the credentials context and don't need to be +specified when creating the repository access object in the code. + +## Walkthrough + +As usual, we start with getting access to an OCM context + +```go + ctx := ocm.DefaultContext() +``` + +Then we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04](../04-working-with-config/README.md#standard-configuration-file). + +```go + err := ReadConfiguration(ctx, cfg) + if err != nil { + return err + } +``` + +This function simply applies the config file using the utility function +provided by the config management: + +```go +func ReadConfiguration(ctx ocm.Context, cfg *helper.Config) error { + if cfg.OCMConfig != "" { + fmt.Printf("*** applying config from %s\n", cfg.OCMConfig) + + _, err := utils.Configure(ctx, cfg.OCMConfig) + if err != nil { + return errors.Wrapf(err, "error in ocm config %s", cfg.OCMConfig) + } + } + return nil +} + +``` + +The context acts as factory for various model types based on +specification descriptor serialization formats in YAML or JSON. +Access method specifications and repository specification are +examples for this feature. + +Now, we use the repository specification serialization format to +determine the target repository for a transport from our yaml +configuration file. + +```go + fmt.Printf("target repository is %s\n", string(cfg.Target)) + target, err := ctx.RepositoryForConfig(cfg.Target, nil) + if err != nil { + return errors.Wrapf(err, "cannot open repository") + } + defer target.Close() +``` + +For our source we just use the component version provided by the last +examples in a remote repository. +Therefore, we set up the credentials context, as +shown in [tour 03](../03-working-with-credentials/README.md#using-the-credential-management). + +```go + id, err := oci.GetConsumerIdForRef(cfg.Repository) + if err != nil { + return errors.Wrapf(err, "invalid consumer") + } + creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) + ctx.CredentialsContext().SetCredentialsForConsumer(id, creds) +``` + +For the transport, we first get access to the component version +we want to transport, by getting the source repository and looking up +the desired component version. + +```go + spec := ocireg.NewRepositorySpec(cfg.Repository, nil) + repo, err := ctx.RepositoryForSpec(spec, creds) + if err != nil { + return err + } + defer repo.Close() + + cv, err := repo.LookupComponentVersion("acme.org/example03", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "added version not found") + } + defer cv.Close() +``` + +We could just add this version to the target repository, but this +would not be a real transport, but just a copy of the component descriptor +and the associated local resources. Transport potentially means more, all +the described artifacts should also be copied into the target environment. + +Such an action is done by the library function `transfer.Transfer`. +It takes several settings influencing the transport mode, +for example transitive or value transport. +Here, all resources are transported per value, all external +references will be inlined as `localBlob`s and imported into +the target environment, applying blob upload handlers +where possible. For a CTF archive as target, there are no +configured handlers by default. Therefore, all resources will +be migrated to local blobs. + +```go + err = transfer.Transfer(cv, target, standard.ResourcesByValue(), standard.Overwrite()) + if err != nil { + return errors.Wrapf(err, "transport failed") + } +``` + +Now, we check the result of our transport action in the target +repository. + + +```go + tcv, err := target.LookupComponentVersion("acme.org/example03", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "transported version not found") + } + defer tcv.Close() + fmt.Printf("*** target version in transportation target\n") + err = describeVersion(tcv) + if err != nil { + return errors.Wrapf(err, "describe failed") + } +``` + +Please be aware that all resources in the target now are `localBlob`s, +if the target is a CTF archive. If it is an OCI registry, all the OCI +artifact resources will be uploaded as OCI artifacts into the target +repository and the access specifications are adapted to type `ociArtifact`, +but now referring to OCI artifacts in the target repository. diff --git a/examples/lib/tour/05-transporting-component-versions/common.go b/examples/lib/tour/05-transporting-component-versions/common.go index d5efc5f78c..cdc9171da9 100644 --- a/examples/lib/tour/05-transporting-component-versions/common.go +++ b/examples/lib/tour/05-transporting-component-versions/common.go @@ -104,7 +104,7 @@ data: some very important data required to understand this component // Any blob content provided by an implementation of this // interface can be added as resource. // There are various access implementations for blobs - // taken from the local host, for example, from the filesystem, + // taken from the local host, for example, from the file system, // or from other repositories (for example by mapping // an access type specification into a blob access). // The most simple form is to directly provide a byte sequence, @@ -146,7 +146,7 @@ data: some very important data required to understand this component } // There are even more complex blob sources, for example - // for helm charts stored in the filesystem, or even for images + // for Helm charts stored in the file system, or even for images // generated by docker builds. // Here, we just compose a multi-platform image built with buildx // from these sources (components/ocmcli) featuring two flavors. diff --git a/examples/lib/tour/05-transporting-component-versions/example.go b/examples/lib/tour/05-transporting-component-versions/example.go index e22b66d08e..34c5713aa5 100644 --- a/examples/lib/tour/05-transporting-component-versions/example.go +++ b/examples/lib/tour/05-transporting-component-versions/example.go @@ -18,6 +18,7 @@ import ( "github.com/open-component-model/ocm/pkg/errors" ) +// --- begin read config --- func ReadConfiguration(ctx ocm.Context, cfg *helper.Config) error { if cfg.OCMConfig != "" { fmt.Printf("*** applying config from %s\n", cfg.OCMConfig) @@ -30,15 +31,21 @@ func ReadConfiguration(ctx ocm.Context, cfg *helper.Config) error { return nil } +// --- end read config --- + func TransportingComponentVersions(cfg *helper.Config) error { + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // Configure context with optional ocm config. // See OCM config scenario in tour 04. + // --- begin configure --- err := ReadConfiguration(ctx, cfg) if err != nil { return err } + // --- end configure --- // the context acts as factory for various model types based on // specification descriptor serialization formats in YAML or JSON. @@ -46,29 +53,36 @@ func TransportingComponentVersions(cfg *helper.Config) error { // examples for this feature. // // Now, we use the repository specification serialization format to - // determine the target repository for a transport from a yaml + // determine the target repository for a transport from our yaml // configuration file. + // --- begin target --- fmt.Printf("target repository is %s\n", string(cfg.Target)) target, err := ctx.RepositoryForConfig(cfg.Target, nil) if err != nil { return errors.Wrapf(err, "cannot open repository") } defer target.Close() + // --- end target --- // we just use the component version provided by the last examples // in a remote target repository. // Therefore, we set up the credentials context, again, as has // been shown in example 3. + // --- begin set credentials --- id, err := oci.GetConsumerIdForRef(cfg.Repository) if err != nil { return errors.Wrapf(err, "invalid consumer") } creds := ociidentity.SimpleCredentials(cfg.Username, cfg.Password) ctx.CredentialsContext().SetCredentialsForConsumer(id, creds) + // --- end set credentials --- // now, we are ready to determine the transportation source. - // open the source repository. + // For the transport, we first get access to the component version + // we want to transport, by getting the source repository and looking up + // the desired component version. + // --- begin source --- spec := ocireg.NewRepositorySpec(cfg.Repository, nil) repo, err := ctx.RepositoryForSpec(spec, creds) if err != nil { @@ -81,6 +95,7 @@ func TransportingComponentVersions(cfg *helper.Config) error { return errors.Wrapf(err, "added version not found") } defer cv.Close() + // --- end source --- fmt.Printf("*** source version in source repository\n") err = describeVersion(cv) @@ -90,29 +105,37 @@ func TransportingComponentVersions(cfg *helper.Config) error { // transfer the component version with value mode. // Here, all resources are transported per value, all external - // references will be inlined as localBlobs and imported into + // references will be inlined as `localBlob` and imported into // the target environment, applying blob upload handlers - // where possible. For a CTF Archive as target, there are no - // configured handlers, by default. + // where possible. For a CTF archive as target, there are no + // configured handlers by default. + // --- begin transfer --- err = transfer.Transfer(cv, target, standard.ResourcesByValue(), standard.Overwrite()) if err != nil { return errors.Wrapf(err, "transport failed") } + // --- end transfer --- + // now, we check the result of our transport action in the target + // repository + // --- begin verify-a --- tcv, err := target.LookupComponentVersion("acme.org/example03", "v0.1.0") if err != nil { return errors.Wrapf(err, "transported version not found") } defer tcv.Close() + // --- end verify-a --- - // please be aware that the all resources in the target now are localBlobs, + // please be aware that all resources in the target now are localBlobs, // if the target is a CTF archive. If it is an OCI registry, all the OCI // artifact resources will be uploaded as OCI artifacts into the target // repository and the access specifications are adapted to type `ociArtifact`. + // --- begin verify-b --- fmt.Printf("*** target version in transportation target\n") err = describeVersion(tcv) if err != nil { return errors.Wrapf(err, "describe failed") } + // --- end verify-b --- return nil } diff --git a/examples/lib/tour/05-transporting-component-versions/main.go b/examples/lib/tour/05-transporting-component-versions/main.go index 0d8f2dcab7..a1a4b903e6 100644 --- a/examples/lib/tour/05-transporting-component-versions/main.go +++ b/examples/lib/tour/05-transporting-component-versions/main.go @@ -13,7 +13,7 @@ import ( ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" var current_version string diff --git a/examples/lib/tour/06-signing-component-versions/01-basic-signing.go b/examples/lib/tour/06-signing-component-versions/01-basic-signing.go index 0154795279..055e5d05fa 100644 --- a/examples/lib/tour/06-signing-component-versions/01-basic-signing.go +++ b/examples/lib/tour/06-signing-component-versions/01-basic-signing.go @@ -16,30 +16,34 @@ import ( ) func SigningComponentVersions(cfg *helper.Config) error { - + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // Configure context with optional ocm config. // See OCM config scenario in tour 04. + // --- begin configure --- err := ReadConfiguration(ctx, cfg) if err != nil { return err } - - // siginfo := signingattr.Get(ctx) + // --- end configure --- // to sign a component version we need a private key. // for this example, we just create a local keypair. // to be able to verify later, we should save the public key, // but here we do all this in a single program. + // --- begin create keypair --- privkey, pubkey, err := rsa.CreateKeyPair() if err != nil { return errors.Wrapf(err, "cannot create keypair") } + // --- end create keypair --- // now we compose a component version without a repository, again. // see tour02 example b. + // --- begin compose --- cv := composition.NewComponentVersion(ctx, "acme.org/example6", "v0.1.0") // just use the same component version setup again @@ -50,20 +54,24 @@ func SigningComponentVersions(cfg *helper.Config) error { fmt.Printf("*** composition version ***\n") err = describeVersion(cv) + // --- end compose --- // now let's sign the component version. // There might be multiple signatures, therefore every signature // has a name (here acme.org). Keys are always specified for // a dedicated signature name. + // --- begin sign --- _, err = signing.SignComponentVersion(cv, "acme.org", signing.PrivateKey("acme.org", privkey)) if err != nil { return errors.Wrapf(err, "cannot sign component version") } fmt.Printf("*** signed composition version ***\n") err = describeVersion(cv) + // --- end sign --- // now add the signed component to a target repository. - // here, we just reuse the code from tour05 + // here, we just reuse the code from tour02 + // --- begin add version --- fmt.Printf("target repository is %s\n", string(cfg.Target)) target, err := ctx.RepositoryForConfig(cfg.Target, nil) if err != nil { @@ -75,8 +83,10 @@ func SigningComponentVersions(cfg *helper.Config) error { if err != nil { return errors.Wrapf(err, "cannot store signed version") } + // --- end add version --- // let's check the target for the new component version + // --- begin lookup --- tcv, err := target.LookupComponentVersion("acme.org/example6", "v0.1.0") if err != nil { return errors.Wrapf(err, "transported version not found") @@ -89,13 +99,16 @@ func SigningComponentVersions(cfg *helper.Config) error { if err != nil { return errors.Wrapf(err, "describe failed") } + // --- end lookup --- - // new lets verify the signature + // now lets verify the signature + // --- begin verify --- _, err = signing.VerifyComponentVersion(cv, "acme.org", signing.PublicKey("acme.org", pubkey)) if err != nil { return errors.Wrapf(err, "verification failed") } else { fmt.Printf("verification succeeded\n") } + // --- end verify --- return nil } diff --git a/examples/lib/tour/06-signing-component-versions/02-using-context-settings.go b/examples/lib/tour/06-signing-component-versions/02-using-context-settings.go index 9656119a5e..c0e8dd3d0a 100644 --- a/examples/lib/tour/06-signing-component-versions/02-using-context-settings.go +++ b/examples/lib/tour/06-signing-component-versions/02-using-context-settings.go @@ -8,12 +8,15 @@ import ( "fmt" "github.com/open-component-model/ocm/examples/lib/helper" + configcfg "github.com/open-component-model/ocm/pkg/contexts/config/config" "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/composition" "github.com/open-component-model/ocm/pkg/contexts/ocm/signing" "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/runtime" "github.com/open-component-model/ocm/pkg/signing/handlers/rsa" + "github.com/open-component-model/ocm/pkg/signing/signutils" ) func prepareComponentInRepo(ctx ocm.Context, cfg *helper.Config) error { @@ -47,47 +50,63 @@ func prepareComponentInRepo(ctx ocm.Context, cfg *helper.Config) error { } func SigningComponentVersionInRepo(cfg *helper.Config) error { + // --- begin default context --- ctx := ocm.DefaultContext() + // --- end default context --- // Configure context with optional ocm config. // See OCM config scenario in tour 04. + // --- begin configure --- err := ReadConfiguration(ctx, cfg) if err != nil { return err } + // --- end configure --- + // to sign a component version we still need a private key. + // for this example, we just create a local keypair. + // to be able to verify later, we should save the public key, + // but here we do all this in a single program. + + // --- begin create keypair --- + privkey, pubkey, err := rsa.CreateKeyPair() + if err != nil { + return errors.Wrapf(err, "cannot create keypair") + } + // --- end create keypair --- + + // --- begin setup --- err = prepareComponentInRepo(ctx, cfg) if err != nil { return errors.Wrapf(err, "cannot prepare component version in target repo") } + // --- end setup --- // every context features a signing registry, which provides available // signers and hashers, but also keys for various purposes. - // It is always asked, if a key is required for a purpose, which is + // It is always asked if a key is required, which is // not explicitly given to a signing/verification call. + // This context part is implemented as additional attribute stored along + // with the context. Attributes are always implemented as a separate package + // containing the attribute structure, its deserialization and + // a `Get(Context)` function to retrieve the attribute for the context. + // This way new arbitrary attributes for various use cases can be added + // without the need to change the context interface. + // --- begin signing attribute --- siginfo := signingattr.Get(ctx) + // --- end signing attribute --- - // to sign a component version we need a private key. - // for this example, we just create a local keypair. - // to be able to verify later, we should save the public key, - // but here we do all this in a single program. - - privkey, pubkey, err := rsa.CreateKeyPair() - if err != nil { - return errors.Wrapf(err, "cannot create keypair") - } - - // now we add the key to our context. - // this can be done, for example by adding an appropriate - // config object to your config file (see tour04). - // here, we do it manually, just for demonstration + // now we add the key manually to our context. + // --- begin configure keys --- siginfo.RegisterPrivateKey("acme.org", privkey) siginfo.RegisterPublicKey("acme.org", pubkey) + // --- end configure keys --- // now, we are prepared and can sign any component version // in any repository for the signature name acme.org. - // just get a the component version from the prepared repo. + // just get the component version from the prepared repo. + // --- begin lookup component version --- fmt.Printf("repository is %s\n", string(cfg.Target)) repo, err := ctx.RepositoryForConfig(cfg.Target, nil) if err != nil { @@ -100,13 +119,16 @@ func SigningComponentVersionInRepo(cfg *helper.Config) error { return errors.Wrapf(err, "version not found") } defer cv.Close() + // --- end lookup component version --- // we don't need to present they key, here. It is taken from the // context. + // --- begin sign --- _, err = signing.SignComponentVersion(cv, "acme.org") if err != nil { return errors.Wrapf(err, "cannot sign component version") } + // --- end sign --- // please be aware that the signature should be stored. fmt.Printf("*** signed composition version ***\n") @@ -115,11 +137,52 @@ func SigningComponentVersionInRepo(cfg *helper.Config) error { return errors.Wrapf(err, "describe failed") } + // The same way we can just call `VerifyComponentVersion` to + // verify the signature. + // --- begin verify --- _, err = signing.VerifyComponentVersion(cv, "acme.org") if err != nil { return errors.Wrapf(err, "verification failed") } else { fmt.Printf("verification succeeded\n") } + // --- end verify --- + + return createOCMConfig(privkey, pubkey) +} + +func createOCMConfig(privkey signutils.GenericPrivateKey, pubkey signutils.GenericPublicKey) error { + // manually adding keys to the signing attribute + // might simplify the call to possibly multiple signing/verification + // calls, but it does not help to provide keys via an external + // configuration (for example for using the OCM CLI). + // in tour05 we have seen how arbitrary configuration + // possibilities can be added. The signing attribute uses + // this mechanism to configure itself by providing an own + // configuration object, which can be used to feed keys (and certificates) + // into the signing attribute of an OCM context. + + // --- begin create signing config --- + sigcfg := signingattr.New() + // --- end create signing config --- + + // it provides methods to add elements + // like keys and certificates. which convert + // these elements into a (de-)serializable form. + // --- begin add signing config --- + sigcfg.AddPrivateKey("acme.org", privkey) + sigcfg.AddPublicKey("acme.org", pubkey) + + ocmcfg := configcfg.New() + ocmcfg.AddConfig(sigcfg) + // --- end add signing config --- + + // --- begin print signing config --- + data, err := runtime.DefaultYAMLEncoding.Marshal(ocmcfg) + if err != nil { + return err + } + fmt.Printf("ocm config file configuring standard keys:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(data)) + // --- end print signing config --- return nil } diff --git a/examples/lib/tour/06-signing-component-versions/README.md b/examples/lib/tour/06-signing-component-versions/README.md index 0fa356f6ed..6054c957f9 100644 --- a/examples/lib/tour/06-signing-component-versions/README.md +++ b/examples/lib/tour/06-signing-component-versions/README.md @@ -1,3 +1,6 @@ + + + # Signing Component Versions This tour illustrates the basic functionality to @@ -5,9 +8,11 @@ sign and verify signatures. It covers two basic scenarios: - [`sign`](01-basic-signing.go) Create, Sign, Transport and Verify a component version. -- [`repo`](02-using-context-settings.go) Using context settings to configure signing and verification in target repo. +- [`context`](02-using-context-settings.go) Using context settings to configure signing and verification in target repo. -You can just call the main program with some config file option (`--config `) and the name of the scenario. +## Running the examples + +You can call the main program with a config file option (`--config `) and the name of the scenario. The config file should have the following content: ```yaml @@ -19,8 +24,324 @@ targetRepository: ocmConfig: ``` -The actual version of the example just works with the filesystem +The actual version of the example just works with the file system target, because it is not possible to specify credentials for the target repository in this simple config file. But, if you specific an [OCM config file](../04-working-with-config/README.md) you can add more credential settings to make target repositories possible requiring credentials. + +## Walkthrough + +### Create, Sign, Transport and Verify a component version + +As usual, we start with getting access to an OCM context + +```go + ctx := ocm.DefaultContext() +``` +Then, we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04](../04-working-with-config/README.md#standard-configuration-file). + +```go + err := ReadConfiguration(ctx, cfg) + if err != nil { + return err + } +``` + +To sign a component version we need a private key. +For this example, we just create a local keypair. +To be able to verify later, we should save the public key, +but here we do all this in a single program. + +```go + privkey, pubkey, err := rsa.CreateKeyPair() + if err != nil { + return errors.Wrapf(err, "cannot create keypair") + } +``` + + +And we need a component version to sign. +We again compose a component version without a repository +(see [tour02 example 2](../02-composing-a-component-version/README.md#composition-environment)). + +```go + cv := composition.NewComponentVersion(ctx, "acme.org/example6", "v0.1.0") + + // just use the same component version setup again + err = setupVersion(cv) + if err != nil { + return errors.Wrapf(err, "version composition") + } + + fmt.Printf("*** composition version ***\n") + err = describeVersion(cv) +``` + +Now, let's sign the component version. +There might be multiple signatures, therefore every signature +has a name (here `acme.org`). Keys are always specified for +a dedicated signature name. The signing process can be influenced by +several options. Here, we just provide the private key to be used in an ad-hoc manner. +[Later](#using-context-settings-to-configure-signing), we will see how everything can be preconfigured in a *signing context*. + +```go + _, err = signing.SignComponentVersion(cv, "acme.org", signing.PrivateKey("acme.org", privkey)) + if err != nil { + return errors.Wrapf(err, "cannot sign component version") + } + fmt.Printf("*** signed composition version ***\n") + err = describeVersion(cv) +``` + +Now, we add the signed component version to a target repository. +Here, we just reuse the code from [tour02](../02-composing-a-component-version/README.md#composition-environment) + +```go + fmt.Printf("target repository is %s\n", string(cfg.Target)) + target, err := ctx.RepositoryForConfig(cfg.Target, nil) + if err != nil { + return errors.Wrapf(err, "cannot open repository") + } + defer target.Close() + + err = target.AddComponentVersion(cv, true) + if err != nil { + return errors.Wrapf(err, "cannot store signed version") + } +``` + +Let's check the target for the new component version. + +```go + tcv, err := target.LookupComponentVersion("acme.org/example6", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "transported version not found") + } + defer tcv.Close() + + // please be aware that the signature should be stored. + fmt.Printf("*** target version in transportation target\n") + err = describeVersion(tcv) + if err != nil { + return errors.Wrapf(err, "describe failed") + } +``` + +Please note, that the version now contains a signature. + +Finally, we check whether the signature is still valid for the +target version. + +```go + _, err = signing.VerifyComponentVersion(cv, "acme.org", signing.PublicKey("acme.org", pubkey)) + if err != nil { + return errors.Wrapf(err, "verification failed") + } else { + fmt.Printf("verification succeeded\n") + } +``` + +### Using Context Settings to Configure Signing + +Instead of providing all signing relevant information directly with +the signing or verification calls, it is possible to preconfigure +various information at the OCM context. + +As usual, we start with getting access to an OCM context + +```go + ctx := ocm.DefaultContext() +``` + +Then, we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04](../04-working-with-config/README.md#standard-configuration-file). + +```go + err := ReadConfiguration(ctx, cfg) + if err != nil { + return err + } +``` + +To sign a component version we need a private key. +For this example, we again just create a local keypair. +To be able to verify later, we should save the public key, +but here we do all this in a single program. + +```go + privkey, pubkey, err := rsa.CreateKeyPair() + if err != nil { + return errors.Wrapf(err, "cannot create keypair") + } +``` + +Finally, we create a component version in our target repository. The called +function + +```go + err = prepareComponentInRepo(ctx, cfg) + if err != nil { + return errors.Wrapf(err, "cannot prepare component version in target repo") + } +``` + +executes the same coding already shown in the [previous](#tour06-compose) example. + +#### Signing Using Manual Context Settings + +After this preparation we now configure the signing part of the OCM context. +Every OCM context features a signing registry, which provides available +signers and hashers, but also keys and certificates for various purposes. +It is always asked if a key is required, which is +not explicitly given to a signing/verification call. + +This context part is implemented as additional attribute stored along +with the context. Attributes are always implemented as a separate package +containing the attribute structure, its deserialization and +a `Get(Context)` function to retrieve the attribute for the context. +This way new arbitrary attributes for various use cases can be added +without the need to change the context interface. + +```go + siginfo := signingattr.Get(ctx) +``` + +Now, we manually add the keys to our context. + +```go + siginfo.RegisterPrivateKey("acme.org", privkey) + siginfo.RegisterPublicKey("acme.org", pubkey) +``` + +We are prepared now and can sign any component version without specifying further options +in any repository for the signature name `acme.org`. + +Therefore, we just get the component version from the prepared repository + +```go + fmt.Printf("repository is %s\n", string(cfg.Target)) + repo, err := ctx.RepositoryForConfig(cfg.Target, nil) + if err != nil { + return errors.Wrapf(err, "cannot open repository") + } + defer repo.Close() + + cv, err := repo.LookupComponentVersion("acme.org/example6", "v0.1.0") + if err != nil { + return errors.Wrapf(err, "version not found") + } + defer cv.Close() +``` + +and finally sign it. We don't need to present the key, here. It is taken from the +context. + +```go + _, err = signing.SignComponentVersion(cv, "acme.org") + if err != nil { + return errors.Wrapf(err, "cannot sign component version") + } +``` + +The same way we can just call `VerifyComponentVersion` to +verify the signature. + +```go + _, err = signing.VerifyComponentVersion(cv, "acme.org") + if err != nil { + return errors.Wrapf(err, "verification failed") + } else { + fmt.Printf("verification succeeded\n") + } +``` + +#### Configuring Keys with OCM Configuration File + +Manually adding keys to the signing attribute +might simplify the call to possibly multiple signing/verification +calls, but it does not help to provide keys via an external +configuration (for example for using the OCM CLI). +In [tour04](../04-working-with-config/README.md#providing-new-config-object-types) +we have seen how arbitrary configuration +possibilities can be added. The signing attribute uses +this mechanism to configure itself by providing an own +configuration object, which can be used to feed keys (and certificates) +into the signing attribute of an OCM context. + +```go + sigcfg := signingattr.New() +``` + +It provides methods to add elements +like keys and certificates, which convert +these elements into a (de-)serializable form. + +```go + sigcfg.AddPrivateKey("acme.org", privkey) + sigcfg.AddPublicKey("acme.org", pubkey) + + ocmcfg := configcfg.New() + ocmcfg.AddConfig(sigcfg) +``` + +By adding this config to a generic configuration object you get +an OCM config usable to predefine keys for your CLI. + +```go + data, err := runtime.DefaultYAMLEncoding.Marshal(ocmcfg) + if err != nil { + return err + } + fmt.Printf("ocm config file configuring standard keys:\n--- begin ocmconfig ---\n%s--- end ocmconfig ---\n", string(data)) +``` + +And here is a sample output containing the public and private key. + +```yaml +configurations: +- privateKeys: + acme.org: + stringdata: | + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEAqcTYOwDJe1osQVw1pbVTwwvpqME/q0ACkAlsK3BtQaUo+SUI + /B31DcsooMdM+SVBVLDx3Pa30j1Ud6Kf5l1tGTeGycT6X+l8hAP5KM2GKAa0Z8gl + egE5lPejlx8++SXEqSMZv39EjMuWPQwATTS6ws0/SMP9H+RbSaeHhscL4UBCQpPg + b1ANkpl37opx5zEx/NhhCesDffE30h7fXD7Uu57lLhkP2HJDTjf7mBfmWVQ8QDDK + cILCshOw8qZvpdhWYaSaUsCQ9W0NJ7sKhQZHIf9MoX3EsY+7g2V1USbNfNXsQskq + +Wvv9mY5gKGP+j4h6tKeTeBNNoiGHPTk6rdCLQIDAQABAoIBAD2oSkgTnxl3xH7w + eGN4mbVLDE/H79HIa6XYZjrYmDWxQFJMSxkV4DxkPps2BxStnS6fHRh9WoG22Iii + vaQy5j60VfXN3okbCagAsWtKSaEb3kWbAVFwRHOABSALrxlZyDUNlHpiRIlGH4iI + ZUulDPdXB9brp3D/xM/ZUnV2sS/bSPF/5/5dLmUZTJi+rC5eZypNDGbWflQ10aJM + QC9cP3nZFZGZx93PWN4iWjmRetlsOqJjaTUAQehdNd9SGHv/tDEPHY3c3hOBwgwr + z+fCQEgfRXX6YuLP01vHqn6lezYAYJNpbA1oRKzlD7+qcoAQCMrUu4+5df9OwoyB + 6xgDMAECgYEAwxApR0xz6UURxA1NBD4kJQEXV8eFdtUoRq26baFclBzc6slpq6N2 + hhQHs1LnTexJ+bODPLJADyuFdoXzrKUMcZb8Q4KEf2ayEsYME2ZdIMeI24cyS2ZI + CjnGcjPplLM3g+9pXq+QS/G2dTdJczmY3h4LSKJ1ActLkkNqNvlDOgECgYEA3s3S + oT4qf1rkPtFO7Z3scgMWQR/Gs4AQOvzKzxdPoDaNq4yCM/ZwO/76Ts9242GgtLNg + GP/q0eRlKVxfN0SYsiskFYoE1Fr0pz48NFKWDDw5KXrWQI6k6yJlXDrnHZp+OJ3z + wUmeLvcuPkm1MBBQ2OdI+dtKiWIb/+HXAQpGEC0CgYBbuF2wiOJ37WJNLXPpas7U + F49CVy8KkXA+y7G9mwJNIsU+ITbu3g39Pa3hRDo/Cbw/DYnIIIi+mVhIQvQxWepf + /v7fP5/NyBwzd6x18swXfbt8fjXH/nAhXslRKdfLc/nGr+x7+VGAZEfHFhgTdiHL + T5U+siUSkuUWAV0QPGTAAQKBgQDQLplEuIWVAiSK3aBWPl2UGnZM25gaWOrRcys9 + XZa1KMQvKtbuHrK4HINd6FQ6GhrDPWfpdBbBkBtGDl2Zkqrqr4zD43anxWUcb/Zp + HVG+lPcEXxaas649VqJHD3KsIpMV6+C7FkKLt8KpyM1X36brRRDXBaQbwmRPL4Jq + ImNc8QKBgAshe+bZFGFV2kqhwh6Vc0OrIrMwnncsZhzHfJVRm74WjuxfqH9u3drt + 6Ep8Xb6L0hlbviN3jRCG29TYL6QVZbyRiO8P4z4oXkZuOWqd93dksAKTbffcuRCF + i20zdlJ/ClTWN0EwmiYgitAJ7e1ZyVZgdt0qjbclqyOlV7Or8qma + -----END RSA PRIVATE KEY----- + publicKeys: + acme.org: + stringdata: | + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAqcTYOwDJe1osQVw1pbVTwwvpqME/q0ACkAlsK3BtQaUo+SUI/B31 + DcsooMdM+SVBVLDx3Pa30j1Ud6Kf5l1tGTeGycT6X+l8hAP5KM2GKAa0Z8glegE5 + lPejlx8++SXEqSMZv39EjMuWPQwATTS6ws0/SMP9H+RbSaeHhscL4UBCQpPgb1AN + kpl37opx5zEx/NhhCesDffE30h7fXD7Uu57lLhkP2HJDTjf7mBfmWVQ8QDDKcILC + shOw8qZvpdhWYaSaUsCQ9W0NJ7sKhQZHIf9MoX3EsY+7g2V1USbNfNXsQskq+Wvv + 9mY5gKGP+j4h6tKeTeBNNoiGHPTk6rdCLQIDAQAB + -----END RSA PUBLIC KEY----- + type: keys.config.ocm.software +type: generic.config.ocm.software +``` \ No newline at end of file diff --git a/examples/lib/tour/06-signing-component-versions/common.go b/examples/lib/tour/06-signing-component-versions/common.go index 0bb09b78cf..f4057b1096 100644 --- a/examples/lib/tour/06-signing-component-versions/common.go +++ b/examples/lib/tour/06-signing-component-versions/common.go @@ -107,7 +107,7 @@ data: some very important data required to understand this component // Any blob content provided by an implementation of this // interface can be added as resource. // There are various access implementations for blobs - // taken from the local host, for example, from the filesystem, + // taken from the local host, for example, from the file system, // or from other repositories (for example by mapping // an access type specification into a blob access). // The most simple form is to directly provide a byte sequence, @@ -149,7 +149,7 @@ data: some very important data required to understand this component } // There are even more complex blob sources, for example - // for helm charts stored in the filesystem, or even for images + // for Helm charts stored in the file system, or even for images // generated by docker builds. // Here, we just compose a multi-platform image built with buildx // from these sources (components/ocmcli) featuring two flavors. diff --git a/examples/lib/tour/06-signing-component-versions/main.go b/examples/lib/tour/06-signing-component-versions/main.go index e2b91e6d83..b253613de3 100644 --- a/examples/lib/tour/06-signing-component-versions/main.go +++ b/examples/lib/tour/06-signing-component-versions/main.go @@ -10,17 +10,21 @@ import ( "strings" "github.com/open-component-model/ocm/examples/lib/helper" + "github.com/open-component-model/ocm/pkg/signing/handlers/rsa" ) // CFG is the path to the file containing the credentials -var CFG = "../examples/lib/cred.yaml" +var CFG = "examples/lib/cred.yaml" var current_version string func init() { data, err := os.ReadFile("VERSION") if err != nil { - panic("VERSION not found") + data, err = os.ReadFile("../../../../../VERSION") + if err != nil { + panic("VERSION not found") + } } current_version = strings.TrimSpace(string(data)) } @@ -48,8 +52,13 @@ func main() { switch cmd { case "sign": err = SigningComponentVersions(cfg) - case "repo": + case "context": err = SigningComponentVersionInRepo(cfg) + case "config": + privkey, pubkey, err := rsa.CreateKeyPair() + if err == nil { + err = createOCMConfig(privkey, pubkey) + } default: err = fmt.Errorf("unknown example %q", cmd) } diff --git a/examples/lib/tour/06-signing-component-versions/settings.yaml b/examples/lib/tour/06-signing-component-versions/settings.yaml new file mode 100644 index 0000000000..513e95095a --- /dev/null +++ b/examples/lib/tour/06-signing-component-versions/settings.yaml @@ -0,0 +1,4 @@ +username: mandelsoft +password: ghp_xyz +component: github.com/mandelsoft/examples/cred1 +version: 0.1.0 \ No newline at end of file diff --git a/examples/lib/tour/README.md b/examples/lib/tour/README.md index 58ac1d1da1..3046d42686 100644 --- a/examples/lib/tour/README.md +++ b/examples/lib/tour/README.md @@ -1,3 +1,6 @@ + + + # A Tour through Usage Scenarios of the OCM Library This tour guides you from a very basic usage of the diff --git a/examples/lib/tour/doc.go b/examples/lib/tour/doc.go new file mode 100644 index 0000000000..9b7a6be504 --- /dev/null +++ b/examples/lib/tour/doc.go @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdref --headings --list docsrc . + +package tour diff --git a/examples/lib/tour/docsrc/01-getting-started/README.md b/examples/lib/tour/docsrc/01-getting-started/README.md new file mode 100644 index 0000000000..d402bd05fb --- /dev/null +++ b/examples/lib/tour/docsrc/01-getting-started/README.md @@ -0,0 +1,200 @@ +# Basic Usage of OCM Repositories +{{getting-started}} + +This [tour](example.go) illustrates the basic usage of the API to +access component versions in an OCM repository. + +## Running the example + +You can call the main program with a config file argument +(`--config `), where the config file has the following content: + +```yaml +component: github.com/mandelsoft/examples/cred1 +repository: ghcr.io/mandelsoft/ocm +version: 0.1.0 +``` + +{{getting-started-walkthrough}} +## Walkthrough + +The basic entry point for using the OCM library is always +an [OCM Context object](../../contexts.md). It bundles all +configuration settings and type registrations, like +access methods, repository types, etc, and +configuration settings, like credentials, +which should be used when working with the OCM +ecosystem. + +Therefore, the first step is always to get access to such +a context object. Our example uses the default context +provided by the library, which covers the complete +type registration contained in the executable. + +It can be accessed by a function of the `pkg/contexts/ocm` package. + +```go +{{include}{../../01-getting-started/example.go}{default context}} +``` + +The context acts as the central entry +point to get access to OCM elements. +First, we get a repository, to look for +component versions. We use the OCM +repository hosted on `ghcr.io`, which is providing the standard OCM +components. + +For every storage technology used to store +OCM components, there is a serializable +descriptor object, the *repository specification*. +It describes the information required to access +the repository and can be used to store the serialized +form as part of other resources, for example +Kubernetes resources or configuration settings. +The available repository implementations can be found +under `.../pkg/contexts/ocm/repositories`. + +```go +{{include}{../../01-getting-started/example.go}{repository spec}} +``` + +The context can now be used to map the descriptor +into a repository object, which then provides access +to the OCM elements stored in this repository. + +```go +{{include}{../../01-getting-started/example.go}{repository}} +``` + +To release potentially allocated temporary resources, many objects +must be closed, if they are not used anymore. +This is typically done by a `defer` statement placed after a +successful object retrieval. + +```go +{{include}{../../01-getting-started/example.go}{close}} +``` + +Now we look for the versions of the component +available in this repository. + +```go +{{include}{../../01-getting-started/example.go}{versions}} +``` + +OCM version names must follow the *SemVer* rules. +Therefore, we can simply order the versions and print them. + +```go +{{include}{../../01-getting-started/example.go}{semver}} +``` + +Now, we have a look at the latest version. It is +the last one in the list. + +```go +{{include}{../../01-getting-started/example.go}{lookup version}} +``` + +{{describe-version}} + +The component version object provides access +to the component descriptor + +```go +{{include}{../../01-getting-started/example.go}{component descriptor}} +``` + +and the resources described by the component version. + +```go +{{include}{../../01-getting-started/example.go}{resources}} +``` + +This results in the following output (the shown version might +differ, because the code always describes the latest version): + +``` +{{execute}{go}{run}{../../01-getting-started}{}{version}} +``` + +Resources have some metadata, like their identity and a resource type. +And, most importantly, they describe how the content of the resource +(as blob) can be accessed. +This is done by an *access specification*, again a serializable descriptor, +like the repository specification. + +The component version used here contains the executables for the OCM CLI +for various platforms. The next step is to +get the executable for the actual environment. +The identity of a resource described by a component version +consists of a set of properties. The property `name` is mandatory. But there may be more identity attributes +finally stored as ``extraIdentity` in the component descriptor. + +A convention is to use dedicated identity properties to indicate the +operating system and the architecture for executables. + +```go +{{include}{../../01-getting-started/example.go}{find executable}} +``` + +Now we want to retrieve the executable. The library provides two +basic ways to do this. + +First, there is the direct way to gain access to the blob by using +the basic model operations to get a reader for the resource blob. +Therefore, in a first step we get the access method for the resource + +```go +{{include}{../../01-getting-started/example.go}{getting access}} +{{include}{../../01-getting-started/example.go}{closing access}} +``` + +The method needs to be closed, because the method +object may cache the technical blob representation +generated by accessing the underlying access technology. +(for example, accessing an OCI image requires a sequence of +backend requests for the manifest, the layers, etc, which will +then be packaged into a tar archive returned as blob). +This caching may not be required, if the backend directly +returns a blob. + +Now, we get access to the reader providing the blob content. +The blob features a mime type, which can be used to understand +the format of the blob. Here, we have a plain octet stream. + +```go +{{include}{../../01-getting-started/example.go}{getting reader}} +``` + +Because this code sequence is a common operation, there is a +utility function handling this sequence. A shorter way to get +a resource reader is as follows: + +```go +{{include}{../../01-getting-started/example.go}{utility function}} +``` + +Before we download the content we check the error and prepare +closing the reader, again + +```go +{{include}{../../01-getting-started/example.go}{closing reader}} +``` + +Now, we just read the content and copy it to the intended +output file. + +```go +{{include}{../../01-getting-started/example.go}{copy}} +``` + +Another way to download a resource is to use registered *downloaders*. +`download.DownloadResource` is used to download resources with specific handlers for +selected resource and mime type combinations. +The executable downloader is registered by default and automatically +sets the `X` flag for the written file. + +```go +{{include}{../../01-getting-started/example.go}{download}} +``` diff --git a/examples/lib/tour/docsrc/02-composing-a-component-version/README.md b/examples/lib/tour/docsrc/02-composing-a-component-version/README.md new file mode 100644 index 0000000000..5e8afce9b2 --- /dev/null +++ b/examples/lib/tour/docsrc/02-composing-a-component-version/README.md @@ -0,0 +1,249 @@ +{{compose-compvers}} +# Composing a Component Version + +This tour illustrates the basic usage of the API to +create/compose component versions. + +It covers two basic scenarios: +- [`basic`](01-basic-componentversion-creation.go) Create a component version stored in the file system +- [`compose`](02-composition-version.go) Create a component version stored in memory using a non-persistent composition version. + +## Running the example + +You can call the main program with the scenario as argument. Configuration is not required. + +## Walkthrough + +### Basic Component Version Creation + +The first variant just creates a new component version +in an OCM repository. To avoid the requirement for +credentials a file system based repository is created, using +the *Common Transport Format* (CTF). + +As usual, we start with getting access to an OCM context +object: + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{default context}} +``` + +To compose and store a new component version +we need some OCM repository to +store the component. The most simple +external repository could be the file system. +For this purpose OCM defines a distribution format, the +*Common Transport Format* (CTF), +which is an extension of the OCI distribution +specification. +There are three flavors, *Directory*, *Tar* or *TGZ*. +The implementation provides a regular OCM repository +interface, like the one used in the previous example. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{create ctf}} +``` + +Once we have a repository we can compose a new version. +First, we create a new version backed by this repository. +The result is a memory based representation, which is not yet persisted. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{new version}} +``` + +Now, we can configure the component version. It only exists in memory +so far, but is already connected to the repository. + +The setup of the component version is put into a +separate method (`setupVersion`), so it can be reused for the second variant. + +First, we configure the component version provider. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup provider}} +``` + +The provider is a structure with a name and some labels. +We just set the name here by directly setting the `Name` attribute. + +Now, we fill the component version with content. +First, we add some resource already located in +an external registry. We use an OCI image here. +A resources has some metadata, like an identity +and a type. +The identity is just a set of string properties, +at least containing the `name` property. +Additional identity properties can be added via +options. +The type represents the logical meaning of the +resource, here an `ociImage`. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup resource meta}} +``` + +In this example, we just use the `name` property +without any extra identity. + +And most importantly, a resource requires content. +Content can already be present in some external +repository. As long, as there is an access type +for this kind of repository, we can just refer to it. +Here, we just use an image provided by the +OCM ecosystem. +Supported access types can be found under +.../pkg/contexts/ocm/accessmethods. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup image access}} +``` + +Once we have both, the metadata and the content specification, +we can now add the resource to our component version. +The `SetResource` methods will replace an existing resource with the same +identity, or add the resource, if no such resource exists in the component +version. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup resource}} +``` + +Now, we will add a second resource, some unspecific yaml data. +Therefore, we use the generic YAML resource type. +In practice, you should always use a resource type describing +the real meaning of the content, for example something like +`kubernetesManifest`. This enables tools working with specific content +to understand the resource set of a component version. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup second meta}} +``` + +Besides referring to external resources, another possibility +to add content is to directly provide the content blob. The +used abstraction here is `blobaccess.BlobAccess`. + +Any blob content, which can be provided by an implementation of this +interface, can be added as resource to a component version. +The library provides various access implementations for blobs +taken from the local host or from other repositories. +For example, this could be some file system content. +To describe blobs taken from external repositories +an access type specification can be mapped to a blob access. +Hereby, blobs are stored along with the component descriptor +instead of storing a reference to content in an external repository. + +The most simple form is to directly provide a byte sequence, +for example some YAML data. +A blob always must provide a mime type, describing the +technical format of the blob's byte sequence. This is different +from the resource type. A logical resource, like a *Helm chart* can be +represented in different technical formats, for example a Helm chart +archive or as OCI image archive. While the type described the +logical content, the meaning of the resource, its mime type +described the technical blob format used to represent +the resource as byte sequence. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{string blob access}} +``` + +When storing the blob, it is possible to provide some +optional additional information: +- a name of the resource described by the blob, which could + be used to do a later upload into an external repository + (for example the image repository of an OCI image stored + as local blob) +- an additional access type, which provides an alternative + global technology specific access to the same content + (we don't use it, here). + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup by blob access}} +``` + +Resources added by blobs will be stored along with the component +version metadata in the same repository, no external +repository is required. + +The above blob example describes the basic operations, +which can be used to compose any kind of resource +from any kind of source. +For selected use cases there are convenience helpers available, +which can be used to compose a resource access object. +This is basically the same interface returned by `GetResource` +functions on the component version from the last example. +Such objects can directly be used to add/modify a resource in a +component version. + +The above case could also be written as follows: + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup by access}} +``` + +The resource access is an abstraction of external access via access +methods or direct blob access objects and additionally +contain all the required resource metadata. + +There are even more complex blob sources, for example +for Helm charts stored in the file system, or even for images +generated by docker builds. +Here, we just compose a multi-platform image built with `buildx` +from these sources (components/ocmcli) featuring two flavors. +(you have to execute `make image.multi` in components/ocmcli +before executing this example. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{setup by docker}} +``` + +{{composition-environment}} +### Composition Environment + +The second variant just creates a new component version +in a memory based composition environment, no persistence is +required. Like all component versions, such component versions +can be added to any repository later. + +As usual, we start with getting access to an OCM context +object: + +```go +{{include}{../../02-composing-a-component-version/02-composition-version.go}{default context}} +``` + +Now, we can create a new component version in the composition +environment. This does not require a repository or component object. + +```go +{{include}{../../02-composing-a-component-version/02-composition-version.go}{new version}} +``` + +To configure the component version, we can just reuse the coding +from the example above, the component version interface is just the same. +We just call the `setupVersion` function for the created component version access. + +```go +{{include}{../../02-composing-a-component-version/02-composition-version.go}{setup version}} +``` + +The resulting component version can be added to any OCM repository, +like the one from the previous example. +Here, we use another feature of the composition environment. It also provides +complete memory based OCM repositories. +It has no storage backend and can be used to internally compose +a set of component versions, which can then be transferred +to any other repository (see [tour 05]({{transport}})) + +```go +{{include}{../../02-composing-a-component-version/02-composition-version.go}{create composition repository}} +``` + +This repository object behaves like any other OCM repository object. We can just +add the new component version. + +```go +{{include}{../../02-composing-a-component-version/02-composition-version.go}{add version}} +``` \ No newline at end of file diff --git a/examples/lib/tour/docsrc/03-working-with-credentials/README.md b/examples/lib/tour/docsrc/03-working-with-credentials/README.md new file mode 100644 index 0000000000..7598a5bb4e --- /dev/null +++ b/examples/lib/tour/docsrc/03-working-with-credentials/README.md @@ -0,0 +1,363 @@ +{{credentials}} +# Working with Credentials + +This tour illustrates the basic handling of credentials +using the OCM library. The library provides +an extensible framework to bring together credential providers +and credential consunmers in a technology-agnostic way. + +It covers four basic scenarios: +- [`basic`](01-using-credentials.go) Writing to a repository with directly specified credentials. +- [`context`](02-basic-credential-management.go) Using credentials via the credential management to publish a component version. +- [`read`](02-basic-credential-management.go) Read the previously created component version using the credential management. +- [`credrepo`](03-credential-repositories.go) Providing credentials via credential repositories. + +## Running the example + +You can call the main program with a config file option (`--config `) and the name of the scenario. +The config file should have content similar to: + +```yaml +repository: ghcr.io/mandelsoft/ocm +username: +password: +``` + +Set your favorite OCI registry and don't forget to add the repository prefix for your OCM repository hosted in this registry. + +## Walkthrough + +### Writing to a repository with directly specified credentials. + +As usual, we start with getting access to an OCM context +object. + +```go +{{include}{../../02-composing-a-component-version/01-basic-componentversion-creation.go}{default context}} +``` + +So far, we just used memory or file system based +OCM repositories to create component versions. +If we want to store something in a remotely accessible +repository typically some credentials are required +for write access. + +The OCM library uses a generic abstraction for credentials. +It is just set of properties. To offer various credential sources +there is an interface `credentials.Credentials` provided, +whose implementations provide access to those properties. +A simple property based implementation is `credentials.DirectCredentials. + + +The most simple use case is to provide the credentials +directly for the repository access creation. +The example config file provides such credentials +for an OCI registry. + +```go +{{include}{../../03-working-with-credentials/01-using-credentials.go}{new credentials}} +``` + +Now, we can use the OCI repository access creation from the [first tour](../01-getting-started/README.md#walkthrough), +but we pass the credentials as additional parameter. +To give you the chance to specify your own registry, the URL +is taken from the config file. + +```go +{{include}{../../03-working-with-credentials/01-using-credentials.go}{repository access}} +``` + +If registry name and credentials are fine, we should be able +now to add a new component version to this repository using the coding +from the previous examples, but now we use a public repository, instead +of a memory or file system based one. This coding is in function `addVersion` +in `common.go` (It is shared by the other examples, also). + +```go +{{include}{../../03-working-with-credentials/common.go}{create version}} +``` + +In contrast to our [first tour]({{getting-started-walkthrough}}) +we cannot list components, here. +OCI registries do not support component listers, therefore we +just look up the actually added version to verify the result. + +```go +{{include}{../../03-working-with-credentials/01-using-credentials.go}{lookup}} +``` + +The coding for `describeVersion` is similar to the one shown in the [first tour]({{describe-version}}). + +{{using-cred-management}} +### Using the Credential Management + +Passing credentials directly at the repository +is fine, as long only the component version +will be accessed. But as soon as described +resource content will be read, the required +credentials and credential types are dependent +on the concrete component version, because +it might contain any kind of access method +referring to any kind of resource repository +type. + +To solve this problem of passing any set +of credentials the OCM context object is +used to store credentials. This is handled +by a sub context, the *Credentials context*. + +As usual, we start with the default OCM context. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{default context}} +``` + +It is now used to gain access to the appropriate +credential context. + + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{cred context}} +``` + +The credentials context brings together +providers of credentials, for example a +Vault or a local Docker config.json +and credential consumers like GitHub or +OCI registries. +It must be able to distinguish various kinds +of consumers. This is done by identifying +a dedicated consumer with a set of properties +called `credentials.ConsumerId`. It consists +at least of a consumer type property and a +consumer type specific set of properties +describing the concrete instance of such +a consumer, for example an OCI artifact in +an OCI registry is identified by a host and +a repository path. + +A credential provider like a vault just provides +named credential sets and typically does not +know anything about the use case for these sets. +The task of the credential context is to +provide credentials for a dedicated consumer. +Therefore, it maintains a configurable +mapping of credential sources (credentials in +a credential repository) and a dedicated consumer. + +This mapping defines a use case, also based on +a property set and dedicated credentials. +If credentials are required for a dedicated +consumer, it matches the defined mappings and +returned the best matching entry. + +Matching? Let's take the GitHub OCI registry as an +example. There are different owners for +different repository paths (the GitHub org/user). +Therefore, different credentials need to be provided +for different repository paths. +For example, credentials for `ghcr.io/acme` can be used +for a repository `ghcr.io/acme/ocm/myimage`. + +To start with the credentials context we just +provide an explicit mapping for our use case. + +First, we create our credentials object as before. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{new credentials}} +``` + +Then we determine the consumer id for our use case. +The repository implementation provides a function +for this task. It provides the most general property +set (no repository path) for an OCI based OCM repository. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{consumer id}} +``` + +The used functions above are just convenience wrappers +around the core type ConsumerId, which might be provided +for dedicated repository/consumer technologies. +Everything can be done directly with the core interface +and property name constants provided by the dedicted technologies. + +Once we have the id we can finally set the credentials for this +id. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{set credentials}} +``` + +Now, the context is prepared to provide credentials +for any usage of our OCI registry +Let's test, whether it could provide credentials +for storing our component version. + +First, we get the repository object for our OCM repository. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{get repository}} +``` + +Second, we determine the consumer id for our intended repository acccess. +A credential consumer may provide consumer id information +for a dedicated sub user context. +This is supported by the OCM repo implementation for OCI registries. +The usage context is here the component name. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{get access id}} +``` + +Third, we ask the credential context for appropriate credentials. +The basic context method `credctx.GetCredentialsForConsumer` returns +a credentials source interface able to provide credentials +for a changing credentials source. Here, we use a convenience +function, which directly provides a credentials interface for the +actually valid credentials. +An error is only provided if something went wrong while determining +the credentials. Delivering NO credentials is a valid result. +The returned interface then offers access to the credential properties. +via various methods. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{get credentials}} +``` + +Now, we can continue with our basic component version composition +from the last example, or we just display the content. + +The following code snipped shows the code for the `context` variant +creating a new version, the `read` variant just omits the version creation. +The rest of the example is identical. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{add version}} +``` + +Let's verify the created content and list the versions as known from tour 1. +OCI registries do not support component listers, therefore we +just get and describe the actually added version. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{show version}} +``` + +As we can see in the resource list, our image artifact has been +uploaded to the OCI registry as OCI artifact and the access method has be changed +to `ociArtifact`. It is not longer a local blob. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{examine cli}} +``` + +This resource access effectively points to the same OCI registry, +but a completely different repository. +If you are using *ghcr.io*, this freshly created repo is private, +therefore, we need credentials for accessing the content. +An access method also acts as credential consumer, which +tries to get required credentials from the credential context. +Optionally, an access method can act as provider for a consumer id, so that +it is possible to query the used consumer id from the method object. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{image credentials}} +``` + +Because the credentials context now knows the required credentials, +the access method as credential consumer can access the blob. + +```go +{{include}{../../03-working-with-credentials/02-basic-credential-management.go}{image access}} +``` + +### Providing credentials via credential repositories + +The OCM toolset embraces multiple storage +backend technologies, for OCM metadata as well +as for artifacts described by a component version. +All those technologies typically have their own +way to configure credentials for command line +tools or servers. + +The credential management provides so-called +credential repositories. Such a repository +is able to provide any number of named +credential sets. This way any special +credential store can be connected to the +OCM credential management just by providing +an own implementation for the repository interface. + +One such case is the docker config json, a config +file used by docker login to store +credentials for dedicated OCI registries. + +We start again by providing access to the +OCM context and the connected credential context. + + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{context}} +``` + +In package `.../contexts/credentials/repositories` you can find +packages for predefined implementations for some standard credential repositories, +for example `dockerconfig`. + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{docker config}} +``` + +There are general credential stores, like a HashiCorp Vault +or type-specific ones, like the docker config json +used to configure credentials for the docker client. +(working with OCI registries). +Those specialized repository implementations are not only able to +provide credential sets, they also know about the usage context +of the provided credentials. +Therefore, such repository implementations are also able to provide +credential mappings for consumer ids. This is supported by the credential +repository API provided by this library. + +The docker config is such a case, so we can instruct the +repository to automatically propagate appropriate the consumer id +mappings. This feature is typically enabled by a dedicated specfication +option. + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{propagation}} +``` + +Implementations for more generic credential repositories can also use this +feature, if the repository allows adding arbitrary metadata. This is for +example used by the `vault` implementation. It uses dedicated attributes +to allow the user to configure intended consumer id properties. + +Now, we can just add the repository for this specification to +the credential context by getting the repository object for our +specification. + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{add repo}} +``` + +We are not interested in the repository object, so we just ignore +the result. + +So, if you have done the appropriate docker login for your +OCI registry, it should be possible now to get the credentials +for the configured repository. + +We first query the consumer id for the repository, again. + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{get consumer id}} +``` + +and then get the credentials from the credentials context like in the previous example. + +```go +{{include}{../../03-working-with-credentials/03-credential-repositories.go}{get credentials}} +``` diff --git a/examples/lib/tour/docsrc/04-working-with-config/README.md b/examples/lib/tour/docsrc/04-working-with-config/README.md new file mode 100644 index 0000000000..74572d52bb --- /dev/null +++ b/examples/lib/tour/docsrc/04-working-with-config/README.md @@ -0,0 +1,603 @@ +{{config}} +# Working with Configurations + +This tour illustrates the basic configuration management +included in the OCM library. The library provides +an extensible framework to bring together configuration settings +and configurable objects. + +It covers five basic scenarios: +- [`basic`](01-basic-config-management.go) Basic configuration management illustrating the configuration of credentials. +- [`generic`](02-handle-arbitrary-config.go) Handling of arbitrary configuration. +- [`ocm`](03-using-ocm-config.go) Central configuration +- [`provide`](04-write-config-type.go) Providing new config object types +- [`consume`](05-write-config-consumer.go) Preparing objects to be configured by the config management + +## Running the example + +You can call the main program with a config file option (`--config `) and the name of the scenario. +The config file should have the following content: + +```yaml +repository: ghcr.io/mandelsoft/ocm +username: +password: +``` + +Set your favorite OCI registry and don't forget to add the repository prefix for your OCM repository hosted in this registry. + +## Walkthrough + +### Basic Configuration Management + +Similar to the other context areas, Configuration is handled by the configuration contexts. +Therefore, for the example, we just get the default configuration context. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{default context}} +``` + +The configuration context handles configuration objects. +A configuration object is any object implementing +the `config.Config` interface. The task of a config object +is to apply configuration to some target object. + +One such object is the configuration object for +credentials provided by the credentials context. +It finally applies settings to a credential context. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{cred config}} +``` + +Here, we can configure credential settings: +credential repositories and consumer id mappings. +We do this by setting the credentials provided +by our config file for the consumer id used +by our configured OCI registry. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{configure creds}} +``` + +(Credential) Configuration objects are typically serializable and deserializable. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{marshal}} +``` + +Like all the other manifest based descriptions this format always includes +a type field, which can be used to deserialize a specification into +the appropriate object. +This can be done by the config context. It accepts YAML or JSON. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{unmarshal}} +``` + +Regardless what variant is used (direct specification object or descriptor) +the config object can be added to a config context. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{apply config}} +``` + +Every config object implements the +`ApplyTo(ctx config.Context, target interface{}) error` method. +It takes an object, which wants to be configured. +The config object then decides, whether it provides +settings for the given object and calls the appropriate +methods on this object (after a type cast). + +Here is the code snippet from the apply method of the credential +config object ([.../pkg/contexts/credentials/config/type.go](../../../../../pkg/contexts/credentials/config/type.go)): + +```go +{{include}{../../../../../pkg/contexts/credentials/config/type.go}{apply}} + ... +``` + +This way the config mechanism reverts the configuration +request, it does not actively configure something, instead +an object, which wants to be configured calls the config +context to apply pending configs. +To do this the config context manages a queue of config objects +and applies them to an object to be configured. + +If the credential context is asked now for credentials, +it asks the config context for pending config objects +and applies them. +Therefore, we now should be able to get the configured credentials. + +```go +{{include}{../../04-working-with-config/01-basic-config-management.go}{get credentials}} +``` + +### Handling of Arbitrary Configuration + +The config management not only manages configuration objects for any +other configurable object, it also provides a configuration object of +its own. The task of the object is to handle other configuration objects +to be applied to a configuration object. + +```go +{{include}{../../04-working-with-config/02-handle-arbitrary-config.go}{config config}} +``` + +The generic config object holds a list of any other config objects, +or their specification formats. +Additionally, it is possible to configure named sets +of configurations, which can later be enabled +on-demand by their name at the config context. + +We recycle our credential config from the last example to get +a config object to be added to our generic config object. + +```go +{{include}{../../04-working-with-config/02-handle-arbitrary-config.go}{sub config}} +``` + +Now, we can add this credential config object to +our generic config list. + +```go +{{include}{../../04-working-with-config/02-handle-arbitrary-config.go}{add config}} +``` + +As we have seen in our previous example, config objects are typically +serializable and deserializable. This also holds for the generic config +object of the config context. + +```go +{{include}{../../04-working-with-config/02-handle-arbitrary-config.go}{serialized}} +``` + +The result is a config object hosting a list (with 1 entry) +of other config object specifications. + +The generic config object can be added to a config context, again, like +any other config object. If it is asked to configure a configuration +context it uses the methods of the configuration context to apply the +contained list of config objects (and the named set of config lists). +Therefore, all config objects applied to a configuration context are +asked to configure the configuration context itself when queued to the +list of applied configuration objects. + +If we now ask the default credential context (which uses the default +configuration context to configure itself) for credentials for our OCI registry, +the credential mapping provided by the config object added to the generic one, +will be found. + +```go +{{include}{../../04-working-with-config/02-handle-arbitrary-config.go}{query}} +``` + +The very same mechanism is used to provide central configuration in a +configuration file for the OCM ecosystem, as will be shown in the next example. + +### Central Configuration + +Although the configuration of an OCM context can +be done by a sequence of explicit calls according to the mechanisms +shown in the examples before, a simple convenience +library function is provided, which can be used to configure an OCM +context and all related other contexts with a single call +based on a central configuration file (`~/.ocmconfig`) + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{central config}} +``` + +This file typically contains the serialization of such a generic +configuration specification (or any other serialized configuration object), +enriched with specialized config specifications for +credentials, default repositories, signing keys and any +other configuration specification. + +{{ocm-config-file}} +#### Standard Configuration File + +Most important are here the credentials. +Because OCM embraces lots of storage technologies for artifact +storage as well as storing OCM component version metadata, +there are typically multiple technology specific ways +to configure credentials for command line tools. +Using the credentials settings shown in the previous tour, +it is possible to specify credentials for all +required purposes, and the configuration management provides +an extensible way to embed native technology specific ways +to provide credentials just by adding an appropriate type +of credential repository, which reads the specialized storage and +feeds it into the credential context. Those specifications +can be added via the credential configuration object to +the central configuration. + +One such repository type is the Docker config type. It +reads a `dockerconfig.json` file and feeds in the credentials. +Because it is used for a dedicated purpose (credentials for +OCI registries), it not only can feed the credentials, but +also their mapping to consumer ids. + +We first create the specification for a new credential repository of +type `dockerconfig` describing the default location +of the standard Docker config file. + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{docker config}} +``` + +By adding the default location for the standard Docker config +file, all credentials provided by the `docker login` command +are available in the OCM toolset, also. + +A typical minimal .ocmconfig file can be composed as follows. +We add this config object to an empty generic configuration object +and print the serialized form. The result can be used as +default initial OCM configuration file. + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{default config}} +``` + +The result should look similar to (but with reordered fields): +```yaml +type: generic.config.ocm.software +configurations: + - type: credentials.config.ocm.software + repositories: + - repository: + type: DockerConfig + dockerConfigFile: ~/.docker/config.json + propagateConsumerIdentity: true +``` + +Because of the ordered map keys the actual output looks a little bit confusing: + +```yaml +{{execute}{go}{run}{../../04-working-with-config}{--config}{settings.yaml}{ocm}{}{ocmconfig}} +``` + +Besides from a file, such a config can be provided as data, also, +taken from any other source, for example from a Kubernetes secret. + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{by data}} +``` + +If you have provided your OCI credentials with +`docker login`, they should now be available. + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{query}} +``` + +#### Templating + +The configuration library function does not only read the +ocm config file, it also applies [*spiff*](github.com/mandelsoft/spiff) +processing to the provided YAML/JSON content. *Spiff* is an +in-domain yaml-based templating engine. Therefore, you can use +any spiff dynaml expression to define values or even complete +sub structures. + +```go +{{include}{../../04-working-with-config/03-using-ocm-config.go}{spiff}} +``` + +This config object is not directly usable, because the cert value is not +a valid certificate. We use it here just to generate the serialized form. + +```yaml +{{execute}{go}{run}{../../04-working-with-config}{--config}{settings.yaml}{ocm}{}{spiffocmconfig}} +``` + +If this is used with the above library functions, the finally generated +config object will contain the read file content, which is hopefully a +valid certificate. + +{{tour04-arbitrary}} +### Providing new config object types + +So far, we just used existing config types to configure existing objects. +But the configuration management is highly extensible, and it is quite +simple to provide new config types, which can be used to configure +any new or existing object, which is prepared to consume configuration. + +The next [chapter]({{consume-config}}) will show how to prepare an +object to be automatically configurable by +the configuration management. Here, we focus on the implementation of +new config object types. Therefore, we want to configure the +credential context by a new configuration object. + +#### The Configuration Object Type + +Typically, every kind of configuration object lives in its own package, +which always have the same layout. + +A configuration object has a *type*, the configuration type. Therefore, +the package declares a constant `TYPE`. + +It is the name of our new configuration object type. +To be globally unique, it should always end with a +DNS domain owned by the provider of the new type. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{type name}} +``` + +Next, we need a Go type. `ExampleConfigSpec` is the new Go type for the +config specification covering our example configuration. +It just encapsulates our simple configuration structure +used to configure the examples of our tour. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{config type}} +``` + +Every config type structure must contain a field (and the appropriate methods) +for storing the config type name. This is done by embedding the +type `runtime.ObjectVersionedType` from the `runtime` package. This package +contains everything to work with specification objects and +serialization/deserialization. + +As second field we just embed the config structure used to read the tour +config. This way any kind of configuration information can be mapped +to the configuration management. + +A config type typically provide a constructor for a config object of +this type: + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{constructor}} +``` + +Additional setters can be used to configure the configuration object. +Here, programmatic objects (like an `ocm.RepositorySpec`) are +converted to a form storable in the configuration object. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{setters}} +``` + +The utility function `runtime.CheckSpecification` can be used to +check a byte sequence to be a valid specification. +It just checks for a valid YAML document featuring a non-empty +`type` field: + +```go +{{include}{../../../../../pkg/runtime/utils.go}{check}} +``` + +The most important method to implement is `ApplyTo(_ cpi.Context, tgt interface{}) error`, +which must be implemented by all configuration objects. +Its task is to apply the described configuration settings to a dedicated +object. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{method apply}} +``` + +Therefore, it decides, whether it is able to handle a dedicated type of target +object and how to configure it. This way a configuration object +may apply is settings or even parts of its setting to any kind of target object. + +Our configuration object supports two kinds of target objects: +if the target is a credentials context +it configures the credentials to be used for the +described OCI repository similar to our [credential management example]({{using-cred-management}}). + +But we want to accept more types of target objects. Therefore, we +introduce an own interface declaring the methods required for applying +some configuration settings. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{config interface}} +``` + +By checking the target object against this interface, we are able +to configure any kind of object, as long as it provides the necessary +configuration methods. + +Now, we are nearly prepared to use our new configuration, there is just one step +missing. To enable the automatic recognition of our new type (for example +in the ocm config file), we have to tell the configuration management +about the new type. This is done by an `init()` function in our config package. + +Here, we call a registration function, +which gets called with a dedicated type object for the new config type. +A *type object* describes the config type, its type name, how +it is serialized and deserialized and some description. +We use a standard type object, here, instead of implementing +an own one. It is parameterized by the Go pointer type (`*ExampleConfigSpec`) for +our specification object. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{init}} +``` + +#### Using our new Config Object + +After preparing a new special config type +we can feed it into the config management. +Because of the registration the config management +now knows about this new type. + +A usual, we gain access to our required contexts. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{default context}} +``` + +To setup our environment we create our new config based on the actual settings +and apply it to the config context. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{apply}} +``` + +Now, we should be prepared to get the credentials +the usual way. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{query credentials}} +``` + +#### Using in the OCM Configuration + +Because of the new credential type, such a specification can +now be added to the ocm config, also. +So, we could use our special tour config file content +directly as part of the ocm config. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{in ocmconfig}} +``` + +The resulting config file looks as follows: + +```yaml +{{execute}{go}{run}{../../04-working-with-config}{--config}{settings.yaml}{provide}{}{ocmconfig}} +``` + +#### Applying to our Configuration Interface + +Above, we added a new kind of target, the `RepositoryTarget` interface. +By providing an implementation for this interface, we can +configure such an object using the config management. +We just provide a simple implementation for this interface, just storing the configured +repository specification. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{demo target}} +``` + +The context management now is able to apply our config to such an object. + +```go +{{include}{../../04-working-with-config/04-write-config-type.go}{apply interface}} +``` + +This way any specialized configuration object can be added +by a user of the OCM library. It can be used to configure +existing objects or even new object types, even in combination. + +What is still required is a way +to implement new config targets, objects, which wants +to be configured and which autoconfigure themselves when +used. Our simple repository target is just an example +for some kind of ad-hoc configuration. +A complete scenario is shown in the next example. + +{{consume-config}} +### Preparing Objects to be Configured by the Config Management + +We already have our new acme.org config object type, +and a target interface which must be implemented by a target +object to be configurable. The last example showed how +such an object can be configured in an ad-hoc manner +by directly requesting it to be configured by the config +management. + +Now, we want to provide an object, which configures +itself when used. +Therefore, we introduce a Go type `RepositoryProvider`, +which should be an object, which is +able to provide an OCI repository reference. +It has a setter and a getter (the setter is +provided by our ad-hoc `SimpleRepositoryTarget`). + +To be able to configure itself, the object must know about +the config context it should use to configure itself. + +Therefore, our type contains an additional field `updater`. +Its type `cpi.Updater` is a utility provided by the configuration +management, which holds a reference to a configuration context +and is able to +configure an object based on a managed configuration +watermark. It remembers which config objects from the +config queue are already applied, and replays +the config objects applied to the config context +after the last update. + +Finally, a mutex field is contained, which is used to +synchronize updates later. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{type}} +``` + +For this type a constructor is provided, which initializes +the `updater` field with the desired configuration context. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{constructor}} +``` + +The magic now happens in the methods provided +by our configurable object. +The first step for methods of configurable objects +dependent on potential configuration is always +to update itself using the embedded updater. + +Please note, the config management reverses the +request direction. Applying a config object to +the config context does not configure dependent objects, +it just manages a config queue, which is used by potential +configuration targets to configure themselves. +The actual configuration action is always initiated +by the object, which want to be configured. +The reason for this is to avoid references from the +management to managed objects. This would prohibit +the garbage collection of all configurable objects +as long as the configuration context exists. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{method}} +``` + +After defining our repository provider type we can now start to use it +together with the configuration management and out configuration object. + +As usual, we first determine out context to use. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{default context}} +``` + +New, we create our provide configurable object by binding it +to the config context. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{object}} +``` + +If we ask now for a repository we will get the empty +answer, because nothing is configured, yet. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{initial query}} +``` + +Now, we apply our config from the last example. Therefore, we create and initialize +the config object with our program settings and apply it to the config +context. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{apply config}} +``` + +Without any further action, asking for a repository now will return the +configured ref. The configurable object automatically catches the +new configuration from the config context. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{query}} +``` + +Now, we should also be prepared to get the credentials, +our config object configures the provider as well as +the credential context. + +```go +{{include}{../../04-working-with-config/05-write-config-consumer.go}{credentials}} +``` \ No newline at end of file diff --git a/examples/lib/tour/docsrc/04-working-with-config/settings.yaml b/examples/lib/tour/docsrc/04-working-with-config/settings.yaml new file mode 100644 index 0000000000..513e95095a --- /dev/null +++ b/examples/lib/tour/docsrc/04-working-with-config/settings.yaml @@ -0,0 +1,4 @@ +username: mandelsoft +password: ghp_xyz +component: github.com/mandelsoft/examples/cred1 +version: 0.1.0 \ No newline at end of file diff --git a/examples/lib/tour/docsrc/05-transporting-component-versions/README.md b/examples/lib/tour/docsrc/05-transporting-component-versions/README.md new file mode 100644 index 0000000000..93964a9077 --- /dev/null +++ b/examples/lib/tour/docsrc/05-transporting-component-versions/README.md @@ -0,0 +1,128 @@ +{{transport}} +# Transporting Component Versions + +This [tour](example.go) illustrates the basic support for +transporting content from one environment into another. + +## Running the example + +You can call the main program with a config file option (`--config `). +The config file should have the following content: + +```yaml +repository: ghcr.io/mandelsoft/ocm +targetRepository: + type: CommonTransportFormat + filePath: /tmp/example05.target.ctf + fileFormat: directory + accessMode: 2 +username: +password: +``` + +Any supported kind of target repository can be specified by using its +specification type. An OCI regisztry target would look like this: + +```yaml +repository: ghcr.io/mandelsoft/ocm +username: +password: +targetRepository: + type: OCIRegistry + baseUrl: ghcr.io/mandelsoft/targetocm +ocmConfig: +``` + +The actual version of the example just works with the file system +target, because it is not possible to specify credentials for the +target repository in this simple config file. But, if you specify an [OCM config file]({{ocm-config-file}}) you can +add more predefined credential settings to make it possible to use +target repositories requiring credentials. The credentials are +automatically taken from the credentials context and don't need to be +specified when creating the repository access object in the code. + +## Walkthrough + +As usual, we start with getting access to an OCM context + +```go +{{include}{../../05-transporting-component-versions/example.go}{default context}} +``` + +Then we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04]({{ocm-config-file}}). + +```go +{{include}{../../05-transporting-component-versions/example.go}{configure}} +``` + +This function simply applies the config file using the utility function +provided by the config management: + +```go +{{include}{../../05-transporting-component-versions/example.go}{read config}} +``` + +The context acts as factory for various model types based on +specification descriptor serialization formats in YAML or JSON. +Access method specifications and repository specification are +examples for this feature. + +Now, we use the repository specification serialization format to +determine the target repository for a transport from our yaml +configuration file. + +```go +{{include}{../../05-transporting-component-versions/example.go}{target}} +``` + +For our source we just use the component version provided by the last +examples in a remote repository. +Therefore, we set up the credentials context, as +shown in [tour 03]({{using-cred-management}}). + +```go +{{include}{../../05-transporting-component-versions/example.go}{set credentials}} +``` + +For the transport, we first get access to the component version +we want to transport, by getting the source repository and looking up +the desired component version. + +```go +{{include}{../../05-transporting-component-versions/example.go}{source}} +``` + +We could just add this version to the target repository, but this +would not be a real transport, but just a copy of the component descriptor +and the associated local resources. Transport potentially means more, all +the described artifacts should also be copied into the target environment. + +Such an action is done by the library function `transfer.Transfer`. +It takes several settings influencing the transport mode, +for example transitive or value transport. +Here, all resources are transported per value, all external +references will be inlined as `localBlob`s and imported into +the target environment, applying blob upload handlers +where possible. For a CTF archive as target, there are no +configured handlers by default. Therefore, all resources will +be migrated to local blobs. + +```go +{{include}{../../05-transporting-component-versions/example.go}{transfer}} +``` + +Now, we check the result of our transport action in the target +repository. + + +```go +{{include}{../../05-transporting-component-versions/example.go}{verify-a}} +{{include}{../../05-transporting-component-versions/example.go}{verify-b}} +``` + +Please be aware that all resources in the target now are `localBlob`s, +if the target is a CTF archive. If it is an OCI registry, all the OCI +artifact resources will be uploaded as OCI artifacts into the target +repository and the access specifications are adapted to type `ociArtifact`, +but now referring to OCI artifacts in the target repository. diff --git a/examples/lib/tour/docsrc/06-signing-component-versions/README.md b/examples/lib/tour/docsrc/06-signing-component-versions/README.md new file mode 100644 index 0000000000..ca27322f0e --- /dev/null +++ b/examples/lib/tour/docsrc/06-signing-component-versions/README.md @@ -0,0 +1,220 @@ +{{signing}} +# Signing Component Versions + +This tour illustrates the basic functionality to +sign and verify signatures. + +It covers two basic scenarios: +- [`sign`](01-basic-signing.go) Create, Sign, Transport and Verify a component version. +- [`context`](02-using-context-settings.go) Using context settings to configure signing and verification in target repo. + +## Running the examples + +You can call the main program with a config file option (`--config `) and the name of the scenario. +The config file should have the following content: + +```yaml +targetRepository: + type: CommonTransportFormat + filePath: /tmp/example06.target.ctf + fileFormat: directory + accessMode: 2 +ocmConfig: +``` + +The actual version of the example just works with the file system +target, because it is not possible to specify credentials for the +target repository in this simple config file. But, if you specific an [OCM config file](../04-working-with-config/README.md) you can +add more credential settings to make target repositories possible +requiring credentials. + +## Walkthrough + +### Create, Sign, Transport and Verify a component version + +As usual, we start with getting access to an OCM context + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{default context}} +``` +Then, we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04]({{ocm-config-file}}). + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{configure}} +``` + +To sign a component version we need a private key. +For this example, we just create a local keypair. +To be able to verify later, we should save the public key, +but here we do all this in a single program. + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{create keypair}} +``` + +{{tour06-compose}} +And we need a component version to sign. +We again compose a component version without a repository +(see [tour02 example 2]({{composition-environment}})). + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{compose}} +``` + +Now, let's sign the component version. +There might be multiple signatures, therefore every signature +has a name (here `acme.org`). Keys are always specified for +a dedicated signature name. The signing process can be influenced by +several options. Here, we just provide the private key to be used in an ad-hoc manner. +[Later]({{signing-context}}), we will see how everything can be preconfigured in a *signing context*. + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{sign}} +``` + +Now, we add the signed component version to a target repository. +Here, we just reuse the code from [tour02]({{composition-environment}}) + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{add version}} +``` + +Let's check the target for the new component version. + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{lookup}} +``` + +Please note, that the version now contains a signature. + +Finally, we check whether the signature is still valid for the +target version. + +```go +{{include}{../../06-signing-component-versions/01-basic-signing.go}{verify}} +``` + +{{signing-context}} +### Using Context Settings to Configure Signing + +Instead of providing all signing relevant information directly with +the signing or verification calls, it is possible to preconfigure +various information at the OCM context. + +As usual, we start with getting access to an OCM context + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{default context}} +``` + +Then, we configure this context with optional ocm config defined in our config file. +See [OCM config scenario in tour 04]({{ocm-config-file}}). + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{configure}} +``` + +To sign a component version we need a private key. +For this example, we again just create a local keypair. +To be able to verify later, we should save the public key, +but here we do all this in a single program. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{create keypair}} +``` + +Finally, we create a component version in our target repository. The called +function + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{setup}} +``` + +executes the same coding already shown in the [previous]({{tour06-compose}}) example. + +#### Signing Using Manual Context Settings + +After this preparation we now configure the signing part of the OCM context. +Every OCM context features a signing registry, which provides available +signers and hashers, but also keys and certificates for various purposes. +It is always asked if a key is required, which is +not explicitly given to a signing/verification call. + +This context part is implemented as additional attribute stored along +with the context. Attributes are always implemented as a separate package +containing the attribute structure, its deserialization and +a `Get(Context)` function to retrieve the attribute for the context. +This way new arbitrary attributes for various use cases can be added +without the need to change the context interface. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{signing attribute}} +``` + +Now, we manually add the keys to our context. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{configure keys}} +``` + +We are prepared now and can sign any component version without specifying further options +in any repository for the signature name `acme.org`. + +Therefore, we just get the component version from the prepared repository + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{lookup component version}} +``` + +and finally sign it. We don't need to present the key, here. It is taken from the +context. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{sign}} +``` + +The same way we can just call `VerifyComponentVersion` to +verify the signature. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{verify}} +``` + +#### Configuring Keys with OCM Configuration File + +Manually adding keys to the signing attribute +might simplify the call to possibly multiple signing/verification +calls, but it does not help to provide keys via an external +configuration (for example for using the OCM CLI). +In [tour04]({{tour04-arbitrary}}) +we have seen how arbitrary configuration +possibilities can be added. The signing attribute uses +this mechanism to configure itself by providing an own +configuration object, which can be used to feed keys (and certificates) +into the signing attribute of an OCM context. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{create signing config}} +``` + +It provides methods to add elements +like keys and certificates, which convert +these elements into a (de-)serializable form. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{add signing config}} +``` + +By adding this config to a generic configuration object you get +an OCM config usable to predefine keys for your CLI. + +```go +{{include}{../../06-signing-component-versions/02-using-context-settings.go}{print signing config}} +``` + +And here is a sample output containing the public and private key. + +```yaml +{{execute}{go}{run}{../../06-signing-component-versions}{--config}{settings.yaml}{config}{}{ocmconfig}} +``` \ No newline at end of file diff --git a/examples/lib/tour/docsrc/06-signing-component-versions/settings.yaml b/examples/lib/tour/docsrc/06-signing-component-versions/settings.yaml new file mode 100644 index 0000000000..513e95095a --- /dev/null +++ b/examples/lib/tour/docsrc/06-signing-component-versions/settings.yaml @@ -0,0 +1,4 @@ +username: mandelsoft +password: ghp_xyz +component: github.com/mandelsoft/examples/cred1 +version: 0.1.0 \ No newline at end of file diff --git a/examples/lib/tour/docsrc/README.md b/examples/lib/tour/docsrc/README.md new file mode 100644 index 0000000000..8f870f96e4 --- /dev/null +++ b/examples/lib/tour/docsrc/README.md @@ -0,0 +1,15 @@ +# A Tour through Usage Scenarios of the OCM Library + +This tour guides you from a very basic usage of the +OCM repository API to more complex scenarios +handling credentials and configurations. + +So far, it does not cover the implementation +of extension points of the library. + +- [Basic Usage of OCM Repositories]({{getting-started}}) +- [Composing Component Versions]({{compose-compvers}}) +- [Working with Credentials]({{credentials}}) +- [Working with Configuration]({{config}}) +- [Transporting Component Versions]({{transport}}) +- [Signing Component Versions]({{signing}}) \ No newline at end of file diff --git a/go.mod b/go.mod index 304bc1b9c8..876486557a 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( require ( github.com/InfiniteLoopSpace/go_S-MIME v0.0.0-20181221134359-3f58f9a4b2b6 + github.com/containerd/log v0.1.0 github.com/distribution/reference v0.5.0 github.com/imdario/mergo v0.3.16 github.com/mandelsoft/vfs v0.4.0 @@ -156,7 +157,6 @@ require ( github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/cloudflare/circl v1.3.5 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect github.com/containers/ocicrypt v1.1.6 // indirect diff --git a/hack/Makefile b/hack/Makefile index 8949ef69d3..9bf5a73b48 100644 --- a/hack/Makefile +++ b/hack/Makefile @@ -45,7 +45,7 @@ ifneq ("v$(GO_BINDATA)",$(GO_BINDATA_VERSION)) endif .PHONY: install-requirements -install-requirements: $(deps) $(GOPATH)/bin/goimports +install-requirements: $(deps) $(GOPATH)/bin/goimports mdref # @$(REPO_ROOT)/hack/install-requirements.sh .PHONY: golangci-lint @@ -65,6 +65,11 @@ go-bindata: $(GOPATH)/bin/goimports: go install -v golang.org/x/tools/cmd/goimports@latest +.PHONY: mdref +mdref: + go install -v github.com/mandelsoft/mdref@master + + Linux_jq: $(info -> jq is missing) $(info - sudo apt-get install jq / sudo dnf install jq / sudo zypper install jq / sudo pacman -S jq) diff --git a/hack/install-requirements.sh b/hack/install-requirements.sh index 0c5a7b15e1..fd5b4f764c 100755 --- a/hack/install-requirements.sh +++ b/hack/install-requirements.sh @@ -16,6 +16,7 @@ GO111MODULE=off go get -u github.com/go-bindata/go-bindata/... go install github.com/go-bindata/go-bindata/v3/go-bindata@v3.1.3 go install golang.org/x/tools/cmd/goimports@latest go install github.com/daixiang0/gci@v0.7.0 +go install github.com/mandelsoft/mdref@latest echo "> Install Registry test binaries" diff --git a/pkg/contexts/credentials/config/type.go b/pkg/contexts/credentials/config/type.go index 2ba1eee2e4..cfed970ff0 100644 --- a/pkg/contexts/credentials/config/type.go +++ b/pkg/contexts/credentials/config/type.go @@ -126,6 +126,8 @@ func (a *Config) AddAlias(name string, repo cpi.RepositorySpec, creds ...cpi.Cre return nil } +// --- begin apply --- + func (a *Config) ApplyTo(ctx cfgcpi.Context, target interface{}) error { list := errors.ErrListf("applying config") t, ok := target.(cpi.Context) @@ -135,6 +137,7 @@ func (a *Config) ApplyTo(ctx cfgcpi.Context, target interface{}) error { for _, e := range a.Consumers { t.SetCredentialsForConsumer(e.Identity, CredentialsChain(e.Credentials...)) } + // --- end apply --- sub := errors.ErrListf("applying aliases") for n, e := range a.Aliases { sub.Add(t.SetAlias(n, &e.Repository, CredentialsChain(e.Credentials...))) diff --git a/pkg/contexts/credentials/interface.go b/pkg/contexts/credentials/interface.go index 054260cd2c..c7b9d83b42 100644 --- a/pkg/contexts/credentials/interface.go +++ b/pkg/contexts/credentials/interface.go @@ -9,6 +9,7 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/internal" + "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/directcreds" "github.com/open-component-model/ocm/pkg/runtime" ) @@ -92,6 +93,14 @@ func CredentialsFromList(props ...string) Credentials { return creds } +func CredentialsSpecFromList(props ...string) CredentialsSpec { + creds := DirectCredentials{} + for i := 1; i < len(props); i += 2 { + creds[props[i-1]] = props[i] + } + return directcreds.NewCredentials(creds.Properties()) +} + func ToGenericCredentialsSpec(spec CredentialsSpec) (*GenericCredentialsSpec, error) { return internal.ToGenericCredentialsSpec(spec) } diff --git a/pkg/contexts/ocm/attrs/signingattr/config.go b/pkg/contexts/ocm/attrs/signingattr/config.go index 81bbb6f9d7..d5ed6bce2c 100644 --- a/pkg/contexts/ocm/attrs/signingattr/config.go +++ b/pkg/contexts/ocm/attrs/signingattr/config.go @@ -8,6 +8,7 @@ import ( "crypto/x509/pkix" "encoding/base64" "encoding/json" + "encoding/pem" "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" @@ -17,6 +18,7 @@ import ( "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/runtime" "github.com/open-component-model/ocm/pkg/signing" + "github.com/open-component-model/ocm/pkg/signing/signutils" "github.com/open-component-model/ocm/pkg/utils" ) @@ -148,19 +150,35 @@ func (a *Config) AddIssuer(name string, issuer *pkix.Name) { a.Issuers[name] = i } -func (a *Config) addKey(set *map[string]KeySpec, name string, key interface{}) { +func (a *Config) addKey(set *map[string]KeySpec, name string, key interface{}, conv func(interface{}) *pem.Block) error { if *set == nil { *set = map[string]KeySpec{} } - (*set)[name] = KeySpec{Parsed: key} + switch data := key.(type) { + case []byte: + (*set)[name] = KeySpec{Data: data} + case string: + (*set)[name] = KeySpec{StringData: data} + default: + if conv != nil { + block := conv(key) + if block == nil { + return errors.ErrUnknown("format") + } + (*set)[name] = KeySpec{Parsed: key, StringData: string(pem.EncodeToMemory(block))} + } else { + (*set)[name] = KeySpec{Parsed: key} + } + } + return nil } -func (a *Config) AddPublicKey(name string, key interface{}) { - a.addKey(&a.PublicKeys, name, key) +func (a *Config) AddPublicKey(name string, key interface{}) error { + return a.addKey(&a.PublicKeys, name, key, func(key interface{}) *pem.Block { return signutils.PemBlockForPublicKey(key) }) } -func (a *Config) AddPrivateKey(name string, key interface{}) { - a.addKey(&a.PrivateKeys, name, key) +func (a *Config) AddPrivateKey(name string, key interface{}) error { + return a.addKey(&a.PrivateKeys, name, key, signutils.PemBlockForPrivateKey) } func (a *Config) addKeyFile(set *map[string]KeySpec, name, path string, fss ...vfs.FileSystem) { diff --git a/pkg/runtime/utils.go b/pkg/runtime/utils.go index dab38b0e52..2f8ec1a6d5 100644 --- a/pkg/runtime/utils.go +++ b/pkg/runtime/utils.go @@ -5,6 +5,7 @@ package runtime import ( + "fmt" "reflect" "sort" "strings" @@ -102,3 +103,22 @@ func Nil[T any]() T { var _nil T return _nil } + +// --- begin check --- + +// CheckSpecification checks a byte sequence to describe a +// valid minimum specification object. +func CheckSpecification(data []byte) error { + var obj ObjectTypedObject + + err := DefaultYAMLEncoding.Unmarshal(data, &obj) + if err != nil { + return errors.ErrInvalidWrap(err, "repository specification", string(data)) + } + if obj.GetType() == "" { + return errors.ErrInvalidWrap(fmt.Errorf("non-empty type field required"), "repository specification", string(data)) + } + return nil +} + +// --- end check --- diff --git a/pkg/signing/signutils/utils.go b/pkg/signing/signutils/utils.go index dac87429b1..2e65df8b70 100644 --- a/pkg/signing/signutils/utils.go +++ b/pkg/signing/signutils/utils.go @@ -9,7 +9,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "log" "os" "runtime" "strings" @@ -46,7 +45,6 @@ func PemBlockForPrivateKey(priv interface{}) *pem.Block { } return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} default: - log.Fatal("invalid key") return nil } }