feat: jfrog plugin + helm upload handler (#1167)

<!-- markdownlint-disable MD041 -->
#### What this PR does / why we need it

Implements a JFrog Plugin that contains an uploader that is able to
upload to JFrog Helm Chart Repositories (different from OCI).

Example Reference Upload configuration:

```yaml
- type: credentials.config.ocm.software
  consumers:
  - identity:
      type: JFrogHelm
      hostname: int.repositories.cloud.sap
    credentials:
    - type: Credentials/v1
      properties:
        username: "your-user-here"
        password: "your-token-here"
- type: uploader.ocm.config.ocm.software
  registrations:
  - name: plugin/jfrog/JFrogHelm
    artifactType: helmChart
    priority: 200
    config:
      type: JFrogHelm/v1alpha1
      url: "your-artifactory-url-here"
      repository: "your-repository-here"
      # reindexAfterUpload: true # in case you want to force a reindex, requires admin repository credentials, off by default
```

The plugin will be installable with 

`ocm install plugin
ghcr.io/open-component-model/ocm//ocm.software/plugins/jfrogplugin`

as it is added to our parallel build list.

Since during development (while this PR does not get merged and we dont
have an RC) you cannot use this command, you can choose to either push
your own version of the plugin (there are commands in the new makefile),
or you can run (also in the new makefile):

```
make -C components/jfrogplugin install
```

which will do a build and install locally so you can use it without
fetching it remotely

Once added, you can run something like

```
ocm plugin get jfrog -oyaml
---
element:
  description: "ALPHA GRADE plugin providing custom functions related to interacting
    with JFrog Repositories (e.g. Artifactory).\n\nThis plugin is solely for interacting
    with JFrog Servers and cannot be used for generic repository types.\nThus, you
    should only consider this plugin if\n- You need to use a JFrog specific API\n-
    You cannot use any of the generic (non-jfrog) implementations.\n\nExamples:\n\nYou
    can configure the JFrog plugin as an Uploader in an ocm config file with:\n\n-
    type: uploader.ocm..config.ocm.software\n  registrations:\n  - name: plugin/jfrog/JFrogHelm\n
    \   artifactType: helmChart\n    priority: 200 # must be > 100 to be used over
    the default handler\n    config:\n      type: JFrogHelm/v1alpha1\n      # this
    is only a sample JFrog Server URL, do NOT append /artifactory\n      url: int.repositories.ocm.software
    \n      repository: ocm-helm-test\n"
  forwardLogging: true
  pluginName: jfrog
  pluginVersion: 0.20.0-dev+962ef1469035fbd7b855dff1ccb6ddfc06269745
  shortDescription: jfrog plugin
  uploaders:
  - constraints:
    - artifactType: helmChart
      contextType: ""
      mediaType: ""
      repositoryType: ""
    description: upload artifacts to JFrog HELM repositories by using the JFrog REST
      API.
    name: JFrogHelm
  version: v1
```

to introspect it.

The plugin is now able to be used by OCM.


The plugin registers itself for the mediaTypes of a Helm Chart TGZ as
well as OCI artifacts to convert them. Notably, the OCI artifact
conversion is lossy because the provenance data is omitted, so back and
forth conversion while maintaining digests might not always be possible
in a fully trusted environment

#### Which issue(s) this PR fixes
<!--
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
-->

fix #1116

---------

Co-authored-by: Gergely Brautigam <182850+skarlso@users.noreply.github.com>

.github/workflows/components.yaml

@@ -30,7 +30,7 @@ permissions:
 env:
   REF: ${{ inputs.ref == '' && github.ref || inputs.ref }}
   CTF_TYPE: directory
-  components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]'
+  components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin", "jfrogplugin"]'
   IMAGE_PLATFORMS: 'linux/amd64 linux/arm64'
   PLATFORMS: 'windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64'
   BUILDX_CACHE_PUSH: false

api/ocm/extensions/blobhandler/handlers/generic/plugin/blobhandler.go

@@ -70,6 +70,7 @@ func (b *pluginHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, glo
 		"uploader", b.name,
 		"arttype", artType,
 		"mediatype", blob.MimeType(),
+		"digest", blob.Digest(),
 		"hint", hint,
 		"target", string(target),
 	)
@@ -85,5 +86,5 @@ func (b *pluginHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, glo
 	r := accessio.NewOndemandReader(blob)
 	defer errors.PropagateError(&err, r.Close)
 
-	return b.plugin.Put(b.name, r, artType, blob.MimeType(), hint, creddata, target)
+	return b.plugin.Put(b.name, r, artType, blob.MimeType(), hint, string(blob.Digest()), creddata, target)
 }

api/ocm/extensions/blobhandler/handlers/generic/plugin/testdata/test

@@ -13,6 +13,7 @@ media=
 artifact=
 hint=
 creds=
+digest=
 args=( )
 
 parseArgs() {
@@ -21,6 +22,9 @@ parseArgs() {
     --mediaType|-m)
         media="$2"
         shift 2;;
+    --digest|-d)
+        digest="$2"
+        shift 2;;
     --artifactType|-a)
         artifact="$2"
         shift 2;;

api/ocm/plugin/cache/updater.go

@@ -296,44 +296,57 @@ func (o *PluginUpdater) download(session ocm.Session, cv ocm.ComponentVersionAcc
 				return nil
 			}
 		}
+
 		dir := plugindirattr.Get(o.Context)
-		if dir != "" {
-			lock, err := filelock.LockDir(dir)
+		if dir == "" {
+			home, err := os.UserHomeDir() // use home if provided
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to determine home directory to determine default plugin directory: %w", err)
 			}
-			defer lock.Close()
-
-			target := filepath.Join(dir, desc.PluginName)
-
-			verb := "installing"
-			if ok, _ := vfs.FileExists(fs, target); ok {
-				if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) {
-					return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir)
-				}
-				if o.UpdateMode {
-					verb = "updating"
-				}
-				fs.Remove(target)
+			dir = filepath.Join(home, plugindirattr.DEFAULT_PLUGIN_DIR)
+			if err := os.Mkdir(dir, os.ModePerm|os.ModeDir); err != nil {
+				return fmt.Errorf("failed to create default plugin directory: %w", err)
 			}
-			o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir)
-			dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755)
-			if err != nil {
-				return errors.Wrapf(err, "cannot create plugin file %s", target)
+			if err := plugindirattr.Set(o.Context, dir); err != nil {
+				return fmt.Errorf("failed to set plugin dir after defaulting: %w", err)
 			}
-			src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0)
-			if err != nil {
-				dst.Close()
-				return errors.Wrapf(err, "cannot open plugin executable %s", file.Name())
+		}
+
+		lock, err := filelock.LockDir(dir)
+		if err != nil {
+			return err
+		}
+		defer lock.Close()
+
+		target := filepath.Join(dir, desc.PluginName)
+
+		verb := "installing"
+		if ok, _ := vfs.FileExists(fs, target); ok {
+			if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) {
+				return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir)
 			}
-			_, err = io.Copy(dst, src)
-			dst.Close()
-			utils.IgnoreError(src.Close())
-			utils.IgnoreError(os.Remove(file.Name()))
-			utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName))
-			if err != nil {
-				return errors.Wrapf(err, "cannot copy plugin file %s", target)
+			if o.UpdateMode {
+				verb = "updating"
 			}
+			fs.Remove(target)
+		}
+		o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir)
+		dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755)
+		if err != nil {
+			return errors.Wrapf(err, "cannot create plugin file %s", target)
+		}
+		src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0)
+		if err != nil {
+			dst.Close()
+			return errors.Wrapf(err, "cannot open plugin executable %s", file.Name())
+		}
+		_, err = io.Copy(dst, src)
+		dst.Close()
+		utils.IgnoreError(src.Close())
+		utils.IgnoreError(os.Remove(file.Name()))
+		utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName))
+		if err != nil {
+			return errors.Wrapf(err, "cannot copy plugin file %s", target)
 		}
 	}
 	return nil

api/ocm/plugin/plugin.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/mandelsoft/goutils/errors"
 	"github.com/mandelsoft/goutils/finalizer"
+	mlog "github.com/mandelsoft/logging"
 	"github.com/mandelsoft/vfs/pkg/vfs"
 
 	"ocm.software/ocm/api/credentials"
@@ -112,11 +114,32 @@ func (p *pluginImpl) Exec(r io.Reader, w io.Writer, args ...string) (result []by
 		args = append([]string{"--" + ppi.OptPlugingLogConfig, string(data)}, args...)
 	}
 
-	if len(p.config) == 0 {
-		p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args)
-	} else {
-		p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args, "config", p.config)
+	if p.ctx.Logger(TAG).Enabled(mlog.DebugLevel) {
+		// Plainly kill any credentials found in the logger.
+		// Stupidly match for "credentials" arg.
+		// Not totally safe, but better than nothing.
+		logargs := make([]string, len(args))
+		for i, arg := range args {
+			if logargs[i] != "" {
+				continue
+			}
+			if strings.Contains(arg, "credentials") {
+				if strings.Contains(arg, "=") {
+					logargs[i] = "***"
+				} else if i+1 < len(args)-1 {
+					logargs[i+1] = "***"
+				}
+			}
+			logargs[i] = arg
+		}
+
+		if len(p.config) == 0 {
+			p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs)
+		} else {
+			p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs, "config", p.config)
+		}
 	}
+
 	data, err := cache.Exec(p.Path(), p.config, r, w, args...)
 
 	if logfile != nil {
@@ -293,7 +316,7 @@ func (p *pluginImpl) Get(w io.Writer, creds, spec json.RawMessage) error {
 	return err
 }
 
-func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint string, creds, target json.RawMessage) (ocm.AccessSpec, error) {
+func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint, digest string, creds, target json.RawMessage) (ocm.AccessSpec, error) {
 	args := []string{upload.Name, put.Name, name, string(target)}
 
 	if creds != nil {
@@ -308,6 +331,9 @@ func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint strin
 	if artType != "" {
 		args = append(args, "--"+put.OptArt, artType)
 	}
+	if digest != "" {
+		args = append(args, "--"+put.OptDigest, digest)
+	}
 	result, err := p.Exec(r, nil, args...)
 	if err != nil {
 		return nil, err

api/ocm/plugin/ppi/cmds/common/const.go

@@ -7,4 +7,5 @@ const (
 	OptArt       = "artifactType"
 	OptConfig    = "config"
 	OptCliConfig = "cli-config"
+	OptDigest    = "digest"
 )

api/ocm/plugin/ppi/cmds/upload/put/cmd.go

@@ -3,8 +3,6 @@ package put
 import (
 	"encoding/json"
 	"fmt"
-	"io"
-	"os"
 
 	"github.com/mandelsoft/goutils/errors"
 	"github.com/spf13/cobra"
@@ -19,11 +17,12 @@ import (
 )
 
 const (
-	Name     = "put"
-	OptCreds = common.OptCreds
-	OptHint  = common.OptHint
-	OptMedia = common.OptMedia
-	OptArt   = common.OptArt
+	Name      = "put"
+	OptCreds  = common.OptCreds
+	OptHint   = common.OptHint
+	OptMedia  = common.OptMedia
+	OptArt    = common.OptArt
+	OptDigest = common.OptDigest
 )
 
 func New(p ppi.Plugin) *cobra.Command {
@@ -58,6 +57,7 @@ type Options struct {
 	Credentials  credentials.DirectCredentials
 	MediaType    string
 	ArtifactType string
+	Digest       string
 
 	Hint string
 }
@@ -68,42 +68,49 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVarP(&o.MediaType, OptMedia, "m", "", "media type of input blob")
 	fs.StringVarP(&o.ArtifactType, OptArt, "a", "", "artifact type of input blob")
 	fs.StringVarP(&o.Hint, OptHint, "H", "", "reference hint for storing blob")
+	fs.StringVarP(&o.Digest, OptDigest, "d", "", "digest of the blob")
 }
 
 func (o *Options) Complete(args []string) error {
 	o.Name = args[0]
 	if err := runtime.DefaultYAMLEncoding.Unmarshal([]byte(args[1]), &o.Specification); err != nil {
-		return errors.Wrapf(err, "invalid repository specification")
+		return fmt.Errorf("invalid repository specification: %w", err)
 	}
 	return nil
 }
 
-func Command(p ppi.Plugin, cmd *cobra.Command, opts *Options) error {
-	spec, err := p.DecodeUploadTargetSpecification(opts.Specification)
-	if err != nil {
-		return errors.Wrapf(err, "target specification")
+func Command(p ppi.Plugin, cmd *cobra.Command, opts *Options) (err error) {
+	var spec ppi.UploadTargetSpec
+	if spec, err = p.DecodeUploadTargetSpecification(opts.Specification); err != nil {
+		return fmt.Errorf("error decoding upload target specification: %w", err)
 	}
 
 	u := p.GetUploader(opts.Name)
 	if u == nil {
 		return errors.ErrNotFound(descriptor.KIND_UPLOADER, fmt.Sprintf("%s:%s", opts.ArtifactType, opts.MediaType))
 	}
-	w, h, err := u.Writer(p, opts.ArtifactType, opts.MediaType, opts.Hint, spec, opts.Credentials)
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(w, os.Stdin)
-	if err != nil {
-		w.Close()
-		return err
-	}
-	err = w.Close()
-	if err != nil {
-		return err
+
+	reader := cmd.InOrStdin()
+
+	var provider ppi.AccessSpecProvider
+	if provider, err = u.Upload(
+		cmd.Context(),
+		p,
+		opts.ArtifactType,
+		opts.MediaType,
+		opts.Hint,
+		opts.Digest,
+		spec,
+		opts.Credentials,
+		reader,
+	); err != nil {
+		return fmt.Errorf("upload failed: %w", err)
 	}
-	acc := h()
-	data, err := json.Marshal(acc)
-	if err == nil {
+
+	acc := provider()
+
+	var data []byte
+	if data, err = json.Marshal(acc); err == nil {
 		cmd.Printf("%s\n", string(data))
 	}
 	return err

api/ocm/plugin/ppi/interface.go

@@ -1,6 +1,7 @@
 package ppi
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 
@@ -113,7 +114,7 @@ type Uploader interface {
 	Description() string
 
 	ValidateSpecification(p Plugin, spec UploadTargetSpec) (info *UploadTargetSpecInfo, err error)
-	Writer(p Plugin, arttype, mediatype string, hint string, spec UploadTargetSpec, creds credentials.Credentials) (io.WriteCloser, AccessSpecProvider, error)
+	Upload(ctx context.Context, p Plugin, arttype, mediatype, hint, digest string, spec UploadTargetSpec, creds credentials.Credentials, reader io.Reader) (AccessSpecProvider, error)
 }
 
 type UploadTargetSpec = runtime.TypedObject