From 7fb4a50cdbf6cb5831505d5fb5176691bc725a40 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 24 Jan 2024 11:28:06 +0100 Subject: [PATCH 01/51] the go way to 'publish' a NPM package original gist is from: https://gist.github.com/cloverstd/7355e95424d59256123a1093f76f78a6 --- components/npm-publish/publish.go | 252 ++++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 components/npm-publish/publish.go diff --git a/components/npm-publish/publish.go b/components/npm-publish/publish.go new file mode 100644 index 0000000000..a59f2d3f59 --- /dev/null +++ b/components/npm-publish/publish.go @@ -0,0 +1,252 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" +) + +func login(registry, username, password string, email string) (string, error) { + data := map[string]interface{}{ + "_id": "org.couchdb.user:" + username, + "name": username, + "email": email, + "password": password, + "type": "user", + } + marshal, err := json.Marshal(data) + if err != nil { + return "", err + } + req, err := http.NewRequest(http.MethodPut, registry+"/-/user/org.couchdb.user:"+url.PathEscape(username), bytes.NewReader(marshal)) + if err != nil { + return "", err + } + req.SetBasicAuth(username, password) + req.Header.Set("content-type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusBadRequest { + all, _ := ioutil.ReadAll(resp.Body) + return "", fmt.Errorf("%d, %s", resp.StatusCode, string(all)) + } + var token struct { + Token string `json:"token"` + } + err = json.NewDecoder(resp.Body).Decode(&token) + if err != nil { + return "", err + } + return token.Token, nil +} + +type Package struct { + pkg map[string]interface{} + Name string + Version string + Readme string + ReadmeFilename string + GitHead string + Description string + ID string + NodeVersion string + NpmVersion string + Dist struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` + } +} + +func (p *Package) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &p.pkg) +} + +func (p *Package) MarshalJSON() ([]byte, error) { + p.pkg["readme"] = p.Readme + p.pkg["readmeFilename"] = p.ReadmeFilename + p.pkg["gitHead"] = p.GitHead + p.pkg["_id"] = p.ID + p.pkg["_nodeVersion"] = p.NodeVersion + p.pkg["_npmVersion"] = p.NpmVersion + p.pkg["dist"] = p.Dist + return json.Marshal(p.pkg) +} + +type Attachment struct { + ContentType string `json:"content_type"` + Data []byte `json:"data"` + Length int `json:"length"` +} + +type Body struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags struct { + Latest string `json:"latest"` + } `json:"dist-tags"` + Versions map[string]*Package `json:"versions"` + Readme string `json:"readme"` + Attachments map[string]*Attachment `json:"_attachments"` +} + +func NewAttachment(data []byte) *Attachment { + return &Attachment{ + ContentType: "application/octet-stream", + Data: data, + Length: len(data), + } +} + +func createIntegrity(data []byte) string { + hash := sha512.New() + hash.Write(data) + return "sha512-" + base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} + +func createShasum(data []byte) string { + hash := sha1.New() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} + +func prepare(data []byte) (*Package, error) { + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + tr := tar.NewReader(gz) + var ( + pkgData []byte + readme []byte + ) + for { + thr, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if pkgData != nil && readme != nil { + break + } + switch thr.Name { + case "package/package.json": + pkgData, err = ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("read package.json failed, %w", err) + } + case "package/README.md": + readme, err = ioutil.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("read README.md failed, %w", err) + } + } + } + if len(pkgData) == 0 { + return nil, fmt.Errorf("package.json is empty") + } + var pkg Package + err = json.Unmarshal(pkgData, &pkg) + if err != nil { + return nil, fmt.Errorf("read package.json failed, %w", err) + } + var meta struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + } + if err := json.Unmarshal(pkgData, &meta); err != nil { + return nil, fmt.Errorf("read package.json version and name failed, %w", err) + } + if meta.Name == "" { + return nil, fmt.Errorf("package.json's name is empty") + } + if meta.Version == "" { + return nil, fmt.Errorf("package.json's version is empty") + } + pkg.Version = meta.Version + pkg.Description = meta.Description + pkg.Name = meta.Name + pkg.Readme = string(readme) + pkg.ReadmeFilename = "README.md" + pkg.ID = meta.Name + meta.Version + pkg.Dist.Shasum = createShasum(data) + pkg.Dist.Integrity = createIntegrity(data) + return &pkg, nil +} + +func main() { + // FIXME: make registry configurable + registry := "https://...FIXME" + + // FIXME: make login configurable + token, err := login(registry, "FIXME user", "FIXME token/password", "FIXME@FIXME.com") + if err != nil { + panic(err) + } + + // FIXME: make URL/File configurable - tar -czf ocm-website.tgz package/ + data, err := ioutil.ReadFile("FIXME.tgz") + if err != nil { + panic(err) + } + pkg, err := prepare(data) + if err != nil { + panic(err) + } + body := Body{ + ID: pkg.Name, + Name: pkg.Name, + Description: pkg.Description, + } + body.DistTags.Latest = pkg.Version + body.Versions = map[string]*Package{ + pkg.Version: pkg, + } + body.Readme = pkg.Readme + tbName := pkg.Name + "-" + pkg.Version + ".tgz" + body.Attachments = map[string]*Attachment{ + tbName: NewAttachment(data), + } + pkg.Dist.Tarball = registry + pkg.Name + "/-/" + tbName + + client := http.Client{} + marshal, err := json.Marshal(body) + if err != nil { + panic(err) + } + req, err := http.NewRequest(http.MethodPut, registry+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) + if err != nil { + panic(err) + } + req.Header.Set("authorization", "Bearer " + token) + req.Header.Set("content-type", "application/json") + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + all, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + fmt.Println(string(all)) + } +} From 7d32ed1327b62f52ed2ec6034fc34f1a86251c6a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 29 Jan 2024 16:11:52 +0100 Subject: [PATCH 02/51] npmjs further drafting --- .../handlers/generic/npmjs/attr.go | 198 ++++++++++++++++++ .../handlers/generic/npmjs/blobhandler.go | 111 ++++++++++ .../handlers/generic/npmjs/registration.go | 61 ++++++ 3 files changed, 370 insertions(+) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go new file mode 100644 index 0000000000..a0d1e353fb --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go @@ -0,0 +1,198 @@ +package npmjs + +import ( + "bytes" + "fmt" + "strings" + "sync" + + "github.com/open-component-model/ocm/pkg/contexts/datacontext" + "github.com/open-component-model/ocm/pkg/contexts/oci" + ocicpi "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/runtime" +) + +const ( + ATTR_KEY = "github.com/open-component-model/ocm/npmjs" + ATTR_SHORT = "npmjs" +) + +func init() { + err := datacontext.RegisterAttributeType(ATTR_KEY, AttributeType{}, ATTR_SHORT) + if err != nil { + return + } +} + +type AttributeType struct{} + +func (a AttributeType) Name() string { + return ATTR_KEY +} + +func (a AttributeType) Description() string { + return ` +*npmjs repository ref* +Upload local npmjs artifact blobs to a dedicated repository. +` +} + +func (a AttributeType) Encode(v interface{}, marshaller runtime.Marshaler) ([]byte, error) { + if _, ok := v.(*Attribute); !ok { + return nil, fmt.Errorf("npmjs Upload Attribute structure required") + } + return marshaller.Marshal(v) +} + +func (a AttributeType) Decode(data []byte, unmarshaler runtime.Unmarshaler) (interface{}, error) { + var value Attribute + err := unmarshaler.Unmarshal(data, &value) + if err == nil { + if value.Repository != nil { + if value.Repository.GetType() == "" { + return nil, errors.ErrInvalidWrap(errors.Newf("missing repository type"), string(data)) + } + return &value, nil + } + if value.Ref == "" { + return nil, errors.ErrInvalidWrap(errors.Newf("missing repository or ref"), string(data)) + } + data = []byte(value.Ref) + } + ref, err := oci.ParseRef(string(data)) + if err != nil { + return nil, errors.ErrInvalidWrap(err, string(data)) + } + if ref.Tag != nil || ref.Digest != nil { + return nil, errors.ErrInvalidWrap(err, string(data)) + } + return &Attribute{Ref: strings.Trim(string(data), "\"")}, nil +} + +type Attribute struct { + Ref string `json:"ociRef,omitempty"` + Repository *ocicpi.GenericRepositorySpec `json:"repository,omitempty"` + NamespacePrefix string `json:"namespacePrefix,omitempty"` + + lock sync.Mutex + ref *oci.RefSpec + spec []byte + + repo oci.Repository + prefix string +} + +func AttributeDescription() map[string]string { + return map[string]string{ + "ociRef": "an OCI repository reference", + "repository": "an OCI repository specification for the target OCI registry", + "namespacePrefix": "a namespace prefix used for the uploaded artifacts", + } +} + +func New(ref string) *Attribute { + return &Attribute{Ref: ref} +} + +func (a *Attribute) reset() { + a.repo = nil + a.prefix = "" + a.ref = nil + a.spec = nil +} + +func (a *Attribute) Close() error { + a.lock.Lock() + defer a.lock.Unlock() + if a.repo != nil { + defer a.reset() + return a.repo.Close() + } + return nil +} + +func (a *Attribute) GetInfo(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { + if a.Ref != "" { + return a.getByRef(ctx) + } + if a.Repository != nil { + return a.getBySpec(ctx) + } + return nil, nil, "", errors.ErrInvalid("ociuploadspec") +} + +func (a *Attribute) getBySpec(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { + data, _ := a.Repository.MarshalJSON() + + spec, err := a.Repository.Evaluate(ctx.OCIContext()) + if err != nil { + return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, string(data)) + } + + a.lock.Lock() + defer a.lock.Unlock() + + if a.spec == nil || bytes.Equal(a.spec, data) { + if a.repo != nil { + a.repo.Close() + a.reset() + } + + a.repo, err = ctx.OCIContext().RepositoryForSpec(spec) + if err != nil { + return nil, nil, "", err + } + + a.prefix = a.NamespacePrefix + a.spec = data + a.ref = &oci.RefSpec{UniformRepositorySpec: *spec.UniformRepositorySpec()} + ctx.Finalizer().Close(a) + } + return a.repo, &a.ref.UniformRepositorySpec, a.prefix, nil +} + +func (a *Attribute) getByRef(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { + ref, err := oci.ParseRef(a.Ref) + if err != nil { + return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, a.Ref) + } + if ref.Tag != nil || ref.Digest != nil { + return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, a.Ref) + } + + a.lock.Lock() + defer a.lock.Unlock() + if a.ref == nil || ref != *a.ref { + if a.repo != nil { + a.repo.Close() + a.reset() + } + + spec, err := ctx.OCIContext().MapUniformRepositorySpec(&ref.UniformRepositorySpec) + if err != nil { + return nil, nil, "", err + } + a.repo, err = ctx.OCIContext().RepositoryForSpec(spec) + if err != nil { + return nil, nil, "", err + } + a.prefix = ref.Repository + a.ref = &ref + ctx.Finalizer().Close(a) + } + return a.repo, &a.ref.UniformRepositorySpec, a.prefix, nil +} + +func Get(ctx datacontext.Context) *Attribute { + a := ctx.GetAttributes().GetAttribute(ATTR_KEY) + if a == nil { + return nil + } + return a.(*Attribute) +} + +func Set(ctx datacontext.Context, attr *Attribute) error { + return ctx.GetAttributes().SetAttribute(ATTR_KEY, attr) +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go new file mode 100644 index 0000000000..b7fa4b8516 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -0,0 +1,111 @@ +package npmjs + +import ( + "encoding/json" + "path" + "strings" + + "github.com/open-component-model/ocm/pkg/blobaccess" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/oci" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" + artifact "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/utils" +) + +func init() { + cpi.RegisterBlobHandler(NewArtifactHandler(), cpi.ForMimeType("npmjs"), cpi.WithPrio(10)) +} + +type artifactHandler struct { + spec *Attribute +} + +func NewArtifactHandler(repospec ...*Attribute) cpi.BlobHandler { + return &artifactHandler{utils.Optional(repospec...)} +} + +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, global cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { + attr := b.spec + if attr == nil { + return nil, nil + } + + mediaType := blob.MimeType() + if !strings.HasSuffix(mediaType, "+tar") && !strings.HasSuffix(mediaType, "+tar+gzip") { + return nil, nil + } + + repo, base, prefix, err := attr.GetInfo(ctx.GetContext()) + if err != nil { + return nil, err + } + + target, err := json.Marshal(repo.GetSpecification()) + if err != nil { + return nil, errors.Wrapf(err, "cannot marshal target specification") + } + values := []interface{}{ + "arttype", artType, + "mediatype", mediaType, + "hint", hint, + "target", string(target), + } + if m, ok := blob.(blobaccess.AnnotatedBlobAccess[cpi.AccessMethod]); ok { + // prepare for optimized point to point implementation + cpi.BlobHandlerLogger(ctx.GetContext()).Debug("npmjs generic artifact handler with ocm access source", + generics.AppendedSlice[any](values, "sourcetype", m.Source().AccessSpec().GetType())..., + ) + } else { + cpi.BlobHandlerLogger(ctx.GetContext()).Debug("npmjs generic artifact handler", values...) + } + + var namespace oci.NamespaceAccess + var version string + var name string + var tag string + + if hint == "" { + name = path.Join(prefix, ctx.TargetComponentName()) + } else { + i := strings.LastIndex(hint, ":") + if i > 0 { + version = hint[i:] + name = path.Join(prefix, hint[:i]) + tag = version[1:] // remove colon + } else { + name = hint + } + } + namespace, err = repo.LookupNamespace(name) + if err != nil { + return nil, errors.Wrapf(err, "lookup namespace %s in target repository %s", name, attr.Ref) + } + defer namespace.Close() + + set, err := artifactset.OpenFromBlob(accessobj.ACC_READONLY, blob) + if err != nil { + return nil, err + } + defer set.Close() + digest := set.GetMain() + if version == "" { + version = "@" + digest.String() + } + art, err := set.GetArtifact(digest.String()) + if err != nil { + return nil, err + } + defer art.Close() + + err = artifactset.TransferArtifact(art, namespace, oci.AsTags(tag)...) + if err != nil { + return nil, err + } + + ref := base.ComposeRef(namespace.GetNamespace() + version) + return artifact.New(ref), nil +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go new file mode 100644 index 0000000000..37a27611b3 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -0,0 +1,61 @@ +package npmjs + +import ( + "fmt" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/registrations" +) + +type Config = Attribute + +func init() { + cpi.RegisterBlobHandlerRegistrationHandler("npmjs/package", &RegistrationHandler{}) +} + +type RegistrationHandler struct{} + +var _ cpi.BlobHandlerRegistrationHandler = (*RegistrationHandler)(nil) + +func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, config cpi.BlobHandlerConfig, olist ...cpi.BlobHandlerOption) (bool, error) { + if handler != "" { + return true, fmt.Errorf("invalid npmjsArtifact handler %q", handler) + } + if config == nil { + return true, fmt.Errorf("npmjs target specification required") + } + attr, err := registrations.DecodeConfig[Config](config, AttributeType{}.Decode) + if err != nil { + return true, errors.Wrapf(err, "blob handler configuration") + } + + var mimes []string + opts := cpi.NewBlobHandlerOptions(olist...) + if opts.MimeType == "npmjs" { + mimes = append(mimes, opts.MimeType) + } + + h := NewArtifactHandler(attr) + for _, m := range mimes { + opts.MimeType = m + ctx.BlobHandlers().Register(h, opts) + } + + return true, nil +} + +func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { + return registrations.NewLeafHandlerInfo("downloading npmjs artifacts", ` +The npmjsArtifacts downloader is able to download npmjs artifacts +as artifact archive according to the npmjs package spec. +The following artifact media types are supported: npmjs +By default, it is registered for these mimetypes. + +It accepts a config with the following fields: +`+listformat.FormatMapElements("", AttributeDescription())+` +Alternatively, a single string value can be given representing a npmjs repository +reference.`, + ) +} From 6d862c8db65e253da8c60ac7629a61a7a217e2e9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Jan 2024 09:07:05 +0100 Subject: [PATCH 03/51] init npmjs --- pkg/contexts/ocm/blobhandler/handlers/init.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/init.go b/pkg/contexts/ocm/blobhandler/handlers/init.go index 35619193a5..9aed77d000 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/init.go +++ b/pkg/contexts/ocm/blobhandler/handlers/init.go @@ -1,10 +1,7 @@ -// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package handlers import ( + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/ocm/comparch" From b5b28731639f12513a167bb73dffefefb1170b2e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 30 Jan 2024 17:39:06 +0100 Subject: [PATCH 04/51] BlobHandler, Identity, RegistrationHandler for npmjs --- pkg/contexts/credentials/cpi/const.go | 5 +- pkg/contexts/credentials/internal/const.go | 5 +- .../handlers/generic/npmjs/attr.go | 198 ------------------ .../handlers/generic/npmjs/blobhandler.go | 137 ++++++------ .../handlers/generic/npmjs/identity.go | 70 +++++++ .../handlers/generic/npmjs/registration.go | 39 ++-- 6 files changed, 157 insertions(+), 297 deletions(-) delete mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go diff --git a/pkg/contexts/credentials/cpi/const.go b/pkg/contexts/credentials/cpi/const.go index 86ebf23c33..0c65870f1d 100644 --- a/pkg/contexts/credentials/cpi/const.go +++ b/pkg/contexts/credentials/cpi/const.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package cpi import ( @@ -13,6 +9,7 @@ const ( ATTR_TYPE = internal.ATTR_TYPE ATTR_USERNAME = internal.ATTR_USERNAME + ATTR_EMAIL = internal.ATTR_EMAIL ATTR_PASSWORD = internal.ATTR_PASSWORD ATTR_SERVER_ADDRESS = internal.ATTR_SERVER_ADDRESS ATTR_TOKEN = internal.ATTR_TOKEN diff --git a/pkg/contexts/credentials/internal/const.go b/pkg/contexts/credentials/internal/const.go index 967ceaf227..cb14b92598 100644 --- a/pkg/contexts/credentials/internal/const.go +++ b/pkg/contexts/credentials/internal/const.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package internal const ( @@ -9,6 +5,7 @@ const ( ATTR_TYPE = "type" ATTR_USERNAME = "username" + ATTR_EMAIL = "email" ATTR_PASSWORD = "password" ATTR_CERTIFICATE_AUTHORITY = "certificateAuthority" ATTR_CERTIFICATE = "certificate" diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go deleted file mode 100644 index a0d1e353fb..0000000000 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/attr.go +++ /dev/null @@ -1,198 +0,0 @@ -package npmjs - -import ( - "bytes" - "fmt" - "strings" - "sync" - - "github.com/open-component-model/ocm/pkg/contexts/datacontext" - "github.com/open-component-model/ocm/pkg/contexts/oci" - ocicpi "github.com/open-component-model/ocm/pkg/contexts/oci/cpi" - "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" - "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/runtime" -) - -const ( - ATTR_KEY = "github.com/open-component-model/ocm/npmjs" - ATTR_SHORT = "npmjs" -) - -func init() { - err := datacontext.RegisterAttributeType(ATTR_KEY, AttributeType{}, ATTR_SHORT) - if err != nil { - return - } -} - -type AttributeType struct{} - -func (a AttributeType) Name() string { - return ATTR_KEY -} - -func (a AttributeType) Description() string { - return ` -*npmjs repository ref* -Upload local npmjs artifact blobs to a dedicated repository. -` -} - -func (a AttributeType) Encode(v interface{}, marshaller runtime.Marshaler) ([]byte, error) { - if _, ok := v.(*Attribute); !ok { - return nil, fmt.Errorf("npmjs Upload Attribute structure required") - } - return marshaller.Marshal(v) -} - -func (a AttributeType) Decode(data []byte, unmarshaler runtime.Unmarshaler) (interface{}, error) { - var value Attribute - err := unmarshaler.Unmarshal(data, &value) - if err == nil { - if value.Repository != nil { - if value.Repository.GetType() == "" { - return nil, errors.ErrInvalidWrap(errors.Newf("missing repository type"), string(data)) - } - return &value, nil - } - if value.Ref == "" { - return nil, errors.ErrInvalidWrap(errors.Newf("missing repository or ref"), string(data)) - } - data = []byte(value.Ref) - } - ref, err := oci.ParseRef(string(data)) - if err != nil { - return nil, errors.ErrInvalidWrap(err, string(data)) - } - if ref.Tag != nil || ref.Digest != nil { - return nil, errors.ErrInvalidWrap(err, string(data)) - } - return &Attribute{Ref: strings.Trim(string(data), "\"")}, nil -} - -type Attribute struct { - Ref string `json:"ociRef,omitempty"` - Repository *ocicpi.GenericRepositorySpec `json:"repository,omitempty"` - NamespacePrefix string `json:"namespacePrefix,omitempty"` - - lock sync.Mutex - ref *oci.RefSpec - spec []byte - - repo oci.Repository - prefix string -} - -func AttributeDescription() map[string]string { - return map[string]string{ - "ociRef": "an OCI repository reference", - "repository": "an OCI repository specification for the target OCI registry", - "namespacePrefix": "a namespace prefix used for the uploaded artifacts", - } -} - -func New(ref string) *Attribute { - return &Attribute{Ref: ref} -} - -func (a *Attribute) reset() { - a.repo = nil - a.prefix = "" - a.ref = nil - a.spec = nil -} - -func (a *Attribute) Close() error { - a.lock.Lock() - defer a.lock.Unlock() - if a.repo != nil { - defer a.reset() - return a.repo.Close() - } - return nil -} - -func (a *Attribute) GetInfo(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { - if a.Ref != "" { - return a.getByRef(ctx) - } - if a.Repository != nil { - return a.getBySpec(ctx) - } - return nil, nil, "", errors.ErrInvalid("ociuploadspec") -} - -func (a *Attribute) getBySpec(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { - data, _ := a.Repository.MarshalJSON() - - spec, err := a.Repository.Evaluate(ctx.OCIContext()) - if err != nil { - return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, string(data)) - } - - a.lock.Lock() - defer a.lock.Unlock() - - if a.spec == nil || bytes.Equal(a.spec, data) { - if a.repo != nil { - a.repo.Close() - a.reset() - } - - a.repo, err = ctx.OCIContext().RepositoryForSpec(spec) - if err != nil { - return nil, nil, "", err - } - - a.prefix = a.NamespacePrefix - a.spec = data - a.ref = &oci.RefSpec{UniformRepositorySpec: *spec.UniformRepositorySpec()} - ctx.Finalizer().Close(a) - } - return a.repo, &a.ref.UniformRepositorySpec, a.prefix, nil -} - -func (a *Attribute) getByRef(ctx cpi.Context) (oci.Repository, *oci.UniformRepositorySpec, string, error) { - ref, err := oci.ParseRef(a.Ref) - if err != nil { - return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, a.Ref) - } - if ref.Tag != nil || ref.Digest != nil { - return nil, nil, "", errors.ErrInvalidWrap(err, oci.KIND_OCI_REFERENCE, a.Ref) - } - - a.lock.Lock() - defer a.lock.Unlock() - if a.ref == nil || ref != *a.ref { - if a.repo != nil { - a.repo.Close() - a.reset() - } - - spec, err := ctx.OCIContext().MapUniformRepositorySpec(&ref.UniformRepositorySpec) - if err != nil { - return nil, nil, "", err - } - a.repo, err = ctx.OCIContext().RepositoryForSpec(spec) - if err != nil { - return nil, nil, "", err - } - a.prefix = ref.Repository - a.ref = &ref - ctx.Finalizer().Close(a) - } - return a.repo, &a.ref.UniformRepositorySpec, a.prefix, nil -} - -func Get(ctx datacontext.Context) *Attribute { - a := ctx.GetAttributes().GetAttribute(ATTR_KEY) - if a == nil { - return nil - } - return a.(*Attribute) -} - -func Set(ctx datacontext.Context, attr *Attribute) error { - return ctx.GetAttributes().SetAttribute(ATTR_KEY, attr) -} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index b7fa4b8516..01970a3a30 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -1,31 +1,24 @@ package npmjs import ( + "bytes" "encoding/json" - "path" - "strings" + "fmt" + "io" + "net/http" + "net/url" - "github.com/open-component-model/ocm/pkg/blobaccess" - "github.com/open-component-model/ocm/pkg/common/accessobj" - "github.com/open-component-model/ocm/pkg/contexts/oci" - "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" - artifact "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" - "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/generics" - "github.com/open-component-model/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/mime" ) -func init() { - cpi.RegisterBlobHandler(NewArtifactHandler(), cpi.ForMimeType("npmjs"), cpi.WithPrio(10)) -} - type artifactHandler struct { - spec *Attribute + spec *Config } -func NewArtifactHandler(repospec ...*Attribute) cpi.BlobHandler { - return &artifactHandler{utils.Optional(repospec...)} +func NewArtifactHandler(repospec *Config) cpi.BlobHandler { + return &artifactHandler{repospec} } func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, global cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { @@ -34,78 +27,82 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g return nil, nil } - mediaType := blob.MimeType() - if !strings.HasSuffix(mediaType, "+tar") && !strings.HasSuffix(mediaType, "+tar+gzip") { + mimeType := blob.MimeType() + if mime.MIME_TGZ != mimeType && mime.MIME_TGZ_ALT != mimeType { return nil, nil } - repo, base, prefix, err := attr.GetInfo(ctx.GetContext()) + blobReader, err := blob.Reader() if err != nil { - return nil, err + panic(err) } + defer blobReader.Close() - target, err := json.Marshal(repo.GetSpecification()) + data, err := io.ReadAll(blobReader) + + // read package.json from tarball to get package name and version + var pkg *Package + pkg, err = prepare(data) if err != nil { - return nil, errors.Wrapf(err, "cannot marshal target specification") - } - values := []interface{}{ - "arttype", artType, - "mediatype", mediaType, - "hint", hint, - "target", string(target), + panic(err) } - if m, ok := blob.(blobaccess.AnnotatedBlobAccess[cpi.AccessMethod]); ok { - // prepare for optimized point to point implementation - cpi.BlobHandlerLogger(ctx.GetContext()).Debug("npmjs generic artifact handler with ocm access source", - generics.AppendedSlice[any](values, "sourcetype", m.Source().AccessSpec().GetType())..., - ) - } else { - cpi.BlobHandlerLogger(ctx.GetContext()).Debug("npmjs generic artifact handler", values...) - } - - var namespace oci.NamespaceAccess - var version string - var name string - var tag string - if hint == "" { - name = path.Join(prefix, ctx.TargetComponentName()) - } else { - i := strings.LastIndex(hint, ":") - if i > 0 { - version = hint[i:] - name = path.Join(prefix, hint[:i]) - tag = version[1:] // remove colon - } else { - name = hint - } + // use user+pass+mail from credentials to login and retrieve bearer token + cred := GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) + username := cred[ATTR_USERNAME] + password := cred[ATTR_PASSWORD] + email := cred[ATTR_EMAIL] + if username == "" || password == "" || email == "" { + return nil, fmt.Errorf("username, password or email missing") } - namespace, err = repo.LookupNamespace(name) + token, err := login(b.spec.Url, username, password, email) if err != nil { - return nil, errors.Wrapf(err, "lookup namespace %s in target repository %s", name, attr.Ref) + panic(err) } - defer namespace.Close() - set, err := artifactset.OpenFromBlob(accessobj.ACC_READONLY, blob) - if err != nil { - return nil, err + // prepare body for upload + body := Body{ + ID: pkg.Name, + Name: pkg.Name, + Description: pkg.Description, + } + body.DistTags.Latest = pkg.Version + body.Versions = map[string]*Package{ + pkg.Version: pkg, } - defer set.Close() - digest := set.GetMain() - if version == "" { - version = "@" + digest.String() + body.Readme = pkg.Readme + tbName := pkg.Name + "-" + pkg.Version + ".tgz" + body.Attachments = map[string]*Attachment{ + tbName: NewAttachment(data), } - art, err := set.GetArtifact(digest.String()) + pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName + marshal, err := json.Marshal(body) if err != nil { - return nil, err + panic(err) } - defer art.Close() - err = artifactset.TransferArtifact(art, namespace, oci.AsTags(tag)...) + // prepare request + req, err := http.NewRequest(http.MethodPut, b.spec.Url+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) if err != nil { - return nil, err + panic(err) + } + req.Header.Set("authorization", "Bearer "+token) + req.Header.Set("content-type", "application/json") + + // send request + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + all, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + fmt.Println(string(all)) } - ref := base.ComposeRef(namespace.GetNamespace() + version) - return artifact.New(ref), nil + return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go new file mode 100644 index 0000000000..dcda858047 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go @@ -0,0 +1,70 @@ +package npmjs + +import ( + "net/url" + "path" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/listformat" +) + +// CONSUMER_TYPE is the npmjs repository type. +const CONSUMER_TYPE = "Registry.npmjs.com" + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_EMAIL, "npmjs registries, require an email address", + }) + + cpi.RegisterStandardIdentity(CONSUMER_TYPE, IdentityMatcher, `Npmjs repository + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +var identityMatcher = hostpath.IdentityMatcher("") + +func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { + return identityMatcher(pattern, cur, id) +} + +const ( + ATTR_USERNAME = cpi.ATTR_USERNAME + ATTR_PASSWORD = cpi.ATTR_PASSWORD + ATTR_EMAIL = cpi.ATTR_EMAIL +) + +func SimpleCredentials(user, passwd string, email string) cpi.Credentials { + return cpi.DirectCredentials{ + ATTR_USERNAME: user, + ATTR_PASSWORD: passwd, + ATTR_EMAIL: email, + } +} + +func GetConsumerId(repourl string, pkgname string) cpi.ConsumerIdentity { + u, err := url.Parse(repourl) + if err != nil { + return nil + } + + u.Path = path.Join(u.Path, pkgname) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, u.String()) +} + +func GetCredentials(ctx cpi.ContextProvider, repourl string, pkgname string) common.Properties { + id := GetConsumerId(repourl, pkgname) + if id == nil { + return nil + } + creds, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) + if creds == nil || err != nil { + return nil + } + return creds.Properties() +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go index 37a27611b3..8e577c92c3 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -4,15 +4,18 @@ import ( "fmt" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/mime" "github.com/open-component-model/ocm/pkg/registrations" ) -type Config = Attribute +type Config struct { + Url string `json:"url"` +} func init() { - cpi.RegisterBlobHandlerRegistrationHandler("npmjs/package", &RegistrationHandler{}) + cpi.RegisterBlobHandlerRegistrationHandler("ocm/npmPackage", &RegistrationHandler{}) } type RegistrationHandler struct{} @@ -26,36 +29,30 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co if config == nil { return true, fmt.Errorf("npmjs target specification required") } - attr, err := registrations.DecodeConfig[Config](config, AttributeType{}.Decode) + cfg, err := registrations.DecodeConfig[Config](config) if err != nil { return true, errors.Wrapf(err, "blob handler configuration") } - var mimes []string - opts := cpi.NewBlobHandlerOptions(olist...) - if opts.MimeType == "npmjs" { - mimes = append(mimes, opts.MimeType) - } - - h := NewArtifactHandler(attr) - for _, m := range mimes { - opts.MimeType = m - ctx.BlobHandlers().Register(h, opts) - } + ctx.BlobHandlers().Register(NewArtifactHandler(cfg), + cpi.ForArtifactType(resourcetypes.NPM_PACKAGE), + cpi.ForMimeType(mime.MIME_TGZ), + cpi.NewBlobHandlerOptions(olist...), + ) return true, nil } func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { - return registrations.NewLeafHandlerInfo("downloading npmjs artifacts", ` -The npmjsArtifacts downloader is able to download npmjs artifacts + return registrations.NewLeafHandlerInfo("uploading npmjs artifacts", ` +The npmjsArtifacts uploader is able to upload npmjs artifacts as artifact archive according to the npmjs package spec. -The following artifact media types are supported: npmjs +The following artifact media types are supported: `+mime.MIME_TGZ+` By default, it is registered for these mimetypes. It accepts a config with the following fields: -`+listformat.FormatMapElements("", AttributeDescription())+` -Alternatively, a single string value can be given representing a npmjs repository -reference.`, +'url': the URL of the npmjs repository. +If not given, the default npmjs.com repository. +`, ) } From 1b7cd7259c06b62b4ba5dc48e92077343f906a7b Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 31 Jan 2024 09:09:40 +0100 Subject: [PATCH 05/51] publish --- .../handlers/generic/npmjs}/publish.go | 69 ++----------------- 1 file changed, 4 insertions(+), 65 deletions(-) rename {components/npm-publish => pkg/contexts/ocm/blobhandler/handlers/generic/npmjs}/publish.go (73%) diff --git a/components/npm-publish/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go similarity index 73% rename from components/npm-publish/publish.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index a59f2d3f59..a147a7801f 100644 --- a/components/npm-publish/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -1,4 +1,4 @@ -package main +package npmjs import ( "archive/tar" @@ -11,7 +11,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/url" ) @@ -40,7 +39,7 @@ func login(registry, username, password string, email string) (string, error) { } defer resp.Body.Close() if resp.StatusCode >= http.StatusBadRequest { - all, _ := ioutil.ReadAll(resp.Body) + all, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("%d, %s", resp.StatusCode, string(all)) } var token struct { @@ -147,12 +146,12 @@ func prepare(data []byte) (*Package, error) { } switch thr.Name { case "package/package.json": - pkgData, err = ioutil.ReadAll(tr) + pkgData, err = io.ReadAll(tr) if err != nil { return nil, fmt.Errorf("read package.json failed, %w", err) } case "package/README.md": - readme, err = ioutil.ReadAll(tr) + readme, err = io.ReadAll(tr) if err != nil { return nil, fmt.Errorf("read README.md failed, %w", err) } @@ -190,63 +189,3 @@ func prepare(data []byte) (*Package, error) { pkg.Dist.Integrity = createIntegrity(data) return &pkg, nil } - -func main() { - // FIXME: make registry configurable - registry := "https://...FIXME" - - // FIXME: make login configurable - token, err := login(registry, "FIXME user", "FIXME token/password", "FIXME@FIXME.com") - if err != nil { - panic(err) - } - - // FIXME: make URL/File configurable - tar -czf ocm-website.tgz package/ - data, err := ioutil.ReadFile("FIXME.tgz") - if err != nil { - panic(err) - } - pkg, err := prepare(data) - if err != nil { - panic(err) - } - body := Body{ - ID: pkg.Name, - Name: pkg.Name, - Description: pkg.Description, - } - body.DistTags.Latest = pkg.Version - body.Versions = map[string]*Package{ - pkg.Version: pkg, - } - body.Readme = pkg.Readme - tbName := pkg.Name + "-" + pkg.Version + ".tgz" - body.Attachments = map[string]*Attachment{ - tbName: NewAttachment(data), - } - pkg.Dist.Tarball = registry + pkg.Name + "/-/" + tbName - - client := http.Client{} - marshal, err := json.Marshal(body) - if err != nil { - panic(err) - } - req, err := http.NewRequest(http.MethodPut, registry+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) - if err != nil { - panic(err) - } - req.Header.Set("authorization", "Bearer " + token) - req.Header.Set("content-type", "application/json") - resp, err := client.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - all, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err) - } - fmt.Println(string(all)) - } -} From bcfec0e189d099dd76dcf5eaa8f735313b818381 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 31 Jan 2024 10:47:37 +0100 Subject: [PATCH 06/51] config test --- .../handlers/generic/npmjs/config_test.go | 20 +++++++++++++++++++ .../handlers/generic/npmjs/identity.go | 2 +- .../handlers/generic/npmjs/registration.go | 18 +++++++++++++++++ .../handlers/generic/npmjs/suite_test.go | 13 ++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go new file mode 100644 index 0000000000..56ab30be82 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go @@ -0,0 +1,20 @@ +package npmjs_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs" + "github.com/open-component-model/ocm/pkg/registrations" + . "github.com/open-component-model/ocm/pkg/testutils" +) + +var _ = Describe("Config deserialization Test Environment", func() { + + It("deserialize string", func() { + cfg := Must(registrations.DecodeConfig[npmjs.Config]("test")) + + Expect(cfg).To(Equal(&npmjs.Config{"test"})) + }) + +}) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go index dcda858047..330fcd6de7 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go @@ -27,7 +27,7 @@ the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -var identityMatcher = hostpath.IdentityMatcher("") +var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { return identityMatcher(pattern, cur, id) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go index 8e577c92c3..d838ff2e25 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -1,6 +1,7 @@ package npmjs import ( + "encoding/json" "fmt" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" @@ -14,6 +15,23 @@ type Config struct { Url string `json:"url"` } +type rawConfig Config + +func (c *Config) UnmarshalJSON(data []byte) error { + err := json.Unmarshal(data, &c.Url) + if err == nil { + return nil + } + var raw rawConfig + err = json.Unmarshal(data, &raw) + if err != nil { + return err + } + *c = Config(raw) + + return nil +} + func init() { cpi.RegisterBlobHandlerRegistrationHandler("ocm/npmPackage", &RegistrationHandler{}) } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go new file mode 100644 index 0000000000..c159ea140f --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go @@ -0,0 +1,13 @@ +package npmjs_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "npmjs Upload Repository Attribute") +} From 7fc42b2deb489169951f5fd4506d66107a5cfba4 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 31 Jan 2024 15:54:59 +0100 Subject: [PATCH 07/51] some cleanup and test --- pkg/contexts/ocm/accessmethods/npm/README.md | 10 ++--- pkg/contexts/ocm/accessmethods/npm/method.go | 12 ++---- .../handlers/generic/npmjs/config_test.go | 6 ++- .../handlers/generic/npmjs/identity.go | 37 ++++++------------- 4 files changed, 23 insertions(+), 42 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/npm/README.md b/pkg/contexts/ocm/accessmethods/npm/README.md index 0ef8f0926c..f5dbab6942 100644 --- a/pkg/contexts/ocm/accessmethods/npm/README.md +++ b/pkg/contexts/ocm/accessmethods/npm/README.md @@ -1,5 +1,4 @@ -# `npm` - NPM packages in an NPM registry - +# `npm` - NPM packages in a NPM registry (e.g. npmjs.com) ### Synopsis ``` @@ -12,7 +11,6 @@ Provided blobs use the following media type: `application/x-tgz` This method implements the access of an NPM package from an NPM registry. - ### Specification Versions Supported specification version is `v1` @@ -23,7 +21,7 @@ The type specific specification fields are: - **`registry`** *string* - Base URL of the NPM registry. + Base URL of the npmjs registry. - **`package`** *string* @@ -31,6 +29,4 @@ The type specific specification fields are: - **`version`** *string* - The version name of the NPM package. - - + The version of the NPM package. diff --git a/pkg/contexts/ocm/accessmethods/npm/method.go b/pkg/contexts/ocm/accessmethods/npm/method.go index d4f01f2250..e85c4237bb 100644 --- a/pkg/contexts/ocm/accessmethods/npm/method.go +++ b/pkg/contexts/ocm/accessmethods/npm/method.go @@ -28,11 +28,7 @@ import ( "github.com/open-component-model/ocm/pkg/runtime" ) -// TODO: open questions -// - authentication??? -// - writing packages - -// Type is the access type of NPM registry. +// Type is the access type of npmjs registry. const ( Type = "npm" TypeV1 = Type + runtime.VersionSeparator + "v1" @@ -43,7 +39,7 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } -// AccessSpec describes the access for a NPM registry. +// AccessSpec describes the access for a npmjs registry. type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` @@ -57,7 +53,7 @@ type AccessSpec struct { var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) -// New creates a new NPM registry access spec version v1. +// New creates a new npmjs registry access spec version v1. func New(registry, pkg, version string) *AccessSpec { return &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), @@ -120,8 +116,6 @@ func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) { return &metadata, nil } -//////////////////////////////////////////////////////////////////////////////// - func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { factory := func() (blobaccess.BlobAccess, error) { meta, err := a.getPackageMeta(c.GetContext()) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go index 56ab30be82..d10a88de0c 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go @@ -11,9 +11,13 @@ import ( var _ = Describe("Config deserialization Test Environment", func() { - It("deserialize string", func() { + It("deserializes string", func() { cfg := Must(registrations.DecodeConfig[npmjs.Config]("test")) + Expect(cfg).To(Equal(&npmjs.Config{"test"})) + }) + It("deserializes struct", func() { + cfg := Must(registrations.DecodeConfig[npmjs.Config](`{"Url":"test"}`)) Expect(cfg).To(Equal(&npmjs.Config{"test"})) }) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go index 330fcd6de7..39e493c886 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go @@ -1,9 +1,10 @@ package npmjs import ( - "net/url" "path" + . "net/url" + "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" @@ -20,51 +21,37 @@ func init() { ATTR_EMAIL, "npmjs registries, require an email address", }) - cpi.RegisterStandardIdentity(CONSUMER_TYPE, IdentityMatcher, `Npmjs repository + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `Npmjs repository It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE) - -func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool { - return identityMatcher(pattern, cur, id) -} - const ( ATTR_USERNAME = cpi.ATTR_USERNAME ATTR_PASSWORD = cpi.ATTR_PASSWORD ATTR_EMAIL = cpi.ATTR_EMAIL ) -func SimpleCredentials(user, passwd string, email string) cpi.Credentials { - return cpi.DirectCredentials{ - ATTR_USERNAME: user, - ATTR_PASSWORD: passwd, - ATTR_EMAIL: email, - } -} - -func GetConsumerId(repourl string, pkgname string) cpi.ConsumerIdentity { - u, err := url.Parse(repourl) +func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { + url, err := Parse(rawURL) if err != nil { return nil } - u.Path = path.Join(u.Path, pkgname) - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, u.String()) + url.Path = path.Join(url.Path, pkgName) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) } -func GetCredentials(ctx cpi.ContextProvider, repourl string, pkgname string) common.Properties { - id := GetConsumerId(repourl, pkgname) +func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { + id := GetConsumerId(repoUrl, pkgName) if id == nil { return nil } - creds, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) - if creds == nil || err != nil { + credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) + if credentials == nil || err != nil { return nil } - return creds.Properties() + return credentials.Properties() } From 16f766c9ea3fcc789ed910f6cffcacc277b1575e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 31 Jan 2024 17:39:25 +0100 Subject: [PATCH 08/51] fix some lint issues --- .../handlers/generic/npmjs/blobhandler.go | 23 +++++++++++-------- .../handlers/generic/npmjs/const.go | 13 +++++++++++ .../handlers/generic/npmjs/identity.go | 9 -------- .../handlers/generic/npmjs/publish.go | 17 +++++++------- .../handlers/generic/npmjs/registration.go | 2 +- 5 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 01970a3a30..52c0038e6c 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -2,6 +2,7 @@ package npmjs import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -22,8 +23,7 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { } func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, global cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { - attr := b.spec - if attr == nil { + if b.spec == nil { return nil, nil } @@ -34,17 +34,20 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g blobReader, err := blob.Reader() if err != nil { - panic(err) + return nil, err } defer blobReader.Close() data, err := io.ReadAll(blobReader) + if err != nil { + return nil, err + } // read package.json from tarball to get package name and version var pkg *Package pkg, err = prepare(data) if err != nil { - panic(err) + return nil, err } // use user+pass+mail from credentials to login and retrieve bearer token @@ -57,7 +60,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } token, err := login(b.spec.Url, username, password, email) if err != nil { - panic(err) + return nil, err } // prepare body for upload @@ -78,13 +81,13 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName marshal, err := json.Marshal(body) if err != nil { - panic(err) + return nil, err } // prepare request - req, err := http.NewRequest(http.MethodPut, b.spec.Url+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, b.spec.Url+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) if err != nil { - panic(err) + return nil, err } req.Header.Set("authorization", "Bearer "+token) req.Header.Set("content-type", "application/json") @@ -93,13 +96,13 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g client := http.Client{} resp, err := client.Do(req) if err != nil { - panic(err) + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { all, err := io.ReadAll(resp.Body) if err != nil { - panic(err) + return nil, err } fmt.Println(string(all)) } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go new file mode 100644 index 0000000000..721e269e51 --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go @@ -0,0 +1,13 @@ +package npmjs + +import "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + +// CONSUMER_TYPE is the npmjs repository type. +const CONSUMER_TYPE = "Registry.npmjs.com" +const BLOB_HANDLER_NAME = "ocm/npmPackage" + +const ( + ATTR_USERNAME = cpi.ATTR_USERNAME + ATTR_PASSWORD = cpi.ATTR_PASSWORD + ATTR_EMAIL = cpi.ATTR_EMAIL +) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go index 39e493c886..8a84d95eb7 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go @@ -11,9 +11,6 @@ import ( "github.com/open-component-model/ocm/pkg/listformat" ) -// CONSUMER_TYPE is the npmjs repository type. -const CONSUMER_TYPE = "Registry.npmjs.com" - func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ ATTR_USERNAME, "the basic auth user name", @@ -28,12 +25,6 @@ the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } -const ( - ATTR_USERNAME = cpi.ATTR_USERNAME - ATTR_PASSWORD = cpi.ATTR_PASSWORD - ATTR_EMAIL = cpi.ATTR_EMAIL -) - func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { url, err := Parse(rawURL) if err != nil { diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index a147a7801f..e6fd7038e9 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "context" "crypto/sha1" "crypto/sha512" "encoding/base64" @@ -27,7 +28,7 @@ func login(registry, username, password string, email string) (string, error) { if err != nil { return "", err } - req, err := http.NewRequest(http.MethodPut, registry+"/-/user/org.couchdb.user:"+url.PathEscape(username), bytes.NewReader(marshal)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, registry+"/-/user/org.couchdb.user:"+url.PathEscape(username), bytes.NewReader(marshal)) if err != nil { return "", err } @@ -134,25 +135,25 @@ func prepare(data []byte) (*Package, error) { readme []byte ) for { - thr, err := tr.Next() - if err != nil { - if err == io.EOF { + thr, e := tr.Next() + if e != nil { + if e == io.EOF { break } - return nil, err + return nil, e } if pkgData != nil && readme != nil { break } switch thr.Name { case "package/package.json": - pkgData, err = io.ReadAll(tr) + pkgData, e = io.ReadAll(tr) if err != nil { return nil, fmt.Errorf("read package.json failed, %w", err) } case "package/README.md": - readme, err = io.ReadAll(tr) - if err != nil { + readme, e = io.ReadAll(tr) + if e != nil { return nil, fmt.Errorf("read README.md failed, %w", err) } } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go index d838ff2e25..d8cc1cad6b 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -33,7 +33,7 @@ func (c *Config) UnmarshalJSON(data []byte) error { } func init() { - cpi.RegisterBlobHandlerRegistrationHandler("ocm/npmPackage", &RegistrationHandler{}) + cpi.RegisterBlobHandlerRegistrationHandler(BLOB_HANDLER_NAME, &RegistrationHandler{}) } type RegistrationHandler struct{} From b93d2d564b1cbeaa08d0257fe331974a60bcd2b7 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 12:45:03 +0100 Subject: [PATCH 09/51] use Logger instead --- .../ocm/blobhandler/handlers/generic/npmjs/blobhandler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 52c0038e6c..2b7060c498 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -11,6 +11,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + log "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" ) @@ -104,7 +105,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if err != nil { return nil, err } - fmt.Println(string(all)) + log.Context().Logger().Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) } return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil From 14908b53c90ee3ca994ed6380d0aa55295fe2f4e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 12:46:07 +0100 Subject: [PATCH 10/51] //nolint:gosec --- .../ocm/blobhandler/handlers/generic/npmjs/publish.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index e6fd7038e9..6b001ba090 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -113,13 +113,13 @@ func NewAttachment(data []byte) *Attachment { } func createIntegrity(data []byte) string { - hash := sha512.New() + hash := sha512.New() //nolint:gosec hash.Write(data) return "sha512-" + base64.StdEncoding.EncodeToString(hash.Sum(nil)) } func createShasum(data []byte) string { - hash := sha1.New() + hash := sha1.New() //nolint:gosec hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } From b82ec71e0a972f7c83d9e7760f58ce6fe113cd8e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 12:46:59 +0100 Subject: [PATCH 11/51] use errors.Is(... --- .../ocm/blobhandler/handlers/generic/npmjs/publish.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index 6b001ba090..d931a3978e 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -10,6 +10,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -137,7 +138,7 @@ func prepare(data []byte) (*Package, error) { for { thr, e := tr.Next() if e != nil { - if e == io.EOF { + if errors.Is(e, io.EOF) { break } return nil, e @@ -147,13 +148,13 @@ func prepare(data []byte) (*Package, error) { } switch thr.Name { case "package/package.json": - pkgData, e = io.ReadAll(tr) + pkgData, err = io.ReadAll(tr) if err != nil { return nil, fmt.Errorf("read package.json failed, %w", err) } case "package/README.md": - readme, e = io.ReadAll(tr) - if e != nil { + readme, err = io.ReadAll(tr) + if err != nil { return nil, fmt.Errorf("read README.md failed, %w", err) } } From ad34a99bd8d010e60341d3ead1d41825ef45262e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 13:55:27 +0100 Subject: [PATCH 12/51] nolint:gosec --- .../ocm/blobhandler/handlers/generic/npmjs/publish.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index d931a3978e..3270a47a96 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "context" + //nolint:gosec // older npm (prior to v5) uses sha1 "crypto/sha1" "crypto/sha512" "encoding/base64" @@ -114,13 +115,13 @@ func NewAttachment(data []byte) *Attachment { } func createIntegrity(data []byte) string { - hash := sha512.New() //nolint:gosec + hash := sha512.New() hash.Write(data) return "sha512-" + base64.StdEncoding.EncodeToString(hash.Sum(nil)) } func createShasum(data []byte) string { - hash := sha1.New() //nolint:gosec + hash := sha1.New() //nolint:gosec // older npm (prior to v5) uses sha1 hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } From fc34e07f821a80907b2d0981c46909dae8239373 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 16:16:44 +0100 Subject: [PATCH 13/51] clean Package struct --- .../handlers/generic/npmjs/blobhandler.go | 12 ++-- .../handlers/generic/npmjs/publish.go | 69 ++++++------------- 2 files changed, 28 insertions(+), 53 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 2b7060c498..902df84de4 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -11,7 +11,7 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" - log "github.com/open-component-model/ocm/pkg/logging" + "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" ) @@ -44,12 +44,14 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g return nil, err } - // read package.json from tarball to get package name and version + // read package.json from tarball to get name, version, etc. var pkg *Package pkg, err = prepare(data) if err != nil { return nil, err } + tbName := pkg.Name + "-" + pkg.Version + ".tgz" + pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName // use user+pass+mail from credentials to login and retrieve bearer token cred := GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) @@ -70,16 +72,14 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g Name: pkg.Name, Description: pkg.Description, } - body.DistTags.Latest = pkg.Version body.Versions = map[string]*Package{ pkg.Version: pkg, } + body.DistTags.Latest = pkg.Version body.Readme = pkg.Readme - tbName := pkg.Name + "-" + pkg.Version + ".tgz" body.Attachments = map[string]*Attachment{ tbName: NewAttachment(data), } - pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName marshal, err := json.Marshal(body) if err != nil { return nil, err @@ -105,7 +105,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if err != nil { return nil, err } - log.Context().Logger().Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) + logging.Context().Logger().Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) } return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index 3270a47a96..db807dfb86 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -56,38 +56,17 @@ func login(registry, username, password string, email string) (string, error) { } type Package struct { - pkg map[string]interface{} - Name string - Version string - Readme string - ReadmeFilename string - GitHead string - Description string - ID string - NodeVersion string - NpmVersion string - Dist struct { + Name string + Version string + Readme string + Description string + Dist struct { Integrity string `json:"integrity"` Shasum string `json:"shasum"` Tarball string `json:"tarball"` } } -func (p *Package) UnmarshalJSON(data []byte) error { - return json.Unmarshal(data, &p.pkg) -} - -func (p *Package) MarshalJSON() ([]byte, error) { - p.pkg["readme"] = p.Readme - p.pkg["readmeFilename"] = p.ReadmeFilename - p.pkg["gitHead"] = p.GitHead - p.pkg["_id"] = p.ID - p.pkg["_nodeVersion"] = p.NodeVersion - p.pkg["_npmVersion"] = p.NpmVersion - p.pkg["dist"] = p.Dist - return json.Marshal(p.pkg) -} - type Attachment struct { ContentType string `json:"content_type"` Data []byte `json:"data"` @@ -114,18 +93,19 @@ func NewAttachment(data []byte) *Attachment { } } -func createIntegrity(data []byte) string { +func createSha512(data []byte) string { hash := sha512.New() hash.Write(data) return "sha512-" + base64.StdEncoding.EncodeToString(hash.Sum(nil)) } -func createShasum(data []byte) string { +func createSha1(data []byte) string { hash := sha1.New() //nolint:gosec // older npm (prior to v5) uses sha1 hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } +// read package.json and README.md from tarball to create Package object func prepare(data []byte) (*Package, error) { gz, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { @@ -160,35 +140,30 @@ func prepare(data []byte) (*Package, error) { } } } + + // fetch some information from package.json if len(pkgData) == 0 { return nil, fmt.Errorf("package.json is empty") } - var pkg Package - err = json.Unmarshal(pkgData, &pkg) + var pkgJson map[string]string + err = json.Unmarshal(pkgData, &pkgJson) if err != nil { return nil, fmt.Errorf("read package.json failed, %w", err) } - var meta struct { - Name string `json:"name"` - Version string `json:"version"` - Description string `json:"description"` - } - if err := json.Unmarshal(pkgData, &meta); err != nil { - return nil, fmt.Errorf("read package.json version and name failed, %w", err) - } - if meta.Name == "" { + if pkgJson["name"] == "" { return nil, fmt.Errorf("package.json's name is empty") } - if meta.Version == "" { + if pkgJson["version"] == "" { return nil, fmt.Errorf("package.json's version is empty") } - pkg.Version = meta.Version - pkg.Description = meta.Description - pkg.Name = meta.Name + + // create package object + var pkg Package + pkg.Name = pkgJson["name"] + pkg.Version = pkgJson["version"] + pkg.Description = pkgJson["description"] pkg.Readme = string(readme) - pkg.ReadmeFilename = "README.md" - pkg.ID = meta.Name + meta.Version - pkg.Dist.Shasum = createShasum(data) - pkg.Dist.Integrity = createIntegrity(data) + pkg.Dist.Shasum = createSha1(data) + pkg.Dist.Integrity = createSha512(data) return &pkg, nil } From a4f84d1285ba5979971983db280783a6f48f8033 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 5 Feb 2024 16:28:40 +0100 Subject: [PATCH 14/51] Update publish.go --- pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index db807dfb86..b6d0a4d11a 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -105,7 +105,7 @@ func createSha1(data []byte) string { return hex.EncodeToString(hash.Sum(nil)) } -// read package.json and README.md from tarball to create Package object +// Read package.json and README.md from tarball to create Package object. func prepare(data []byte) (*Package, error) { gz, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { From 8177ccceb8742ff9fae2921393bd849437a70992 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 6 Feb 2024 09:17:11 +0100 Subject: [PATCH 15/51] gofumpt --- .../ocm/blobhandler/handlers/generic/npmjs/const.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go index 721e269e51..fd8fcf8452 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go @@ -2,12 +2,15 @@ package npmjs import "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" -// CONSUMER_TYPE is the npmjs repository type. -const CONSUMER_TYPE = "Registry.npmjs.com" -const BLOB_HANDLER_NAME = "ocm/npmPackage" - const ( + // CONSUMER_TYPE is the npmjs repository type. + CONSUMER_TYPE = "Registry.npmjs.com" + BLOB_HANDLER_NAME = "ocm/npmPackage" + + // ATTR_USERNAME is the username attribute. Required for login at any npmjs registry. ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any npmjs registry. ATTR_PASSWORD = cpi.ATTR_PASSWORD - ATTR_EMAIL = cpi.ATTR_EMAIL + // ATTR_EMAIL is the email attribute. Required for login at any npmjs registry. + ATTR_EMAIL = cpi.ATTR_EMAIL ) From 581d6d6586fddc667c2a218a44b03d7a3e3568bf Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 6 Feb 2024 17:16:43 +0100 Subject: [PATCH 16/51] don't re-upload same artifacts --- .../handlers/generic/npmjs/blobhandler.go | 23 +++- .../handlers/generic/npmjs/publish.go | 127 ++++++++++++------ 2 files changed, 105 insertions(+), 45 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 902df84de4..34e09e2770 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -45,6 +45,8 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } // read package.json from tarball to get name, version, etc. + log := logging.Context().Logger() + log.Debug("reading package.json from tarball") var pkg *Package pkg, err = prepare(data) if err != nil { @@ -52,6 +54,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } tbName := pkg.Name + "-" + pkg.Version + ".tgz" pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName + log.Debug("identified package: %s@%s", pkg.Name, pkg.Version) // use user+pass+mail from credentials to login and retrieve bearer token cred := GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) @@ -61,11 +64,22 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if username == "" || password == "" || email == "" { return nil, fmt.Errorf("username, password or email missing") } + log.Debug("logging in as %s on %s", username, b.spec.Url) token, err := login(b.spec.Url, username, password, email) if err != nil { return nil, err } + // check if package exists + exists, err := packageExists(b.spec.Url, *pkg, token) + if err != nil { + return nil, err + } + if exists { + log.Debug("package %s@%s already exists, skipping upload", pkg.Name, pkg.Version) + return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil + } + // prepare body for upload body := Body{ ID: pkg.Name, @@ -85,7 +99,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g return nil, err } - // prepare request + // prepare PUT request req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, b.spec.Url+"/"+url.PathEscape(pkg.Name), bytes.NewReader(marshal)) if err != nil { return nil, err @@ -93,8 +107,9 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g req.Header.Set("authorization", "Bearer "+token) req.Header.Set("content-type", "application/json") - // send request + // send PUT request - upload tgz client := http.Client{} + log.Debug("uploading package %s@%s to %s", pkg.Name, pkg.Version, b.spec.Url) resp, err := client.Do(req) if err != nil { return nil, err @@ -105,8 +120,8 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if err != nil { return nil, err } - logging.Context().Logger().Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) + log.Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) } - + log.Debug("package %s@%s uploaded to %s", pkg.Name, pkg.Version, b.spec.Url) return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go index b6d0a4d11a..da1c8425e3 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go @@ -18,43 +18,6 @@ import ( "net/url" ) -func login(registry, username, password string, email string) (string, error) { - data := map[string]interface{}{ - "_id": "org.couchdb.user:" + username, - "name": username, - "email": email, - "password": password, - "type": "user", - } - marshal, err := json.Marshal(data) - if err != nil { - return "", err - } - req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, registry+"/-/user/org.couchdb.user:"+url.PathEscape(username), bytes.NewReader(marshal)) - if err != nil { - return "", err - } - req.SetBasicAuth(username, password) - req.Header.Set("content-type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode >= http.StatusBadRequest { - all, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("%d, %s", resp.StatusCode, string(all)) - } - var token struct { - Token string `json:"token"` - } - err = json.NewDecoder(resp.Body).Decode(&token) - if err != nil { - return "", err - } - return token.Token, nil -} - type Package struct { Name string Version string @@ -93,6 +56,44 @@ func NewAttachment(data []byte) *Attachment { } } +// Login to npmjs registry (URL) and retrieve bearer token. +func login(registry string, username string, password string, email string) (string, error) { + data := map[string]interface{}{ + "_id": "org.couchdb.user:" + username, + "name": username, + "email": email, + "password": password, + "type": "user", + } + marshal, err := json.Marshal(data) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, registry+"/-/user/org.couchdb.user:"+url.PathEscape(username), bytes.NewReader(marshal)) + if err != nil { + return "", err + } + req.SetBasicAuth(username, password) + req.Header.Set("content-type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusBadRequest { + all, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("%d, %s", resp.StatusCode, string(all)) + } + var token struct { + Token string `json:"token"` + } + err = json.NewDecoder(resp.Body).Decode(&token) + if err != nil { + return "", err + } + return token.Token, nil +} + func createSha512(data []byte) string { hash := sha512.New() hash.Write(data) @@ -145,7 +146,7 @@ func prepare(data []byte) (*Package, error) { if len(pkgData) == 0 { return nil, fmt.Errorf("package.json is empty") } - var pkgJson map[string]string + var pkgJson map[string]interface{} err = json.Unmarshal(pkgData, &pkgJson) if err != nil { return nil, fmt.Errorf("read package.json failed, %w", err) @@ -159,11 +160,55 @@ func prepare(data []byte) (*Package, error) { // create package object var pkg Package - pkg.Name = pkgJson["name"] - pkg.Version = pkgJson["version"] - pkg.Description = pkgJson["description"] + pkg.Name = pkgJson["name"].(string) + pkg.Version = pkgJson["version"].(string) + pkg.Description = pkgJson["description"].(string) pkg.Readme = string(readme) pkg.Dist.Shasum = createSha1(data) pkg.Dist.Integrity = createSha512(data) return &pkg, nil } + +// Check if package already exists in npmjs registry. If it does, checks if it's the same. +func packageExists(repoUrl string, pkg Package, token string) (bool, error) { + client := http.Client{} + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, repoUrl+"/"+url.PathEscape(pkg.Name)+"/"+url.PathEscape(pkg.Version), nil) + if err != nil { + return false, err + } + req.Header.Set("authorization", "Bearer "+token) + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + // artifact doesn't exist, it's safe to upload + return false, nil + } + + // artifact exists, let's check if it's the same + all, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("http (%d) - %s", resp.StatusCode, string(all)) + } + var data map[string]interface{} + err = json.Unmarshal(all, &data) + if err != nil { + return false, err + } + dist := data["dist"].(map[string]interface{}) + if pkg.Dist.Integrity == dist["integrity"] { + // sha-512 sum is the same, we can skip the upload + return true, nil + } + if pkg.Dist.Shasum == dist["shasum"] { + // sha-1 sum is the same, we can skip the upload + return true, nil + } + + return false, fmt.Errorf("artifact already exists but has different shasum or integrity") +} From 19055b5e8b5fbb3d4170190de36396e5f547e052 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 6 Feb 2024 17:19:17 +0100 Subject: [PATCH 17/51] return error --- .../ocm/blobhandler/handlers/generic/npmjs/blobhandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 34e09e2770..0958a916c9 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -120,7 +120,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if err != nil { return nil, err } - log.Error("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) + return nil, fmt.Errorf("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) } log.Debug("package %s@%s uploaded to %s", pkg.Name, pkg.Version, b.spec.Url) return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil From 9eaf836893c2c56a433305f0bbd7ca818a4f2c8c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 12:30:18 +0100 Subject: [PATCH 18/51] logging --- .../handlers/generic/npmjs/blobhandler.go | 14 ++++++++------ .../blobhandler/handlers/generic/npmjs/const.go | 8 +++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go index 0958a916c9..5979ca2abf 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go @@ -45,7 +45,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } // read package.json from tarball to get name, version, etc. - log := logging.Context().Logger() + log := logging.Context().WithContext(NPM_REALM).Logger() log.Debug("reading package.json from tarball") var pkg *Package pkg, err = prepare(data) @@ -54,7 +54,8 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } tbName := pkg.Name + "-" + pkg.Version + ".tgz" pkg.Dist.Tarball = b.spec.Url + pkg.Name + "/-/" + tbName - log.Debug("identified package: %s@%s", pkg.Name, pkg.Version) + log = log.WithValues("package", pkg.Name, "version", pkg.Version) + log.Debug("identified") // use user+pass+mail from credentials to login and retrieve bearer token cred := GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) @@ -64,7 +65,8 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g if username == "" || password == "" || email == "" { return nil, fmt.Errorf("username, password or email missing") } - log.Debug("logging in as %s on %s", username, b.spec.Url) + log = log.WithValues("user", username, "repo", b.spec.Url) + log.Debug("login") token, err := login(b.spec.Url, username, password, email) if err != nil { return nil, err @@ -76,7 +78,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g return nil, err } if exists { - log.Debug("package %s@%s already exists, skipping upload", pkg.Name, pkg.Version) + log.Debug("package+version already exists, skipping upload") return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil } @@ -109,7 +111,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g // send PUT request - upload tgz client := http.Client{} - log.Debug("uploading package %s@%s to %s", pkg.Name, pkg.Version, b.spec.Url) + log.Debug("uploading") resp, err := client.Do(req) if err != nil { return nil, err @@ -122,6 +124,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, g } return nil, fmt.Errorf("http (%d) - failed to upload package: %s", resp.StatusCode, string(all)) } - log.Debug("package %s@%s uploaded to %s", pkg.Name, pkg.Version, b.spec.Url) + log.Debug("successfully uploaded") return npm.New(b.spec.Url, pkg.Name, pkg.Version), nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go index fd8fcf8452..8c1461b806 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go @@ -1,6 +1,9 @@ package npmjs -import "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" +import ( + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/logging" +) const ( // CONSUMER_TYPE is the npmjs repository type. @@ -14,3 +17,6 @@ const ( // ATTR_EMAIL is the email attribute. Required for login at any npmjs registry. ATTR_EMAIL = cpi.ATTR_EMAIL ) + +// logging Realm +var NPM_REALM = logging.DefineSubRealm("NPM registry", "NPM") From 16167c24b1e020a73d53b3643798caebdc4b17cc Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 12:34:20 +0100 Subject: [PATCH 19/51] npmjs -> NPM --- pkg/contexts/ocm/accessmethods/npm/README.md | 2 +- pkg/contexts/ocm/accessmethods/npm/method.go | 8 +++++--- .../ocm/blobhandler/handlers/generic/npmjs/identity.go | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/npm/README.md b/pkg/contexts/ocm/accessmethods/npm/README.md index f5dbab6942..dc69976a8c 100644 --- a/pkg/contexts/ocm/accessmethods/npm/README.md +++ b/pkg/contexts/ocm/accessmethods/npm/README.md @@ -21,7 +21,7 @@ The type specific specification fields are: - **`registry`** *string* - Base URL of the npmjs registry. + Base URL of the NPM registry. - **`package`** *string* diff --git a/pkg/contexts/ocm/accessmethods/npm/method.go b/pkg/contexts/ocm/accessmethods/npm/method.go index e85c4237bb..f64fabfd00 100644 --- a/pkg/contexts/ocm/accessmethods/npm/method.go +++ b/pkg/contexts/ocm/accessmethods/npm/method.go @@ -28,7 +28,7 @@ import ( "github.com/open-component-model/ocm/pkg/runtime" ) -// Type is the access type of npmjs registry. +// Type is the access type of NPM registry. const ( Type = "npm" TypeV1 = Type + runtime.VersionSeparator + "v1" @@ -39,7 +39,7 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](TypeV1, accspeccpi.WithFormatSpec(formatV1), accspeccpi.WithConfigHandler(ConfigHandler()))) } -// AccessSpec describes the access for a npmjs registry. +// AccessSpec describes the access for a NPM registry. type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` @@ -53,7 +53,7 @@ type AccessSpec struct { var _ accspeccpi.AccessSpec = (*AccessSpec)(nil) -// New creates a new npmjs registry access spec version v1. +// New creates a new NPM registry access spec version v1. func New(registry, pkg, version string) *AccessSpec { return &AccessSpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), @@ -116,6 +116,8 @@ func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) { return &metadata, nil } +//////////////////////////////////////////////////////////////////////////////// + func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) { factory := func() (blobaccess.BlobAccess, error) { meta, err := a.getPackageMeta(c.GetContext()) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go index 8a84d95eb7..7410a25c91 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go @@ -15,10 +15,10 @@ func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ ATTR_USERNAME, "the basic auth user name", ATTR_PASSWORD, "the basic auth password", - ATTR_EMAIL, "npmjs registries, require an email address", + ATTR_EMAIL, "NPM registry, require an email address", }) - cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `Npmjs repository + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, From 2d846cb44d63fafd11110aaee4c7137426d8c47c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 12:45:07 +0100 Subject: [PATCH 20/51] description --- .../ocm/blobhandler/handlers/generic/npmjs/registration.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go index d8cc1cad6b..2d2298af94 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -63,12 +63,11 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { return registrations.NewLeafHandlerInfo("uploading npmjs artifacts", ` -The npmjsArtifacts uploader is able to upload npmjs artifacts +The `+BLOB_HANDLER_NAME+` uploader is able to upload npmjs artifacts as artifact archive according to the npmjs package spec. -The following artifact media types are supported: `+mime.MIME_TGZ+` -By default, it is registered for these mimetypes. +If registered the default mime type is: `+mime.MIME_TGZ+` -It accepts a config with the following fields: +It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npmjs repository. If not given, the default npmjs.com repository. `, From 5342d8d643d68ce5353d73d63120f47ae1744062 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 12:58:49 +0100 Subject: [PATCH 21/51] description --- .../ocm/blobhandler/handlers/generic/npmjs/registration.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go index 2d2298af94..8cd913e969 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go @@ -69,7 +69,6 @@ If registered the default mime type is: `+mime.MIME_TGZ+` It accepts a plain string for the URL or a config with the following field: 'url': the URL of the npmjs repository. -If not given, the default npmjs.com repository. `, ) } From c3dadad5057aab82040175950b0334970cb7b446 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 13:03:34 +0100 Subject: [PATCH 22/51] rename package --- .../handlers/generic/{npmjs => npm}/blobhandler.go | 2 +- .../handlers/generic/{npmjs => npm}/config_test.go | 12 ++++++------ .../handlers/generic/{npmjs => npm}/const.go | 12 ++++++------ .../handlers/generic/{npmjs => npm}/identity.go | 2 +- .../handlers/generic/{npmjs => npm}/publish.go | 6 +++--- .../handlers/generic/{npmjs => npm}/registration.go | 12 ++++++------ .../handlers/generic/{npmjs => npm}/suite_test.go | 4 ++-- pkg/contexts/ocm/blobhandler/handlers/init.go | 2 +- 8 files changed, 26 insertions(+), 26 deletions(-) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/blobhandler.go (99%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/config_test.go (59%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/const.go (76%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/identity.go (98%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/publish.go (97%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/registration.go (86%) rename pkg/contexts/ocm/blobhandler/handlers/generic/{npmjs => npm}/suite_test.go (67%) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go similarity index 99% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 5979ca2abf..cb6a463474 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -1,4 +1,4 @@ -package npmjs +package npm import ( "bytes" diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go similarity index 59% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go index d10a88de0c..e6b891198a 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/config_test.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go @@ -1,10 +1,10 @@ -package npmjs_test +package npm_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs" + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" "github.com/open-component-model/ocm/pkg/registrations" . "github.com/open-component-model/ocm/pkg/testutils" ) @@ -12,13 +12,13 @@ import ( var _ = Describe("Config deserialization Test Environment", func() { It("deserializes string", func() { - cfg := Must(registrations.DecodeConfig[npmjs.Config]("test")) - Expect(cfg).To(Equal(&npmjs.Config{"test"})) + cfg := Must(registrations.DecodeConfig[npm.Config]("test")) + Expect(cfg).To(Equal(&npm.Config{"test"})) }) It("deserializes struct", func() { - cfg := Must(registrations.DecodeConfig[npmjs.Config](`{"Url":"test"}`)) - Expect(cfg).To(Equal(&npmjs.Config{"test"})) + cfg := Must(registrations.DecodeConfig[npm.Config](`{"Url":"test"}`)) + Expect(cfg).To(Equal(&npm.Config{"test"})) }) }) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go similarity index 76% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go index 8c1461b806..1bbfbedef4 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go @@ -1,4 +1,4 @@ -package npmjs +package npm import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" @@ -6,15 +6,15 @@ import ( ) const ( - // CONSUMER_TYPE is the npmjs repository type. - CONSUMER_TYPE = "Registry.npmjs.com" + // CONSUMER_TYPE is the npm repository type. + CONSUMER_TYPE = "Registry.npm.com" BLOB_HANDLER_NAME = "ocm/npmPackage" - // ATTR_USERNAME is the username attribute. Required for login at any npmjs registry. + // ATTR_USERNAME is the username attribute. Required for login at any npm registry. ATTR_USERNAME = cpi.ATTR_USERNAME - // ATTR_PASSWORD is the password attribute. Required for login at any npmjs registry. + // ATTR_PASSWORD is the password attribute. Required for login at any npm registry. ATTR_PASSWORD = cpi.ATTR_PASSWORD - // ATTR_EMAIL is the email attribute. Required for login at any npmjs registry. + // ATTR_EMAIL is the email attribute. Required for login at any npm registry. ATTR_EMAIL = cpi.ATTR_EMAIL ) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go similarity index 98% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go index 7410a25c91..d6b5bb02e1 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/identity.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go @@ -1,4 +1,4 @@ -package npmjs +package npm import ( "path" diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/publish.go similarity index 97% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/publish.go index da1c8425e3..de1d309be4 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/publish.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/publish.go @@ -1,4 +1,4 @@ -package npmjs +package npm import ( "archive/tar" @@ -56,7 +56,7 @@ func NewAttachment(data []byte) *Attachment { } } -// Login to npmjs registry (URL) and retrieve bearer token. +// Login to npm registry (URL) and retrieve bearer token. func login(registry string, username string, password string, email string) (string, error) { data := map[string]interface{}{ "_id": "org.couchdb.user:" + username, @@ -169,7 +169,7 @@ func prepare(data []byte) (*Package, error) { return &pkg, nil } -// Check if package already exists in npmjs registry. If it does, checks if it's the same. +// Check if package already exists in npm registry. If it does, checks if it's the same. func packageExists(repoUrl string, pkg Package, token string) (bool, error) { client := http.Client{} req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, repoUrl+"/"+url.PathEscape(pkg.Name)+"/"+url.PathEscape(pkg.Version), nil) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go similarity index 86% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go index 8cd913e969..c998198884 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go @@ -1,4 +1,4 @@ -package npmjs +package npm import ( "encoding/json" @@ -45,7 +45,7 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co return true, fmt.Errorf("invalid npmjsArtifact handler %q", handler) } if config == nil { - return true, fmt.Errorf("npmjs target specification required") + return true, fmt.Errorf("npm target specification required") } cfg, err := registrations.DecodeConfig[Config](config) if err != nil { @@ -62,13 +62,13 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co } func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { - return registrations.NewLeafHandlerInfo("uploading npmjs artifacts", ` -The `+BLOB_HANDLER_NAME+` uploader is able to upload npmjs artifacts -as artifact archive according to the npmjs package spec. + return registrations.NewLeafHandlerInfo("uploading npm artifacts", ` +The `+BLOB_HANDLER_NAME+` uploader is able to upload npm artifacts +as artifact archive according to the npm package spec. If registered the default mime type is: `+mime.MIME_TGZ+` It accepts a plain string for the URL or a config with the following field: -'url': the URL of the npmjs repository. +'url': the URL of the npm repository. `, ) } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/suite_test.go similarity index 67% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go rename to pkg/contexts/ocm/blobhandler/handlers/generic/npm/suite_test.go index c159ea140f..90a3bb8d13 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs/suite_test.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/suite_test.go @@ -1,4 +1,4 @@ -package npmjs_test +package npm_test import ( "testing" @@ -9,5 +9,5 @@ import ( func TestConfig(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "npmjs Upload Repository Attribute") + RunSpecs(t, "npm Upload Repository Attribute") } diff --git a/pkg/contexts/ocm/blobhandler/handlers/init.go b/pkg/contexts/ocm/blobhandler/handlers/init.go index 9aed77d000..d19447156c 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/init.go +++ b/pkg/contexts/ocm/blobhandler/handlers/init.go @@ -1,7 +1,7 @@ package handlers import ( - _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npmjs" + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/oci/ocirepo" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/ocm/comparch" From 4b9c7dbd06694ed8a34dac17a8eeea2f16f1762c Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 13:10:03 +0100 Subject: [PATCH 23/51] unused parameters --- pkg/contexts/ocm/accessmethods/npm/method.go | 6 +++--- .../ocm/blobhandler/handlers/generic/npm/blobhandler.go | 2 +- .../ocm/blobhandler/handlers/generic/npm/registration.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/contexts/ocm/accessmethods/npm/method.go b/pkg/contexts/ocm/accessmethods/npm/method.go index f64fabfd00..7698cd9920 100644 --- a/pkg/contexts/ocm/accessmethods/npm/method.go +++ b/pkg/contexts/ocm/accessmethods/npm/method.go @@ -63,7 +63,7 @@ func New(registry, pkg, version string) *AccessSpec { } } -func (a *AccessSpec) Describe(ctx accspeccpi.Context) string { +func (a *AccessSpec) Describe(_ accspeccpi.Context) string { return fmt.Sprintf("NPM package %s:%s in registry %s", a.Package, a.Version, a.Registry) } @@ -71,11 +71,11 @@ func (_ *AccessSpec) IsLocal(accspeccpi.Context) bool { return false } -func (a *AccessSpec) GlobalAccessSpec(ctx accspeccpi.Context) accspeccpi.AccessSpec { +func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpec { return a } -func (a *AccessSpec) GetReferenceHint(cv accspeccpi.ComponentVersionAccess) string { +func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string { return a.Package + ":" + a.Version } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index cb6a463474..2e64b740ed 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -23,7 +23,7 @@ func NewArtifactHandler(repospec *Config) cpi.BlobHandler { return &artifactHandler{repospec} } -func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, global cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { +func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ cpi.AccessSpec, ctx cpi.StorageContext) (cpi.AccessSpec, error) { if b.spec == nil { return nil, nil } diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go index c998198884..ba729fe4e4 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration.go @@ -61,7 +61,7 @@ func (r *RegistrationHandler) RegisterByName(handler string, ctx cpi.Context, co return true, nil } -func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { +func (r *RegistrationHandler) GetHandlers(_ cpi.Context) registrations.HandlerInfos { return registrations.NewLeafHandlerInfo("uploading npm artifacts", ` The `+BLOB_HANDLER_NAME+` uploader is able to upload npm artifacts as artifact archive according to the npm package spec. From aca0895d03a207f4391ca55be69047fdfaa55fd6 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 13:11:44 +0100 Subject: [PATCH 24/51] doc has to end with . --- pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go index 1bbfbedef4..09dfa9e1f0 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go @@ -18,5 +18,5 @@ const ( ATTR_EMAIL = cpi.ATTR_EMAIL ) -// logging Realm +// Logging Realm. var NPM_REALM = logging.DefineSubRealm("NPM registry", "NPM") From c151abb66c394096f3990ee2583966cb6d8a6ceb Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 14:04:16 +0100 Subject: [PATCH 25/51] check if URL is configured --- .../ocm/blobhandler/handlers/generic/npm/blobhandler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 2e64b740ed..b212c995aa 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -33,6 +33,10 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c return nil, nil } + if b.spec.Url == "" { + return nil, fmt.Errorf("NPM registry url not provided") + } + blobReader, err := blob.Reader() if err != nil { return nil, err From b41d870d1145c6ac43f6fd452f56f2806f17f6ef Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 15:44:00 +0100 Subject: [PATCH 26/51] REALM --- .../ocm/blobhandler/handlers/generic/npm/blobhandler.go | 2 +- pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index b212c995aa..2c244cdd1f 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -49,7 +49,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c } // read package.json from tarball to get name, version, etc. - log := logging.Context().WithContext(NPM_REALM).Logger() + log := logging.Context().Logger(REALM) log.Debug("reading package.json from tarball") var pkg *Package pkg, err = prepare(data) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go index 09dfa9e1f0..9ef991c683 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go @@ -19,4 +19,4 @@ const ( ) // Logging Realm. -var NPM_REALM = logging.DefineSubRealm("NPM registry", "NPM") +var REALM = logging.DefineSubRealm("NPM registry", "NPM") From a224c3638b09cd404b7a043a91d038f773200092 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 7 Feb 2024 16:05:02 +0100 Subject: [PATCH 27/51] npmjs.com --- pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go index 9ef991c683..e8f1271879 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go @@ -7,7 +7,7 @@ import ( const ( // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npm.com" + CONSUMER_TYPE = "Registry.npmjs.com" BLOB_HANDLER_NAME = "ocm/npmPackage" // ATTR_USERNAME is the username attribute. Required for login at any npm registry. From 97503f8f9bff3d44785b3ee84f88af2c9fc8f3b6 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 13 Feb 2024 15:29:58 +0100 Subject: [PATCH 28/51] draft --- pkg/contexts/credentials/builtin/init.go | 11 +- .../builtin}/npm/identity.go | 13 +- .../credentials/repositories/npm/config.go | 106 +++++++++++++ .../repositories}/npm/config_test.go | 8 + .../credentials/repositories/npm/provider.go | 71 +++++++++ .../repositories/npm/repository.go | 142 ++++++++++++++++++ .../repositories/npm/testdata/.npmrc | 3 + .../handlers/generic/npm/blobhandler.go | 2 +- .../blobhandler/handlers/generic/npm/const.go | 2 +- 9 files changed, 345 insertions(+), 13 deletions(-) rename pkg/contexts/{ocm/blobhandler/handlers/generic => credentials/builtin}/npm/identity.go (65%) create mode 100644 pkg/contexts/credentials/repositories/npm/config.go rename pkg/contexts/{ocm/blobhandler/handlers/generic => credentials/repositories}/npm/config_test.go (63%) create mode 100644 pkg/contexts/credentials/repositories/npm/provider.go create mode 100644 pkg/contexts/credentials/repositories/npm/repository.go create mode 100644 pkg/contexts/credentials/repositories/npm/testdata/.npmrc diff --git a/pkg/contexts/credentials/builtin/init.go b/pkg/contexts/credentials/builtin/init.go index 7e0e1032eb..578f38cc04 100644 --- a/pkg/contexts/credentials/builtin/init.go +++ b/pkg/contexts/credentials/builtin/init.go @@ -1,7 +1,8 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package builtin -import _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/github" +import ( + _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/github" + _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/helm/identity" + _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm" + _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" +) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go b/pkg/contexts/credentials/builtin/npm/identity.go similarity index 65% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go rename to pkg/contexts/credentials/builtin/npm/identity.go index d6b5bb02e1..62e29749c9 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity.go @@ -8,19 +8,20 @@ import ( "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" "github.com/open-component-model/ocm/pkg/listformat" ) func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - ATTR_USERNAME, "the basic auth user name", - ATTR_PASSWORD, "the basic auth password", - ATTR_EMAIL, "NPM registry, require an email address", + npm.ATTR_USERNAME, "the basic auth user name", + npm.ATTR_PASSWORD, "the basic auth password", + npm.ATTR_EMAIL, "NPM registry, require an email address", }) - cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository + cpi.RegisterStandardIdentity(npm.CONSUMER_TYPE, hostpath.IdentityMatcher(npm.CONSUMER_TYPE), `NPM repository -It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +It matches the `+npm.CONSUMER_TYPE+` consumer type and additionally acts like the `+hostpath.IDENTITY_TYPE+` type.`, attrs) } @@ -32,7 +33,7 @@ func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { } url.Path = path.Join(url.Path, pkgName) - return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) + return hostpath.GetConsumerIdentity(npm.CONSUMER_TYPE, url.String()) } func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go new file mode 100644 index 0000000000..d7c4d71bcb --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -0,0 +1,106 @@ +package npm + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/runtime" +) + +const ( + // REPOSITORY_TYPE is the type of the NPMConfig. + REPOSITORY_TYPE = "NPMConfig" + REPOSITORY_TYPE_v1 = REPOSITORY_TYPE + runtime.VersionSeparator + "v1" +) + +func init() { + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](REPOSITORY_TYPE)) + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](REPOSITORY_TYPE_v1, cpi.WithDescription(usage), cpi.WithFormatSpec(format))) +} + +var usage = ` +This repository type can be used to access credentials stored in a file +following the NPM config format (~/.npmrc). It take into account the +credentials helper section, also. If enabled, the described +credentials will be automatically assigned to appropriate consumer ids. +` + +var format = `The repository specification supports the following fields: +` + listformat.FormatListElements("", listformat.StringElementDescriptionList{ + "npmrcFile", "*string*: the file path to a NPM config file", + "propagateConsumerIdentity", "*bool*(optional): enable consumer id propagation", +}) + +// RepositorySpec describes a docker config based credential repository interface. +type RepositorySpec struct { + runtime.ObjectVersionedType `json:",inline"` + NpmrcFile string `json:"npmrcFile,omitempty"` +} + +// NewRepositorySpec creates a new memory RepositorySpec. +func NewRepositorySpec(path string, prop ...bool) *RepositorySpec { + p := false + for _, e := range prop { + p = p || e + } + if path == "" { + path = "~/.npmrc" + } + return &RepositorySpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(REPOSITORY_TYPE), + NpmrcFile: path, + } +} + +func (rs *RepositorySpec) GetType() string { + return REPOSITORY_TYPE +} + +func (rs *RepositorySpec) Repository(ctx cpi.Context, creds cpi.Credentials) (cpi.Repository, error) { + // r := ctx.GetAttributes().GetOrCreateAttribute(ATTR_REPOS, newRepositories) + // repos, ok := r.(*Repositories) + // if !ok { + // return nil, fmt.Errorf("failed to assert type %T to Repositories", r) + // } + // return repos.GetRepository(ctx, a.DockerConfigFile, a.DockerConfig, true) + return nil, fmt.Errorf("not implemented") +} + +// ReadNpmConfigFile reads "~/.npmrc" file line by line, parse it and return the result as a map. +func ReadNpmConfigFile(path string) (map[string]string, error) { + // Open the file + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + // Create a new scanner and read the file line by line + scanner := bufio.NewScanner(file) + config := make(map[string]string) + for scanner.Scan() { + line := scanner.Text() + line, authFound := strings.CutPrefix(line, "//") + if !authFound { + // e.g. 'global=false' + continue + } + // Split the line into key and value + parts := strings.SplitN(line, "/:_authToken=", 2) + if len(parts) == 2 { + // TODO: should we directly add here the prefix 'https://'? + config[parts[0]] = parts[1] + } + } + + // Check for errors + if err = scanner.Err(); err != nil { + return nil, err + } + + return config, nil +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go similarity index 63% rename from pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go rename to pkg/contexts/credentials/repositories/npm/config_test.go index e6b891198a..5ff0df0748 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -4,6 +4,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + npm2 "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" "github.com/open-component-model/ocm/pkg/registrations" . "github.com/open-component-model/ocm/pkg/testutils" @@ -21,4 +22,11 @@ var _ = Describe("Config deserialization Test Environment", func() { Expect(cfg).To(Equal(&npm.Config{"test"})) }) + It("read .npmrc", func() { + cfg, err := npm2.ReadNpmConfigFile("testdata/.npmrc") + Expect(err).To(BeNil()) + Expect(cfg).ToNot(BeNil()) + Expect(cfg).To(HaveKeyWithValue("npm.registry.acme.com/api/npm", "bearer_TOKEN")) + Expect(cfg).To(HaveKeyWithValue("registry.npm.org", "npm_TOKEN")) + }) }) diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go new file mode 100644 index 0000000000..689ab78f3e --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -0,0 +1,71 @@ +package npm + +import ( + dockercred "github.com/docker/cli/cli/config/credentials" + + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/utils" +) + +const PROVIDER = "ocm.software/credentialprovider/" + REPOSITORY_TYPE + +type ConsumerProvider struct { + cfg string +} + +var _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) + +func (p *ConsumerProvider) Unregister(id cpi.ProviderIdentity) { +} + +func (p *ConsumerProvider) Match(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { + return p.get(req, cur, m) +} + +func (p *ConsumerProvider) Get(req cpi.ConsumerIdentity) (cpi.CredentialsSource, bool) { + creds, _ := p.get(req, nil, cpi.CompleteMatch) + return creds, creds != nil +} + +func (p *ConsumerProvider) get(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { + cfg := p.cfg + all, err := ReadNpmConfigFile(cfg) + if err != nil { + panic(err) + return nil, nil + } + + var creds cpi.CredentialsSource + + for key, value := range all { + hostname, port, _ := utils.SplitLocator(key) + attrs := []string{identity.ID_HOSTNAME, hostname} + if port != "" { + attrs = append(attrs, identity.ID_PORT, port) + } + id := cpi.NewConsumerIdentity(identity.CONSUMER_TYPE, attrs...) + if m(req, cur, id) { + if IsEmptyAuthConfig(value) { + } else { + creds = newCredentials(value) + } + cur = id + } + } + for h, helper := range cfg.CredentialHelpers { + hostname := dockercred.ConvertToHostname(h) + if hostname == "index.docker.io" { + hostname = "docker.io" + } + id := cpi.ConsumerIdentity{ + cpi.ATTR_TYPE: identity.CONSUMER_TYPE, + identity.ID_HOSTNAME: hostname, + } + if m(req, cur, id) { + creds = NewCredentials(cfg, h, dockercred.NewNativeStore(cfg, helper)) + cur = id + } + } + return creds, cur +} diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go new file mode 100644 index 0000000000..f4abd67ca0 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -0,0 +1,142 @@ +package npm + +import ( + "bytes" + "fmt" + "os" + "strings" + "sync" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/config/types" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/utils" +) + +type Repository struct { + lock sync.RWMutex + ctx cpi.Context + propagate bool + path string + data []byte + config *configfile.ConfigFile +} + +func NewRepository(ctx cpi.Context, path string, data []byte, propagate bool) (*Repository, error) { + r := &Repository{ + ctx: ctx, + propagate: propagate, + path: path, + data: data, + } + if path != "" && len(data) > 0 { + return nil, fmt.Errorf("only config file or config data possible") + } + err := r.Read(true) + return r, err +} + +var _ cpi.Repository = &Repository{} + +func (r *Repository) ExistsCredentials(name string) (bool, error) { + err := r.Read(false) + if err != nil { + return false, err + } + r.lock.RLock() + defer r.lock.RUnlock() + + _, err = r.config.GetAuthConfig(name) + return err != nil, err +} + +func (r *Repository) LookupCredentials(name string) (cpi.Credentials, error) { + err := r.Read(false) + if err != nil { + return nil, err + } + r.lock.RLock() + defer r.lock.RUnlock() + + auth, err := r.config.GetAuthConfig(name) + if err != nil { + return nil, err + } + return newCredentials(auth), nil +} + +func (r *Repository) WriteCredentials(name string, creds cpi.Credentials) (cpi.Credentials, error) { + return nil, errors.ErrNotSupported("write", "credentials", REPOSITORY_TYPE) +} + +func (r *Repository) Read(force bool) error { + r.lock.Lock() + defer r.lock.Unlock() + if !force && r.config != nil { + return nil + } + var ( + data []byte + err error + id finalizer.ObjectIdentity + ) + if r.path != "" { + path, err := utils.ResolvePath(r.path) + if err != nil { + return errors.Wrapf(err, "cannot resolve path %q", r.path) + } + data, err = os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file '%s': %w", path, err) + } + id = cpi.ProviderIdentity(PROVIDER + "/" + path) + } else if len(r.data) > 0 { + data = r.data + id = finalizer.NewObjectIdentity(PROVIDER) + } + + cfg, err := config.LoadFromReader(bytes.NewBuffer(data)) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + if r.propagate { + r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{cfg}) + } + r.config = cfg + return nil +} + +func newCredentials(auth types.AuthConfig) cpi.Credentials { + props := common.Properties{ + cpi.ATTR_USERNAME: norm(auth.Username), + cpi.ATTR_PASSWORD: norm(auth.Password), + } + props.SetNonEmptyValue("auth", auth.Auth) + props.SetNonEmptyValue(cpi.ATTR_SERVER_ADDRESS, norm(auth.ServerAddress)) + props.SetNonEmptyValue(cpi.ATTR_IDENTITY_TOKEN, norm(auth.IdentityToken)) + props.SetNonEmptyValue(cpi.ATTR_REGISTRY_TOKEN, norm(auth.RegistryToken)) + return cpi.NewCredentials(props) +} + +func norm(s string) string { + for strings.HasSuffix(s, "\n") { + s = s[:len(s)-1] + } + return s +} + +// IsEmptyAuthConfig validates if the resulting auth config contains credentials. +func IsEmptyAuthConfig(auth types.AuthConfig) bool { + if len(auth.Auth) != 0 { + return false + } + if len(auth.Username) != 0 { + return false + } + return true +} diff --git a/pkg/contexts/credentials/repositories/npm/testdata/.npmrc b/pkg/contexts/credentials/repositories/npm/testdata/.npmrc new file mode 100644 index 0000000000..00ea1baa8f --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/testdata/.npmrc @@ -0,0 +1,3 @@ +global=false +//npm.registry.acme.com/api/npm/:_authToken=bearer_TOKEN +//registry.npmjs.org/:_authToken=npm_TOKEN diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 2c244cdd1f..fcba4afc1d 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -62,7 +62,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c log.Debug("identified") // use user+pass+mail from credentials to login and retrieve bearer token - cred := GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) + cred := npmCredentials.GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) username := cred[ATTR_USERNAME] password := cred[ATTR_PASSWORD] email := cred[ATTR_EMAIL] diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go index e8f1271879..9ef991c683 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go @@ -7,7 +7,7 @@ import ( const ( // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npmjs.com" + CONSUMER_TYPE = "Registry.npm.com" BLOB_HANDLER_NAME = "ocm/npmPackage" // ATTR_USERNAME is the username attribute. Required for login at any npm registry. From b86eab1087e92ff6418817526219747828e35d78 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 14 Feb 2024 13:31:54 +0100 Subject: [PATCH 29/51] revert --- pkg/contexts/credentials/builtin/init.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/credentials/builtin/init.go b/pkg/contexts/credentials/builtin/init.go index 578f38cc04..7e0e1032eb 100644 --- a/pkg/contexts/credentials/builtin/init.go +++ b/pkg/contexts/credentials/builtin/init.go @@ -1,8 +1,7 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + package builtin -import ( - _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/github" - _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/helm/identity" - _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm" - _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" -) +import _ "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/github" From a104900976c066081e5685913dea52a700699759 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 14 Feb 2024 13:34:07 +0100 Subject: [PATCH 30/51] identity --- .../credentials/builtin/npm/identity.go | 49 --------------- .../builtin/npm/identity/identity.go | 63 +++++++++++++++++++ 2 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 pkg/contexts/credentials/builtin/npm/identity.go create mode 100644 pkg/contexts/credentials/builtin/npm/identity/identity.go diff --git a/pkg/contexts/credentials/builtin/npm/identity.go b/pkg/contexts/credentials/builtin/npm/identity.go deleted file mode 100644 index 62e29749c9..0000000000 --- a/pkg/contexts/credentials/builtin/npm/identity.go +++ /dev/null @@ -1,49 +0,0 @@ -package npm - -import ( - "path" - - . "net/url" - - "github.com/open-component-model/ocm/pkg/common" - "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" - "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" - "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" - "github.com/open-component-model/ocm/pkg/listformat" -) - -func init() { - attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ - npm.ATTR_USERNAME, "the basic auth user name", - npm.ATTR_PASSWORD, "the basic auth password", - npm.ATTR_EMAIL, "NPM registry, require an email address", - }) - - cpi.RegisterStandardIdentity(npm.CONSUMER_TYPE, hostpath.IdentityMatcher(npm.CONSUMER_TYPE), `NPM repository - -It matches the `+npm.CONSUMER_TYPE+` consumer type and additionally acts like -the `+hostpath.IDENTITY_TYPE+` type.`, - attrs) -} - -func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { - url, err := Parse(rawURL) - if err != nil { - return nil - } - - url.Path = path.Join(url.Path, pkgName) - return hostpath.GetConsumerIdentity(npm.CONSUMER_TYPE, url.String()) -} - -func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { - id := GetConsumerId(repoUrl, pkgName) - if id == nil { - return nil - } - credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) - if credentials == nil || err != nil { - return nil - } - return credentials.Properties() -} diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go new file mode 100644 index 0000000000..da0d7dd16a --- /dev/null +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -0,0 +1,63 @@ +package identity + +import ( + "path" + + . "net/url" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" + "github.com/open-component-model/ocm/pkg/listformat" +) + +const ( + // CONSUMER_TYPE is the npm repository type. + CONSUMER_TYPE = "Registry.npm.com" + + // ATTR_USERNAME is the username attribute. Required for login at any npm registry. + ATTR_USERNAME = cpi.ATTR_USERNAME + // ATTR_PASSWORD is the password attribute. Required for login at any npm registry. + ATTR_PASSWORD = cpi.ATTR_PASSWORD + // ATTR_EMAIL is the email attribute. Required for login at any npm registry. + ATTR_EMAIL = cpi.ATTR_EMAIL + // ATTR_TOKEN is the token attribute. May exist after login at any npm registry. + ATTR_TOKEN = cpi.ATTR_TOKEN +) + +func init() { + attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ + ATTR_USERNAME, "the basic auth user name", + ATTR_PASSWORD, "the basic auth password", + ATTR_EMAIL, "NPM registry, require an email address", + ATTR_TOKEN, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!", + }) + + cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository + +It matches the `+CONSUMER_TYPE+` consumer type and additionally acts like +the `+hostpath.IDENTITY_TYPE+` type.`, + attrs) +} + +func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity { + url, err := Parse(rawURL) + if err != nil { + return nil + } + + url.Path = path.Join(url.Path, pkgName) + return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String()) +} + +func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties { + id := GetConsumerId(repoUrl, pkgName) + if id == nil { + return nil + } + credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id) + if credentials == nil || err != nil { + return nil + } + return credentials.Properties() +} From 61b4e90019bfd14c0540efbfde75c3d52c59a756 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 14 Feb 2024 14:51:45 +0100 Subject: [PATCH 31/51] gone --- .../blobhandler/handlers/generic/npm/const.go | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go deleted file mode 100644 index 9ef991c683..0000000000 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/const.go +++ /dev/null @@ -1,22 +0,0 @@ -package npm - -import ( - "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" - "github.com/open-component-model/ocm/pkg/logging" -) - -const ( - // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npm.com" - BLOB_HANDLER_NAME = "ocm/npmPackage" - - // ATTR_USERNAME is the username attribute. Required for login at any npm registry. - ATTR_USERNAME = cpi.ATTR_USERNAME - // ATTR_PASSWORD is the password attribute. Required for login at any npm registry. - ATTR_PASSWORD = cpi.ATTR_PASSWORD - // ATTR_EMAIL is the email attribute. Required for login at any npm registry. - ATTR_EMAIL = cpi.ATTR_EMAIL -) - -// Logging Realm. -var REALM = logging.DefineSubRealm("NPM registry", "NPM") From 35481c28ee50342da338402a6c3332960790217e Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 14 Feb 2024 16:17:00 +0100 Subject: [PATCH 32/51] npm credential repository --- .../credentials/repositories/npm/cache.go | 33 +++++ .../credentials/repositories/npm/config.go | 36 +++--- .../repositories/npm/config_test.go | 15 ++- .../credentials/repositories/npm/provider.go | 52 +++----- .../repositories/npm/repository.go | 119 +++++------------- .../repositories/npm/suite_test.go | 13 ++ .../handlers/generic/npm/blobhandler.go | 12 +- 7 files changed, 129 insertions(+), 151 deletions(-) create mode 100644 pkg/contexts/credentials/repositories/npm/cache.go create mode 100644 pkg/contexts/credentials/repositories/npm/suite_test.go diff --git a/pkg/contexts/credentials/repositories/npm/cache.go b/pkg/contexts/credentials/repositories/npm/cache.go new file mode 100644 index 0000000000..e1d3d06363 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/cache.go @@ -0,0 +1,33 @@ +package npm + +import ( + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/datacontext" +) + +type Cache struct { + repos map[string]*Repository +} + +func createCache(_ datacontext.Context) interface{} { + return &Cache{ + repos: map[string]*Repository{}, + } +} + +func (r *Cache) GetRepository(ctx cpi.Context, name string) (*Repository, error) { + var ( + err error = nil + repo *Repository + ) + if name != "" { + repo = r.repos[name] + } + if repo == nil { + repo, err = NewRepository(ctx, name) + if err == nil { + r.repos[name] = repo + } + } + return repo, err +} diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index d7c4d71bcb..5d83b8d4de 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -24,18 +24,20 @@ func init() { var usage = ` This repository type can be used to access credentials stored in a file -following the NPM config format (~/.npmrc). It take into account the +following the NPM npmrc format (~/.npmrc). It take into account the credentials helper section, also. If enabled, the described credentials will be automatically assigned to appropriate consumer ids. ` var format = `The repository specification supports the following fields: ` + listformat.FormatListElements("", listformat.StringElementDescriptionList{ - "npmrcFile", "*string*: the file path to a NPM config file", + "npmrcFile", "*string*: the file path to a NPM npmrc file", "propagateConsumerIdentity", "*bool*(optional): enable consumer id propagation", }) -// RepositorySpec describes a docker config based credential repository interface. +type npmConfig map[string]string + +// RepositorySpec describes a docker npmrc based credential repository interface. type RepositorySpec struct { runtime.ObjectVersionedType `json:",inline"` NpmrcFile string `json:"npmrcFile,omitempty"` @@ -60,18 +62,17 @@ func (rs *RepositorySpec) GetType() string { return REPOSITORY_TYPE } -func (rs *RepositorySpec) Repository(ctx cpi.Context, creds cpi.Credentials) (cpi.Repository, error) { - // r := ctx.GetAttributes().GetOrCreateAttribute(ATTR_REPOS, newRepositories) - // repos, ok := r.(*Repositories) - // if !ok { - // return nil, fmt.Errorf("failed to assert type %T to Repositories", r) - // } - // return repos.GetRepository(ctx, a.DockerConfigFile, a.DockerConfig, true) - return nil, fmt.Errorf("not implemented") +func (rs *RepositorySpec) Repository(ctx cpi.Context, _ cpi.Credentials) (cpi.Repository, error) { + r := ctx.GetAttributes().GetOrCreateAttribute(".npmrc", createCache) + cache, ok := r.(*Cache) + if !ok { + return nil, fmt.Errorf("failed to assert type %T to Cache", r) + } + return cache.GetRepository(ctx, rs.NpmrcFile) } -// ReadNpmConfigFile reads "~/.npmrc" file line by line, parse it and return the result as a map. -func ReadNpmConfigFile(path string) (map[string]string, error) { +// readNpmConfigFile reads "~/.npmrc" file line by line, parse it and return the result as a npmConfig. +func readNpmConfigFile(path string) (npmConfig, error) { // Open the file file, err := os.Open(path) if err != nil { @@ -81,7 +82,7 @@ func ReadNpmConfigFile(path string) (map[string]string, error) { // Create a new scanner and read the file line by line scanner := bufio.NewScanner(file) - config := make(map[string]string) + cfg := make(map[string]string) for scanner.Scan() { line := scanner.Text() line, authFound := strings.CutPrefix(line, "//") @@ -90,10 +91,9 @@ func ReadNpmConfigFile(path string) (map[string]string, error) { continue } // Split the line into key and value - parts := strings.SplitN(line, "/:_authToken=", 2) + parts := strings.SplitN(line, ":_authToken=", 2) if len(parts) == 2 { - // TODO: should we directly add here the prefix 'https://'? - config[parts[0]] = parts[1] + cfg["https://"+parts[0]] = parts[1] } } @@ -102,5 +102,5 @@ func ReadNpmConfigFile(path string) (map[string]string, error) { return nil, err } - return config, nil + return cfg, nil } diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index 5ff0df0748..f73ba174e1 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -1,10 +1,9 @@ -package npm_test +package npm import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - npm2 "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" "github.com/open-component-model/ocm/pkg/registrations" . "github.com/open-component-model/ocm/pkg/testutils" @@ -14,19 +13,19 @@ var _ = Describe("Config deserialization Test Environment", func() { It("deserializes string", func() { cfg := Must(registrations.DecodeConfig[npm.Config]("test")) - Expect(cfg).To(Equal(&npm.Config{"test"})) + Expect(cfg).To(Equal(&npm.Config{Url: "test"})) }) It("deserializes struct", func() { cfg := Must(registrations.DecodeConfig[npm.Config](`{"Url":"test"}`)) - Expect(cfg).To(Equal(&npm.Config{"test"})) + Expect(cfg).To(Equal(&npm.Config{Url: "test"})) }) - It("read .npmrc", func() { - cfg, err := npm2.ReadNpmConfigFile("testdata/.npmrc") + FIt("read .npmrc", func() { + cfg, err := readNpmConfigFile("testdata/.npmrc") Expect(err).To(BeNil()) Expect(cfg).ToNot(BeNil()) - Expect(cfg).To(HaveKeyWithValue("npm.registry.acme.com/api/npm", "bearer_TOKEN")) - Expect(cfg).To(HaveKeyWithValue("registry.npm.org", "npm_TOKEN")) + Expect(cfg["https://npm.registry.acme.com/api/npm/"]).To(Equal("bearer_TOKEN")) + Expect(cfg["https://registry.npm.org/"]).To(Equal("npm_TOKEN")) }) }) diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index 689ab78f3e..0a5393085a 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -1,22 +1,19 @@ package npm import ( - dockercred "github.com/docker/cli/cli/config/credentials" + "net/url" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" - "github.com/open-component-model/ocm/pkg/utils" ) -const PROVIDER = "ocm.software/credentialprovider/" + REPOSITORY_TYPE - type ConsumerProvider struct { - cfg string + npmrcPath string } var _ cpi.ConsumerProvider = (*ConsumerProvider)(nil) -func (p *ConsumerProvider) Unregister(id cpi.ProviderIdentity) { +func (p *ConsumerProvider) Unregister(_ cpi.ProviderIdentity) { } func (p *ConsumerProvider) Match(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { @@ -28,9 +25,8 @@ func (p *ConsumerProvider) Get(req cpi.ConsumerIdentity) (cpi.CredentialsSource, return creds, creds != nil } -func (p *ConsumerProvider) get(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { - cfg := p.cfg - all, err := ReadNpmConfigFile(cfg) +func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { + all, err := readNpmConfigFile(p.npmrcPath) if err != nil { panic(err) return nil, nil @@ -39,33 +35,21 @@ func (p *ConsumerProvider) get(req cpi.ConsumerIdentity, cur cpi.ConsumerIdentit var creds cpi.CredentialsSource for key, value := range all { - hostname, port, _ := utils.SplitLocator(key) - attrs := []string{identity.ID_HOSTNAME, hostname} - if port != "" { - attrs = append(attrs, identity.ID_PORT, port) - } - id := cpi.NewConsumerIdentity(identity.CONSUMER_TYPE, attrs...) - if m(req, cur, id) { - if IsEmptyAuthConfig(value) { - } else { - creds = newCredentials(value) - } - cur = id - } - } - for h, helper := range cfg.CredentialHelpers { - hostname := dockercred.ConvertToHostname(h) - if hostname == "index.docker.io" { - hostname = "docker.io" + u, e := url.Parse(key) + if e != nil { + return nil, nil } - id := cpi.ConsumerIdentity{ - cpi.ATTR_TYPE: identity.CONSUMER_TYPE, - identity.ID_HOSTNAME: hostname, + + attrs := []string{identity.ID_HOSTNAME, u.Hostname()} + if u.Port() != "" { + attrs = append(attrs, identity.ID_PORT, u.Port()) } - if m(req, cur, id) { - creds = NewCredentials(cfg, h, dockercred.NewNativeStore(cfg, helper)) - cur = id + id := cpi.NewConsumerIdentity(identity.CONSUMER_TYPE, attrs...) + if m(requested, currentFound, id) { + creds = newCredentials(value) + currentFound = id } } - return creds, cur + + return creds, currentFound } diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index f4abd67ca0..73f08bccd9 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -1,41 +1,28 @@ package npm import ( - "bytes" "fmt" - "os" - "strings" - "sync" - - "github.com/docker/cli/cli/config" - "github.com/docker/cli/cli/config/configfile" - "github.com/docker/cli/cli/config/types" "github.com/open-component-model/ocm/pkg/common" + npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/finalizer" "github.com/open-component-model/ocm/pkg/utils" ) +const PROVIDER = "ocm.software/credentialprovider/" + REPOSITORY_TYPE + type Repository struct { - lock sync.RWMutex - ctx cpi.Context - propagate bool - path string - data []byte - config *configfile.ConfigFile + ctx cpi.Context + path string + npmrc npmConfig } -func NewRepository(ctx cpi.Context, path string, data []byte, propagate bool) (*Repository, error) { +func NewRepository(ctx cpi.Context, path string) (*Repository, error) { r := &Repository{ - ctx: ctx, - propagate: propagate, - path: path, - data: data, - } - if path != "" && len(data) > 0 { - return nil, fmt.Errorf("only config file or config data possible") + ctx: ctx, + path: path, } err := r.Read(true) return r, err @@ -48,95 +35,51 @@ func (r *Repository) ExistsCredentials(name string) (bool, error) { if err != nil { return false, err } - r.lock.RLock() - defer r.lock.RUnlock() - - _, err = r.config.GetAuthConfig(name) - return err != nil, err + return r.npmrc[name] != "", nil } func (r *Repository) LookupCredentials(name string) (cpi.Credentials, error) { - err := r.Read(false) + exists, err := r.ExistsCredentials(name) if err != nil { return nil, err } - r.lock.RLock() - defer r.lock.RUnlock() - - auth, err := r.config.GetAuthConfig(name) - if err != nil { - return nil, err + if !exists { + return nil, errors.ErrNotFound("credentials", name, REPOSITORY_TYPE) } - return newCredentials(auth), nil + return newCredentials(r.npmrc[name]), nil } -func (r *Repository) WriteCredentials(name string, creds cpi.Credentials) (cpi.Credentials, error) { +func (r *Repository) WriteCredentials(_ string, _ cpi.Credentials) (cpi.Credentials, error) { return nil, errors.ErrNotSupported("write", "credentials", REPOSITORY_TYPE) } func (r *Repository) Read(force bool) error { - r.lock.Lock() - defer r.lock.Unlock() - if !force && r.config != nil { + if !force && r.npmrc != nil { return nil } - var ( - data []byte - err error - id finalizer.ObjectIdentity - ) - if r.path != "" { - path, err := utils.ResolvePath(r.path) - if err != nil { - return errors.Wrapf(err, "cannot resolve path %q", r.path) - } - data, err = os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read file '%s': %w", path, err) - } - id = cpi.ProviderIdentity(PROVIDER + "/" + path) - } else if len(r.data) > 0 { - data = r.data - id = finalizer.NewObjectIdentity(PROVIDER) - } - cfg, err := config.LoadFromReader(bytes.NewBuffer(data)) + if r.path == "" { + return fmt.Errorf("npmrc path not provided") + } + path, err := utils.ResolvePath(r.path) if err != nil { - return fmt.Errorf("failed to load config: %w", err) + return errors.Wrapf(err, "cannot resolve path %q", r.path) } - if r.propagate { - r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{cfg}) + cfg, err := readNpmConfigFile(path) + if err != nil { + return fmt.Errorf("failed to load npmrc: %w", err) } - r.config = cfg + var id finalizer.ObjectIdentity + id = cpi.ProviderIdentity(PROVIDER + "/" + path) + + r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{path}) + r.npmrc = cfg return nil } -func newCredentials(auth types.AuthConfig) cpi.Credentials { +func newCredentials(token string) cpi.Credentials { props := common.Properties{ - cpi.ATTR_USERNAME: norm(auth.Username), - cpi.ATTR_PASSWORD: norm(auth.Password), + npmCredentials.ATTR_TOKEN: token, } - props.SetNonEmptyValue("auth", auth.Auth) - props.SetNonEmptyValue(cpi.ATTR_SERVER_ADDRESS, norm(auth.ServerAddress)) - props.SetNonEmptyValue(cpi.ATTR_IDENTITY_TOKEN, norm(auth.IdentityToken)) - props.SetNonEmptyValue(cpi.ATTR_REGISTRY_TOKEN, norm(auth.RegistryToken)) return cpi.NewCredentials(props) } - -func norm(s string) string { - for strings.HasSuffix(s, "\n") { - s = s[:len(s)-1] - } - return s -} - -// IsEmptyAuthConfig validates if the resulting auth config contains credentials. -func IsEmptyAuthConfig(auth types.AuthConfig) bool { - if len(auth.Auth) != 0 { - return false - } - if len(auth.Username) != 0 { - return false - } - return true -} diff --git a/pkg/contexts/credentials/repositories/npm/suite_test.go b/pkg/contexts/credentials/repositories/npm/suite_test.go new file mode 100644 index 0000000000..e4947738d0 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/suite_test.go @@ -0,0 +1,13 @@ +package npm_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "NPM Repository tests") +} diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index fcba4afc1d..cf422277d1 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -9,12 +9,18 @@ import ( "net/http" "net/url" + npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm" "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" "github.com/open-component-model/ocm/pkg/logging" "github.com/open-component-model/ocm/pkg/mime" ) +const BLOB_HANDLER_NAME = "ocm/npmPackage" + +// Logging Realm. +var REALM = logging.DefineSubRealm("NPM registry", "NPM") + type artifactHandler struct { spec *Config } @@ -63,9 +69,9 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c // use user+pass+mail from credentials to login and retrieve bearer token cred := npmCredentials.GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) - username := cred[ATTR_USERNAME] - password := cred[ATTR_PASSWORD] - email := cred[ATTR_EMAIL] + username := cred[npmCredentials.ATTR_USERNAME] + password := cred[npmCredentials.ATTR_PASSWORD] + email := cred[npmCredentials.ATTR_EMAIL] if username == "" || password == "" || email == "" { return nil, fmt.Errorf("username, password or email missing") } From d1ec79e38a8c5634d112ccd98f52b26ecd9f129d Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Wed, 14 Feb 2024 16:40:20 +0100 Subject: [PATCH 33/51] fix lint and test issues --- pkg/contexts/credentials/repositories/npm/config_test.go | 5 +++-- pkg/contexts/credentials/repositories/npm/provider.go | 5 ++++- pkg/contexts/credentials/repositories/npm/repository.go | 4 +--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index f73ba174e1..6712a3e653 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -21,11 +21,12 @@ var _ = Describe("Config deserialization Test Environment", func() { Expect(cfg).To(Equal(&npm.Config{Url: "test"})) }) - FIt("read .npmrc", func() { + It("read .npmrc", func() { cfg, err := readNpmConfigFile("testdata/.npmrc") Expect(err).To(BeNil()) Expect(cfg).ToNot(BeNil()) + Expect(cfg).ToNot(BeEmpty()) + Expect(cfg["https://registry.npmjs.org/"]).To(Equal("npm_TOKEN")) Expect(cfg["https://npm.registry.acme.com/api/npm/"]).To(Equal("bearer_TOKEN")) - Expect(cfg["https://registry.npm.org/"]).To(Equal("npm_TOKEN")) }) }) diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index 0a5393085a..01a986828f 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -5,6 +5,8 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" + "github.com/open-component-model/ocm/pkg/logging" ) type ConsumerProvider struct { @@ -28,7 +30,8 @@ func (p *ConsumerProvider) Get(req cpi.ConsumerIdentity) (cpi.CredentialsSource, func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { all, err := readNpmConfigFile(p.npmrcPath) if err != nil { - panic(err) + log := logging.Context().Logger(npm.REALM) + log.LogError(err, "Failed to read npmrc file", "path", p.npmrcPath) return nil, nil } diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index 73f08bccd9..e42970db38 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -7,7 +7,6 @@ import ( npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/finalizer" "github.com/open-component-model/ocm/pkg/utils" ) @@ -69,8 +68,7 @@ func (r *Repository) Read(force bool) error { if err != nil { return fmt.Errorf("failed to load npmrc: %w", err) } - var id finalizer.ObjectIdentity - id = cpi.ProviderIdentity(PROVIDER + "/" + path) + id := cpi.ProviderIdentity(PROVIDER + "/" + path) r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{path}) r.npmrc = cfg From 4626c2f635ad23a3b1c933b13f91be75518e9b29 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 08:58:24 +0100 Subject: [PATCH 34/51] obsolete... useless leftover from copy&paste --- pkg/contexts/credentials/repositories/npm/config.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index 5d83b8d4de..bb31ffbbfe 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -45,10 +45,6 @@ type RepositorySpec struct { // NewRepositorySpec creates a new memory RepositorySpec. func NewRepositorySpec(path string, prop ...bool) *RepositorySpec { - p := false - for _, e := range prop { - p = p || e - } if path == "" { path = "~/.npmrc" } From 3982246c3c95e30a0d8d8e647b3e8821dc7a601a Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 09:00:55 +0100 Subject: [PATCH 35/51] obsolete... useless leftover from copy&paste --- pkg/contexts/credentials/repositories/npm/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index bb31ffbbfe..763d60eb9c 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -44,7 +44,7 @@ type RepositorySpec struct { } // NewRepositorySpec creates a new memory RepositorySpec. -func NewRepositorySpec(path string, prop ...bool) *RepositorySpec { +func NewRepositorySpec(path string) *RepositorySpec { if path == "" { path = "~/.npmrc" } From a804a5f0a5106d48ca5ae5984776fc7a3faa0680 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 09:02:47 +0100 Subject: [PATCH 36/51] check for existing token and reuse it, instead of login --- .../handlers/generic/npm/blobhandler.go | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index cf422277d1..21349e6bff 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -67,19 +67,34 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c log = log.WithValues("package", pkg.Name, "version", pkg.Version) log.Debug("identified") - // use user+pass+mail from credentials to login and retrieve bearer token + // get credentials cred := npmCredentials.GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) - username := cred[npmCredentials.ATTR_USERNAME] - password := cred[npmCredentials.ATTR_PASSWORD] - email := cred[npmCredentials.ATTR_EMAIL] - if username == "" || password == "" || email == "" { - return nil, fmt.Errorf("username, password or email missing") - } - log = log.WithValues("user", username, "repo", b.spec.Url) - log.Debug("login") - token, err := login(b.spec.Url, username, password, email) - if err != nil { - return nil, err + if cred == nil { + return nil, fmt.Errorf("No credentials found for %s. Couldn't upload '%s'.", b.spec.Url, pkg.Name) + } + log.Debug("found credentials") + + // check if token exists, if not login and retrieve token + token := cred[npmCredentials.ATTR_TOKEN] + if token == "" { + // use user+pass+mail from credentials to login and retrieve bearer token + username := cred[npmCredentials.ATTR_USERNAME] + password := cred[npmCredentials.ATTR_PASSWORD] + email := cred[npmCredentials.ATTR_EMAIL] + if username == "" || password == "" || email == "" { + return nil, fmt.Errorf("No credentials for %s are invalid. Username, password or email missing! Couldn't upload '%s'.", b.spec.Url, pkg.Name) + } + log = log.WithValues("user", username, "repo", b.spec.Url) + log.Debug("login") + + // TODO: check different kinds of .npmrc content + token, err = login(b.spec.Url, username, password, email) + if err != nil { + return nil, err + } + cred[npmCredentials.ATTR_TOKEN] = token // Frage: @mandelsoft funktioniert das? + } else { + log.Debug("token found, skipping login") } // check if package exists From 4c8e2f9caf0675df477f0695b8422472763426ec Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 11:42:03 +0100 Subject: [PATCH 37/51] moved logging REALM --- pkg/contexts/credentials/builtin/npm/identity/identity.go | 6 +++++- .../ocm/blobhandler/handlers/generic/npm/blobhandler.go | 8 ++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/contexts/credentials/builtin/npm/identity/identity.go b/pkg/contexts/credentials/builtin/npm/identity/identity.go index da0d7dd16a..32f9a16e8c 100644 --- a/pkg/contexts/credentials/builtin/npm/identity/identity.go +++ b/pkg/contexts/credentials/builtin/npm/identity/identity.go @@ -9,11 +9,12 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath" "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/logging" ) const ( // CONSUMER_TYPE is the npm repository type. - CONSUMER_TYPE = "Registry.npm.com" + CONSUMER_TYPE = "Registry.npmjs.com" // ATTR_USERNAME is the username attribute. Required for login at any npm registry. ATTR_USERNAME = cpi.ATTR_USERNAME @@ -25,6 +26,9 @@ const ( ATTR_TOKEN = cpi.ATTR_TOKEN ) +// Logging Realm. +var REALM = logging.DefineSubRealm("NPM registry", "NPM") + func init() { attrs := listformat.FormatListElements("", listformat.StringElementDescriptionList{ ATTR_USERNAME, "the basic auth user name", diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go index 21349e6bff..5e69bab218 100644 --- a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/blobhandler.go @@ -18,9 +18,6 @@ import ( const BLOB_HANDLER_NAME = "ocm/npmPackage" -// Logging Realm. -var REALM = logging.DefineSubRealm("NPM registry", "NPM") - type artifactHandler struct { spec *Config } @@ -55,7 +52,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c } // read package.json from tarball to get name, version, etc. - log := logging.Context().Logger(REALM) + log := logging.Context().Logger(npmCredentials.REALM) log.Debug("reading package.json from tarball") var pkg *Package pkg, err = prepare(data) @@ -67,7 +64,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c log = log.WithValues("package", pkg.Name, "version", pkg.Version) log.Debug("identified") - // get credentials + // get credentials and TODO cache it cred := npmCredentials.GetCredentials(ctx.GetContext(), b.spec.Url, pkg.Name) if cred == nil { return nil, fmt.Errorf("No credentials found for %s. Couldn't upload '%s'.", b.spec.Url, pkg.Name) @@ -92,7 +89,6 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, _ string, _ string, _ c if err != nil { return nil, err } - cred[npmCredentials.ATTR_TOKEN] = token // Frage: @mandelsoft funktioniert das? } else { log.Debug("token found, skipping login") } From 9eacd28214661c1510c6e18387944e997ae92e98 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 11:47:42 +0100 Subject: [PATCH 38/51] npm.CONSUMER_TYPE --- pkg/contexts/credentials/repositories/npm/provider.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index 01a986828f..c0beecdf90 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -3,9 +3,9 @@ package npm import ( "net/url" + npm "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" - "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" "github.com/open-component-model/ocm/pkg/logging" ) @@ -47,7 +47,10 @@ func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi. if u.Port() != "" { attrs = append(attrs, identity.ID_PORT, u.Port()) } - id := cpi.NewConsumerIdentity(identity.CONSUMER_TYPE, attrs...) + if u.Path != "" { + attrs = append(attrs, identity.ID_PATHPREFIX, u.Path) + } + id := cpi.NewConsumerIdentity(npm.CONSUMER_TYPE, attrs...) if m(requested, currentFound, id) { creds = newCredentials(value) currentFound = id From 23ecab9fbec03b45fd91cdbfefac94fa1a36eff9 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 11:48:31 +0100 Subject: [PATCH 39/51] init --- pkg/contexts/credentials/repositories/init.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/credentials/repositories/init.go b/pkg/contexts/credentials/repositories/init.go index 84d8ad4bc8..fbdbb91bc1 100644 --- a/pkg/contexts/credentials/repositories/init.go +++ b/pkg/contexts/credentials/repositories/init.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package repositories import ( @@ -11,5 +7,6 @@ import ( _ "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/gardenerconfig" _ "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/memory" _ "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/memory/config" + _ "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" _ "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/vault" ) From 0702046605f513f6f1b0e7f03c058f6527a7ed29 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Fri, 16 Feb 2024 12:20:46 +0100 Subject: [PATCH 40/51] fix import cycle in test --- .../repositories/npm/config_test.go | 13 ----------- .../handlers/generic/npm/registration_test.go | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration_test.go diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index 6712a3e653..aad85dc5b8 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -4,23 +4,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" - "github.com/open-component-model/ocm/pkg/registrations" . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("Config deserialization Test Environment", func() { - - It("deserializes string", func() { - cfg := Must(registrations.DecodeConfig[npm.Config]("test")) - Expect(cfg).To(Equal(&npm.Config{Url: "test"})) - }) - - It("deserializes struct", func() { - cfg := Must(registrations.DecodeConfig[npm.Config](`{"Url":"test"}`)) - Expect(cfg).To(Equal(&npm.Config{Url: "test"})) - }) - It("read .npmrc", func() { cfg, err := readNpmConfigFile("testdata/.npmrc") Expect(err).To(BeNil()) diff --git a/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration_test.go b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration_test.go new file mode 100644 index 0000000000..d97f412c3a --- /dev/null +++ b/pkg/contexts/ocm/blobhandler/handlers/generic/npm/registration_test.go @@ -0,0 +1,23 @@ +package npm_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/blobhandler/handlers/generic/npm" + "github.com/open-component-model/ocm/pkg/registrations" + . "github.com/open-component-model/ocm/pkg/testutils" +) + +var _ = Describe("Config deserialization Test Environment", func() { + + It("deserializes string", func() { + cfg := Must(registrations.DecodeConfig[npm.Config]("test")) + Expect(cfg).To(Equal(&npm.Config{Url: "test"})) + }) + + It("deserializes struct", func() { + cfg := Must(registrations.DecodeConfig[npm.Config](`{"Url":"test"}`)) + Expect(cfg).To(Equal(&npm.Config{Url: "test"})) + }) +}) From b6058b0d54f2d54dd9ad6dc5754c2be90eef5cb0 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 16 Feb 2024 22:33:21 +0100 Subject: [PATCH 41/51] fix cred names + harmonize propagation handling --- .../repositories/dockerconfig/type.go | 20 +++--- .../repositories/gardenerconfig/type.go | 10 +-- .../credentials/repositories/npm/a_usage.go | 22 +++++++ .../credentials/repositories/npm/cache.go | 4 +- .../credentials/repositories/npm/config.go | 65 ++----------------- .../repositories/npm/config_test.go | 39 ++++++++--- .../repositories/npm/repository.go | 28 +++++--- .../credentials/repositories/npm/type.go | 55 ++++++++++++++++ 8 files changed, 150 insertions(+), 93 deletions(-) create mode 100644 pkg/contexts/credentials/repositories/npm/a_usage.go create mode 100644 pkg/contexts/credentials/repositories/npm/type.go diff --git a/pkg/contexts/credentials/repositories/dockerconfig/type.go b/pkg/contexts/credentials/repositories/dockerconfig/type.go index bba00f371b..0d01494890 100644 --- a/pkg/contexts/credentials/repositories/dockerconfig/type.go +++ b/pkg/contexts/credentials/repositories/dockerconfig/type.go @@ -9,7 +9,9 @@ import ( "fmt" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/generics" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) const ( @@ -27,19 +29,19 @@ type RepositorySpec struct { runtime.ObjectVersionedType `json:",inline"` DockerConfigFile string `json:"dockerConfigFile,omitempty"` DockerConfig json.RawMessage `json:"dockerConfig,omitempty"` - PropgateConsumerIdentity bool `json:"propagateConsumerIdentity,omitempty"` + PropgateConsumerIdentity *bool `json:"propagateConsumerIdentity,omitempty"` } func (s RepositorySpec) WithConsumerPropagation(propagate bool) *RepositorySpec { - s.PropgateConsumerIdentity = propagate + s.PropgateConsumerIdentity = &propagate return &s } // NewRepositorySpec creates a new memory RepositorySpec. func NewRepositorySpec(path string, prop ...bool) *RepositorySpec { - p := false - for _, e := range prop { - p = p || e + var p *bool + if len(prop) > 0 { + p = generics.Pointer(utils.Optional(prop...)) } if path == "" { path = "~/.docker/config.json" @@ -52,9 +54,9 @@ func NewRepositorySpec(path string, prop ...bool) *RepositorySpec { } func NewRepositorySpecForConfig(data []byte, prop ...bool) *RepositorySpec { - p := false - for _, e := range prop { - p = p || e + var p *bool + if len(prop) > 0 { + p = generics.Pointer(utils.Optional(prop...)) } return &RepositorySpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), @@ -73,5 +75,5 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds cpi.Credentials) (cpi if !ok { return nil, fmt.Errorf("failed to assert type %T to Repositories", r) } - return repos.GetRepository(ctx, a.DockerConfigFile, a.DockerConfig, a.PropgateConsumerIdentity) + return repos.GetRepository(ctx, a.DockerConfigFile, a.DockerConfig, utils.AsBool(a.PropgateConsumerIdentity, true)) } diff --git a/pkg/contexts/credentials/repositories/gardenerconfig/type.go b/pkg/contexts/credentials/repositories/gardenerconfig/type.go index e57fd9f4d0..b7421da486 100644 --- a/pkg/contexts/credentials/repositories/gardenerconfig/type.go +++ b/pkg/contexts/credentials/repositories/gardenerconfig/type.go @@ -11,7 +11,9 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/credentials/internal" gardenercfgcpi "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/gardenerconfig/cpi" "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/gardenerconfig/identity" + "github.com/open-component-model/ocm/pkg/generics" "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" ) const ( @@ -30,19 +32,19 @@ type RepositorySpec struct { URL string `json:"url"` ConfigType gardenercfgcpi.ConfigType `json:"configType"` Cipher Cipher `json:"cipher"` - PropagateConsumerIdentity bool `json:"propagateConsumerIdentity"` + PropagateConsumerIdentity *bool `json:"propagateConsumerIdentity,omitempty"` } var _ cpi.ConsumerIdentityProvider = (*RepositorySpec)(nil) // NewRepositorySpec creates a new memory RepositorySpec. -func NewRepositorySpec(url string, configType gardenercfgcpi.ConfigType, cipher Cipher, propagateConsumerIdentity bool) *RepositorySpec { +func NewRepositorySpec(url string, configType gardenercfgcpi.ConfigType, cipher Cipher, propagateConsumerIdentity ...bool) *RepositorySpec { return &RepositorySpec{ ObjectVersionedType: runtime.NewVersionedTypedObject(Type), URL: url, ConfigType: configType, Cipher: cipher, - PropagateConsumerIdentity: propagateConsumerIdentity, + PropagateConsumerIdentity: generics.Pointer(utils.OptionalDefaultedBool(true, propagateConsumerIdentity...)), } } @@ -62,7 +64,7 @@ func (a *RepositorySpec) Repository(ctx cpi.Context, creds cpi.Credentials) (cpi return nil, fmt.Errorf("unable to get key from context: %w", err) } - return repos.GetRepository(ctx, a.URL, a.ConfigType, a.Cipher, key, a.PropagateConsumerIdentity) + return repos.GetRepository(ctx, a.URL, a.ConfigType, a.Cipher, key, utils.AsBool(a.PropagateConsumerIdentity, true)) } func (a *RepositorySpec) GetConsumerId(uctx ...internal.UsageContext) internal.ConsumerIdentity { diff --git a/pkg/contexts/credentials/repositories/npm/a_usage.go b/pkg/contexts/credentials/repositories/npm/a_usage.go new file mode 100644 index 0000000000..b200f35a4e --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/a_usage.go @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package npm + +import ( + "github.com/open-component-model/ocm/pkg/listformat" +) + +var usage = ` +This repository type can be used to access credentials stored in a file +following the NPM npmrc format (~/.npmrc). It take into account the +credentials helper section, also. If enabled, the described +credentials will be automatically assigned to appropriate consumer ids. +` + +var format = `The repository specification supports the following fields: +` + listformat.FormatListElements("", listformat.StringElementDescriptionList{ + "npmrcFile", "*string*: the file path to a NPM npmrc file", + "propagateConsumerIdentity", "*bool*(optional): enable consumer id propagation", +}) diff --git a/pkg/contexts/credentials/repositories/npm/cache.go b/pkg/contexts/credentials/repositories/npm/cache.go index e1d3d06363..15b2f0e74b 100644 --- a/pkg/contexts/credentials/repositories/npm/cache.go +++ b/pkg/contexts/credentials/repositories/npm/cache.go @@ -15,7 +15,7 @@ func createCache(_ datacontext.Context) interface{} { } } -func (r *Cache) GetRepository(ctx cpi.Context, name string) (*Repository, error) { +func (r *Cache) GetRepository(ctx cpi.Context, name string, prop bool) (*Repository, error) { var ( err error = nil repo *Repository @@ -24,7 +24,7 @@ func (r *Cache) GetRepository(ctx cpi.Context, name string) (*Repository, error) repo = r.repos[name] } if repo == nil { - repo, err = NewRepository(ctx, name) + repo, err = NewRepository(ctx, name, prop) if err == nil { r.repos[name] = repo } diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index 763d60eb9c..1b79737415 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -2,71 +2,12 @@ package npm import ( "bufio" - "fmt" "os" "strings" - - "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" - "github.com/open-component-model/ocm/pkg/listformat" - "github.com/open-component-model/ocm/pkg/runtime" -) - -const ( - // REPOSITORY_TYPE is the type of the NPMConfig. - REPOSITORY_TYPE = "NPMConfig" - REPOSITORY_TYPE_v1 = REPOSITORY_TYPE + runtime.VersionSeparator + "v1" ) -func init() { - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](REPOSITORY_TYPE)) - cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](REPOSITORY_TYPE_v1, cpi.WithDescription(usage), cpi.WithFormatSpec(format))) -} - -var usage = ` -This repository type can be used to access credentials stored in a file -following the NPM npmrc format (~/.npmrc). It take into account the -credentials helper section, also. If enabled, the described -credentials will be automatically assigned to appropriate consumer ids. -` - -var format = `The repository specification supports the following fields: -` + listformat.FormatListElements("", listformat.StringElementDescriptionList{ - "npmrcFile", "*string*: the file path to a NPM npmrc file", - "propagateConsumerIdentity", "*bool*(optional): enable consumer id propagation", -}) - type npmConfig map[string]string -// RepositorySpec describes a docker npmrc based credential repository interface. -type RepositorySpec struct { - runtime.ObjectVersionedType `json:",inline"` - NpmrcFile string `json:"npmrcFile,omitempty"` -} - -// NewRepositorySpec creates a new memory RepositorySpec. -func NewRepositorySpec(path string) *RepositorySpec { - if path == "" { - path = "~/.npmrc" - } - return &RepositorySpec{ - ObjectVersionedType: runtime.NewVersionedTypedObject(REPOSITORY_TYPE), - NpmrcFile: path, - } -} - -func (rs *RepositorySpec) GetType() string { - return REPOSITORY_TYPE -} - -func (rs *RepositorySpec) Repository(ctx cpi.Context, _ cpi.Credentials) (cpi.Repository, error) { - r := ctx.GetAttributes().GetOrCreateAttribute(".npmrc", createCache) - cache, ok := r.(*Cache) - if !ok { - return nil, fmt.Errorf("failed to assert type %T to Cache", r) - } - return cache.GetRepository(ctx, rs.NpmrcFile) -} - // readNpmConfigFile reads "~/.npmrc" file line by line, parse it and return the result as a npmConfig. func readNpmConfigFile(path string) (npmConfig, error) { // Open the file @@ -89,7 +30,11 @@ func readNpmConfigFile(path string) (npmConfig, error) { // Split the line into key and value parts := strings.SplitN(line, ":_authToken=", 2) if len(parts) == 2 { - cfg["https://"+parts[0]] = parts[1] + if strings.HasSuffix(parts[0], "/") { + cfg["https://"+parts[0][:len(parts[0])-1]] = parts[1] + } else { + cfg["https://"+parts[0]] = parts[1] + } } } diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index aad85dc5b8..deca26ad15 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -1,19 +1,42 @@ -package npm +package npm_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials" + "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" + "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" . "github.com/open-component-model/ocm/pkg/testutils" ) var _ = Describe("Config deserialization Test Environment", func() { It("read .npmrc", func() { - cfg, err := readNpmConfigFile("testdata/.npmrc") - Expect(err).To(BeNil()) - Expect(cfg).ToNot(BeNil()) - Expect(cfg).ToNot(BeEmpty()) - Expect(cfg["https://registry.npmjs.org/"]).To(Equal("npm_TOKEN")) - Expect(cfg["https://npm.registry.acme.com/api/npm/"]).To(Equal("bearer_TOKEN")) + ctx := credentials.New() + + repo := Must(npm.NewRepository(ctx, "testdata/.npmrc")) + Expect(Must(repo.LookupCredentials("https://registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"})) + Expect(Must(repo.LookupCredentials("https://npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"})) + }) + + It("propagates credentials", func() { + ctx := credentials.New() + + spec := npm.NewRepositorySpec("testdata/.npmrc") + + _ = Must(ctx.RepositoryForSpec(spec)) + id := identity.GetConsumerId("https://registry.npmjs.org", "pkg") + + creds := Must(credentials.CredentialsForConsumer(ctx, id)) + Expect(creds).NotTo(BeNil()) + Expect(creds.GetProperty(identity.ATTR_TOKEN)).To(Equal("npm_TOKEN")) }) + + It("has description", func() { + ctx := credentials.New() + t := ctx.RepositoryTypes().GetType(npm.TypeV1) + Expect(t).NotTo(BeNil()) + Expect(t.Description()).NotTo(Equal("")) + }) + }) diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index e42970db38..282f6063bc 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -10,18 +10,24 @@ import ( "github.com/open-component-model/ocm/pkg/utils" ) -const PROVIDER = "ocm.software/credentialprovider/" + REPOSITORY_TYPE +const PROVIDER = "ocm.software/credentialprovider/" + Type type Repository struct { - ctx cpi.Context - path string - npmrc npmConfig + ctx cpi.Context + path string + propagate bool + npmrc npmConfig } -func NewRepository(ctx cpi.Context, path string) (*Repository, error) { +func NewRepository(ctx cpi.Context, path string, prop ...bool) (*Repository, error) { + return newRepository(ctx, path, utils.OptionalDefaultedBool(true, prop...)) +} + +func newRepository(ctx cpi.Context, path string, prop bool) (*Repository, error) { r := &Repository{ - ctx: ctx, - path: path, + ctx: ctx, + path: path, + propagate: prop, } err := r.Read(true) return r, err @@ -43,13 +49,13 @@ func (r *Repository) LookupCredentials(name string) (cpi.Credentials, error) { return nil, err } if !exists { - return nil, errors.ErrNotFound("credentials", name, REPOSITORY_TYPE) + return nil, errors.ErrNotFound("credentials", name, Type) } return newCredentials(r.npmrc[name]), nil } func (r *Repository) WriteCredentials(_ string, _ cpi.Credentials) (cpi.Credentials, error) { - return nil, errors.ErrNotSupported("write", "credentials", REPOSITORY_TYPE) + return nil, errors.ErrNotSupported("write", "credentials", Type) } func (r *Repository) Read(force bool) error { @@ -70,7 +76,9 @@ func (r *Repository) Read(force bool) error { } id := cpi.ProviderIdentity(PROVIDER + "/" + path) - r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{path}) + if r.propagate { + r.ctx.RegisterConsumerProvider(id, &ConsumerProvider{r.path}) + } r.npmrc = cfg return nil } diff --git a/pkg/contexts/credentials/repositories/npm/type.go b/pkg/contexts/credentials/repositories/npm/type.go new file mode 100644 index 0000000000..0982b2d9e6 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/type.go @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package npm + +import ( + "fmt" + + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/runtime" + "github.com/open-component-model/ocm/pkg/utils" +) + +const ( + // Type is the type of the NPMConfig. + Type = "NPMConfig" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) + +func init() { + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](Type)) + cpi.RegisterRepositoryType(cpi.NewRepositoryType[*RepositorySpec](TypeV1, cpi.WithDescription(usage), cpi.WithFormatSpec(format))) +} + +// RepositorySpec describes a docker npmrc based credential repository interface. +type RepositorySpec struct { + runtime.ObjectVersionedType `json:",inline"` + NpmrcFile string `json:"npmrcFile,omitempty"` + PropgateConsumerIdentity *bool `json:"propagateConsumerIdentity,omitempty"` +} + +// NewRepositorySpec creates a new memory RepositorySpec. +func NewRepositorySpec(path string) *RepositorySpec { + if path == "" { + path = "~/.npmrc" + } + return &RepositorySpec{ + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + NpmrcFile: path, + } +} + +func (rs *RepositorySpec) GetType() string { + return Type +} + +func (rs *RepositorySpec) Repository(ctx cpi.Context, _ cpi.Credentials) (cpi.Repository, error) { + r := ctx.GetAttributes().GetOrCreateAttribute(".npmrc", createCache) + cache, ok := r.(*Cache) + if !ok { + return nil, fmt.Errorf("failed to assert type %T to Cache", r) + } + return cache.GetRepository(ctx, rs.NpmrcFile, utils.AsBool(rs.PropgateConsumerIdentity, true)) +} From 300ccaf50d1af0a6786588251315d1a153ac5e6b Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Fri, 16 Feb 2024 12:53:13 +0100 Subject: [PATCH 42/51] improve description handling --- .../utils/defaultconfigregistry/configure.go | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/contexts/ocm/utils/defaultconfigregistry/configure.go b/pkg/contexts/ocm/utils/defaultconfigregistry/configure.go index 30fed05799..67f51b24c7 100644 --- a/pkg/contexts/ocm/utils/defaultconfigregistry/configure.go +++ b/pkg/contexts/ocm/utils/defaultconfigregistry/configure.go @@ -43,6 +43,20 @@ func (r *defaultConfigurationRegistry) Get() []DefaultConfigHandler { return result } +func (r *defaultConfigurationRegistry) Description() string { + var result []string + + r.lock.Lock() + defer r.lock.Unlock() + + for _, h := range defaultConfigRegistry.list { + if h.desc != "" { + result = append(result, strings.TrimSpace(h.desc)) + } + } + return listformat.FormatDescriptionList("", result...) +} + var defaultConfigRegistry = &defaultConfigurationRegistry{} func RegisterDefaultConfigHandler(h DefaultConfigHandler, desc string) { @@ -54,12 +68,5 @@ func Get() []DefaultConfigHandler { } func Description() string { - var result []string - - for _, h := range defaultConfigRegistry.list { - if h.desc != "" { - result = append(result, strings.TrimSpace(h.desc)) - } - } - return listformat.FormatDescriptionList("", result...) + return defaultConfigRegistry.Description() } From 3f468d19fb7bec507743315445ef0afc2107c420 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Sat, 17 Feb 2024 11:40:09 +0100 Subject: [PATCH 43/51] generate cli docu (includes npm upload) --- docs/reference/ocm.md | 2 +- docs/reference/ocm_credential-handling.md | 28 +++++++++++++++++++ docs/reference/ocm_get_credentials.md | 13 +++++++++ docs/reference/ocm_logging.md | 1 + docs/reference/ocm_ocm-uploadhandlers.md | 17 ++++++++--- .../ocm_transfer_commontransportarchive.md | 17 ++++++++--- .../ocm_transfer_componentversions.md | 17 ++++++++--- 7 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/reference/ocm.md b/docs/reference/ocm.md index 863d813065..407aff355b 100644 --- a/docs/reference/ocm.md +++ b/docs/reference/ocm.md @@ -50,7 +50,7 @@ location a default configuration is composed according to known type specific configuration files. The following configuration sources are used: - - The docker configuration file at ~/.docker/config.jaon is + - The docker configuration file at ~/.docker/config.json is read to feed in the configured credentials for OCI registries. diff --git a/docs/reference/ocm_credential-handling.md b/docs/reference/ocm_credential-handling.md index 049c39a3dc..fa503e5457 100644 --- a/docs/reference/ocm_credential-handling.md +++ b/docs/reference/ocm_credential-handling.md @@ -170,6 +170,19 @@ The following credential consumer types are used/supported: - certificateAuthority: the certificate authority certificate used to verify certificates + - Registry.npmjs.com: NPM repository + + It matches the Registry.npmjs.com consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - email: NPM registry, require an email address + - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + + - S3: S3 credential matcher This matcher is a hostpath matcher. @@ -306,6 +319,21 @@ behaviours are described in the following list: is read. +- Credential provider NPMConfig + + This repository type can be used to access credentials stored in a file + following the NPM npmrc format (~/.npmrc). It take into account the + credentials helper section, also. If enabled, the described + credentials will be automatically assigned to appropriate consumer ids. + + The following versions are supported: + - Version v1 + + The repository specification supports the following fields: + - npmrcFile: *string*: the file path to a NPM npmrc file + - propagateConsumerIdentity: *bool*(optional): enable consumer id propagation + + ### SEE ALSO diff --git a/docs/reference/ocm_get_credentials.md b/docs/reference/ocm_get_credentials.md index 8d56179552..e6b09c0373 100644 --- a/docs/reference/ocm_get_credentials.md +++ b/docs/reference/ocm_get_credentials.md @@ -96,6 +96,19 @@ Matchers exist for the following usage contexts or consumer types: - certificateAuthority: the certificate authority certificate used to verify certificates + - Registry.npmjs.com: NPM repository + + It matches the Registry.npmjs.com consumer type and additionally acts like + the hostpath type. + + Credential consumers of the consumer type Registry.npmjs.com evaluate the following credential properties: + + - username: the basic auth user name + - password: the basic auth password + - email: NPM registry, require an email address + - token: the token attribute. May exist after login at any npm registry. Check your .npmrc file! + + - S3: S3 credential matcher This matcher is a hostpath matcher. diff --git a/docs/reference/ocm_logging.md b/docs/reference/ocm_logging.md index 78950872ce..e1de158350 100644 --- a/docs/reference/ocm_logging.md +++ b/docs/reference/ocm_logging.md @@ -17,6 +17,7 @@ The following *tags* are used by the command line tool: The following *realms* are used by the command line tool: - ocm: general realm used for the ocm go library. + - ocm/NPM: NPM registry - ocm/accessmethod/ociartifact: access method ociArtifact - ocm/compdesc: component descriptor handling - ocm/config: configuration management diff --git a/docs/reference/ocm_ocm-uploadhandlers.md b/docs/reference/ocm_ocm-uploadhandlers.md index 161043fd17..767da82a6d 100644 --- a/docs/reference/ocm_ocm-uploadhandlers.md +++ b/docs/reference/ocm_ocm-uploadhandlers.md @@ -36,10 +36,6 @@ resource blob), it is possible to pass a target configuration controlling the exact behaviour of the handler for selected artifacts. The following handler names are possible: - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/ociArtifacts: downloading OCI artifacts The ociArtifacts downloader is able to download OCI artifacts @@ -64,6 +60,19 @@ The following handler names are possible: Alternatively, a single string value can be given representing an OCI repository reference. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/npmPackage: uploading npm artifacts + + The ocm/npmPackage uploader is able to upload npm artifacts + as artifact archive according to the npm package spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the npm repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_commontransportarchive.md b/docs/reference/ocm_transfer_commontransportarchive.md index 43a638641f..faa7bfe6a1 100644 --- a/docs/reference/ocm_transfer_commontransportarchive.md +++ b/docs/reference/ocm_transfer_commontransportarchive.md @@ -110,10 +110,6 @@ are configured for the operation. It has the following format The uploader name may be a path expression with the following possibilities: - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/ociArtifacts: downloading OCI artifacts The ociArtifacts downloader is able to download OCI artifacts @@ -138,6 +134,19 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/npmPackage: uploading npm artifacts + + The ocm/npmPackage uploader is able to upload npm artifacts + as artifact archive according to the npm package spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the npm repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using diff --git a/docs/reference/ocm_transfer_componentversions.md b/docs/reference/ocm_transfer_componentversions.md index 31bea4db5d..cbbae63fe8 100644 --- a/docs/reference/ocm_transfer_componentversions.md +++ b/docs/reference/ocm_transfer_componentversions.md @@ -167,10 +167,6 @@ are configured for the operation. It has the following format The uploader name may be a path expression with the following possibilities: - - plugin: [downloaders provided by plugins] - - sub namespace of the form <plugin name>/<handler> - - ocm/ociArtifacts: downloading OCI artifacts The ociArtifacts downloader is able to download OCI artifacts @@ -195,6 +191,19 @@ The uploader name may be a path expression with the following possibilities: Alternatively, a single string value can be given representing an OCI repository reference. + - plugin: [downloaders provided by plugins] + + sub namespace of the form <plugin name>/<handler> + + - ocm/npmPackage: uploading npm artifacts + + The ocm/npmPackage uploader is able to upload npm artifacts + as artifact archive according to the npm package spec. + If registered the default mime type is: application/x-tgz + + It accepts a plain string for the URL or a config with the following field: + 'url': the URL of the npm repository. + See [ocm ocm-uploadhandlers](ocm_ocm-uploadhandlers.md) for further details on using From ac88604d6585a4ed016afac0744d92a0d6a7579f Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Sat, 17 Feb 2024 11:56:41 +0100 Subject: [PATCH 44/51] default config location + handle default config --- docs/reference/ocm.md | 3 + .../credentials/repositories/npm/default.go | 57 +++++++++++++++++++ .../credentials/repositories/npm/type.go | 18 ++++-- 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 pkg/contexts/credentials/repositories/npm/default.go diff --git a/docs/reference/ocm.md b/docs/reference/ocm.md index 407aff355b..77ee2b50e7 100644 --- a/docs/reference/ocm.md +++ b/docs/reference/ocm.md @@ -53,6 +53,9 @@ The following configuration sources are used: - The docker configuration file at ~/.docker/config.json is read to feed in the configured credentials for OCI registries. + - The npm configuration file at ~/.npmrc is + read to feed in the configured credentials for NPM registries. + With the option --cred it is possible to specify arbitrary credentials diff --git a/pkg/contexts/credentials/repositories/npm/default.go b/pkg/contexts/credentials/repositories/npm/default.go new file mode 100644 index 0000000000..0f823432b1 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/default.go @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package npm + +import ( + "fmt" + "os" + + "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/mandelsoft/vfs/pkg/osfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/contexts/config" + credcfg "github.com/open-component-model/ocm/pkg/contexts/credentials/config" + "github.com/open-component-model/ocm/pkg/contexts/ocm/utils/defaultconfigregistry" + "github.com/open-component-model/ocm/pkg/errors" +) + +const ( + ConfigFileName = ".npmrc" +) + +func init() { + defaultconfigregistry.RegisterDefaultConfigHandler(DefaultConfigHandler, desc) +} + +func DefaultConfig() (string, error) { + d, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(d, ConfigFileName), nil +} + +func DefaultConfigHandler(cfg config.Context) error { + // use docker config as default config for ocm cli + d, err := DefaultConfig() + if err != nil { + return nil + } + if ok, err := vfs.FileExists(osfs.New(), d); ok && err == nil { + ccfg := credcfg.New() + ccfg.AddRepository(NewRepositorySpec(d, true)) + err = cfg.ApplyConfig(ccfg, d) + if err != nil { + return errors.Wrapf(err, "cannot apply npm config %q", d) + } + } + return nil +} + +var desc = fmt.Sprintf(` +The npm configuration file at ~/%s is +read to feed in the configured credentials for NPM registries. +`, ConfigFileName) diff --git a/pkg/contexts/credentials/repositories/npm/type.go b/pkg/contexts/credentials/repositories/npm/type.go index 0982b2d9e6..eee3973827 100644 --- a/pkg/contexts/credentials/repositories/npm/type.go +++ b/pkg/contexts/credentials/repositories/npm/type.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + "github.com/open-component-model/ocm/pkg/generics" "github.com/open-component-model/ocm/pkg/runtime" "github.com/open-component-model/ocm/pkg/utils" ) @@ -31,13 +32,22 @@ type RepositorySpec struct { } // NewRepositorySpec creates a new memory RepositorySpec. -func NewRepositorySpec(path string) *RepositorySpec { +func NewRepositorySpec(path string, propagate ...bool) *RepositorySpec { + var p *bool if path == "" { - path = "~/.npmrc" + d, err := DefaultConfig() + if err == nil { + path = d + } } + if len(propagate) > 0 { + p = generics.Pointer(utils.OptionalDefaultedBool(true, propagate...)) + } + return &RepositorySpec{ - ObjectVersionedType: runtime.NewVersionedTypedObject(Type), - NpmrcFile: path, + ObjectVersionedType: runtime.NewVersionedTypedObject(Type), + NpmrcFile: path, + PropgateConsumerIdentity: p, } } From 9f6eb3ba1d51a135e4c231195be663f68a80cb52 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 19 Feb 2024 16:05:02 +0100 Subject: [PATCH 45/51] fix '/' desaster move 'https://' prefix --- .../credentials/repositories/npm/a_usage.go | 4 ---- .../credentials/repositories/npm/config.go | 4 ++-- .../credentials/repositories/npm/config_test.go | 7 ++++--- .../credentials/repositories/npm/default.go | 4 ---- .../credentials/repositories/npm/provider.go | 16 +--------------- .../credentials/repositories/npm/type.go | 4 ---- 6 files changed, 7 insertions(+), 32 deletions(-) diff --git a/pkg/contexts/credentials/repositories/npm/a_usage.go b/pkg/contexts/credentials/repositories/npm/a_usage.go index b200f35a4e..a8b77fb9d7 100644 --- a/pkg/contexts/credentials/repositories/npm/a_usage.go +++ b/pkg/contexts/credentials/repositories/npm/a_usage.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package npm import ( diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index 1b79737415..26228eddbf 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -31,9 +31,9 @@ func readNpmConfigFile(path string) (npmConfig, error) { parts := strings.SplitN(line, ":_authToken=", 2) if len(parts) == 2 { if strings.HasSuffix(parts[0], "/") { - cfg["https://"+parts[0][:len(parts[0])-1]] = parts[1] + cfg[parts[0][:len(parts[0])-1]] = parts[1] } else { - cfg["https://"+parts[0]] = parts[1] + cfg[parts[0]] = parts[1] } } } diff --git a/pkg/contexts/credentials/repositories/npm/config_test.go b/pkg/contexts/credentials/repositories/npm/config_test.go index deca26ad15..e8cc8e28b3 100644 --- a/pkg/contexts/credentials/repositories/npm/config_test.go +++ b/pkg/contexts/credentials/repositories/npm/config_test.go @@ -3,6 +3,7 @@ package npm_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" @@ -15,8 +16,8 @@ var _ = Describe("Config deserialization Test Environment", func() { ctx := credentials.New() repo := Must(npm.NewRepository(ctx, "testdata/.npmrc")) - Expect(Must(repo.LookupCredentials("https://registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"})) - Expect(Must(repo.LookupCredentials("https://npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"})) + Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"})) + Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"})) }) It("propagates credentials", func() { @@ -25,7 +26,7 @@ var _ = Describe("Config deserialization Test Environment", func() { spec := npm.NewRepositorySpec("testdata/.npmrc") _ = Must(ctx.RepositoryForSpec(spec)) - id := identity.GetConsumerId("https://registry.npmjs.org", "pkg") + id := identity.GetConsumerId("registry.npmjs.org", "pkg") creds := Must(credentials.CredentialsForConsumer(ctx, id)) Expect(creds).NotTo(BeNil()) diff --git a/pkg/contexts/credentials/repositories/npm/default.go b/pkg/contexts/credentials/repositories/npm/default.go index 0f823432b1..14c569f842 100644 --- a/pkg/contexts/credentials/repositories/npm/default.go +++ b/pkg/contexts/credentials/repositories/npm/default.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package npm import ( diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index c0beecdf90..3c18c41487 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -1,10 +1,7 @@ package npm import ( - "net/url" - npm "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" - "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" "github.com/open-component-model/ocm/pkg/logging" ) @@ -38,19 +35,8 @@ func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi. var creds cpi.CredentialsSource for key, value := range all { - u, e := url.Parse(key) - if e != nil { - return nil, nil - } + id := npm.GetConsumerId("https://"+key, "") - attrs := []string{identity.ID_HOSTNAME, u.Hostname()} - if u.Port() != "" { - attrs = append(attrs, identity.ID_PORT, u.Port()) - } - if u.Path != "" { - attrs = append(attrs, identity.ID_PATHPREFIX, u.Path) - } - id := cpi.NewConsumerIdentity(npm.CONSUMER_TYPE, attrs...) if m(requested, currentFound, id) { creds = newCredentials(value) currentFound = id diff --git a/pkg/contexts/credentials/repositories/npm/type.go b/pkg/contexts/credentials/repositories/npm/type.go index eee3973827..32cf05b2f6 100644 --- a/pkg/contexts/credentials/repositories/npm/type.go +++ b/pkg/contexts/credentials/repositories/npm/type.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package npm import ( From f7c262e14a8c7005c34d4570d941bae1e7318dc6 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Mon, 19 Feb 2024 16:06:29 +0100 Subject: [PATCH 46/51] more tests for RepositorySpec --- .../repositories/npm/repository_test.go | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 pkg/contexts/credentials/repositories/npm/repository_test.go diff --git a/pkg/contexts/credentials/repositories/npm/repository_test.go b/pkg/contexts/credentials/repositories/npm/repository_test.go new file mode 100644 index 0000000000..8de30de2b9 --- /dev/null +++ b/pkg/contexts/credentials/repositories/npm/repository_test.go @@ -0,0 +1,78 @@ +package npm_test + +import ( + "encoding/json" + "reflect" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/credentials" + npmCredentials "github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity" + "github.com/open-component-model/ocm/pkg/contexts/credentials/cpi" + local "github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm" + "github.com/open-component-model/ocm/pkg/finalizer" + . "github.com/open-component-model/ocm/pkg/testutils" +) + +var _ = Describe("NPM config - .npmrc", func() { + + props := common.Properties{ + npmCredentials.ATTR_TOKEN: "npm_TOKEN", + } + + props2 := common.Properties{ + npmCredentials.ATTR_TOKEN: "bearer_TOKEN", + } + + var DefaultContext credentials.Context + + BeforeEach(func() { + DefaultContext = credentials.New() + }) + + specdata := "{\"type\":\"NPMConfig\",\"npmrcFile\":\"testdata/.npmrc\"}" + + It("serializes repo spec", func() { + spec := local.NewRepositorySpec("testdata/.npmrc") + data := Must(json.Marshal(spec)) + Expect(data).To(Equal([]byte(specdata))) + }) + + It("deserializes repo spec", func() { + spec := Must(DefaultContext.RepositorySpecForConfig([]byte(specdata), nil)) + Expect(reflect.TypeOf(spec).String()).To(Equal("*npm.RepositorySpec")) + Expect(spec.(*local.RepositorySpec).NpmrcFile).To(Equal("testdata/.npmrc")) + }) + + It("resolves repository", func() { + repo := Must(DefaultContext.RepositoryForConfig([]byte(specdata), nil)) + Expect(reflect.TypeOf(repo).String()).To(Equal("*npm.Repository")) + }) + + It("retrieves credentials", func() { + repo := Must(DefaultContext.RepositoryForConfig([]byte(specdata), nil)) + + creds := Must(repo.LookupCredentials("registry.npmjs.org")) + Expect(creds.Properties()).To(Equal(props)) + + creds = Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")) + Expect(creds.Properties()).To(Equal(props2)) + }) + + It("can access the default context", func() { + ctx := credentials.New() + + r := finalizer.GetRuntimeFinalizationRecorder(ctx) + Expect(r).NotTo(BeNil()) + + Must(ctx.RepositoryForConfig([]byte(specdata), nil)) + + ci := cpi.NewConsumerIdentity(npmCredentials.CONSUMER_TYPE) + Expect(ci).NotTo(BeNil()) + credentials := Must(cpi.CredentialsForConsumer(ctx.CredentialsContext(), ci)) + Expect(credentials).NotTo(BeNil()) + Expect(credentials.Properties()).To(Equal(props)) + }) +}) From 803a49b7e0b4312adec3806186a9fd0b6c9b12c4 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 20 Feb 2024 10:22:54 +0100 Subject: [PATCH 47/51] replace os.Getenv("HOME") with os.UserHomeDir() --- pkg/contexts/config/configutils/configure.go | 12 +++++------- pkg/contexts/ocm/attrs/plugindirattr/attr.go | 6 +----- pkg/contexts/ocm/utils/configure.go | 9 ++++----- pkg/utils/path.go | 8 ++------ 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/pkg/contexts/config/configutils/configure.go b/pkg/contexts/config/configutils/configure.go index 9cb7068f06..433aac865e 100644 --- a/pkg/contexts/config/configutils/configure.go +++ b/pkg/contexts/config/configutils/configure.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package configutils import ( @@ -9,14 +5,13 @@ import ( "os" "strings" - _ "github.com/open-component-model/ocm/pkg/contexts/datacontext/config" - "github.com/mandelsoft/spiff/features" "github.com/mandelsoft/spiff/spiffing" "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/open-component-model/ocm/pkg/contexts/config" + _ "github.com/open-component-model/ocm/pkg/contexts/datacontext/config" "github.com/open-component-model/ocm/pkg/errors" ) @@ -27,7 +22,10 @@ func Configure(path string) error { func ConfigureContext(ctxp config.ContextProvider, path string) error { ctx := config.FromProvider(ctxp) - h := os.Getenv("HOME") + h, err := os.UserHomeDir() + if err != nil { + return errors.Wrapf(err, "cannot determine home directory") + } if path == "" { if h != "" { cfg := h + "/.ocmconfig" diff --git a/pkg/contexts/ocm/attrs/plugindirattr/attr.go b/pkg/contexts/ocm/attrs/plugindirattr/attr.go index faab56d98f..d33a758993 100644 --- a/pkg/contexts/ocm/attrs/plugindirattr/attr.go +++ b/pkg/contexts/ocm/attrs/plugindirattr/attr.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package plugindirattr import ( @@ -30,7 +26,7 @@ func init() { } func DefaultDir(fs vfs.FileSystem) string { - home := os.Getenv("HOME") + home, _ := os.UserHomeDir() if home != "" { dir := filepath.Join(home, DEFAULT_PLUGIN_DIR) if ok, err := vfs.DirExists(fs, dir); ok && err == nil { diff --git a/pkg/contexts/ocm/utils/configure.go b/pkg/contexts/ocm/utils/configure.go index 467f9ff718..a40faee717 100644 --- a/pkg/contexts/ocm/utils/configure.go +++ b/pkg/contexts/ocm/utils/configure.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package utils import ( @@ -28,7 +24,10 @@ func Configure(ctx ocm.Context, path string, fss ...vfs.FileSystem) (ocm.Context if ctx == nil { ctx = ocm.DefaultContext() } - h := os.Getenv("HOME") + h, err := os.UserHomeDir() + if err != nil { + return nil, errors.Wrapf(err, "cannot determine home directory") + } if path == "" { if h != "" { cfg := h + "/" + DEFAULT_OCM_CONFIG diff --git a/pkg/utils/path.go b/pkg/utils/path.go index f56227a72a..1f1c14f724 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -1,7 +1,3 @@ -// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. -// -// SPDX-License-Identifier: Apache-2.0 - package utils import ( @@ -19,8 +15,8 @@ import ( // ResolvePath handles the ~ notation for the home directory. func ResolvePath(path string) (string, error) { if strings.HasPrefix(path, "~"+string(os.PathSeparator)) { - home := os.Getenv("HOME") - if home == "" { + home, err := os.UserHomeDir() + if home == "" || err != nil { return path, fmt.Errorf("HOME not set") } path = home + path[1:] From 82ad97f2306a0780ebc8c53919cbec1e9ca075e3 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 20 Feb 2024 10:27:29 +0100 Subject: [PATCH 48/51] use utils.ResolvePath() in readNpmConfigFile() --- .../credentials/repositories/npm/config.go | 16 ++++++++++++---- .../credentials/repositories/npm/provider.go | 4 ++-- .../credentials/repositories/npm/repository.go | 6 +----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/contexts/credentials/repositories/npm/config.go b/pkg/contexts/credentials/repositories/npm/config.go index 26228eddbf..b6b750b9ea 100644 --- a/pkg/contexts/credentials/repositories/npm/config.go +++ b/pkg/contexts/credentials/repositories/npm/config.go @@ -4,16 +4,24 @@ import ( "bufio" "os" "strings" + + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/utils" ) type npmConfig map[string]string // readNpmConfigFile reads "~/.npmrc" file line by line, parse it and return the result as a npmConfig. -func readNpmConfigFile(path string) (npmConfig, error) { +func readNpmConfigFile(path string) (npmConfig, string, error) { + path, err := utils.ResolvePath(path) + if err != nil { + return nil, path, errors.Wrapf(err, "cannot resolve path %q", path) + } + // Open the file file, err := os.Open(path) if err != nil { - return nil, err + return nil, path, err } defer file.Close() @@ -40,8 +48,8 @@ func readNpmConfigFile(path string) (npmConfig, error) { // Check for errors if err = scanner.Err(); err != nil { - return nil, err + return nil, path, err } - return cfg, nil + return cfg, path, nil } diff --git a/pkg/contexts/credentials/repositories/npm/provider.go b/pkg/contexts/credentials/repositories/npm/provider.go index 3c18c41487..010dd12ac5 100644 --- a/pkg/contexts/credentials/repositories/npm/provider.go +++ b/pkg/contexts/credentials/repositories/npm/provider.go @@ -25,10 +25,10 @@ func (p *ConsumerProvider) Get(req cpi.ConsumerIdentity) (cpi.CredentialsSource, } func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi.ConsumerIdentity, m cpi.IdentityMatcher) (cpi.CredentialsSource, cpi.ConsumerIdentity) { - all, err := readNpmConfigFile(p.npmrcPath) + all, path, err := readNpmConfigFile(p.npmrcPath) if err != nil { log := logging.Context().Logger(npm.REALM) - log.LogError(err, "Failed to read npmrc file", "path", p.npmrcPath) + log.LogError(err, "Failed to read npmrc file", "path", path) return nil, nil } diff --git a/pkg/contexts/credentials/repositories/npm/repository.go b/pkg/contexts/credentials/repositories/npm/repository.go index 282f6063bc..c02d7d5971 100644 --- a/pkg/contexts/credentials/repositories/npm/repository.go +++ b/pkg/contexts/credentials/repositories/npm/repository.go @@ -66,11 +66,7 @@ func (r *Repository) Read(force bool) error { if r.path == "" { return fmt.Errorf("npmrc path not provided") } - path, err := utils.ResolvePath(r.path) - if err != nil { - return errors.Wrapf(err, "cannot resolve path %q", r.path) - } - cfg, err := readNpmConfigFile(path) + cfg, path, err := readNpmConfigFile(r.path) if err != nil { return fmt.Errorf("failed to load npmrc: %w", err) } From d74999c58964b770d22a82df38a5729496e83116 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Tue, 20 Feb 2024 11:02:31 +0100 Subject: [PATCH 49/51] Update configure.go --- pkg/contexts/config/configutils/configure.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/contexts/config/configutils/configure.go b/pkg/contexts/config/configutils/configure.go index 433aac865e..aeb25982af 100644 --- a/pkg/contexts/config/configutils/configure.go +++ b/pkg/contexts/config/configutils/configure.go @@ -5,13 +5,14 @@ import ( "os" "strings" + _ "github.com/open-component-model/ocm/pkg/contexts/datacontext/config" + "github.com/mandelsoft/spiff/features" "github.com/mandelsoft/spiff/spiffing" "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/open-component-model/ocm/pkg/contexts/config" - _ "github.com/open-component-model/ocm/pkg/contexts/datacontext/config" "github.com/open-component-model/ocm/pkg/errors" ) From 83658b25ebfd4626f6f1f775fd05238a70f7e0f1 Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 22 Feb 2024 09:51:24 +0100 Subject: [PATCH 50/51] ignore err --- pkg/contexts/config/configutils/configure.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/config/configutils/configure.go b/pkg/contexts/config/configutils/configure.go index aeb25982af..b28c2a0920 100644 --- a/pkg/contexts/config/configutils/configure.go +++ b/pkg/contexts/config/configutils/configure.go @@ -23,10 +23,7 @@ func Configure(path string) error { func ConfigureContext(ctxp config.ContextProvider, path string) error { ctx := config.FromProvider(ctxp) - h, err := os.UserHomeDir() - if err != nil { - return errors.Wrapf(err, "cannot determine home directory") - } + h, _ := os.UserHomeDir() if path == "" { if h != "" { cfg := h + "/.ocmconfig" From 90f04258cbcea14d77d234f318fbc8da8b885deb Mon Sep 17 00:00:00 2001 From: Hilmar Falkenberg Date: Thu, 22 Feb 2024 09:54:42 +0100 Subject: [PATCH 51/51] ignore err --- pkg/contexts/ocm/utils/configure.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/contexts/ocm/utils/configure.go b/pkg/contexts/ocm/utils/configure.go index a40faee717..6a6cf297db 100644 --- a/pkg/contexts/ocm/utils/configure.go +++ b/pkg/contexts/ocm/utils/configure.go @@ -24,10 +24,7 @@ func Configure(ctx ocm.Context, path string, fss ...vfs.FileSystem) (ocm.Context if ctx == nil { ctx = ocm.DefaultContext() } - h, err := os.UserHomeDir() - if err != nil { - return nil, errors.Wrapf(err, "cannot determine home directory") - } + h, _ := os.UserHomeDir() if path == "" { if h != "" { cfg := h + "/" + DEFAULT_OCM_CONFIG