diff --git a/api/porch/types.go b/api/porch/types.go index c903ce10..3f041283 100644 --- a/api/porch/types.go +++ b/api/porch/types.go @@ -448,7 +448,6 @@ type NameMeta struct { Namespace string `json:"namespace,omitempty"` } - // PackageRevisionResources // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/porch/v1alpha1/types.go b/api/porch/v1alpha1/types.go index 96903c86..1d5dd3f3 100644 --- a/api/porch/v1alpha1/types.go +++ b/api/porch/v1alpha1/types.go @@ -523,7 +523,6 @@ type PackageRevisionResourcesStatus struct { RenderStatus RenderStatus `json:"renderStatus,omitempty"` } - // Package // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -559,4 +558,4 @@ type PackageStatus struct { // LatestRevision identifies the package revision that is the latest // published package revision belonging to this package LatestRevision string `json:"latestRevision,omitempty"` -} \ No newline at end of file +} diff --git a/api/porchconfig/v1alpha1/types.go b/api/porchconfig/v1alpha1/types.go index 5c7239f1..32457723 100644 --- a/api/porchconfig/v1alpha1/types.go +++ b/api/porchconfig/v1alpha1/types.go @@ -46,7 +46,7 @@ const ( type RepositoryContent string const ( - RepositoryContentPackage RepositoryContent = "Package" + RepositoryContentPackage RepositoryContent = "Package" ) // RepositorySpec defines the desired state of Repository @@ -60,12 +60,12 @@ type RepositorySpec struct { Deployment bool `json:"deployment,omitempty"` // Type of the repository (i.e. git, OCI) Type RepositoryType `json:"type,omitempty"` - // The Content field is deprecated, please do not specify it in new manifests. + // The Content field is deprecated, please do not specify it in new manifests. // For partial backward compatibility it is still recognized, but its only valid value is "Package", and if not specified its default value is also "Package". // +kubebuilder:validation:XValidation:message="The 'content' field is deprecated, its only valid value is 'Package'",rule="self == '' || self == 'Package'" // +kubebuilder:default="Package" Content *RepositoryContent `json:"content,omitempty"` - + // Git repository details. Required if `type` is `git`. Ignored if `type` is not `git`. Git *GitRepository `json:"git,omitempty"` // OCI repository details. Required if `type` is `oci`. Ignored if `type` is not `oci`. diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index bc09554c..7cb15142 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -27,10 +27,11 @@ import ( internalapi "github.com/nephio-project/porch/internal/api/porchinternal/v1alpha1" "github.com/nephio-project/porch/internal/kpt/fnruntime" "github.com/nephio-project/porch/pkg/cache" - "github.com/nephio-project/porch/pkg/cache/dbcache" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/engine" "github.com/nephio-project/porch/pkg/meta" "github.com/nephio-project/porch/pkg/registry/porch" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" "google.golang.org/api/option" "google.golang.org/api/sts/v1" corev1 "k8s.io/api/core/v1" @@ -75,13 +76,13 @@ func init() { // ExtraConfig holds custom apiserver config type ExtraConfig struct { - CoreAPIKubeconfigPath string - CacheDirectory string - FunctionRunnerAddress string - DefaultImagePrefix string - RepoSyncFrequency time.Duration - UseGitCaBundle bool - MaxGrpcMessageSize int + CoreAPIKubeconfigPath string + CacheDirectory string + FunctionRunnerAddress string + DefaultImagePrefix string + RepoSyncFrequency time.Duration + UseUserDefinedCaBundle bool + MaxGrpcMessageSize int } // Config defines the config for the apiserver @@ -94,7 +95,7 @@ type Config struct { type PorchServer struct { GenericAPIServer *genericapiserver.GenericAPIServer coreClient client.WithWatch - cache cache.Cache + cache cachetypes.Cache PeriodicRepoSyncFrequency time.Duration } @@ -227,22 +228,22 @@ func (c completedConfig) New() (*PorchServer, error) { userInfoProvider := &porch.ApiserverUserInfoProvider{} watcherMgr := engine.NewWatcherManager() - /* - cacheImpl := memorycache.NewCache(c.ExtraConfig.CacheDirectory, c.ExtraConfig.RepoSyncFrequency, c.ExtraConfig.UseGitCaBundle, memorycache.CacheOptions{ - CredentialResolver: credentialResolver, - UserInfoProvider: userInfoProvider, - MetadataStore: metadataStore, - ObjectNotifier: watcherMgr, + cacheImpl, err := cache.CreateCacheImpl( + context.TODO(), + cachetypes.CacheOptions{ + RepoImplOptions: repoimpltypes.RepoImplOptions{ + LocalDirectory: c.ExtraConfig.CacheDirectory, + UseUserDefinedCaBundle: c.ExtraConfig.UseUserDefinedCaBundle, + CredentialResolver: credentialResolver, + UserInfoProvider: userInfoProvider, + }, + RepoSyncFrequency: c.ExtraConfig.RepoSyncFrequency, + MetadataStore: metadataStore, + RepoPRChangeNotifier: watcherMgr, }) - */ - cacheImpl := dbcache.NewCache(c.ExtraConfig.CacheDirectory, c.ExtraConfig.RepoSyncFrequency, c.ExtraConfig.UseGitCaBundle, dbcache.CacheOptions{ - Driver: "pgx", - DataSource: "postgresql://porch:porch@172.18.255.202:55432/porch", - CredentialResolver: credentialResolver, - UserInfoProvider: userInfoProvider, - MetadataStore: metadataStore, - ObjectNotifier: watcherMgr, - }) + if err != nil { + return nil, fmt.Errorf("failed to creeate repository cache: %w", err) + } runnerOptionsResolver := func(namespace string) fnruntime.RunnerOptions { runnerOptions := fnruntime.RunnerOptions{} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index a9de7b67..854bcdd2 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -17,12 +17,18 @@ package cache import ( "context" - configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/repository" + memorycache "github.com/nephio-project/porch/pkg/cache/memorycache" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" ) -type Cache interface { - OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (repository.Repository, error) - UpdateRepository(ctx context.Context, repositorySpec *configapi.Repository) error - CloseRepository(ctx context.Context, repositorySpec *configapi.Repository, allRepos []configapi.Repository) error +var tracer = otel.Tracer("cache") + +func CreateCacheImpl(ctx context.Context, options cachetypes.CacheOptions) (cachetypes.Cache, error) { + ctx, span := tracer.Start(ctx, "Repository::RepositoryFactory", trace.WithAttributes()) + defer span.End() + + var cacheFactory = new(memorycache.MemoryCacheFactory) + return cacheFactory.NewCache(ctx, options) } diff --git a/pkg/cache/dbcache/cache_test.go b/pkg/cache/dbcache/cache_test.go deleted file mode 100644 index 5bef4d50..00000000 --- a/pkg/cache/dbcache/cache_test.go +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2022, 2024 The kpt and Nephio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dbcache - -import ( - "context" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - api "github.com/nephio-project/porch/api/porch/v1alpha1" - "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - - fakecache "github.com/nephio-project/porch/pkg/cache/fake" - "github.com/nephio-project/porch/pkg/git" - "github.com/nephio-project/porch/pkg/meta" - fakemeta "github.com/nephio-project/porch/pkg/meta/fake" - "github.com/nephio-project/porch/pkg/repository" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/yaml" -) - -func TestLatestPackages(t *testing.T) { - ctx := context.Background() - testPath := filepath.Join("..", "..", "git", "testdata") - - cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") - - wantLatest := map[string]string{ - "sample": "v2", - "catalog/empty": "v1", - "catalog/gcp/bucket": "v1", - "catalog/namespace/basens": "v3", - "catalog/namespace/istions": "v3", - } - revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{}) - if err != nil { - t.Fatalf("ListPackageRevisions failed: %v", err) - } - - gotLatest := map[string]string{} - for _, pr := range revisions { - rev, err := pr.GetPackageRevision(ctx) - if err != nil { - t.Errorf("didn't expect error, but got %v", err) - } - - if latest, ok := rev.Labels[api.LatestPackageRevisionKey]; ok { - if got, want := latest, api.LatestPackageRevisionValue; got != want { - t.Errorf("%s label value: got %q, want %q", api.LatestPackageRevisionKey, got, want) - continue - } - - if existing, ok := gotLatest[rev.Spec.PackageName]; ok { - t.Errorf("Multiple latest package revisions for package %q: %q and %q", - rev.Spec.PackageName, rev.Spec.Revision, existing) - } - - // latest package - gotLatest[rev.Spec.PackageName] = rev.Spec.Revision - } - } - - if !cmp.Equal(wantLatest, gotLatest) { - t.Errorf("Latest package revisions differ (-want,+got): %s", cmp.Diff(wantLatest, gotLatest)) - } -} - -func TestPublishedLatest(t *testing.T) { - ctx := context.Background() - testPath := filepath.Join("..", "..", "git", "testdata") - cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") - - revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ - Package: "catalog/gcp/bucket", - WorkspaceName: "v2", - }) - if err != nil { - t.Fatalf("ListPackageRevisions failed: %v", err) - } - - // Expect a single result - if got, want := len(revisions), 1; got != want { - t.Fatalf("ListPackageRevisions returned %d packages; want %d", got, want) - } - - bucket := revisions[0] - // Expect draft package - if got, want := bucket.Lifecycle(), api.PackageRevisionLifecycleDraft; got != want { - t.Fatalf("Bucket package lifecycle: got %s, want %s", got, want) - } - - update, err := cachedRepo.UpdatePackageRevision(ctx, bucket) - if err != nil { - t.Fatalf("UpdatePackaeg(%s) failed: %v", bucket.Key(), err) - } - if err := update.UpdateLifecycle(ctx, api.PackageRevisionLifecyclePublished); err != nil { - t.Fatalf("UpdateLifecycle failed; %v", err) - } - closed, err := cachedRepo.ClosePackageRevisionDraft(ctx, update, "") - if err != nil { - t.Fatalf("Close failed: %v", err) - } - resource, err := closed.GetPackageRevision(ctx) - if err != nil { - t.Errorf("didn't expect error, but got %v", err) - } - if got, ok := resource.Labels[api.LatestPackageRevisionKey]; !ok { - t.Errorf("Label %s not found as expected", api.LatestPackageRevisionKey) - } else if want := api.LatestPackageRevisionValue; got != want { - t.Errorf("Latest label: got %s, want %s", got, want) - } -} - -func TestDeletePublishedMain(t *testing.T) { - ctx := context.Background() - testPath := filepath.Join("../..", "git", "testdata") - cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") - - revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ - Package: "catalog/gcp/bucket", - WorkspaceName: "v2", - }) - if err != nil { - t.Fatalf("ListPackageRevisions failed: %v", err) - } - - // Expect a single result - if got, want := len(revisions), 1; got != want { - t.Fatalf("ListPackageRevisions returned %d packages; want %d", got, want) - } - - bucket := revisions[0] - // Expect draft package - if got, want := bucket.Lifecycle(), api.PackageRevisionLifecycleDraft; got != want { - t.Fatalf("Bucket package lifecycle: got %s, want %s", got, want) - } - - update, err := cachedRepo.UpdatePackageRevision(ctx, bucket) - if err != nil { - t.Fatalf("UpdatePackage(%s) failed: %v", bucket.Key(), err) - } - if err := update.UpdateLifecycle(ctx, api.PackageRevisionLifecyclePublished); err != nil { - t.Fatalf("UpdateLifecycle failed; %v", err) - } - closed, err := cachedRepo.ClosePackageRevisionDraft(ctx, update, "") - if err != nil { - t.Fatalf("Close failed: %v", err) - } - _, err = closed.GetPackageRevision(ctx) - if err != nil { - t.Errorf("didn't expect error, but got %v", err) - } - - publishedRevisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ - Package: "catalog/gcp/bucket", - WorkspaceName: "v2", - Lifecycle: api.PackageRevisionLifecyclePublished, - Revision: "main", - }) - if err != nil { - t.Fatalf("ListPackageRevisions failed: %v", err) - } - - // Expect a single result - if got, want := len(publishedRevisions), 1; got != want { - t.Fatalf("ListPackageRevisions returned %d packages; want %d", got, want) - } - - approvedBucket := publishedRevisions[0] - - if got, want := approvedBucket.Lifecycle(), api.PackageRevisionLifecyclePublished; got != want { - t.Fatalf("Approved Bucket package lifecycle: got %s, want %s", got, want) - } - - err = approvedBucket.UpdateLifecycle(ctx, api.PackageRevisionLifecycleDeletionProposed) - if err != nil { - t.Fatalf("Deletion proposal for approved Bucket failed; %v", err) - } - err = cachedRepo.DeletePackageRevision(ctx, approvedBucket) - if err != nil { - t.Fatalf("Deleting Main packageRevision failed; %v", err) - } - - postDeletePublishedRevisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ - Package: "catalog/gcp/bucket", - WorkspaceName: "v2", - Lifecycle: api.PackageRevisionLifecyclePublished, - Revision: "main", - }) - - if err != nil { - t.Fatalf("ListPackageRevisions failed: %v", err) - } - - //Expect 0 entries - if got, want := len(postDeletePublishedRevisions), 0; got != want { - t.Fatalf("ListPackageRevisions returned %d packages; want %d", got, want) - } - -} - -func openRepositoryFromArchive(t *testing.T, ctx context.Context, testPath, name string) repository.Repository { - t.Helper() - - tempdir := t.TempDir() - tarfile := filepath.Join(testPath, fmt.Sprintf("%s-repository.tar", name)) - _, address := git.ServeGitRepository(t, tarfile, tempdir) - metadataStore := createMetadataStoreFromArchive(t, fmt.Sprintf("%s-metadata.yaml", name), name) - - cache := NewCache(t.TempDir(), 60*time.Second, true, CacheOptions{ - MetadataStore: metadataStore, - ObjectNotifier: &fakecache.ObjectNotifier{}, - CredentialResolver: &fakecache.CredentialResolver{}, - }) - apiRepo := &v1alpha1.Repository{ - TypeMeta: metav1.TypeMeta{ - Kind: v1alpha1.TypeRepository.Kind, - APIVersion: v1alpha1.TypeRepository.APIVersion(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - }, - Spec: v1alpha1.RepositorySpec{ - Deployment: false, - Type: v1alpha1.RepositoryTypeGit, - Git: &v1alpha1.GitRepository{ - Repo: address, - }, - }, - } - cachedRepo, err := cache.OpenRepository(ctx, apiRepo) - if err != nil { - t.Fatalf("OpenRepository(%q) of %q failed; %v", address, tarfile, err) - } - - t.Cleanup(func() { - err := cache.CloseRepository(ctx, apiRepo, []v1alpha1.Repository{*apiRepo}) - if err != nil { - t.Errorf("CloseRepository(%q) failed: %v", address, err) - } - - // TODO: Uncomment this - //if len(cache.repositories) != 0 { - // t.Errorf("CloseRepository hasn't deleted repository from cache") - //} - }) - return cachedRepo -} - -func createMetadataStoreFromArchive(t *testing.T, testPath, name string) meta.MetadataStore { - t.Helper() - - f := filepath.Join("..", "git", "testdata", testPath) - c, err := os.ReadFile(f) - if err != nil && !os.IsNotExist(err) { - t.Fatalf("Error reading metadata file found for repository %s", name) - } - if os.IsNotExist(err) { - return &fakemeta.MemoryMetadataStore{ - Metas: []metav1.ObjectMeta{}, - } - } - - var metas []metav1.ObjectMeta - if err := yaml.Unmarshal(c, &metas); err != nil { - t.Fatalf("Error unmarshalling metadata file for repository %s", name) - } - - return &fakemeta.MemoryMetadataStore{ - Metas: metas, - } -} diff --git a/pkg/cache/dbcache/dbcache.go b/pkg/cache/dbcache/dbcache.go index 16f9c7e3..84d3ea9c 100644 --- a/pkg/cache/dbcache/dbcache.go +++ b/pkg/cache/dbcache/dbcache.go @@ -16,68 +16,20 @@ package dbcache import ( "context" - "sync" "time" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/cache" - "github.com/nephio-project/porch/pkg/meta" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/repository" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" - "k8s.io/apimachinery/pkg/watch" ) -var _ cache.Cache = &dbCache{} +var _ cachetypes.Cache = &dbCache{} var tracer = otel.Tracer("dbcache") -// Cache allows us to keep state for repositories, rather than querying them every time. -// -// Cache Structure: -// /git/ -// * Caches bare git repositories in directories named based on the repository address. -// /oci/ -// * Caches oci images with further hierarchy underneath -// * We Cache image layers in /oci/layers/ (this might be obsolete with the flattened Cache) -// * We Cache flattened tar files in /oci/ (so we don't need to pull to read resources) -// * We poll the repositories (every minute) and Cache the discovered images in memory. type dbCache struct { - mutex sync.Mutex - cacheDir string - credentialResolver repository.CredentialResolver - userInfoProvider repository.UserInfoProvider - repoSyncFrequency time.Duration - objectNotifier objectNotifier - useGitCaBundle bool -} - -type objectNotifier interface { - NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision) int -} - -type CacheOptions struct { - Driver string - DataSource string - CredentialResolver repository.CredentialResolver - UserInfoProvider repository.UserInfoProvider - MetadataStore meta.MetadataStore - ObjectNotifier objectNotifier -} - -func NewCache(cacheDir string, repoSyncFrequency time.Duration, useGitCaBundle bool, opts CacheOptions) cache.Cache { - - if err := OpenDBConnection(opts); err != nil { - return nil - } - - return &dbCache{ - cacheDir: cacheDir, - credentialResolver: opts.CredentialResolver, - userInfoProvider: opts.UserInfoProvider, - objectNotifier: opts.ObjectNotifier, - repoSyncFrequency: repoSyncFrequency, - useGitCaBundle: useGitCaBundle, - } + options cachetypes.CacheOptions } func (c *dbCache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (repository.Repository, error) { @@ -131,3 +83,10 @@ func (c *dbCache) CloseRepository(ctx context.Context, repositorySpec *configapi return err } } + +func (c *dbCache) GetRepositories(ctx context.Context) []configapi.Repository { + _, span := tracer.Start(ctx, "dbCache::GetRepositories", trace.WithAttributes()) + defer span.End() + + return []configapi.Repository{} +} diff --git a/pkg/cache/dbcache/dbcachefactory.go b/pkg/cache/dbcache/dbcachefactory.go new file mode 100644 index 00000000..fadd953f --- /dev/null +++ b/pkg/cache/dbcache/dbcachefactory.go @@ -0,0 +1,37 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbcache + +import ( + "context" + + cachetypes "github.com/nephio-project/porch/pkg/cache/types" +) + +var _ cachetypes.CacheFactory = &DbCacheFactory{} + +type DbCacheFactory struct { +} + +func (f *DbCacheFactory) NewCache(_ context.Context, options cachetypes.CacheOptions) (cachetypes.Cache, error) { + + if err := OpenDBConnection(options); err != nil { + return nil, err + } + + return &dbCache{ + options: options, + }, nil +} diff --git a/pkg/cache/dbcache/dbconnection.go b/pkg/cache/dbcache/dbconnection.go index a038ea40..11a5d344 100644 --- a/pkg/cache/dbcache/dbconnection.go +++ b/pkg/cache/dbcache/dbconnection.go @@ -20,6 +20,7 @@ import ( "k8s.io/klog/v2" _ "github.com/jackc/pgx/v5/stdlib" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" ) // TODO: Add connection pooling @@ -36,7 +37,7 @@ type DBConnection struct { var dbConnection *DBConnection = nil -func OpenDBConnection(opts CacheOptions) error { +func OpenDBConnection(opts cachetypes.CacheOptions) error { klog.Infof("DBConnection: %s %s", opts.Driver, opts.DataSource) if dbConnection != nil { diff --git a/pkg/cache/memory/cache.go b/pkg/cache/memory/cache.go deleted file mode 100644 index df3c5cc0..00000000 --- a/pkg/cache/memory/cache.go +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2022, 2024 The kpt and Nephio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "context" - "errors" - "fmt" - "path/filepath" - "sync" - "time" - - kptoci "github.com/GoogleContainerTools/kpt/pkg/oci" - configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/cache" - "github.com/nephio-project/porch/pkg/git" - "github.com/nephio-project/porch/pkg/meta" - "github.com/nephio-project/porch/pkg/oci" - "github.com/nephio-project/porch/pkg/repository" - "go.opentelemetry.io/otel/trace" - "k8s.io/apimachinery/pkg/watch" -) - -// Cache allows us to keep state for repositories, rather than querying them every time. -// -// Cache Structure: -// /git/ -// * Caches bare git repositories in directories named based on the repository address. -// /oci/ -// * Caches oci images with further hierarchy underneath -// * We Cache image layers in /oci/layers/ (this might be obsolete with the flattened Cache) -// * We Cache flattened tar files in /oci/ (so we don't need to pull to read resources) -// * We poll the repositories (every minute) and Cache the discovered images in memory. -type Cache struct { - mutex sync.Mutex - repositories map[string]*cachedRepository - cacheDir string - credentialResolver repository.CredentialResolver - userInfoProvider repository.UserInfoProvider - metadataStore meta.MetadataStore - repoSyncFrequency time.Duration - objectNotifier objectNotifier - useGitCaBundle bool -} - -var _ cache.Cache = &Cache{} - -type objectNotifier interface { - NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision) int -} - -type CacheOptions struct { - CredentialResolver repository.CredentialResolver - UserInfoProvider repository.UserInfoProvider - MetadataStore meta.MetadataStore - ObjectNotifier objectNotifier -} - -func NewCache(cacheDir string, repoSyncFrequency time.Duration, useGitCaBundle bool, opts CacheOptions) *Cache { - return &Cache{ - repositories: make(map[string]*cachedRepository), - cacheDir: cacheDir, - credentialResolver: opts.CredentialResolver, - userInfoProvider: opts.UserInfoProvider, - metadataStore: opts.MetadataStore, - objectNotifier: opts.ObjectNotifier, - repoSyncFrequency: repoSyncFrequency, - useGitCaBundle: useGitCaBundle, - } -} - -func getCacheKey(repositorySpec *configapi.Repository) (string, error) { - switch repositoryType := repositorySpec.Spec.Type; repositoryType { - case configapi.RepositoryTypeOCI: - ociSpec := repositorySpec.Spec.Oci - if ociSpec == nil { - return "", fmt.Errorf("oci not configured") - } - return "oci://" + ociSpec.Registry, nil - - case configapi.RepositoryTypeGit: - gitSpec := repositorySpec.Spec.Git - if gitSpec == nil { - return "", errors.New("git property is required") - } - if gitSpec.Repo == "" { - return "", errors.New("git.repo property is required") - } - return fmt.Sprintf("git://%s/%s@%s/%s", gitSpec.Repo, gitSpec.Directory, repositorySpec.Namespace, repositorySpec.Name), nil - - default: - return "", fmt.Errorf("repository type %q not supported", repositoryType) - } -} - -func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (repository.Repository, error) { - ctx, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) - defer span.End() - - key, err := getCacheKey(repositorySpec) - if err != nil { - return nil, err - } - c.mutex.Lock() - defer c.mutex.Unlock() - cachedRepo := c.repositories[key] - - switch repositoryType := repositorySpec.Spec.Type; repositoryType { - case configapi.RepositoryTypeOCI: - ociSpec := repositorySpec.Spec.Oci - if cachedRepo == nil { - cacheDir := filepath.Join(c.cacheDir, "oci") - storage, err := kptoci.NewStorage(cacheDir) - if err != nil { - return nil, err - } - - r, err := oci.OpenRepository(repositorySpec.Name, repositorySpec.Namespace, ociSpec, repositorySpec.Spec.Deployment, storage) - if err != nil { - return nil, err - } - cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) - c.repositories[key] = cachedRepo - } - return cachedRepo, nil - - case configapi.RepositoryTypeGit: - gitSpec := repositorySpec.Spec.Git - if cachedRepo == nil { - var mbs git.MainBranchStrategy - if gitSpec.CreateBranch { - mbs = git.CreateIfMissing - } else { - mbs = git.ErrorIfMissing - } - - r, err := git.OpenRepository(ctx, repositorySpec.Name, repositorySpec.Namespace, gitSpec, repositorySpec.Spec.Deployment, filepath.Join(c.cacheDir, "git"), git.GitRepositoryOptions{ - CredentialResolver: c.credentialResolver, - UserInfoProvider: c.userInfoProvider, - MainBranchStrategy: mbs, - UseGitCaBundle: c.useGitCaBundle, - }) - if err != nil { - return nil, err - } - - cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) - c.repositories[key] = cachedRepo - } else { - // If there is an error from the background refresh goroutine, return it. - if err := cachedRepo.getRefreshError(); err != nil { - return nil, err - } - } - return cachedRepo, nil - - default: - return nil, fmt.Errorf("type %q not supported", repositoryType) - } -} - -func (c *Cache) UpdateRepository(ctx context.Context, repositorySpec *configapi.Repository) error { - return errors.New("update on memory cached repositories is not supported") -} - -func (c *Cache) CloseRepository(ctx context.Context, repositorySpec *configapi.Repository, allRepos []configapi.Repository) error { - _, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) - defer span.End() - - key, err := getCacheKey(repositorySpec) - if err != nil { - return err - } - - // check if repositorySpec shares the underlying cached repo with another repository - for _, r := range allRepos { - if r.Name == repositorySpec.Name && r.Namespace == repositorySpec.Namespace { - continue - } - otherKey, err := getCacheKey(&r) - if err != nil { - return err - } - if otherKey == key { - // do not close cached repo if it is shared - return nil - } - } - - var repository *cachedRepository - { - c.mutex.Lock() - if r, ok := c.repositories[key]; ok { - delete(c.repositories, key) - repository = r - } - c.mutex.Unlock() - } - - if repository != nil { - return repository.Close() - } else { - return nil - } -} diff --git a/pkg/cache/memorycache/cache.go b/pkg/cache/memorycache/cache.go new file mode 100644 index 00000000..c8edb943 --- /dev/null +++ b/pkg/cache/memorycache/cache.go @@ -0,0 +1,134 @@ +// Copyright 2022, 2024 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memorycache + +import ( + "context" + "errors" + "sync" + + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" + "github.com/nephio-project/porch/pkg/repoimpl" + "github.com/nephio-project/porch/pkg/repository" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +var tracer = otel.Tracer("memorycache") + +// Cache allows us to keep state for repositories, rather than querying them every time. +// +// Cache Structure: +// /git/ +// * Caches bare git repositories in directories named based on the repository address. +// /oci/ +// * Caches oci images with further hierarchy underneath +// * We Cache image layers in /oci/layers/ (this might be obsolete with the flattened Cache) +// * We Cache flattened tar files in /oci/ (so we don't need to pull to read resources) +// * We poll the repositories (every minute) and Cache the discovered images in memory. +type Cache struct { + mutex sync.Mutex + repositories map[string]*cachedRepository + options cachetypes.CacheOptions +} + +var _ cachetypes.Cache = &Cache{} + +func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (repository.Repository, error) { + ctx, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) + defer span.End() + + key, err := repoimpl.RepositoryKey(repositorySpec) + if err != nil { + return nil, err + } + + c.mutex.Lock() + defer c.mutex.Unlock() + + if cachedRepo := c.repositories[key]; cachedRepo != nil { + // If there is an error from the background refresh goroutine, return it. + if err := cachedRepo.getRefreshError(); err == nil { + return cachedRepo, nil + } else { + return nil, err + } + } + + repoImpl, err := repoimpl.CreateRepositoryImpl(ctx, repositorySpec, c.options.RepoImplOptions) + if err != nil { + return nil, err + } + + cachedRepo := newRepository(key, repositorySpec, repoImpl, c.options) + c.repositories[key] = cachedRepo + + return cachedRepo, nil +} + +func (c *Cache) UpdateRepository(ctx context.Context, repositorySpec *configapi.Repository) error { + return errors.New("update on memory cached repositories is not supported") +} + +func (c *Cache) CloseRepository(ctx context.Context, repositorySpec *configapi.Repository, allRepos []configapi.Repository) error { + _, span := tracer.Start(ctx, "Cache::CloseRepository", trace.WithAttributes()) + defer span.End() + + key, err := repoimpl.RepositoryKey(repositorySpec) + if err != nil { + return err + } + + // check if repositorySpec shares the underlying cached repo with another repository + for _, r := range allRepos { + if r.Name == repositorySpec.Name && r.Namespace == repositorySpec.Namespace { + continue + } + otherKey, err := repoimpl.RepositoryKey(&r) + if err != nil { + return err + } + if otherKey == key { + // do not close cached repo if it is shared + return nil + } + } + + var repository *cachedRepository + { + c.mutex.Lock() + if r, ok := c.repositories[key]; ok { + delete(c.repositories, key) + repository = r + } + c.mutex.Unlock() + } + + if repository != nil { + return repository.Close() + } else { + return nil + } +} + +func (c *Cache) GetRepositories(ctx context.Context) []configapi.Repository { + repoSlice := []configapi.Repository{} + + for _, repo := range c.repositories { + repoSlice = append(repoSlice, *repo.repoSpec) + } + return repoSlice +} diff --git a/pkg/cache/memory/cache_test.go b/pkg/cache/memorycache/cache_test.go similarity index 91% rename from pkg/cache/memory/cache_test.go rename to pkg/cache/memorycache/cache_test.go index c2523d8d..5b7f77bf 100644 --- a/pkg/cache/memory/cache_test.go +++ b/pkg/cache/memorycache/cache_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package memorycache import ( "context" @@ -27,9 +27,11 @@ import ( "github.com/nephio-project/porch/api/porchconfig/v1alpha1" fakecache "github.com/nephio-project/porch/pkg/cache/fake" - "github.com/nephio-project/porch/pkg/git" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/meta" fakemeta "github.com/nephio-project/porch/pkg/meta/fake" + "github.com/nephio-project/porch/pkg/repoimpl/git" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" "github.com/nephio-project/porch/pkg/repository" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -37,7 +39,7 @@ import ( func TestLatestPackages(t *testing.T) { ctx := context.Background() - testPath := filepath.Join("..", "..", "git", "testdata") + testPath := filepath.Join("..", "..", "repoimpl", "git", "testdata") cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") @@ -83,7 +85,7 @@ func TestLatestPackages(t *testing.T) { func TestPublishedLatest(t *testing.T) { ctx := context.Background() - testPath := filepath.Join("..", "..", "git", "testdata") + testPath := filepath.Join("..", "..", "repoimpl", "git", "testdata") cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ @@ -129,7 +131,7 @@ func TestPublishedLatest(t *testing.T) { func TestDeletePublishedMain(t *testing.T) { ctx := context.Background() - testPath := filepath.Join("../..", "git", "testdata") + testPath := filepath.Join("../..", "repoimpl", "git", "testdata") cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ @@ -223,10 +225,15 @@ func openRepositoryFromArchive(t *testing.T, ctx context.Context, testPath, name _, address := git.ServeGitRepository(t, tarfile, tempdir) metadataStore := createMetadataStoreFromArchive(t, fmt.Sprintf("%s-metadata.yaml", name), name) - cache := NewCache(t.TempDir(), 60*time.Second, true, CacheOptions{ - MetadataStore: metadataStore, - ObjectNotifier: &fakecache.ObjectNotifier{}, - CredentialResolver: &fakecache.CredentialResolver{}, + cache, _ := new(MemoryCacheFactory).NewCache(ctx, cachetypes.CacheOptions{ + RepoImplOptions: repoimpltypes.RepoImplOptions{ + LocalDirectory: t.TempDir(), + UseUserDefinedCaBundle: true, + CredentialResolver: &fakecache.CredentialResolver{}, + }, + RepoSyncFrequency: 60 * time.Second, + MetadataStore: metadataStore, + RepoPRChangeNotifier: &fakecache.ObjectNotifier{}, }) apiRepo := &v1alpha1.Repository{ TypeMeta: metav1.TypeMeta{ @@ -255,7 +262,7 @@ func openRepositoryFromArchive(t *testing.T, ctx context.Context, testPath, name if err != nil { t.Errorf("CloseRepository(%q) failed: %v", address, err) } - if len(cache.repositories) != 0 { + if len(cache.GetRepositories(ctx)) != 0 { t.Errorf("CloseRepository hasn't deleted repository from cache") } }) diff --git a/pkg/cache/memorycache/memorycachefactory.go b/pkg/cache/memorycache/memorycachefactory.go new file mode 100644 index 00000000..7e45813d --- /dev/null +++ b/pkg/cache/memorycache/memorycachefactory.go @@ -0,0 +1,33 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memorycache + +import ( + "context" + + cachetypes "github.com/nephio-project/porch/pkg/cache/types" +) + +var _ cachetypes.CacheFactory = &MemoryCacheFactory{} + +type MemoryCacheFactory struct { +} + +func (f *MemoryCacheFactory) NewCache(_ context.Context, options cachetypes.CacheOptions) (cachetypes.Cache, error) { + return &Cache{ + repositories: make(map[string]*cachedRepository), + options: options, + }, nil +} diff --git a/pkg/cache/memory/package.go b/pkg/cache/memorycache/package.go similarity index 98% rename from pkg/cache/memory/package.go rename to pkg/cache/memorycache/package.go index d02093f2..929937da 100644 --- a/pkg/cache/memory/package.go +++ b/pkg/cache/memorycache/package.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package memorycache import ( "github.com/nephio-project/porch/pkg/repository" diff --git a/pkg/cache/memory/packagerevision.go b/pkg/cache/memorycache/packagerevision.go similarity index 99% rename from pkg/cache/memory/packagerevision.go rename to pkg/cache/memorycache/packagerevision.go index f12f8d62..79371bf2 100644 --- a/pkg/cache/memory/packagerevision.go +++ b/pkg/cache/memorycache/packagerevision.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package memorycache import ( "context" diff --git a/pkg/cache/memory/repository.go b/pkg/cache/memorycache/repository.go similarity index 91% rename from pkg/cache/memory/repository.go rename to pkg/cache/memorycache/repository.go index add34983..969b8a90 100644 --- a/pkg/cache/memory/repository.go +++ b/pkg/cache/memorycache/repository.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package memorycache import ( "context" @@ -22,10 +22,8 @@ import ( "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/git" - "github.com/nephio-project/porch/pkg/meta" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/repository" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,8 +32,6 @@ import ( "k8s.io/klog/v2" ) -var tracer = otel.Tracer("cache") - // We take advantage of the cache having a global view of all the packages // in a repository and compute the latest package revision in the cache // rather than add another level of caching in the repositories themselves. @@ -61,25 +57,22 @@ type cachedRepository struct { // This is returned back by the cache to the background goroutine when it calls periodicall to resync repositories. refreshRevisionsError error - objectNotifier objectNotifier - - metadataStore meta.MetadataStore + options cachetypes.CacheOptions } -func newRepository(id string, repoSpec *configapi.Repository, repo repository.Repository, objectNotifier objectNotifier, metadataStore meta.MetadataStore, repoSyncFrequency time.Duration) *cachedRepository { +func newRepository(id string, repoSpec *configapi.Repository, repo repository.Repository, options cachetypes.CacheOptions) *cachedRepository { ctx, cancel := context.WithCancel(context.Background()) r := &cachedRepository{ - id: id, - repoSpec: repoSpec, - repo: repo, - cancel: cancel, - objectNotifier: objectNotifier, - metadataStore: metadataStore, + id: id, + repoSpec: repoSpec, + repo: repo, + cancel: cancel, + options: options, } // TODO: Should we fetch the packages here? - go r.pollForever(ctx, repoSyncFrequency) + go r.pollForever(ctx, options.RepoSyncFrequency) return r } @@ -150,12 +143,8 @@ func (r *cachedRepository) getCachedPackages(ctx context.Context, forceRefresh b packages = nil packageRevisions = nil - if gitRepo, isGitRepo := r.repo.(git.GitRepository); isGitRepo { - // TODO: Figure out a way to do this without the cache layer - // needing to know what type of repo we are working with. - if err := gitRepo.UpdateDeletionProposedCache(); err != nil { - return nil, nil, err - } + if err := r.repo.Refresh(ctx); err != nil { + return nil, nil, err } } r.mutex.Unlock() @@ -289,13 +278,13 @@ func (r *cachedRepository) createMainPackageRevision(ctx context.Context, update } // Create the package if it doesn't exist - _, err := r.metadataStore.Get(ctx, pkgRevMetaNN) + _, err := r.options.MetadataStore.Get(ctx, pkgRevMetaNN) if errors.IsNotFound(err) { pkgRevMeta := metav1.ObjectMeta{ Name: updatedMain.KubeObjectName(), Namespace: updatedMain.KubeObjectNamespace(), } - _, err := r.metadataStore.Create(ctx, pkgRevMeta, r.repoSpec.Name, updatedMain.UID()) + _, err := r.options.MetadataStore.Create(ctx, pkgRevMeta, r.repoSpec.Name, updatedMain.UID()) if err != nil { klog.Warningf("unable to create PackageRev CR for %s/%s: %v", updatedMain.KubeObjectNamespace(), updatedMain.KubeObjectName(), err) @@ -375,7 +364,7 @@ func (r *cachedRepository) Close() error { // the repository, so we have to just delete the PackageRevision regardless of any // finalizers. klog.Infof("repo %s: deleting packagerev %s/%s because repository is closed", r.id, nn.Namespace, nn.Name) - _, err := r.metadataStore.Delete(context.TODO(), nn, true) + _, err := r.options.MetadataStore.Delete(context.TODO(), nn, true) if err != nil { // There isn't much use in returning an error here, so we just log it // and create a PackageRevisionMeta with just name and namespace. This @@ -383,7 +372,7 @@ func (r *cachedRepository) Close() error { klog.Warningf("repo %s: error deleting packagerev for %s: %v", r.id, nn.Name, err) } klog.Infof("repo %s: successfully deleted packagerev %s/%s", r.id, nn.Namespace, nn.Name) - sent += r.objectNotifier.NotifyPackageRevisionChange(watch.Deleted, pr) + sent += r.options.RepoPRChangeNotifier.NotifyPackageRevisionChange(watch.Deleted, pr) } klog.Infof("repo %s: sent %d notifications for %d package revisions during close", r.id, sent, len(r.cachedPackageRevisions)) return r.repo.Close() @@ -452,7 +441,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re // Look up all existing PackageRevCRs so we an compare those to the // actual Packagerevisions found in git/oci, and add/prune PackageRevCRs // as necessary. - existingPkgRevCRs, err := r.metadataStore.List(ctx, r.repoSpec) + existingPkgRevCRs, err := r.options.MetadataStore.List(ctx, r.repoSpec) if err != nil { return nil, nil, err } @@ -497,7 +486,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re if _, found := newPackageRevisionNames[prm.Name]; !found { klog.Infof("repo %s: deleting PackageRev %s/%s because parent PackageRevision was not found", r.id, prm.Namespace, prm.Name) - if _, err := r.metadataStore.Delete(ctx, types.NamespacedName{ + if _, err := r.options.MetadataStore.Delete(ctx, types.NamespacedName{ Name: prm.Name, Namespace: prm.Namespace, }, true); err != nil { @@ -516,10 +505,10 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re for kname, newPackage := range newPackageRevisionNames { oldPackage := oldPackageRevisionNames[kname] if oldPackage == nil { - addSent += r.objectNotifier.NotifyPackageRevisionChange(watch.Added, newPackage) + addSent += r.options.RepoPRChangeNotifier.NotifyPackageRevisionChange(watch.Added, newPackage) } else { if oldPackage.ResourceVersion() != newPackage.ResourceVersion() { - modSent += r.objectNotifier.NotifyPackageRevisionChange(watch.Modified, newPackage) + modSent += r.options.RepoPRChangeNotifier.NotifyPackageRevisionChange(watch.Modified, newPackage) } } } @@ -532,7 +521,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re Name: pkgRevName, Namespace: r.repoSpec.Namespace, } - if _, err := r.metadataStore.Create(ctx, pkgRevMeta, r.repoSpec.Name, pkgRev.UID()); err != nil { + if _, err := r.options.MetadataStore.Create(ctx, pkgRevMeta, r.repoSpec.Name, pkgRev.UID()); err != nil { // TODO: We should try to find a way to make these errors available through // either the repository CR or the PackageRevision CR. This will be // retried on the next sync. @@ -552,7 +541,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re } klog.Infof("repo %s: deleting PackageRev %s/%s because PackageRevision was removed from SoT", r.id, nn.Namespace, nn.Name) - delSent += r.objectNotifier.NotifyPackageRevisionChange(watch.Deleted, oldPackage) + delSent += r.options.RepoPRChangeNotifier.NotifyPackageRevisionChange(watch.Deleted, oldPackage) } } klog.Infof("repo %s: addSent %d, modSent %d, delSent for %d old and %d new repo packages", r.id, addSent, modSent, len(oldPackageRevisionNames), len(newPackageRevisionNames)) diff --git a/pkg/cache/memory/util.go b/pkg/cache/memorycache/util.go similarity index 99% rename from pkg/cache/memory/util.go rename to pkg/cache/memorycache/util.go index c69d402b..8d9e6acb 100644 --- a/pkg/cache/memory/util.go +++ b/pkg/cache/memorycache/util.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package memory +package memorycache import ( "sort" diff --git a/pkg/cache/types/cachetypes.go b/pkg/cache/types/cachetypes.go new file mode 100644 index 00000000..e824cca9 --- /dev/null +++ b/pkg/cache/types/cachetypes.go @@ -0,0 +1,50 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachetypes + +import ( + "context" + "time" + + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/meta" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" + "github.com/nephio-project/porch/pkg/repository" + "k8s.io/apimachinery/pkg/watch" +) + +type CacheOptions struct { + RepoImplOptions repoimpltypes.RepoImplOptions + RepoSyncFrequency time.Duration + MetadataStore meta.MetadataStore + RepoPRChangeNotifier RepoPRChangeNotifier + Driver string + DataSource string +} + +type Cache interface { + OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (repository.Repository, error) + CloseRepository(ctx context.Context, repositorySpec *configapi.Repository, allRepos []configapi.Repository) error + GetRepositories(ctx context.Context) []configapi.Repository + UpdateRepository(ctx context.Context, repositorySpec *configapi.Repository) error +} + +type CacheFactory interface { + NewCache(ctx context.Context, options CacheOptions) (Cache, error) +} + +type RepoPRChangeNotifier interface { + NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision) int +} diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 27ec82b8..e02f77dc 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -57,7 +57,7 @@ type PorchServerOptions struct { FunctionRunnerAddress string DefaultImagePrefix string RepoSyncFrequency time.Duration - UseGitCaBundle bool + UseUserDefinedCaBundle bool DisableValidatingAdmissionPolicy bool MaxRequestBodySize int @@ -195,13 +195,13 @@ func (o *PorchServerOptions) Config() (*apiserver.Config, error) { config := &apiserver.Config{ GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ - CoreAPIKubeconfigPath: o.CoreAPIKubeconfigPath, - CacheDirectory: o.CacheDirectory, - RepoSyncFrequency: o.RepoSyncFrequency, - FunctionRunnerAddress: o.FunctionRunnerAddress, - DefaultImagePrefix: o.DefaultImagePrefix, - UseGitCaBundle: o.UseGitCaBundle, - MaxGrpcMessageSize: o.MaxRequestBodySize, + CoreAPIKubeconfigPath: o.CoreAPIKubeconfigPath, + CacheDirectory: o.CacheDirectory, + RepoSyncFrequency: o.RepoSyncFrequency, + FunctionRunnerAddress: o.FunctionRunnerAddress, + DefaultImagePrefix: o.DefaultImagePrefix, + UseUserDefinedCaBundle: o.UseUserDefinedCaBundle, + MaxGrpcMessageSize: o.MaxRequestBodySize, }, } return config, nil @@ -248,7 +248,7 @@ func (o *PorchServerOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.DefaultImagePrefix, "default-image-prefix", fnruntime.GCRImagePrefix, "Default prefix for unqualified function names") fs.StringVar(&o.CacheDirectory, "cache-directory", "", "Directory where Porch server stores repository and package caches.") fs.IntVar(&o.MaxRequestBodySize, "max-request-body-size", 6*1024*1024, "Maximum size of the request body in bytes. Keep this in sync with function-runner's corresponding argument.") - fs.BoolVar(&o.UseGitCaBundle, "use-git-cabundle", false, "Determine whether to use a user-defined CaBundle for TLS towards git.") + fs.BoolVar(&o.UseUserDefinedCaBundle, "use-user-cabundle", false, "Determine whether to use a user-defined CaBundle for TLS towards the repository system.") fs.BoolVar(&o.DisableValidatingAdmissionPolicy, "disable-validating-admissions-policy", true, "Determine whether to (dis|en)able the Validating Admission Policy, which requires k8s version >= v1.30") fs.DurationVar(&o.RepoSyncFrequency, "repo-sync-frequency", 10*time.Minute, "Frequency in seconds at which registered repositories will be synced and the background job repository refresh runs.") } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index c80eb5e6..07e62d96 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -23,7 +23,7 @@ import ( api "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - cache "github.com/nephio-project/porch/pkg/cache" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/meta" "github.com/nephio-project/porch/pkg/repository" "github.com/nephio-project/porch/pkg/task" @@ -73,7 +73,7 @@ func NewCaDEngine(opts ...EngineOption) (CaDEngine, error) { } type cadEngine struct { - cache cache.Cache + cache cachetypes.Cache userInfoProvider repository.UserInfoProvider metadataStore meta.MetadataStore diff --git a/pkg/engine/options.go b/pkg/engine/options.go index 21388edb..d5406924 100644 --- a/pkg/engine/options.go +++ b/pkg/engine/options.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/nephio-project/porch/internal/kpt/fnruntime" - "github.com/nephio-project/porch/pkg/cache" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "github.com/nephio-project/porch/pkg/kpt" "github.com/nephio-project/porch/pkg/kpt/fn" "github.com/nephio-project/porch/pkg/meta" @@ -38,7 +38,7 @@ func (f EngineOptionFunc) apply(engine *cadEngine) error { return f(engine) } -func WithCache(cache cache.Cache) EngineOption { +func WithCache(cache cachetypes.Cache) EngineOption { return EngineOptionFunc(func(engine *cadEngine) error { engine.cache = cache return nil diff --git a/pkg/registry/porch/background.go b/pkg/registry/porch/background.go index 3efd4ffd..01c45893 100644 --- a/pkg/registry/porch/background.go +++ b/pkg/registry/porch/background.go @@ -20,7 +20,7 @@ import ( "time" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/cache" + cachetypes "github.com/nephio-project/porch/pkg/cache/types" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func RunBackground(ctx context.Context, coreClient client.WithWatch, cache cache.Cache, PeriodicRepoSyncFrequency time.Duration) { +func RunBackground(ctx context.Context, coreClient client.WithWatch, cache cachetypes.Cache, PeriodicRepoSyncFrequency time.Duration) { b := background{ coreClient: coreClient, cache: cache, @@ -40,7 +40,7 @@ func RunBackground(ctx context.Context, coreClient client.WithWatch, cache cache // background manages background tasks type background struct { coreClient client.WithWatch - cache cache.Cache + cache cachetypes.Cache PeriodicRepoSyncFrequency time.Duration } diff --git a/pkg/registry/porch/watch_test.go b/pkg/registry/porch/watch_test.go index fa21fa95..349872c5 100644 --- a/pkg/registry/porch/watch_test.go +++ b/pkg/registry/porch/watch_test.go @@ -22,8 +22,8 @@ import ( "github.com/nephio-project/porch/api/porch/v1alpha1" "github.com/nephio-project/porch/pkg/engine" + "github.com/nephio-project/porch/pkg/repoimpl/fake" "github.com/nephio-project/porch/pkg/repository" - "github.com/nephio-project/porch/pkg/repository/fake" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/repository/fake/packagerevision.go b/pkg/repoimpl/fake/packagerevision.go similarity index 100% rename from pkg/repository/fake/packagerevision.go rename to pkg/repoimpl/fake/packagerevision.go diff --git a/pkg/repository/fake/repository.go b/pkg/repoimpl/fake/repository.go similarity index 98% rename from pkg/repository/fake/repository.go rename to pkg/repoimpl/fake/repository.go index 864d5e89..37865564 100644 --- a/pkg/repository/fake/repository.go +++ b/pkg/repoimpl/fake/repository.go @@ -88,3 +88,7 @@ func (r *Repository) DeletePackage(_ context.Context, pr repository.Package) err func (r *Repository) Refresh(_ context.Context) error { return nil } + +func (r *Repository) Key() string { + return "" +} diff --git a/pkg/git/annotation.go b/pkg/repoimpl/git/annotation.go similarity index 100% rename from pkg/git/annotation.go rename to pkg/repoimpl/git/annotation.go diff --git a/pkg/git/commit.go b/pkg/repoimpl/git/commit.go similarity index 100% rename from pkg/git/commit.go rename to pkg/repoimpl/git/commit.go diff --git a/pkg/git/commit_test.go b/pkg/repoimpl/git/commit_test.go similarity index 100% rename from pkg/git/commit_test.go rename to pkg/repoimpl/git/commit_test.go diff --git a/pkg/git/dir.go b/pkg/repoimpl/git/dir.go similarity index 100% rename from pkg/git/dir.go rename to pkg/repoimpl/git/dir.go diff --git a/pkg/git/dir_test.go b/pkg/repoimpl/git/dir_test.go similarity index 100% rename from pkg/git/dir_test.go rename to pkg/repoimpl/git/dir_test.go diff --git a/pkg/git/doc.go b/pkg/repoimpl/git/doc.go similarity index 100% rename from pkg/git/doc.go rename to pkg/repoimpl/git/doc.go diff --git a/pkg/git/draft.go b/pkg/repoimpl/git/draft.go similarity index 100% rename from pkg/git/draft.go rename to pkg/repoimpl/git/draft.go diff --git a/pkg/git/git.go b/pkg/repoimpl/git/git.go similarity index 98% rename from pkg/git/git.go rename to pkg/repoimpl/git/git.go index 2454370c..2cbe499b 100644 --- a/pkg/git/git.go +++ b/pkg/repoimpl/git/git.go @@ -39,6 +39,7 @@ import ( "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" kptfilev1 "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" "github.com/nephio-project/porch/pkg/repository" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -68,10 +69,8 @@ const ( ) type GitRepositoryOptions struct { - CredentialResolver repository.CredentialResolver - UserInfoProvider repository.UserInfoProvider + repoimpltypes.RepoImplOptions MainBranchStrategy MainBranchStrategy - UseGitCaBundle bool } func OpenRepository(ctx context.Context, name, namespace string, spec *configapi.GitRepository, deployment bool, root string, opts GitRepositoryOptions) (GitRepository, error) { @@ -136,14 +135,14 @@ func OpenRepository(ctx context.Context, name, namespace string, spec *configapi branch: branch, directory: strings.Trim(spec.Directory, "/"), secret: spec.SecretRef.Name, - credentialResolver: opts.CredentialResolver, - userInfoProvider: opts.UserInfoProvider, + credentialResolver: opts.RepoImplOptions.CredentialResolver, + userInfoProvider: opts.RepoImplOptions.UserInfoProvider, cacheDir: dir, deployment: deployment, } - if opts.UseGitCaBundle { - if caBundle, err := opts.CredentialResolver.ResolveCredential(ctx, namespace, namespace+"-ca-bundle"); err != nil { + if opts.RepoImplOptions.UseUserDefinedCaBundle { + if caBundle, err := opts.RepoImplOptions.CredentialResolver.ResolveCredential(ctx, namespace, namespace+"-ca-bundle"); err != nil { klog.Errorf("failed to obtain caBundle from secret %s/%s: %v", namespace, namespace+"-ca-bundle", err) } else { repository.caBundle = []byte(caBundle.ToString()) @@ -1229,7 +1228,7 @@ func (r *gitRepository) GetResources(hash plumbing.Hash) (map[string]string, err // findLatestPackageCommit returns the latest commit from the history that pertains // to the package given by the packagePath. If no commit is found, it will return nil. -func (r *gitRepository) findLatestPackageCommit(ctx context.Context, startCommit *object.Commit, packagePath string) (*object.Commit, error) { +func (r *gitRepository) findLatestPackageCommit(_ context.Context, startCommit *object.Commit, packagePath string) (*object.Commit, error) { var commit *object.Commit err := r.packageHistoryIterator(startCommit, packagePath, func(c *object.Commit) error { commit = c @@ -1338,7 +1337,7 @@ func (r *gitRepository) GetLifecycle(ctx context.Context, pkgRev *gitPackageRevi return r.getLifecycle(ctx, pkgRev) } -func (r *gitRepository) getLifecycle(ctx context.Context, pkgRev *gitPackageRevision) v1alpha1.PackageRevisionLifecycle { +func (r *gitRepository) getLifecycle(_ context.Context, pkgRev *gitPackageRevision) v1alpha1.PackageRevisionLifecycle { switch ref := pkgRev.ref; { case ref == nil: return r.checkPublishedLifecycle(pkgRev) @@ -1464,7 +1463,7 @@ func (r *gitRepository) UpdateDraftResources(ctx context.Context, draft *gitPack } func (r *gitRepository) ClosePackageRevisionDraft(ctx context.Context, prd repository.PackageRevisionDraft, version string) (repository.PackageRevision, error) { - ctx, span := tracer.Start(ctx, "GitRepository::ClosePackageRevisionDraft", trace.WithAttributes()) + ctx, span := tracer.Start(ctx, "GitRepository::UpdateLifecycle", trace.WithAttributes()) defer span.End() r.mutex.Lock() @@ -1698,7 +1697,11 @@ func (r *gitRepository) discoverPackagesInTree(commit *object.Commit, opt Discov } func (r *gitRepository) Refresh(_ context.Context) error { - return nil + return r.UpdateDeletionProposedCache() +} + +func (r *gitRepository) Key() string { + return fmt.Sprintf("git://%s/%s@%s/%s", r.repo, r.directory, r.namespace, r.name) } // See https://eli.thegreenplace.net/2021/generic-functions-on-slices-with-go-type-parameters/ diff --git a/pkg/git/git_test.go b/pkg/repoimpl/git/git_test.go similarity index 99% rename from pkg/git/git_test.go rename to pkg/repoimpl/git/git_test.go index 7e287a28..afdefb71 100644 --- a/pkg/git/git_test.go +++ b/pkg/repoimpl/git/git_test.go @@ -691,8 +691,6 @@ func (g GitSuite) TestApproveDraft(t *testing.T) { t.Fatalf("UpdateLifecycle failed: %v", err) } - update.UpdateLifecycle(ctx, v1alpha1.PackageRevisionLifecyclePublished) - new, err := git.ClosePackageRevisionDraft(ctx, update, "v1") if err != nil { t.Fatalf("Close failed: %v", err) @@ -757,8 +755,6 @@ func (g GitSuite) TestApproveDraftWithHistory(t *testing.T) { t.Fatalf("UpdateLifecycle failed: %v", err) } - update.UpdateLifecycle(ctx, v1alpha1.PackageRevisionLifecyclePublished) - new, err := git.ClosePackageRevisionDraft(ctx, update, "v1") if err != nil { t.Fatalf("Close failed: %v", err) diff --git a/pkg/repoimpl/git/gitrepofactory.go b/pkg/repoimpl/git/gitrepofactory.go new file mode 100644 index 00000000..7aecf0ca --- /dev/null +++ b/pkg/repoimpl/git/gitrepofactory.go @@ -0,0 +1,56 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "context" + "errors" + "path/filepath" + + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" + "github.com/nephio-project/porch/pkg/repository" +) + +var _ repoimpltypes.RepoImplFactory = &GitRepoFactory{} + +type GitRepoFactory struct { +} + +func (f *GitRepoFactory) NewRepositoryImpl(ctx context.Context, repositorySpec *configapi.Repository, options repoimpltypes.RepoImplOptions) (repository.Repository, error) { + if repositorySpec.Spec.Git == nil { + return nil, errors.New("git property is required") + } + if repositorySpec.Spec.Git.Repo == "" { + return nil, errors.New("git.repo property is required") + } + + var mbs MainBranchStrategy + if repositorySpec.Spec.Git.CreateBranch { + mbs = CreateIfMissing + } else { + mbs = ErrorIfMissing + } + + repo, err := OpenRepository(ctx, repositorySpec.Name, repositorySpec.Namespace, repositorySpec.Spec.Git, repositorySpec.Spec.Deployment, filepath.Join(options.LocalDirectory, "git"), GitRepositoryOptions{ + RepoImplOptions: options, + MainBranchStrategy: mbs, + }) + if err != nil { + return nil, err + } + + return repo, nil +} diff --git a/pkg/git/gogit.go b/pkg/repoimpl/git/gogit.go similarity index 100% rename from pkg/git/gogit.go rename to pkg/repoimpl/git/gogit.go diff --git a/pkg/git/mainbranchstrategy_string.go b/pkg/repoimpl/git/mainbranchstrategy_string.go similarity index 100% rename from pkg/git/mainbranchstrategy_string.go rename to pkg/repoimpl/git/mainbranchstrategy_string.go diff --git a/pkg/git/package.go b/pkg/repoimpl/git/package.go similarity index 100% rename from pkg/git/package.go rename to pkg/repoimpl/git/package.go diff --git a/pkg/git/package_test.go b/pkg/repoimpl/git/package_test.go similarity index 100% rename from pkg/git/package_test.go rename to pkg/repoimpl/git/package_test.go diff --git a/pkg/git/package_tree.go b/pkg/repoimpl/git/package_tree.go similarity index 100% rename from pkg/git/package_tree.go rename to pkg/repoimpl/git/package_tree.go diff --git a/pkg/git/primitives_test.go b/pkg/repoimpl/git/primitives_test.go similarity index 99% rename from pkg/git/primitives_test.go rename to pkg/repoimpl/git/primitives_test.go index c33e8355..d2140fe1 100644 --- a/pkg/git/primitives_test.go +++ b/pkg/repoimpl/git/primitives_test.go @@ -438,7 +438,7 @@ func initRepositoryWithRemote(t *testing.T, dir, address string) *git.Repository return repo } -func createTestCommit(t *testing.T, repo *git.Repository, parent plumbing.Hash, message, name, contents string) plumbing.Hash { +func createTestCommit(t *testing.T, repo *git.Repository, parent plumbing.Hash, _, name, contents string) plumbing.Hash { wt, err := repo.Worktree() if err != nil { t.Fatalf("Failed getting worktree: %v", err) diff --git a/pkg/git/push.go b/pkg/repoimpl/git/push.go similarity index 100% rename from pkg/git/push.go rename to pkg/repoimpl/git/push.go diff --git a/pkg/git/ref.go b/pkg/repoimpl/git/ref.go similarity index 100% rename from pkg/git/ref.go rename to pkg/repoimpl/git/ref.go diff --git a/pkg/git/ref_test.go b/pkg/repoimpl/git/ref_test.go similarity index 100% rename from pkg/git/ref_test.go rename to pkg/repoimpl/git/ref_test.go diff --git a/pkg/git/repo.go b/pkg/repoimpl/git/repo.go similarity index 100% rename from pkg/git/repo.go rename to pkg/repoimpl/git/repo.go diff --git a/pkg/git/repos.go b/pkg/repoimpl/git/repos.go similarity index 100% rename from pkg/git/repos.go rename to pkg/repoimpl/git/repos.go diff --git a/pkg/git/testdata/.gitignore b/pkg/repoimpl/git/testdata/.gitignore similarity index 100% rename from pkg/git/testdata/.gitignore rename to pkg/repoimpl/git/testdata/.gitignore diff --git a/pkg/git/testdata/Makefile b/pkg/repoimpl/git/testdata/Makefile similarity index 100% rename from pkg/git/testdata/Makefile rename to pkg/repoimpl/git/testdata/Makefile diff --git a/pkg/git/testdata/README.md b/pkg/repoimpl/git/testdata/README.md similarity index 100% rename from pkg/git/testdata/README.md rename to pkg/repoimpl/git/testdata/README.md diff --git a/pkg/git/testdata/drafts-repository.md b/pkg/repoimpl/git/testdata/drafts-repository.md similarity index 100% rename from pkg/git/testdata/drafts-repository.md rename to pkg/repoimpl/git/testdata/drafts-repository.md diff --git a/pkg/git/testdata/drafts-repository.tar b/pkg/repoimpl/git/testdata/drafts-repository.tar similarity index 100% rename from pkg/git/testdata/drafts-repository.tar rename to pkg/repoimpl/git/testdata/drafts-repository.tar diff --git a/pkg/git/testdata/empty-repository.md b/pkg/repoimpl/git/testdata/empty-repository.md similarity index 100% rename from pkg/git/testdata/empty-repository.md rename to pkg/repoimpl/git/testdata/empty-repository.md diff --git a/pkg/git/testdata/empty-repository.tar b/pkg/repoimpl/git/testdata/empty-repository.tar similarity index 100% rename from pkg/git/testdata/empty-repository.tar rename to pkg/repoimpl/git/testdata/empty-repository.tar diff --git a/pkg/git/testdata/nested-repository.md b/pkg/repoimpl/git/testdata/nested-repository.md similarity index 100% rename from pkg/git/testdata/nested-repository.md rename to pkg/repoimpl/git/testdata/nested-repository.md diff --git a/pkg/git/testdata/nested-repository.tar b/pkg/repoimpl/git/testdata/nested-repository.tar similarity index 100% rename from pkg/git/testdata/nested-repository.tar rename to pkg/repoimpl/git/testdata/nested-repository.tar diff --git a/pkg/git/testdata/publishinfo-repository.md b/pkg/repoimpl/git/testdata/publishinfo-repository.md similarity index 100% rename from pkg/git/testdata/publishinfo-repository.md rename to pkg/repoimpl/git/testdata/publishinfo-repository.md diff --git a/pkg/git/testdata/publishinfo-repository.tar b/pkg/repoimpl/git/testdata/publishinfo-repository.tar similarity index 100% rename from pkg/git/testdata/publishinfo-repository.tar rename to pkg/repoimpl/git/testdata/publishinfo-repository.tar diff --git a/pkg/git/testdata/simple-repository.md b/pkg/repoimpl/git/testdata/simple-repository.md similarity index 100% rename from pkg/git/testdata/simple-repository.md rename to pkg/repoimpl/git/testdata/simple-repository.md diff --git a/pkg/git/testdata/simple-repository.tar b/pkg/repoimpl/git/testdata/simple-repository.tar similarity index 100% rename from pkg/git/testdata/simple-repository.tar rename to pkg/repoimpl/git/testdata/simple-repository.tar diff --git a/pkg/git/testdata/trivial-repository.md b/pkg/repoimpl/git/testdata/trivial-repository.md similarity index 100% rename from pkg/git/testdata/trivial-repository.md rename to pkg/repoimpl/git/testdata/trivial-repository.md diff --git a/pkg/git/testdata/trivial-repository.tar b/pkg/repoimpl/git/testdata/trivial-repository.tar similarity index 100% rename from pkg/git/testdata/trivial-repository.tar rename to pkg/repoimpl/git/testdata/trivial-repository.tar diff --git a/pkg/git/testing.go b/pkg/repoimpl/git/testing.go similarity index 100% rename from pkg/git/testing.go rename to pkg/repoimpl/git/testing.go diff --git a/pkg/git/testing_repo.go b/pkg/repoimpl/git/testing_repo.go similarity index 100% rename from pkg/git/testing_repo.go rename to pkg/repoimpl/git/testing_repo.go diff --git a/pkg/oci/doc.go b/pkg/repoimpl/oci/doc.go similarity index 100% rename from pkg/oci/doc.go rename to pkg/repoimpl/oci/doc.go diff --git a/pkg/oci/loader.go b/pkg/repoimpl/oci/loader.go similarity index 100% rename from pkg/oci/loader.go rename to pkg/repoimpl/oci/loader.go diff --git a/pkg/oci/mutate.go b/pkg/repoimpl/oci/mutate.go similarity index 100% rename from pkg/oci/mutate.go rename to pkg/repoimpl/oci/mutate.go diff --git a/pkg/oci/oci.go b/pkg/repoimpl/oci/oci.go similarity index 97% rename from pkg/oci/oci.go rename to pkg/repoimpl/oci/oci.go index a9c6be4e..6abcba3e 100644 --- a/pkg/oci/oci.go +++ b/pkg/repoimpl/oci/oci.go @@ -37,17 +37,6 @@ import ( "k8s.io/klog/v2" ) -func OpenRepository(name string, namespace string, spec *configapi.OciRepository, deployment bool, storage *oci.Storage) (repository.Repository, error) { - return &ociRepository{ - name: name, - namespace: namespace, - spec: *spec.DeepCopy(), - deployment: deployment, - storage: storage, - }, nil - -} - type ociRepository struct { name string namespace string @@ -248,6 +237,10 @@ func (r *ociRepository) Refresh(_ context.Context) error { return nil } +func (r *ociRepository) Key() string { + return "oci://" + r.spec.Registry +} + // ToMainPackageRevision implements repository.PackageRevision. func (p *ociPackageRevision) ToMainPackageRevision() repository.PackageRevision { panic("unimplemented") diff --git a/pkg/repoimpl/oci/ocirepofactory.go b/pkg/repoimpl/oci/ocirepofactory.go new file mode 100644 index 00000000..4f52d7e7 --- /dev/null +++ b/pkg/repoimpl/oci/ocirepofactory.go @@ -0,0 +1,52 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oci + +import ( + "context" + "fmt" + "path/filepath" + + kptoci "github.com/GoogleContainerTools/kpt/pkg/oci" + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" + "github.com/nephio-project/porch/pkg/repository" +) + +var _ repoimpltypes.RepoImplFactory = &OciRepoFactory{} + +type OciRepoFactory struct { +} + +func (f *OciRepoFactory) NewRepositoryImpl(ctx context.Context, repositorySpec *configapi.Repository, options repoimpltypes.RepoImplOptions) (repository.Repository, error) { + if repositorySpec.Spec.Oci == nil { + return nil, fmt.Errorf("oci not configured") + } + + ociSpec := repositorySpec.Spec.Oci + localDir := filepath.Join(options.LocalDirectory, "oci") + storage, err := kptoci.NewStorage(localDir) + if err != nil { + return nil, err + } + + return &ociRepository{ + name: repositorySpec.Name, + namespace: repositorySpec.Namespace, + spec: *ociSpec.DeepCopy(), + deployment: repositorySpec.Spec.Deployment, + storage: storage, + }, nil +} diff --git a/pkg/repoimpl/repoimpl.go b/pkg/repoimpl/repoimpl.go new file mode 100644 index 00000000..cbddc988 --- /dev/null +++ b/pkg/repoimpl/repoimpl.go @@ -0,0 +1,75 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repoimpl + +import ( + "context" + "errors" + "fmt" + + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/repoimpl/git" + "github.com/nephio-project/porch/pkg/repoimpl/oci" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" + "github.com/nephio-project/porch/pkg/repository" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +var tracer = otel.Tracer("repoimpl") + +func CreateRepositoryImpl(ctx context.Context, repositorySpec *configapi.Repository, options repoimpltypes.RepoImplOptions) (repository.Repository, error) { + ctx, span := tracer.Start(ctx, "Repository::RepositoryFactory", trace.WithAttributes()) + defer span.End() + + var repoFactory repoimpltypes.RepoImplFactory + + switch repositoryType := repositorySpec.Spec.Type; repositoryType { + case configapi.RepositoryTypeOCI: + repoFactory = new(oci.OciRepoFactory) + + case configapi.RepositoryTypeGit: + repoFactory = new(git.GitRepoFactory) + + default: + return nil, fmt.Errorf("type %q not supported", repositoryType) + } + + return repoFactory.NewRepositoryImpl(ctx, repositorySpec, options) +} + +func RepositoryKey(repositorySpec *configapi.Repository) (string, error) { + switch repositoryType := repositorySpec.Spec.Type; repositoryType { + case configapi.RepositoryTypeOCI: + ociSpec := repositorySpec.Spec.Oci + if ociSpec == nil { + return "", fmt.Errorf("oci not configured") + } + return "oci://" + ociSpec.Registry, nil + + case configapi.RepositoryTypeGit: + gitSpec := repositorySpec.Spec.Git + if gitSpec == nil { + return "", errors.New("git property is required") + } + if gitSpec.Repo == "" { + return "", errors.New("git.repo property is required") + } + return fmt.Sprintf("git://%s/%s@%s/%s", gitSpec.Repo, gitSpec.Directory, repositorySpec.Namespace, repositorySpec.Name), nil + + default: + return "", fmt.Errorf("repository type %q not supported", repositoryType) + } +} diff --git a/pkg/repoimpl/types/repoimpltypes.go b/pkg/repoimpl/types/repoimpltypes.go new file mode 100644 index 00000000..b8fd920c --- /dev/null +++ b/pkg/repoimpl/types/repoimpltypes.go @@ -0,0 +1,33 @@ +// Copyright 2025 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repoimpltypes + +import ( + "context" + + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/repository" +) + +type RepoImplFactory interface { + NewRepositoryImpl(ctx context.Context, repositorySpec *configapi.Repository, options RepoImplOptions) (repository.Repository, error) +} + +type RepoImplOptions struct { + LocalDirectory string + UseUserDefinedCaBundle bool + CredentialResolver repository.CredentialResolver + UserInfoProvider repository.UserInfoProvider +} diff --git a/pkg/task/clone.go b/pkg/task/clone.go index 5e9bbf28..bc6d84e8 100644 --- a/pkg/task/clone.go +++ b/pkg/task/clone.go @@ -23,9 +23,10 @@ import ( api "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" "github.com/nephio-project/porch/internal/kpt/builtins" - "github.com/nephio-project/porch/pkg/git" "github.com/nephio-project/porch/pkg/kpt" v1 "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" + "github.com/nephio-project/porch/pkg/repoimpl/git" + repoimpltypes "github.com/nephio-project/porch/pkg/repoimpl/types" "github.com/nephio-project/porch/pkg/repository" "go.opentelemetry.io/otel/trace" "k8s.io/klog/v2" @@ -155,7 +156,9 @@ func (m *clonePackageMutation) cloneFromGit(ctx context.Context, gitPackage *api defer os.RemoveAll(dir) r, err := git.OpenRepository(ctx, "", "", &spec, false, dir, git.GitRepositoryOptions{ - CredentialResolver: m.credentialResolver, + RepoImplOptions: repoimpltypes.RepoImplOptions{ + CredentialResolver: m.credentialResolver, + }, MainBranchStrategy: git.SkipVerification, // We are only reading so we don't need the main branch to exist. }) if err != nil { diff --git a/pkg/task/clone_test.go b/pkg/task/clone_test.go index 6856d252..6f41ec22 100644 --- a/pkg/task/clone_test.go +++ b/pkg/task/clone_test.go @@ -36,7 +36,7 @@ import ( githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" "github.com/nephio-project/porch/api/porch/v1alpha1" - "github.com/nephio-project/porch/pkg/git" + "github.com/nephio-project/porch/pkg/repoimpl/git" "github.com/nephio-project/porch/pkg/repository" ) diff --git a/pkg/task/edit_test.go b/pkg/task/edit_test.go index bedeb4fd..637aee44 100644 --- a/pkg/task/edit_test.go +++ b/pkg/task/edit_test.go @@ -23,8 +23,8 @@ import ( "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" kptfile "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" + "github.com/nephio-project/porch/pkg/repoimpl/fake" "github.com/nephio-project/porch/pkg/repository" - "github.com/nephio-project/porch/pkg/repository/fake" ) func TestEdit(t *testing.T) { diff --git a/pkg/task/patch_test.go b/pkg/task/patch_test.go index 32961e88..6a2b2e92 100644 --- a/pkg/task/patch_test.go +++ b/pkg/task/patch_test.go @@ -22,8 +22,8 @@ import ( "github.com/google/go-cmp/cmp" api "github.com/nephio-project/porch/api/porch/v1alpha1" kptfile "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" + "github.com/nephio-project/porch/pkg/repoimpl/fake" "github.com/nephio-project/porch/pkg/repository" - "github.com/nephio-project/porch/pkg/repository/fake" ) func TestSomething(t *testing.T) { diff --git a/test/e2e/suite.go b/test/e2e/suite.go index db6c8ffb..59004b7a 100644 --- a/test/e2e/suite.go +++ b/test/e2e/suite.go @@ -37,8 +37,8 @@ import ( configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" internalapi "github.com/nephio-project/porch/internal/api/porchinternal/v1alpha1" internalpkg "github.com/nephio-project/porch/internal/kpt/pkg" - "github.com/nephio-project/porch/pkg/git" kptfilev1 "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" + "github.com/nephio-project/porch/pkg/repoimpl/git" "github.com/nephio-project/porch/pkg/repository" appsv1 "k8s.io/api/apps/v1" coreapi "k8s.io/api/core/v1" diff --git a/test/git/main.go b/test/git/main.go index cd8f0699..fde5fb3e 100644 --- a/test/git/main.go +++ b/test/git/main.go @@ -23,7 +23,7 @@ import ( "os" "os/signal" - "github.com/nephio-project/porch/pkg/git" + "github.com/nephio-project/porch/pkg/repoimpl/git" "k8s.io/klog/v2" )