diff --git a/api/registry/types.go b/api/registry/types.go index a06ad4c3e2..9f2092730f 100644 --- a/api/registry/types.go +++ b/api/registry/types.go @@ -30,6 +30,8 @@ const ( ChartGroupFinalize FinalizerName = "chartgroup" // ChartFinalize is an internal finalizer values to Chart. ChartFinalize FinalizerName = "chart" + // RegistryClientUserAgent is the user agent for tke registry client + RegistryClientUserAgent = "tke-registry-client" ) // +genclient diff --git a/api/registry/v1/types.go b/api/registry/v1/types.go index b2bef83141..c96553a276 100644 --- a/api/registry/v1/types.go +++ b/api/registry/v1/types.go @@ -30,6 +30,8 @@ const ( ChartGroupFinalize FinalizerName = "chartgroup" // ChartFinalize is an internal finalizer values to Chart. ChartFinalize FinalizerName = "chart" + // RegistryClientUserAgent is the user agent for tke registry client + RegistryClientUserAgent = "tke-registry-client" ) // +genclient diff --git a/pkg/registry/apiserver/apiserver.go b/pkg/registry/apiserver/apiserver.go index 15fefdb0aa..377496c788 100644 --- a/pkg/registry/apiserver/apiserver.go +++ b/pkg/registry/apiserver/apiserver.go @@ -19,6 +19,7 @@ package apiserver import ( + "github.com/docker/libtrust" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" @@ -133,6 +134,11 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } } + pk, err := libtrust.LoadKeyFile(c.ExtraConfig.RegistryConfig.Security.TokenPrivateKeyFile) + if err != nil { + return nil, err + } + // The order here is preserved in discovery. restStorageProviders := []storage.RESTStorageProvider{ ®istryrest.StorageProvider{ @@ -146,6 +152,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) PlatformClient: c.ExtraConfig.PlatformClient, RegistryConfig: c.ExtraConfig.RegistryConfig, Authorizer: c.GenericConfig.Authorization.Authorizer, + TokenPrivateKey: pk, }, } m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...) diff --git a/pkg/registry/distribution/auth/auth.go b/pkg/registry/distribution/auth/auth.go index 337b20387e..d5a349abc1 100644 --- a/pkg/registry/distribution/auth/auth.go +++ b/pkg/registry/distribution/auth/auth.go @@ -148,7 +148,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - jwtToken, err := makeToken(username, access, h.expiredHours, h.privateKey) + jwtToken, err := MakeToken(username, access, h.expiredHours, h.privateKey) if err != nil { log.Error("Failed create token for docker registry authentication", log.String("username", username), diff --git a/pkg/registry/distribution/auth/token.go b/pkg/registry/distribution/auth/token.go index 1086e8f9f6..ad7c41d334 100644 --- a/pkg/registry/distribution/auth/token.go +++ b/pkg/registry/distribution/auth/token.go @@ -22,11 +22,12 @@ import ( "crypto" "encoding/base64" "fmt" - "github.com/docker/distribution/registry/auth/token" - "github.com/docker/libtrust" "math/rand" "strings" "time" + + "github.com/docker/distribution/registry/auth/token" + "github.com/docker/libtrust" ) const ( @@ -43,7 +44,7 @@ type Token struct { } // makeToken makes a valid jwt token based on params. -func makeToken(username string, access []*token.ResourceActions, expiredHours int64, privateKey libtrust.PrivateKey) (*Token, error) { +func MakeToken(username string, access []*token.ResourceActions, expiredHours int64, privateKey libtrust.PrivateKey) (*Token, error) { tk, expiresIn, issuedAt, err := makeTokenCore(Issuer, username, Service, expiredHours, access, privateKey) if err != nil { return nil, err @@ -123,3 +124,12 @@ func randString(length int) (string, error) { func base64UrlEncode(b []byte) string { return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") } + +// GetToken returns the content of the token +func (t *Token) GetToken() string { + token := t.Token + if len(token) == 0 { + token = t.AccessToken + } + return token +} diff --git a/pkg/registry/distribution/client/repo.go b/pkg/registry/distribution/client/repo.go new file mode 100644 index 0000000000..0a5002a763 --- /dev/null +++ b/pkg/registry/distribution/client/repo.go @@ -0,0 +1,276 @@ +package client + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "sort" + "strings" + + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/registry/auth/token" + "github.com/docker/libtrust" + "tkestack.io/tke/api/registry" + "tkestack.io/tke/pkg/registry/distribution/auth" + "tkestack.io/tke/pkg/util/log" +) + +var ManifestAccepts = []string{ + manifestlist.MediaTypeManifestList, + schema2.MediaTypeManifest, + schema1.MediaTypeSignedManifest, + schema1.MediaTypeManifest, +} + +// Repository holds information of a repository entity +type Repository struct { + Endpoint *url.URL + client *http.Client + privateKey libtrust.PrivateKey +} + +// NewRepository returns an instance of Repository +func NewRepository(endpoint string, privateKey libtrust.PrivateKey) (*Repository, error) { + u, err := ParseEndpoint(endpoint) + if err != nil { + return nil, err + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + repository := &Repository{ + Endpoint: u, + client: client, + privateKey: privateKey, + } + + return repository, nil +} + +// ParseEndpoint parses endpoint to a URL +func ParseEndpoint(endpoint string) (*url.URL, error) { + endpoint = strings.Trim(endpoint, " ") + endpoint = strings.TrimRight(endpoint, "/") + if len(endpoint) == 0 { + return nil, fmt.Errorf("empty URL") + } + i := strings.Index(endpoint, "://") + if i >= 0 { + scheme := endpoint[:i] + if scheme != "http" && scheme != "https" { + return nil, fmt.Errorf("invalid scheme: %s", scheme) + } + } else { + endpoint = "http://" + endpoint + } + + return url.ParseRequestURI(endpoint) +} + +// DeleteTag ... +func (r *Repository) DeleteTag(repoName, tag, user, tenantID string) error { + digest, exist, err := r.ManifestExist(tag, repoName, tag, user, tenantID) + if err != nil { + return err + } + + if !exist { + log.Warnf("repo: %s:%s manifests not found.", repoName, tag) + return nil + } + + if err := r.DeleteManifest(digest, repoName, tag, user, tenantID); err != nil { + return err + } + return nil +} + +// ListTag ... +func (r *Repository) ListTag(repoName, user, tenantID string) ([]string, error) { + tags := []string{} + req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), repoName), nil) + if err != nil { + return tags, err + } + err = r.withAuthInfo(req, repoName, user, tenantID) + if err != nil { + return tags, err + } + + resp, err := r.client.Do(req) + if err != nil { + return tags, parseError(err) + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + if resp.StatusCode == http.StatusOK { + tagsResp := struct { + Tags []string `json:"tags"` + }{} + + if err := json.Unmarshal(b, &tagsResp); err != nil { + return tags, err + } + sort.Strings(tags) + tags = tagsResp.Tags + + return tags, nil + } else if resp.StatusCode == http.StatusNotFound { + return tags, nil + } + + return tags, &Error{ + Code: resp.StatusCode, + Message: string(b), + } + +} + +// ManifestExist ... +func (r *Repository) ManifestExist(reference, repoName, tag, user, tenantID string) (digest string, exist bool, err error) { + req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), repoName, reference), nil) + if err != nil { + return + } + err = r.withAuthInfo(req, repoName, user, tenantID) + if err != nil { + return + } + + for _, mediaType := range ManifestAccepts { + req.Header.Add("Accept", mediaType) + } + + resp, err := r.client.Do(req) + if err != nil { + err = parseError(err) + return + } + + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + exist = true + digest = resp.Header.Get("Docker-Content-Digest") + return + } + + if resp.StatusCode == http.StatusNotFound { + return + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = &Error{ + Code: resp.StatusCode, + Message: string(b), + } + return +} + +// DeleteManifest ... +func (r *Repository) DeleteManifest(digest, repoName, tag, user, tenantID string) error { + req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), repoName, digest), nil) + if err != nil { + return err + } + err = r.withAuthInfo(req, repoName, user, tenantID) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return parseError(err) + } + + defer resp.Body.Close() + + if resp.StatusCode == http.StatusAccepted { + return nil + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return &Error{ + Code: resp.StatusCode, + Message: string(b), + } +} + +func (r *Repository) withAuthInfo(req *http.Request, repoName, user, tenantID string) error { + access := []*token.ResourceActions{ + { + Type: "repository", + Actions: []string{"*", "pull"}, + // to make token be available, should rename repo name with tenantID + Name: fmt.Sprintf("%s-%s", tenantID, repoName), + }, + } + token, err := auth.MakeToken(user, access, 24, r.privateKey) + if err != nil { + return err + } + log.Infof("token: %s", token.GetToken()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.GetToken())) + // set registry client UA to avoid error cased by reporting event of the deleted repo + req.Header.Set("User-Agent", registry.RegistryClientUserAgent) + return nil +} + +func buildManifestURL(endpoint, repoName, reference string) string { + return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference) +} + +func buildTagListURL(endpoint, repoName string) string { + return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName) +} + +func parseError(err error) error { + if urlErr, ok := err.(*url.Error); ok { + if regErr, ok := urlErr.Err.(*Error); ok { + return regErr + } + } + return err +} + +// Error wrap HTTP status code and message as an error +type Error struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// Error ... +func (e *Error) Error() string { + return fmt.Sprintf("http error: code %d, message %s", e.Code, e.Message) +} + +// String wraps the error msg to the well formatted error message +func (e *Error) String() string { + data, err := json.Marshal(&e) + if err != nil { + return e.Message + } + return string(data) +} diff --git a/pkg/registry/distribution/handler/handler.go b/pkg/registry/distribution/handler/handler.go new file mode 100644 index 0000000000..48a95c2967 --- /dev/null +++ b/pkg/registry/distribution/handler/handler.go @@ -0,0 +1,54 @@ +package handler + +import ( + "context" + "fmt" + "net/http" + "sync" + + distributionClient "tkestack.io/tke/pkg/registry/distribution/client" + "tkestack.io/tke/pkg/util/log" +) + +func DeleteRepo(ctx context.Context, client *distributionClient.Repository, userName, tenantID, repoName string) (err error) { + tags, err := client.ListTag(repoName, userName, tenantID) + if err != nil { + return err + } + var wg sync.WaitGroup + var errsMap sync.Map + for _, tag := range tags { + wg.Add(1) + go func(repoName, tag, user, tenantID string, rc *distributionClient.Repository) { + defer wg.Done() + image := fmt.Sprintf("%s:%s", repoName, tag) + log.Infof("delete label of image at database: %s", image) + if err := rc.DeleteTag(repoName, tag, user, tenantID); err != nil { + if regErr, ok := err.(*distributionClient.Error); ok { + if regErr.Code == http.StatusNotFound { + return + } + } + errsMap.Store(tag, err.Error()) + log.Errorf("failed to delete %s: %v", image, err) + return + } + log.Infof("delete tag at registry: %s", image) + + }(repoName, tag, userName, tenantID, client) + } + wg.Wait() + errMessages := []string{} + for _, tag := range tags { + message, ok := errsMap.Load(tag) + if !ok { + continue + } + errMessages = append(errMessages, fmt.Sprintf("failed to delete %s tag %s: %s", repoName, tag, message.(string))) + } + if len(errMessages) > 0 { + return fmt.Errorf("failed to delete repo: %v", errMessages) + } + return nil + +} diff --git a/pkg/registry/distribution/notification/notification.go b/pkg/registry/distribution/notification/notification.go index d4d502c27b..af25983c9a 100644 --- a/pkg/registry/distribution/notification/notification.go +++ b/pkg/registry/distribution/notification/notification.go @@ -201,7 +201,7 @@ func checkEvent(event *Event) bool { } // if it is pull action, check the user-agent userAgent := strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) - return userAgent != "tke-registry-client" + return userAgent != registry.RegistryClientUserAgent } // ParseRepository splits a repository into three parts: tenantID, namespace and rest diff --git a/pkg/registry/registry/namespace/storage/storage.go b/pkg/registry/registry/namespace/storage/storage.go index 6460578c36..20235b4598 100644 --- a/pkg/registry/registry/namespace/storage/storage.go +++ b/pkg/registry/registry/namespace/storage/storage.go @@ -170,8 +170,11 @@ func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.Va if err != nil { return nil, false, err } + o := obj.(*registryapi.Namespace) + if o.Status.RepoCount > 0 { + return nil, false, fmt.Errorf("%s(%s) cannot be delete, since repositories under this namespace is not empty", o.Spec.Name, o.Name) + } if r.harborClient != nil { - o := obj.(*registryapi.Namespace) // delete harbor project err = harborHandler.DeleteProject(ctx, r.harborClient, nil, fmt.Sprintf("%s-image-%s", o.Spec.TenantID, o.Spec.Name)) if err != nil { diff --git a/pkg/registry/registry/repository/storage/storage.go b/pkg/registry/registry/repository/storage/storage.go index 44a6be0d64..8390ea56d7 100644 --- a/pkg/registry/registry/repository/storage/storage.go +++ b/pkg/registry/registry/repository/storage/storage.go @@ -22,6 +22,7 @@ import ( "context" "fmt" + "github.com/docker/libtrust" metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -30,7 +31,10 @@ import ( "k8s.io/apiserver/pkg/registry/rest" registryinternalclient "tkestack.io/tke/api/client/clientset/internalversion/typed/registry/internalversion" registryapi "tkestack.io/tke/api/registry" + "tkestack.io/tke/pkg/apiserver/authentication" apiserverutil "tkestack.io/tke/pkg/apiserver/util" + "tkestack.io/tke/pkg/registry/distribution/client" + distributionHandler "tkestack.io/tke/pkg/registry/distribution/handler" harbor "tkestack.io/tke/pkg/registry/harbor/client" harborHandler "tkestack.io/tke/pkg/registry/harbor/handler" repositorystrategy "tkestack.io/tke/pkg/registry/registry/repository" @@ -45,7 +49,7 @@ type Storage struct { } // NewStorage returns a Storage object that will work against repositories. -func NewStorage(optsGetter genericregistry.RESTOptionsGetter, registryClient *registryinternalclient.RegistryClient, privilegedUsername string, harborClient *harbor.APIClient) *Storage { +func NewStorage(optsGetter genericregistry.RESTOptionsGetter, registryClient *registryinternalclient.RegistryClient, privilegedUsername, repoEndpoint string, harborClient *harbor.APIClient, tokenPrivateKey libtrust.PrivateKey) *Storage { strategy := repositorystrategy.NewStrategy(registryClient) store := ®istry.Store{ NewFunc: func() runtime.Object { return ®istryapi.Repository{} }, @@ -67,12 +71,16 @@ func NewStorage(optsGetter genericregistry.RESTOptionsGetter, registryClient *re if err := store.CompleteWithOptions(options); err != nil { log.Panic("Failed to create repository etcd rest storage", log.Err(err)) } + distributionClient, err := client.NewRepository(repoEndpoint, tokenPrivateKey) + if err != nil { + log.Panic("Failed to create distribution client", log.Err(err)) + } statusStore := *store statusStore.UpdateStrategy = repositorystrategy.NewStatusStrategy(strategy) return &Storage{ - Repository: &REST{store, privilegedUsername, harborClient, registryClient}, + Repository: &REST{store, privilegedUsername, harborClient, registryClient, distributionClient}, Status: &StatusREST{&statusStore}, } } @@ -119,6 +127,7 @@ type REST struct { privilegedUsername string harborClient *harbor.APIClient registryClient *registryinternalclient.RegistryClient + distributionClient *client.Repository } var _ rest.ShortNamesProvider = &REST{} @@ -158,6 +167,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje // Delete enforces life-cycle rules for cluster termination func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { + userName, _ := authentication.UsernameAndTenantID(ctx) obj, err := ValidateGetObjectAndTenantID(ctx, r.Store, name, &metav1.GetOptions{}) if err != nil { return nil, false, err @@ -174,7 +184,16 @@ func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.Va return nil, false, err } } + if r.distributionClient != nil { + repoName := fmt.Sprintf("%s/%s", o.Spec.NamespaceName, o.Spec.Name) + err := distributionHandler.DeleteRepo(ctx, r.distributionClient, userName, o.Spec.TenantID, repoName) + if err != nil { + return nil, false, err + } + } + UpdateNamespaceRepoCount(ctx, r.registryClient, o.Spec.NamespaceName, o.Spec.TenantID) + return r.Store.Delete(ctx, name, deleteValidation, options) } diff --git a/pkg/registry/registry/rest/rest.go b/pkg/registry/registry/rest/rest.go index 33eac2d6fb..977949050c 100644 --- a/pkg/registry/registry/rest/rest.go +++ b/pkg/registry/registry/rest/rest.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" + "github.com/docker/libtrust" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" @@ -61,6 +62,7 @@ type StorageProvider struct { PlatformClient platformversionedclient.PlatformV1Interface RegistryConfig *registryconfig.RegistryConfiguration Authorizer authorizer.Authorizer + TokenPrivateKey libtrust.PrivateKey } // Implement RESTStorageProvider @@ -124,7 +126,7 @@ func (s *StorageProvider) v1Storage(apiResourceConfigSource serverstorage.APIRes storageMap["namespaces"] = namespaceREST.Namespace storageMap["namespaces/status"] = namespaceREST.Status - repositoryREST := repositorystorage.NewStorage(restOptionsGetter, registryClient, s.PrivilegedUsername, harborClient) + repositoryREST := repositorystorage.NewStorage(restOptionsGetter, registryClient, s.PrivilegedUsername, s.LoopbackClientConfig.Host, harborClient, s.TokenPrivateKey) storageMap["repositories"] = repositoryREST.Repository storageMap["repositories/status"] = repositoryREST.Status