fix(sync): added bearer client for sync

examples/README.md

@@ -936,7 +936,7 @@ Configure each registry sync:
 				]
 			},
 			{
-				"urls": ["https://docker.io/library"],
+				"urls": ["https://index.docker.io"],
 				"onDemand": true,                     # doesn't have content, don't periodically pull, pull just on demand.
 				"tlsVerify": true,
 				"maxRetries": 3,                      

pkg/api/authn.go

@@ -482,7 +482,7 @@ func bearerAuthHandler(ctlr *Controller) mux.MiddlewareFunc {
 					if err != nil {
 						ctlr.Log.Error().Err(err).Msg("failed to parse Authorization header")
 						response.Header().Set("Content-Type", "application/json")
-						zcommon.WriteJSON(response, http.StatusInternalServerError, apiErr.NewError(apiErr.UNSUPPORTED))
+						zcommon.WriteJSON(response, http.StatusUnauthorized, apiErr.NewError(apiErr.UNSUPPORTED))
 
 						return
 					}

pkg/api/controller_test.go

@@ -3114,7 +3114,7 @@ func TestBearerAuth(t *testing.T) {
 			Get(baseURL + "/v2/")
 		So(err, ShouldBeNil)
 		So(resp, ShouldNotBeNil)
-		So(resp.StatusCode(), ShouldEqual, http.StatusInternalServerError)
+		So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
 
 		resp, err = resty.R().SetHeader("Authorization",
 			fmt.Sprintf("Bearer %s", goodToken.AccessToken)).Options(baseURL + "/v2/")

pkg/api/routes.go

@@ -240,7 +240,7 @@ func (rh *RouteHandler) CheckVersionSupport(response http.ResponseWriter, reques
 	response.Header().Set(constants.DistAPIVersion, "registry/2.0")
 	// NOTE: compatibility workaround - return this header in "allowed-read" mode to allow for clients to
 	// work correctly
-	if rh.c.Config.HTTP.Auth != nil {
+	if rh.c.Config.IsBasicAuthnEnabled() || rh.c.Config.IsBearerAuthEnabled() {
 		// don't send auth headers if request is coming from UI
 		if request.Header.Get(constants.SessionClientHeaderName) != constants.SessionClientHeaderValue {
 			if rh.c.Config.HTTP.Auth.Bearer != nil {

pkg/common/http_client.go

@@ -1,18 +1,12 @@
 package common
 
 import (
-	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
-	"errors"
-	"io"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
-
-	"zotregistry.dev/zot/pkg/log"
 )
 
 func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
@@ -107,57 +101,7 @@ func CreateHTTPClient(verifyTLS bool, host string, certDir string) (*http.Client
 	}
 
 	return &http.Client{
-		Timeout:   httpTimeout,
 		Transport: htr,
+		Timeout:   httpTimeout,
 	}, nil
 }
-
-func MakeHTTPGetRequest(ctx context.Context, httpClient *http.Client,
-	username string, password string, resultPtr interface{},
-	blobURL string, mediaType string, log log.Logger,
-) ([]byte, string, int, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobURL, nil) //nolint
-	if err != nil {
-		return nil, "", 0, err
-	}
-
-	if mediaType != "" {
-		req.Header.Set("Accept", mediaType)
-	}
-
-	if username != "" && password != "" {
-		req.SetBasicAuth(username, password)
-	}
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		log.Error().Str("errorType", TypeOf(err)).
-			Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
-
-		return nil, "", -1, err
-	}
-
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Error().Str("errorType", TypeOf(err)).
-			Err(err).Str("blobURL", blobURL).Msg("couldn't get blob")
-
-		return nil, "", resp.StatusCode, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113
-	}
-
-	// read blob
-	if len(body) > 0 {
-		err = json.Unmarshal(body, &resultPtr)
-		if err != nil {
-			return body, "", resp.StatusCode, err
-		}
-	}
-
-	return body, resp.Header.Get("Content-Type"), resp.StatusCode, err
-}

pkg/common/http_client_test.go

@@ -1,19 +1,14 @@
 package common_test
 
 import (
-	"context"
 	"crypto/x509"
 	"os"
 	"path"
 	"testing"
 
-	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	. "github.com/smartystreets/goconvey/convey"
 
-	"zotregistry.dev/zot/pkg/api"
-	"zotregistry.dev/zot/pkg/api/config"
 	"zotregistry.dev/zot/pkg/common"
-	"zotregistry.dev/zot/pkg/log"
 	test "zotregistry.dev/zot/pkg/test/common"
 )
 
@@ -54,30 +49,4 @@ func TestHTTPClient(t *testing.T) {
 		_, err = common.CreateHTTPClient(true, "localhost", tempDir)
 		So(err, ShouldNotBeNil)
 	})
-
-	Convey("test MakeHTTPGetRequest() no permissions on key", t, func() {
-		port := test.GetFreePort()
-		baseURL := test.GetBaseURL(port)
-
-		conf := config.New()
-		conf.HTTP.Port = port
-
-		ctlr := api.NewController(conf)
-		tempDir := t.TempDir()
-		err := test.CopyTestKeysAndCerts(tempDir)
-		So(err, ShouldBeNil)
-		ctlr.Config.Storage.RootDirectory = tempDir
-
-		cm := test.NewControllerManager(ctlr)
-		cm.StartServer()
-		defer cm.StopServer()
-		test.WaitTillServerReady(baseURL)
-
-		var resultPtr interface{}
-		httpClient, err := common.CreateHTTPClient(true, "localhost", tempDir)
-		So(err, ShouldBeNil)
-		_, _, _, err = common.MakeHTTPGetRequest(context.Background(), httpClient, "", "",
-			resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", ""))
-		So(err, ShouldBeNil)
-	})
 }

pkg/extensions/extension_sync.go

@@ -50,6 +50,8 @@ func EnableSyncExtension(config *config.Config, metaDB mTypes.MetaDB,
 
 			service, err := sync.New(registryConfig, credsPath, tmpDir, storeController, metaDB, log)
 			if err != nil {
+				log.Error().Err(err).Msg("failed to initialize sync extension")
+
 				return nil, err
 			}
 

pkg/extensions/sync/httpclient/cache.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"sync"
+)
+
+// Key:Value store for bearer tokens, key is namespace, value is token.
+// We are storing only pull scoped tokens, the http client is for pulling only.
+type TokenCache struct {
+	entries sync.Map
+}
+
+func NewTokenCache() *TokenCache {
+	return &TokenCache{
+		entries: sync.Map{},
+	}
+}
+
+func (c *TokenCache) Set(namespace string, token *bearerToken) {
+	if c == nil || token == nil {
+		return
+	}
+
+	defer c.prune()
+
+	c.entries.Store(namespace, token)
+}
+
+func (c *TokenCache) Get(namespace string) *bearerToken {
+	if c == nil {
+		return nil
+	}
+
+	val, ok := c.entries.Load(namespace)
+	if !ok {
+		return nil
+	}
+
+	bearerToken, ok := val.(*bearerToken)
+	if !ok {
+		return nil
+	}
+
+	return bearerToken
+}
+
+func (c *TokenCache) prune() {
+	c.entries.Range(func(key, val any) bool {
+		bearerToken, ok := val.(*bearerToken)
+		if ok {
+			if bearerToken.isExpired() {
+				c.entries.Delete(key)
+			}
+		}
+
+		return true
+	})
+}