feat(retention): added image retention policies

feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

errors/errors.go

@@ -163,4 +163,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrGCPolicyNotFound               = errors.New("gc: repo/tag policy not found")
 )

examples/config-gc-periodic.json

@@ -4,13 +4,13 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "gcDelay": "1h",
-        "gcInterval": "24h",
+        "gcInterval": "1h",
         "subPaths": {
             "/a": {
                 "rootDirectory": "/tmp/zot1",
                 "gc": true,
                 "gcDelay": "1h",
-                "gcInterval": "24h"
+                "gcInterval": "1h"
             }
         }
     },

examples/config-gc.json

@@ -3,10 +3,48 @@
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": true,
-        "gcReferrers": true,
         "gcDelay": "2h",
         "untaggedImageRetentionDelay": "4h",
-        "gcInterval": "1h"
+        "gcInterval": "1h",
+        "retention": {
+            "dryRun": false,
+            "policies": [
+                {
+                    "repoNames": ["infra/*", "prod/*"],
+                    "deleteReferrers": false,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v2.*", "*-prod"],
+                        "retainAlways": true
+                    },
+                    {
+                        "names": ["v3.*", "*-prod"],
+                        "pulledWithinLastNrDays": 7
+                    }]
+                },
+                {
+                    "repoNames": ["tmp/**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v1.*"],
+                        "pulledWithinLastNrDays": 7,
+                        "pushedWithinLastNrDays": 7   
+                    }]
+                },
+                {
+                    "repoNames": ["**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "mostRecentlyPushedCount": 10,
+                        "mostRecentlyPulledCount": 10,
+                        "pulledWithinLastNrDays": 30,
+                        "pushedWithinLastNrDays": 30
+                    }]
+                }
+            ]
+        }
     },
     "http": {
         "address": "127.0.0.1",

examples/config-sync.json

@@ -35,12 +35,12 @@
 							}
 						},
 						{
-							"prefix": "/repo1/repo",
+							"prefix": "/repo2/repo",
 							"destination": "/repo",
 							"stripPrefix": true
 						},
 						{
-							"prefix": "/repo2/repo"
+							"prefix": "/repo3/**"
 						}
 					]
 				},
@@ -54,7 +54,7 @@
 					"onDemand": false,
 					"content": [
 						{
-							"prefix": "/repo2",
+							"prefix": "**",
 							"tags": {
 								"semver": true
 							}

pkg/api/config/config.go

@@ -27,15 +27,36 @@ type StorageConfig struct {
 	Dedupe                      bool
 	RemoteCache                 bool
 	GC                          bool
-	Commit                      bool
 	GCDelay                     time.Duration
 	GCInterval                  time.Duration
-	GCReferrers                 bool
 	UntaggedImageRetentionDelay time.Duration
+	Retention                   ImageRetention
+	Commit                      bool
 	StorageDriver               map[string]interface{} `mapstructure:",omitempty"`
 	CacheDriver                 map[string]interface{} `mapstructure:",omitempty"`
 }
 
+type ImageRetention struct {
+	DryRun   bool
+	Policies []GCPolicy
+}
+
+type GCPolicy struct {
+	RepoNames       []string
+	DeleteReferrers bool
+	DeleteUntagged  bool
+	TagsRetention   []TagsRetentionPolicy
+}
+
+type TagsRetentionPolicy struct {
+	Names                   []string
+	RetainAlways            bool // default is true
+	PulledWithinLastNrDays  *int
+	PushedWithinLastNrDays  *int
+	MostRecentlyPushedCount *int
+	MostRecentlyPulledCount *int
+}
+
 type TLSConfig struct {
 	Cert   string
 	Key    string
@@ -190,9 +211,12 @@ func New() *Config {
 		BinaryType:      BinaryType,
 		Storage: GlobalStorageConfig{
 			StorageConfig: StorageConfig{
-				GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
+				Dedupe:                      true,
+				GC:                          true,
+				GCDelay:                     storageConstants.DefaultGCDelay,
 				UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
-				GCInterval:                  storageConstants.DefaultGCInterval, Dedupe: true,
+				GCInterval:                  storageConstants.DefaultGCInterval,
+				Retention:                   ImageRetention{},
 			},
 		},
 		HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
@@ -202,7 +226,8 @@ func New() *Config {
 
 func (expConfig StorageConfig) ParamsEqual(actConfig StorageConfig) bool {
 	return expConfig.GC == actConfig.GC && expConfig.Dedupe == actConfig.Dedupe &&
-		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval
+		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval &&
+		expConfig.UntaggedImageRetentionDelay == actConfig.UntaggedImageRetentionDelay
 }
 
 // SameFile compare two files.
@@ -368,6 +393,18 @@ func (c *Config) IsImageTrustEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable
 }
 
+func (c *Config) IsGarbageCollectEnabled() bool {
+	gcEnabled := c.Storage.GC
+
+	for _, subpath := range c.Storage.SubPaths {
+		if subpath.GC {
+			gcEnabled = true
+		}
+	}
+
+	return gcEnabled
+}
+
 func (c *Config) IsCosignEnabled() bool {
 	return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign
 }

pkg/api/controller.go

@@ -261,7 +261,8 @@ func (c *Controller) InitImageStore() error {
 
 func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 	// init metaDB if search is enabled or we need to store user profiles, api keys or signatures
-	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() {
+	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() ||
+		c.Config.IsGarbageCollectEnabled() {
 		driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
 		if err != nil {
 			return err
@@ -277,7 +278,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
-		err = meta.ParseStorage(driver, c.StoreController, c.Log)
+		err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck
 		if err != nil {
 			return err
 		}
@@ -293,10 +294,7 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config.
 	c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl
 
 	// reload periodical gc config
-	c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
 	c.Config.Storage.GC = newConfig.Storage.GC
-	c.Config.Storage.GCDelay = newConfig.Storage.GCDelay
-	c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers
 
 	// reload background tasks
 	if newConfig.Extensions != nil {
@@ -340,9 +338,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 	// Enable running garbage-collect periodically for DefaultStore
 	if c.Config.Storage.GC {
 		gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{
-			Referrers:      c.Config.Storage.GCReferrers,
+			DryRun:         c.Config.Storage.Retention.DryRun,
 			Delay:          c.Config.Storage.GCDelay,
-			RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay,
+			UntaggedDelay:  c.Config.Storage.UntaggedImageRetentionDelay,
+			ImageRetention: c.Config.Storage.Retention,
 		}, c.Log)
 
 		gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler)
@@ -363,9 +362,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 			if storageConfig.GC {
 				gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB,
 					gc.Options{
-						Referrers:      storageConfig.GCReferrers,
+						DryRun:         storageConfig.Retention.DryRun,
 						Delay:          storageConfig.GCDelay,
-						RetentionDelay: storageConfig.UntaggedImageRetentionDelay,
+						UntaggedDelay:  storageConfig.UntaggedImageRetentionDelay,
+						ImageRetention: storageConfig.Retention,
 					}, c.Log)
 
 				gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler)

pkg/api/controller_test.go

@@ -314,9 +314,6 @@ func TestRunAlreadyRunningServer(t *testing.T) {
 		defer cm.StopServer()
 
 		err := ctlr.Init(context.Background())
-		So(err, ShouldBeNil)
-
-		err = ctlr.Run(context.Background())
 		So(err, ShouldNotBeNil)
 	})
 }
@@ -7695,6 +7692,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.GC = true
 			ctlr.Config.Storage.GCDelay = 1 * time.Millisecond
 			ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond
+			ctlr.Config.Storage.Retention = config.ImageRetention{
+				Policies: []config.GCPolicy{
+					{
+						RepoNames:       []string{"**"},
+						DeleteReferrers: true,
+						DeleteUntagged:  true,
+					},
+				},
+			}
 
 			ctlr.Config.Storage.Dedupe = false
 
@@ -7710,11 +7716,10 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
 				gc.Options{
-					Referrers:      ctlr.Config.Storage.GCReferrers,
 					Delay:          ctlr.Config.Storage.GCDelay,
-					RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
-				},
-				ctlr.Log)
+					UntaggedDelay:  ctlr.Config.Storage.UntaggedImageRetentionDelay,
+					ImageRetention: ctlr.Config.Storage.Retention,
+				}, ctlr.Log)
 
 			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
 			So(err, ShouldBeNil)
@@ -7940,6 +7945,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.GC = true
 			ctlr.Config.Storage.GCDelay = 1 * time.Second
 			ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second
+			ctlr.Config.Storage.Retention = config.ImageRetention{
+				Policies: []config.GCPolicy{
+					{
+						RepoNames:       []string{"**"},
+						DeleteReferrers: true,
+						DeleteUntagged:  true,
+					},
+				},
+			}
 
 			err := WriteImageToFileSystem(CreateDefaultImage(), repoName, tag,
 				ociutils.GetDefaultStoreController(dir, ctlr.Log))
@@ -7951,9 +7965,9 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
 				gc.Options{
-					Referrers:      ctlr.Config.Storage.GCReferrers,
 					Delay:          ctlr.Config.Storage.GCDelay,
-					RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
+					UntaggedDelay:  ctlr.Config.Storage.UntaggedImageRetentionDelay,
+					ImageRetention: ctlr.Config.Storage.Retention,
 				}, ctlr.Log)
 
 			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
@@ -8084,8 +8098,13 @@ func TestPeriodicGC(t *testing.T) {
 		subPaths := make(map[string]config.StorageConfig)
 
 		subPaths["/a"] = config.StorageConfig{
-			RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second,
-			UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false,
+			RootDirectory:               subDir,
+			GC:                          true,
+			GCDelay:                     1 * time.Second,
+			UntaggedImageRetentionDelay: 1 * time.Second,
+			GCInterval:                  24 * time.Hour,
+			RemoteCache:                 false,
+			Dedupe:                      false,
 		} //nolint:lll // gofumpt conflicts with lll
 		ctlr.Config.Storage.Dedupe = false
 		ctlr.Config.Storage.SubPaths = subPaths

pkg/api/routes.go

@@ -725,8 +725,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
 	}
 
 	if rh.c.MetaDB != nil {
-		err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB,
-			rh.c.Log)
+		err := meta.OnUpdateManifest(request.Context(), name, reference, mediaType,
+			digest, body, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)