change catalog-specific URL from full path to API endpoint ref

solves #427 and implements phase 1 of the CatalogD expandable interface.
This phase just moves from `status.contentURL` to `status.baseURL`, and
will be used to provide reference to the catalog-specific API instead of
the full path to JSONLines-formatted content, plus tests and
documentation.

Signed-off-by: Jordan Keister <jordan@nimblewidget.com>

README.md

@@ -38,60 +38,59 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
     
     *Example output*
     ```sh
-    Name:         operatorhubio
-    Namespace:    
-    Labels:       olm.operatorframework.io/metadata.name=operatorhubio
-    Annotations:  <none>
-    API Version:  olm.operatorframework.io/v1alpha1
-    Kind:         ClusterCatalog
-    Metadata:
-      Creation Timestamp:  2024-09-12T13:37:04Z
-      Finalizers:
-        olm.operatorframework.io/delete-server-cache
-      Generation:        1
-      Resource Version:  961
-      UID:               fa6bb9cf-1a36-4189-a7a0-83284c3f6f55
-    Spec:
-      Priority:  0
-      Source:
-        Image:
-          Poll Interval:  10m0s
-          Ref:            quay.io/operatorhubio/catalog:latest
-        Type:             Image
-    Status:
-      Conditions:
-        Last Transition Time:  2024-09-12T13:37:53Z
-        Message:               Successfully unpacked and stored content from quay.io/operatorhubio/catalog:latest
-        Reason:                Succeeded
-        Status:                False
-        Type:                  Progressing
-        Last Transition Time:  2024-09-12T13:37:53Z
-        Message:               Content from quay.io/operatorhubio/catalog:latest is being served
-        Reason:                Available
-        Status:                True
-        Type:                  Serving
-      Content URL:             https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
-      Last Unpacked:           2024-09-12T13:37:52Z
-      Observed Generation:     1
-      Resolved Source:
-        Image:
-          Last Poll Attempt:  2024-09-12T13:37:52Z
-          Last Unpacked:      2024-09-12T13:37:52Z
-          Ref:                quay.io/operatorhubio/catalog:latest
-          Resolved Ref:       quay.io/operatorhubio/catalog@sha256:4453a361198d39d0390fd8c1a7f07b5a5a3ae1e8dac9979ef0c4eba46299df16
-        Type:                 Image
-   Events:                   <none>
+Name:         operatorhubio
+Namespace:
+Labels:       olm.operatorframework.io/metadata.name=operatorhubio
+Annotations:  <none>
+API Version:  olm.operatorframework.io/v1alpha1
+Kind:         ClusterCatalog
+Metadata:
+  Creation Timestamp:  2024-10-10T19:25:01Z
+  Finalizers:
+    olm.operatorframework.io/delete-server-cache
+  Generation:        1
+  Resource Version:  10206
+  UID:               8a83a2e2-6f7f-430c-9359-258003483b12
+Spec:
+  Priority:  0
+  Source:
+    Image:
+      Poll Interval:  10m0s
+      Ref:            quay.io/operatorhubio/catalog:latest
+    Type:             Image
+Status:
+  Base URL:  https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api
+  Conditions:
+    Last Transition Time:  2024-10-10T19:25:15Z
+    Message:               Successfully unpacked and stored content from resolved source
+    Observed Generation:   1
+    Reason:                Succeeded
+    Status:                False
+    Type:                  Progressing
+    Last Transition Time:  2024-10-10T19:25:15Z
+    Message:               Serving desired content from resolved source
+    Observed Generation:   1
+    Reason:                Available
+    Status:                True
+    Type:                  Serving
+  Last Unpacked:           2024-10-10T19:25:14Z
+  Resolved Source:
+    Image:
+      Last Successful Poll Attempt:  2024-10-10T20:45:33Z
+      Ref:                           quay.io/operatorhubio/catalog@sha256:7fbc6c61dc36e8225d4cb392cf0ffc754c179a82b12f19b91970c09b431aa61f
+    Type:                            Image
+Events:                              <none>
    ```
 
 1. Port forward the `catalogd-service` service in the `olmv1-system` namespace:
     ```sh
     $ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443
     ```
 
-1. Run the following command to get a list of packages:
+1. Access the `v1/all` service endpoint and filter the results to a list of packages by running the following command:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/vi/all | jq -s '.[] | select(.schema == "olm.package") | .name'
     ```
 
     *Example output*
@@ -118,7 +117,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
 1. Run the following command to get a list of channels for the `ack-acm-controller` package:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
     ```
 
     *Example output*
@@ -132,7 +131,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A
 1. Run the following command to get a list of bundles belonging to the `ack-acm-controller` package:
 
     ```sh
-    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
+    $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
     ```
     
     *Example output*
@@ -153,7 +152,7 @@ Thanks for your interest in contributing to `catalogd`!
 
 `catalogd` is in the very early stages of development and a more in depth contributing guide will come in the near future.
 
-In the mean time, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet).
+In the meantime, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet).
 
 If you have any questions, feel free to reach out to us on the Kubernetes Slack channel [#olm-dev](https://kubernetes.slack.com/archives/C0181L6JYQ2) or [create an issue](https://github.com/operator-framework/catalogd/issues/new)
 ### Testing Local Changes

api/core/v1alpha1/clustercatalog_types.go

@@ -129,11 +129,9 @@ type ClusterCatalogStatus struct {
 	//  type: Image
 	// +optional
 	ResolvedSource *ResolvedCatalogSource `json:"resolvedSource,omitempty"`
-	// contentURL is a cluster-internal URL from which on-cluster components
-	// can read the content of a catalog
+	// baseURL is a cluster-internal URL from which on-cluster components can access the API endpoint for this catalog
 	// +optional
-	ContentURL string `json:"contentURL,omitempty"`
-
+	BaseURL string `json:"baseURL,omitempty"`
 	// lastUnpacked represents the time when the
 	// ClusterCatalog object was last unpacked successfully.
 	// +optional

cmd/manager/main.go

@@ -258,7 +258,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	localStorage = storage.LocalDir{RootDir: storeDir, BaseURL: baseStorageURL}
+	localStorage = storage.LocalDir{RootDir: storeDir, RootURL: baseStorageURL}
 
 	catalogServerConfig := serverutil.CatalogServerConfig{
 		ExternalAddr: externalAddr,

config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml

@@ -130,6 +130,10 @@ spec:
           status:
             description: ClusterCatalogStatus defines the observed state of ClusterCatalog
             properties:
+              baseURL:
+                description: baseURL is a cluster-internal URL from which on-cluster
+                  components can access the API endpoint for this catalog
+                type: string
               conditions:
                 description: |-
                   conditions is a representation of the current state for this ClusterCatalog.
@@ -206,11 +210,6 @@ spec:
                   - type
                   type: object
                 type: array
-              contentURL:
-                description: |-
-                  contentURL is a cluster-internal URL from which on-cluster components
-                  can read the content of a catalog
-                type: string
               lastUnpacked:
                 description: |-
                   lastUnpacked represents the time when the

docs/fetching-catalog-contents.md

@@ -1,8 +1,9 @@
 # `ClusterCatalog` Interface
-`catalogd` serves catalog content via an HTTP(S) endpoint
+`catalogd` serves catalog content via a catalog-specific, versioned HTTP(S) endpoint. Clients access catalog information via this API endpoint and a versioned reference of the desired format. Current support includes only a complete catalog download, indicated by the path "v1/all", for example `https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/vi/all` would receive the complete FBC for the catalog `operatorhubio`.
+
 
 ## Response Format
-`catalogd` responses are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines.
+`catalogd` responses retrieved via the catalog-specific v1 API are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines.
 
 ### Example
 For an example JSON-encoded FBC snippet
@@ -79,30 +80,31 @@ For example purposes we make the following assumption:
 For local development, consider skipping TLS verification, such as `curl -k`, or reference external material
 on self-signed certificate verification.
 
-`ClusterCatalog` CRs have a status.contentURL field whose value is the location where the content 
-of a catalog can be read from:
+`ClusterCatalog` CRs have a status.baseURL field which identifies the catalog-specific API to access the catalog content:
 
 ```yaml
   status:
     .
     .
-    contentURL: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
+    baseURL: https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/
     resolvedSource:
       image:
         ref: quay.io/operatorhubio/catalog@sha256:e53267559addc85227c2a7901ca54b980bc900276fc24d3f4db0549cb38ecf76
       type: Image
 ```
 
-
 ## On cluster
 
-When making a request for the contents of the `operatorhubio` `ClusterCatalog` from within
-the cluster issue a HTTP `GET` request to 
-`https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json`
+When making a request for the complete contents of the `operatorhubio` `ClusterCatalog` from within
+the cluster, clients would combine `status.baseURL` with the desired API service and issue an HTTP GET request for the URL.
+
+For example, to receive the complete catalog data for the `operatorhubio` catalog indicated above, the client would append the service point designator `v1/all`, like:
+
+`https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/v1/all`.
 
 An example command to run a `Pod` to `curl` the catalog contents:
 ```sh
-kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json
+kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api/v1/all
 ```
 
 ## Off cluster
@@ -115,11 +117,11 @@ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443
 ```
 
 Once the service has been successfully forwarded to a localhost port, issue a HTTP `GET`
-request to `https://localhost:8080/catalogs/operatorhubio/all.json`
+request to `https://localhost:8080/catalogs/operatorhubio/api/v1/all`
 
 An example `curl` request that assumes the port-forwarding is mapped to port 8080 on the local machine:
 ```sh
-curl http://localhost:8080/catalogs/operatorhubio/all.json
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all
 ```
 
 # Fetching `ClusterCatalog` contents from the `Catalogd` Service outside of the cluster
@@ -190,7 +192,7 @@ This section outlines a way of exposing the `Catalogd` Service's endpoints outsi
 1. Run the below example `curl` request to retrieve all of the catalog contents:
 
     ```sh
-      $ curl https://<address>/catalogs/operatorhubio/all.json
+      $ curl https://<address>/catalogs/operatorhubio/api/v1/all
     ```
 
     To obtain `address` of the ingress object, you can run the below command and look for the value in the `ADDRESS` field from output: 

hack/scripts/demo-script.sh

@@ -11,6 +11,7 @@ sleep 10
 kubectl apply -f https://github.com/operator-framework/catalogd/releases/latest/download/catalogd.yaml
 kubectl wait --for=condition=Available -n olmv1-system deploy/catalogd-controller-manager --timeout=60s
 sleep 10
+
 # inspect crds (catalog)
 kubectl get crds -A
 
@@ -19,15 +20,15 @@ kubectl apply -f config/samples/core_v1alpha1_catalog.yaml
 # shows catalog-sample
 kubectl get catalog -A 
 # waiting for catalog to report ready status
-time kubectl wait --for=condition=Unpacked catalog/operatorhubio --timeout=1m
+time kubectl wait --for=condition=Serving catalog/operatorhubio --timeout=1m
 
 # port forward the catalogd-service service to interact with the HTTP server serving catalog contents
 (kubectl -n olmv1-system port-forward svc/catalogd-service 8080:80)&
 
 # check what 'packages' are available in this catalog
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.package") | .name'
 # check what channels are included in the wavefront package
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name'
 # check what bundles are included in the wavefront package
-curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name'
+curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name'
 

hack/scripts/gzip-demo-script.sh

@@ -16,13 +16,13 @@ time kubectl wait --for=condition=Unpacked clustercatalog/operatorhubio --timeou
 sleep 5
 
 # retrieve catalog as plaintext JSONlines
-curl -k -vvv https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content.json
+curl -k -vvv https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content.json
 
 # advertise handling of compressed content
-curl -vvv -k https://localhost:8080/catalogs/operatorhubio/all.json -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz
+curl -vvv -k https://localhost:8080/catalogs/operatorhubio/api/v1/all -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz
 
 # let curl handle the compress/decompress for us
-curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content-decompressed.txt
+curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content-decompressed.txt
 
 # show that there's no content change with changed format
 diff /tmp/cat-content.json /tmp/cat-content-decompressed.txt

internal/controllers/core/clustercatalog_controller.go

@@ -304,7 +304,7 @@ func updateStatusProgressing(status *v1alpha1.ClusterCatalogStatus, generation i
 
 func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Result, contentURL string, generation int64) {
 	status.ResolvedSource = result.ResolvedSource
-	status.ContentURL = contentURL
+	status.BaseURL = contentURL
 	status.LastUnpacked = metav1.NewTime(result.UnpackTime)
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:               v1alpha1.TypeServing,
@@ -317,7 +317,7 @@ func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Re
 
 func updateStatusNotServing(status *v1alpha1.ClusterCatalogStatus, generation int64) {
 	status.ResolvedSource = nil
-	status.ContentURL = ""
+	status.BaseURL = ""
 	status.LastUnpacked = metav1.Time{}
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:               v1alpha1.TypeServing,

internal/controllers/core/clustercatalog_controller_test.go

@@ -261,7 +261,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL: "URL",
+					BaseURL: "URL",
 					Conditions: []metav1.Condition{
 						{
 							Type:   catalogdv1alpha1.TypeServing,
@@ -393,7 +393,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL:   "URL",
+					BaseURL:      "URL",
 					LastUnpacked: metav1.Time{},
 					ResolvedSource: &catalogdv1alpha1.ResolvedCatalogSource{
 						Type: catalogdv1alpha1.SourceTypeImage,
@@ -430,7 +430,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL: "",
+					BaseURL: "",
 					Conditions: []metav1.Condition{
 						{
 							Type:   catalogdv1alpha1.TypeServing,
@@ -473,7 +473,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL: "URL",
+					BaseURL: "URL",
 					Conditions: []metav1.Condition{
 						{
 							Type:   catalogdv1alpha1.TypeProgressing,
@@ -503,7 +503,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL: "URL",
+					BaseURL: "URL",
 					Conditions: []metav1.Condition{
 						{
 							Type:   catalogdv1alpha1.TypeProgressing,
@@ -544,7 +544,7 @@ func TestCatalogdControllerReconcile(t *testing.T) {
 					},
 				},
 				Status: catalogdv1alpha1.ClusterCatalogStatus{
-					ContentURL: "URL",
+					BaseURL: "URL",
 					Conditions: []metav1.Condition{
 						{
 							Type:   catalogdv1alpha1.TypeProgressing,
@@ -696,7 +696,7 @@ func TestPollingReconcilerUnpack(t *testing.T) {
 	successfulObservedGeneration := int64(2)
 	successfulUnpackStatus := func(mods ...func(status *catalogdv1alpha1.ClusterCatalogStatus)) catalogdv1alpha1.ClusterCatalogStatus {
 		s := catalogdv1alpha1.ClusterCatalogStatus{
-			ContentURL: "URL",
+			BaseURL: "URL",
 			Conditions: []metav1.Condition{
 				{
 					Type:               catalogdv1alpha1.TypeProgressing,