From e04ea4f7573f933fb006ec081804eb6cee0202e0 Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Wed, 24 Jan 2024 12:43:40 +0530 Subject: [PATCH 01/10] update httprouterefs to routerefs in cc --- .../apis/dp/v1alpha1/api_conversion.go | 16 ++++++++++++++++ common-go-libs/apis/dp/v1alpha2/api_types.go | 6 +++--- .../apis/dp/v1alpha2/api_webhook.go | 11 +++++------ .../apis/dp/v1alpha2/zz_generated.deepcopy.go | 4 ++-- .../config/crd/bases/dp.wso2.com_apis.yaml | 16 ++++++++-------- common-go-libs/go.sum | 1 + helm-charts/crds/dp.wso2.com_apis.yaml | 19 +++++++++---------- 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/common-go-libs/apis/dp/v1alpha1/api_conversion.go b/common-go-libs/apis/dp/v1alpha1/api_conversion.go index 154d6bfbd..85a067880 100644 --- a/common-go-libs/apis/dp/v1alpha1/api_conversion.go +++ b/common-go-libs/apis/dp/v1alpha1/api_conversion.go @@ -39,6 +39,22 @@ func (src *API) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.BasePath = src.Spec.BasePath dst.Spec.Organization = src.Spec.Organization dst.Spec.SystemAPI = src.Spec.SystemAPI + if src.Spec.Production != nil { + src.Spec.Production = []EnvConfig{} + for _, productionRef := range src.Spec.Production { + dst.Spec.Production = append(dst.Spec.Production, v1alpha2.EnvConfig{ + RouteRefs: productionRef.HTTPRouteRefs, + }) + } + } + if src.Spec.Sandbox != nil { + src.Spec.Sandbox = []EnvConfig{} + for _, sandboxRef := range src.Spec.Sandbox { + dst.Spec.Sandbox = append(dst.Spec.Sandbox, v1alpha2.EnvConfig{ + RouteRefs: sandboxRef.HTTPRouteRefs, + }) + } + } // Convert []Property to []v1alpha2.Property var properties []v1alpha2.Property diff --git a/common-go-libs/apis/dp/v1alpha2/api_types.go b/common-go-libs/apis/dp/v1alpha2/api_types.go index 741814b45..4579e5e0b 100644 --- a/common-go-libs/apis/dp/v1alpha2/api_types.go +++ b/common-go-libs/apis/dp/v1alpha2/api_types.go @@ -47,7 +47,7 @@ type APISpec struct { // +optional IsDefaultVersion bool `json:"isDefaultVersion"` - // DefinitionFileRef contains the OpenAPI 3 or Swagger + // DefinitionFileRef contains the // definition of the API in a ConfigMap. // // +optional @@ -124,8 +124,8 @@ type Property struct { // EnvConfig contains the environment specific configuration type EnvConfig struct { - // HTTPRouteRefs denotes the environment of the API. - HTTPRouteRefs []string `json:"httpRouteRefs"` + // RouteRefs denotes the environment of the API. + RouteRefs []string `json:"routeRefs"` } // APIStatus defines the observed state of API diff --git a/common-go-libs/apis/dp/v1alpha2/api_webhook.go b/common-go-libs/apis/dp/v1alpha2/api_webhook.go index e347680a7..19de63456 100644 --- a/common-go-libs/apis/dp/v1alpha2/api_webhook.go +++ b/common-go-libs/apis/dp/v1alpha2/api_webhook.go @@ -91,7 +91,6 @@ func (r *API) ValidateDelete() (admission.Warnings, error) { // validateAPI validate api crd fields func (r *API) validateAPI() error { - loggers.LoggerAPKOperator.Error("validating...") var allErrs field.ErrorList conf := config.ReadConfigs() namespaces := conf.CommonController.Operator.Namespaces @@ -127,17 +126,18 @@ func (r *API) validateAPI() error { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("organization"), "Organization can not be empty")) } - if !(len(r.Spec.Production) > 0 && r.Spec.Production[0].HTTPRouteRefs != nil && len(r.Spec.Production[0].HTTPRouteRefs) > 0) && !(len(r.Spec.Sandbox) > 0 && r.Spec.Sandbox[0].HTTPRouteRefs != nil && len(r.Spec.Sandbox[0].HTTPRouteRefs) > 0) { + if !(len(r.Spec.Production) > 0 && r.Spec.Production[0].RouteRefs != nil && len(r.Spec.Production[0].RouteRefs) > 0) && + !(len(r.Spec.Sandbox) > 0 && r.Spec.Sandbox[0].RouteRefs != nil && len(r.Spec.Sandbox[0].RouteRefs) > 0) { allErrs = append(allErrs, field.Required(field.NewPath("spec"), "both API production and sandbox endpoint references cannot be empty")) } var prodHTTPRoute1, sandHTTPRoute1 []string if len(r.Spec.Production) > 0 { - prodHTTPRoute1 = r.Spec.Production[0].HTTPRouteRefs + prodHTTPRoute1 = r.Spec.Production[0].RouteRefs } if len(r.Spec.Sandbox) > 0 { - sandHTTPRoute1 = r.Spec.Sandbox[0].HTTPRouteRefs + sandHTTPRoute1 = r.Spec.Sandbox[0].RouteRefs } if isEmptyStringsInArray(prodHTTPRoute1) { @@ -263,8 +263,7 @@ func validateGzip(name, namespace string) (string, string) { configMap := &corev1.ConfigMap{} if err := c.Get(context.Background(), types.NamespacedName{Name: string(name), Namespace: namespace}, configMap); err == nil { var apiDef []byte - for key, val := range configMap.BinaryData { - loggers.LoggerAPKOperator.Error(key) + for _, val := range configMap.BinaryData { // config map data key is "swagger.yaml" apiDef = []byte(val) } diff --git a/common-go-libs/apis/dp/v1alpha2/zz_generated.deepcopy.go b/common-go-libs/apis/dp/v1alpha2/zz_generated.deepcopy.go index 0400f7780..5a869b17c 100644 --- a/common-go-libs/apis/dp/v1alpha2/zz_generated.deepcopy.go +++ b/common-go-libs/apis/dp/v1alpha2/zz_generated.deepcopy.go @@ -531,8 +531,8 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvConfig) DeepCopyInto(out *EnvConfig) { *out = *in - if in.HTTPRouteRefs != nil { - in, out := &in.HTTPRouteRefs, &out.HTTPRouteRefs + if in.RouteRefs != nil { + in, out := &in.RouteRefs, &out.RouteRefs *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml index 304529c66..7846de3ca 100644 --- a/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml +++ b/common-go-libs/config/crd/bases/dp.wso2.com_apis.yaml @@ -259,8 +259,8 @@ spec: pattern: ^[/][a-zA-Z0-9~/_.-]*$ type: string definitionFileRef: - description: DefinitionFileRef contains the OpenAPI 3 or Swagger definition - of the API in a ConfigMap. + description: DefinitionFileRef contains the definition of the API + in a ConfigMap. type: string definitionPath: default: /api-definition @@ -285,13 +285,13 @@ spec: items: description: EnvConfig contains the environment specific configuration properties: - httpRouteRefs: - description: HTTPRouteRefs denotes the environment of the API. + routeRefs: + description: RouteRefs denotes the environment of the API. items: type: string type: array required: - - httpRouteRefs + - routeRefs type: object maxItems: 1 nullable: true @@ -302,13 +302,13 @@ spec: items: description: EnvConfig contains the environment specific configuration properties: - httpRouteRefs: - description: HTTPRouteRefs denotes the environment of the API. + routeRefs: + description: RouteRefs denotes the environment of the API. items: type: string type: array required: - - httpRouteRefs + - routeRefs type: object maxItems: 1 nullable: true diff --git a/common-go-libs/go.sum b/common-go-libs/go.sum index b7ff16a46..9db5ea8a8 100644 --- a/common-go-libs/go.sum +++ b/common-go-libs/go.sum @@ -153,6 +153,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/helm-charts/crds/dp.wso2.com_apis.yaml b/helm-charts/crds/dp.wso2.com_apis.yaml index fc4e0a2a9..7846de3ca 100644 --- a/helm-charts/crds/dp.wso2.com_apis.yaml +++ b/helm-charts/crds/dp.wso2.com_apis.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: apis.dp.wso2.com spec: group: dp.wso2.com @@ -260,8 +259,8 @@ spec: pattern: ^[/][a-zA-Z0-9~/_.-]*$ type: string definitionFileRef: - description: DefinitionFileRef contains the OpenAPI 3 or SDL file - in gzipped format. definition of the API in a ConfigMap. + description: DefinitionFileRef contains the definition of the API + in a ConfigMap. type: string definitionPath: default: /api-definition @@ -286,13 +285,13 @@ spec: items: description: EnvConfig contains the environment specific configuration properties: - httpRouteRefs: - description: HTTPRouteRefs denotes the environment of the API. + routeRefs: + description: RouteRefs denotes the environment of the API. items: type: string type: array required: - - httpRouteRefs + - routeRefs type: object maxItems: 1 nullable: true @@ -303,13 +302,13 @@ spec: items: description: EnvConfig contains the environment specific configuration properties: - httpRouteRefs: - description: HTTPRouteRefs denotes the environment of the API. + routeRefs: + description: RouteRefs denotes the environment of the API. items: type: string type: array required: - - httpRouteRefs + - routeRefs type: object maxItems: 1 nullable: true From 5b078af93b8fdbeb5c8267407217e035d9a90ea3 Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Wed, 24 Jan 2024 14:27:24 +0530 Subject: [PATCH 02/10] rename httprouterefs to routerefs --- .../envoyconf/routes_with_clusters_test.go | 12 ++++----- .../operator/controllers/dp/api_controller.go | 12 ++++----- .../dp/ratelimitpolicy_controller.go | 4 +-- .../apis/dp/v1alpha1/api_conversion.go | 26 ------------------- 4 files changed, 14 insertions(+), 40 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go index 03ca8957c..6fae0262d 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go @@ -48,7 +48,7 @@ func TestCreateRoutesWithClustersWithExactAndRegularExpressionRules(t *testing.T BasePath: "/test-api/2.0.0", Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ "test-api-2-prod-http-route", }, }, @@ -218,7 +218,7 @@ func generateSampleAPI(apiName string, apiVersion string, basePath string) synch BasePath: basePath, Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ apiName + "-prod-http-route", }, }, @@ -281,7 +281,7 @@ func TestCreateRoutesWithClustersWithMultiplePathPrefixRules(t *testing.T) { BasePath: "/test-api/1.0.0", Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ "test-api-1-prod-http-route", }, }, @@ -430,7 +430,7 @@ func TestCreateRoutesWithClustersWithBackendTLSConfigs(t *testing.T) { BasePath: "/test-api-3/1.0.0", Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ "test-api-3-prod-http-route", }, }, @@ -553,7 +553,7 @@ func TestCreateRoutesWithClustersDifferentBackendRefs(t *testing.T) { BasePath: "/test-api-different-backendrefs/1.0.0", Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ "test-api-different-backendrefs-prod-http-route", }, }, @@ -645,7 +645,7 @@ func TestCreateRoutesWithClustersSameBackendRefs(t *testing.T) { BasePath: "/test-api-same-backendrefs/1.0.0", Production: []v1alpha2.EnvConfig{ { - HTTPRouteRefs: []string{ + RouteRefs: []string{ "test-api-same-backendrefs-prod-http-route", }, }, diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 352be2868..6d32554fc 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -300,10 +300,10 @@ func (apiReconciler *APIReconciler) applyStartupAPIs() { func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1alpha2.API) (*synchronizer.APIEvent, error) { var prodRouteRefs, sandRouteRefs []string if len(api.Spec.Production) > 0 { - prodRouteRefs = api.Spec.Production[0].HTTPRouteRefs + prodRouteRefs = api.Spec.Production[0].RouteRefs } if len(api.Spec.Sandbox) > 0 { - sandRouteRefs = api.Spec.Sandbox[0].HTTPRouteRefs + sandRouteRefs = api.Spec.Sandbox[0].RouteRefs } apiState := &synchronizer.APIState{ @@ -1484,7 +1484,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { api := rawObj.(*dpv1alpha2.API) var httpRoutes []string if len(api.Spec.Production) > 0 { - for _, ref := range api.Spec.Production[0].HTTPRouteRefs { + for _, ref := range api.Spec.Production[0].RouteRefs { if ref != "" { httpRoutes = append(httpRoutes, types.NamespacedName{ @@ -1495,7 +1495,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { } } if len(api.Spec.Sandbox) > 0 { - for _, ref := range api.Spec.Sandbox[0].HTTPRouteRefs { + for _, ref := range api.Spec.Sandbox[0].RouteRefs { if ref != "" { httpRoutes = append(httpRoutes, types.NamespacedName{ @@ -1515,7 +1515,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { api := rawObj.(*dpv1alpha2.API) var gqlRoutes []string if len(api.Spec.Production) > 0 { - for _, ref := range api.Spec.Production[0].HTTPRouteRefs { + for _, ref := range api.Spec.Production[0].RouteRefs { if ref != "" { gqlRoutes = append(gqlRoutes, types.NamespacedName{ @@ -1526,7 +1526,7 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { } } if len(api.Spec.Sandbox) > 0 { - for _, ref := range api.Spec.Sandbox[0].HTTPRouteRefs { + for _, ref := range api.Spec.Sandbox[0].RouteRefs { if ref != "" { gqlRoutes = append(gqlRoutes, types.NamespacedName{ diff --git a/common-controller/internal/operator/controllers/dp/ratelimitpolicy_controller.go b/common-controller/internal/operator/controllers/dp/ratelimitpolicy_controller.go index b89691bb6..c4de5e578 100644 --- a/common-controller/internal/operator/controllers/dp/ratelimitpolicy_controller.go +++ b/common-controller/internal/operator/controllers/dp/ratelimitpolicy_controller.go @@ -289,7 +289,7 @@ func (ratelimitReconsiler *RateLimitPolicyReconciler) marshelRateLimit(ctx conte if len(api.Spec.Production) > 0 && api.Spec.APIType == "REST" { resolveResourceList, err := ratelimitReconsiler.getHTTPRouteResourceList(ctx, ratelimitKey, ratelimitPolicy, - api.Spec.Production[0].HTTPRouteRefs) + api.Spec.Production[0].RouteRefs) if err != nil { return nil, err } @@ -301,7 +301,7 @@ func (ratelimitReconsiler *RateLimitPolicyReconciler) marshelRateLimit(ctx conte if len(api.Spec.Sandbox) > 0 && api.Spec.APIType == "REST" { resolveResourceList, err := ratelimitReconsiler.getHTTPRouteResourceList(ctx, ratelimitKey, ratelimitPolicy, - api.Spec.Sandbox[0].HTTPRouteRefs) + api.Spec.Sandbox[0].RouteRefs) if err != nil { return nil, err } diff --git a/common-go-libs/apis/dp/v1alpha1/api_conversion.go b/common-go-libs/apis/dp/v1alpha1/api_conversion.go index 85a067880..4de702be0 100644 --- a/common-go-libs/apis/dp/v1alpha1/api_conversion.go +++ b/common-go-libs/apis/dp/v1alpha1/api_conversion.go @@ -63,19 +63,6 @@ func (src *API) ConvertTo(dstRaw conversion.Hub) error { } dst.Spec.APIProperties = properties - // Convert []EnvConfig to []v1alpha2.EnvConfig - var production []v1alpha2.EnvConfig - for _, p := range src.Spec.Production { - production = append(production, v1alpha2.EnvConfig(p)) - } - dst.Spec.Production = production - - var sandbox []v1alpha2.EnvConfig - for _, s := range src.Spec.Sandbox { - sandbox = append(sandbox, v1alpha2.EnvConfig(s)) - } - dst.Spec.Sandbox = sandbox - // Status dst.Status.DeploymentStatus = v1alpha2.DeploymentStatus(src.Status.DeploymentStatus) @@ -107,19 +94,6 @@ func (src *API) ConvertFrom(srcRaw conversion.Hub) error { } src.Spec.APIProperties = properties - // Convert []EnvConfig to []v1alpha1.EnvConfig - var production []EnvConfig - for _, p := range dst.Spec.Production { - production = append(production, EnvConfig(p)) - } - src.Spec.Production = production - - var sandbox []EnvConfig - for _, s := range dst.Spec.Sandbox { - sandbox = append(sandbox, EnvConfig(s)) - } - src.Spec.Sandbox = sandbox - // Status src.Status.DeploymentStatus = DeploymentStatus(dst.Status.DeploymentStatus) From a2a7524e87ef4a9be8fcbda5c857217224f88ee7 Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Wed, 24 Jan 2024 14:39:58 +0530 Subject: [PATCH 03/10] rename httprouterefs to routerefs in crs --- .../integration/tests/resources/base/manifests.yaml | 2 +- .../resources/tests/all-http-methods-for-wildcard.yaml | 2 +- .../tests/resources/tests/api-different-listener.yaml | 2 +- .../resources/tests/api-policy-with-jwt-generator.yaml | 2 +- .../tests/resources/tests/api-with-backend-base-path.yaml | 2 +- .../tests/resources/tests/api-with-cors-policy.yaml | 4 ++-- .../tests/resources/tests/api-with-operational-policy.yaml | 2 +- .../tests/resources/tests/api-with-path-params.yaml | 2 +- .../resources/tests/api-with-request-header-modify.yaml | 2 +- .../resources/tests/api-with-response-header-modify.yaml | 2 +- .../resources/tests/api-without-backend-base-path.yaml | 2 +- .../tests/resources/tests/custom-auth-header.yaml | 2 +- .../tests/resources/tests/custom-policy-ratelimiting.yaml | 2 +- .../tests/default-api-version-ratelimit-resource-level.yaml | 2 +- .../resources/tests/default-api-version-ratelimit.yaml | 2 +- .../tests/resources/tests/default-api-version.yaml | 2 +- .../resources/tests/different-endpoint-with-same-route.yaml | 2 +- .../tests/resources/tests/disable-api-level-jwt.yaml | 6 +++--- .../tests/resources/tests/disable-api-level-security.yaml | 2 +- .../tests/resources/tests/disable-resource-level-jwt.yaml | 6 +++--- .../resources/tests/disable-resource-level-security.yaml | 2 +- .../tests/resources/tests/fetch-api-definition.yaml | 2 +- .../resources/tests/fetch-non-existing-api-definition.yaml | 2 +- .../integration/tests/resources/tests/gql-api.yaml | 2 +- .../tests/resources/tests/interceptors-api-level.yaml | 2 +- .../tests/resources/tests/interceptors-resource-level.yaml | 2 +- .../tests/resources/tests/multiple_port_listener.yaml | 4 ++-- .../tests/resources/tests/prod-and-sand-apis.yaml | 6 +++--- .../tests/resources/tests/ratelimit-priority.yaml | 2 +- .../integration/tests/resources/tests/resource-scopes.yaml | 2 +- .../integration/tests/resources/tests/trailing-slash.yaml | 2 +- 31 files changed, 39 insertions(+), 39 deletions(-) diff --git a/test/integration/integration/tests/resources/base/manifests.yaml b/test/integration/integration/tests/resources/base/manifests.yaml index aa8a03411..5d4ef2460 100644 --- a/test/integration/integration/tests/resources/base/manifests.yaml +++ b/test/integration/integration/tests/resources/base/manifests.yaml @@ -426,6 +426,6 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - all-http-methods-for-wildcard-httproute organization: wso2-org1111 diff --git a/test/integration/integration/tests/resources/tests/all-http-methods-for-wildcard.yaml b/test/integration/integration/tests/resources/tests/all-http-methods-for-wildcard.yaml index 75fc0e4c5..fdbe8313d 100644 --- a/test/integration/integration/tests/resources/tests/all-http-methods-for-wildcard.yaml +++ b/test/integration/integration/tests/resources/tests/all-http-methods-for-wildcard.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - all-http-methods-for-wildcard-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-different-listener.yaml b/test/integration/integration/tests/resources/tests/api-different-listener.yaml index 65f80d101..b8c03f24f 100644 --- a/test/integration/integration/tests/resources/tests/api-different-listener.yaml +++ b/test/integration/integration/tests/resources/tests/api-different-listener.yaml @@ -26,7 +26,7 @@ spec: basePath: /test-api-with-different-listener/v1.0.0 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - test-api-with-different-listener-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-policy-with-jwt-generator.yaml b/test/integration/integration/tests/resources/tests/api-policy-with-jwt-generator.yaml index 8adfb9a2f..bbc22ce45 100644 --- a/test/integration/integration/tests/resources/tests/api-policy-with-jwt-generator.yaml +++ b/test/integration/integration/tests/resources/tests/api-policy-with-jwt-generator.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - api-policy-with-jwt-generator-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-backend-base-path.yaml b/test/integration/integration/tests/resources/tests/api-with-backend-base-path.yaml index acc89d9ff..9fb505524 100644 --- a/test/integration/integration/tests/resources/tests/api-with-backend-base-path.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-backend-base-path.yaml @@ -38,7 +38,7 @@ spec: isDefaultVersion: true #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - test-api-with-backend-base-path-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-cors-policy.yaml b/test/integration/integration/tests/resources/tests/api-with-cors-policy.yaml index e2aca8699..6c71a76c2 100644 --- a/test/integration/integration/tests/resources/tests/api-with-cors-policy.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-cors-policy.yaml @@ -26,7 +26,7 @@ spec: basePath: /cors-policy-api/1.0.0 isDefaultVersion: true production: - - httpRouteRefs: + - routeRefs: - cors-policy-httproute organization: wso2-org --- @@ -95,7 +95,7 @@ spec: apiVersion: 1.0.0 basePath: /no-cors-policy-api/1.0.0 production: - - httpRouteRefs: + - routeRefs: - no-cors-policy-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-operational-policy.yaml b/test/integration/integration/tests/resources/tests/api-with-operational-policy.yaml index be2fbef20..6d350c3b8 100644 --- a/test/integration/integration/tests/resources/tests/api-with-operational-policy.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-operational-policy.yaml @@ -43,7 +43,7 @@ spec: isDefaultVersion: true #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - test-api-with-operatinal-policy-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-path-params.yaml b/test/integration/integration/tests/resources/tests/api-with-path-params.yaml index 81304619e..e4e1e053a 100644 --- a/test/integration/integration/tests/resources/tests/api-with-path-params.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-path-params.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - test-api-with-path-params-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-request-header-modify.yaml b/test/integration/integration/tests/resources/tests/api-with-request-header-modify.yaml index 7ac6a07da..d045c275e 100644 --- a/test/integration/integration/tests/resources/tests/api-with-request-header-modify.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-request-header-modify.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - test-api-with-request-header-modify-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-with-response-header-modify.yaml b/test/integration/integration/tests/resources/tests/api-with-response-header-modify.yaml index 416247d03..6596cd0cd 100644 --- a/test/integration/integration/tests/resources/tests/api-with-response-header-modify.yaml +++ b/test/integration/integration/tests/resources/tests/api-with-response-header-modify.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - test-api-with-response-header-modify-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/api-without-backend-base-path.yaml b/test/integration/integration/tests/resources/tests/api-without-backend-base-path.yaml index 54358cdb7..2353d1ee4 100644 --- a/test/integration/integration/tests/resources/tests/api-without-backend-base-path.yaml +++ b/test/integration/integration/tests/resources/tests/api-without-backend-base-path.yaml @@ -26,7 +26,7 @@ spec: basePath: /no-basepath/v1 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - no-backend-base-path-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/custom-auth-header.yaml b/test/integration/integration/tests/resources/tests/custom-auth-header.yaml index 585359006..3f602c0b0 100644 --- a/test/integration/integration/tests/resources/tests/custom-auth-header.yaml +++ b/test/integration/integration/tests/resources/tests/custom-auth-header.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - custom-auth-header-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/custom-policy-ratelimiting.yaml b/test/integration/integration/tests/resources/tests/custom-policy-ratelimiting.yaml index a41adb3e5..751228c0e 100644 --- a/test/integration/integration/tests/resources/tests/custom-policy-ratelimiting.yaml +++ b/test/integration/integration/tests/resources/tests/custom-policy-ratelimiting.yaml @@ -70,7 +70,7 @@ spec: basePath: /http-bin-api/1.0.8 #definitionFileRef: swagger-definition-http-bin-api production: - - httpRouteRefs: + - routeRefs: - prod-http-route-http-bin-api-basic organization: a3b58ccf-6ecc-4557-b5bb-0a35cce38256 --- diff --git a/test/integration/integration/tests/resources/tests/default-api-version-ratelimit-resource-level.yaml b/test/integration/integration/tests/resources/tests/default-api-version-ratelimit-resource-level.yaml index ab977102e..07eb23cdf 100644 --- a/test/integration/integration/tests/resources/tests/default-api-version-ratelimit-resource-level.yaml +++ b/test/integration/integration/tests/resources/tests/default-api-version-ratelimit-resource-level.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - default-api-version-ratelimit-resource-level-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/default-api-version-ratelimit.yaml b/test/integration/integration/tests/resources/tests/default-api-version-ratelimit.yaml index 95aa3a7bd..3a9b9a5db 100644 --- a/test/integration/integration/tests/resources/tests/default-api-version-ratelimit.yaml +++ b/test/integration/integration/tests/resources/tests/default-api-version-ratelimit.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - default-api-version-ratelimit-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/default-api-version.yaml b/test/integration/integration/tests/resources/tests/default-api-version.yaml index 64a6440b5..b63875b0f 100644 --- a/test/integration/integration/tests/resources/tests/default-api-version.yaml +++ b/test/integration/integration/tests/resources/tests/default-api-version.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - default-api-version-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/different-endpoint-with-same-route.yaml b/test/integration/integration/tests/resources/tests/different-endpoint-with-same-route.yaml index 02fc2318d..a7bda1ef9 100644 --- a/test/integration/integration/tests/resources/tests/different-endpoint-with-same-route.yaml +++ b/test/integration/integration/tests/resources/tests/different-endpoint-with-same-route.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - different-endpoint-with-same-route-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/disable-api-level-jwt.yaml b/test/integration/integration/tests/resources/tests/disable-api-level-jwt.yaml index f28abe7f6..646c908f8 100644 --- a/test/integration/integration/tests/resources/tests/disable-api-level-jwt.yaml +++ b/test/integration/integration/tests/resources/tests/disable-api-level-jwt.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-api-level-jwt-httproute organization: wso2-org --- @@ -88,7 +88,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-api-level-jwt1-httproute organization: wso2-org --- @@ -159,7 +159,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-api-level-jwt2-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/disable-api-level-security.yaml b/test/integration/integration/tests/resources/tests/disable-api-level-security.yaml index 3a112479c..fd4558e0b 100644 --- a/test/integration/integration/tests/resources/tests/disable-api-level-security.yaml +++ b/test/integration/integration/tests/resources/tests/disable-api-level-security.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-api-security-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/disable-resource-level-jwt.yaml b/test/integration/integration/tests/resources/tests/disable-resource-level-jwt.yaml index a6c559be9..f7459d308 100644 --- a/test/integration/integration/tests/resources/tests/disable-resource-level-jwt.yaml +++ b/test/integration/integration/tests/resources/tests/disable-resource-level-jwt.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-resource-level-jwt-httproute organization: wso2-org --- @@ -94,7 +94,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-resource-level-jwt1-httproute organization: wso2-org --- @@ -171,7 +171,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-resource-level-jwt2-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/disable-resource-level-security.yaml b/test/integration/integration/tests/resources/tests/disable-resource-level-security.yaml index d9e11c97a..bd66d7241 100644 --- a/test/integration/integration/tests/resources/tests/disable-resource-level-security.yaml +++ b/test/integration/integration/tests/resources/tests/disable-resource-level-security.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - disable-resoruce-security-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/fetch-api-definition.yaml b/test/integration/integration/tests/resources/tests/fetch-api-definition.yaml index 34a519318..97e5a1d79 100644 --- a/test/integration/integration/tests/resources/tests/fetch-api-definition.yaml +++ b/test/integration/integration/tests/resources/tests/fetch-api-definition.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true definitionFileRef: definition-file-swagger production: - - httpRouteRefs: + - routeRefs: - fetch-api-defintion-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/fetch-non-existing-api-definition.yaml b/test/integration/integration/tests/resources/tests/fetch-non-existing-api-definition.yaml index 60c341572..eb25ac95a 100644 --- a/test/integration/integration/tests/resources/tests/fetch-non-existing-api-definition.yaml +++ b/test/integration/integration/tests/resources/tests/fetch-non-existing-api-definition.yaml @@ -25,7 +25,7 @@ spec: apiVersion: v1.0.0 basePath: /fetch-non-existing-api-definition/v1.0.0 production: - - httpRouteRefs: + - routeRefs: - fetch-non-api-definition-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/gql-api.yaml b/test/integration/integration/tests/resources/tests/gql-api.yaml index 474e9bca4..224dfba25 100644 --- a/test/integration/integration/tests/resources/tests/gql-api.yaml +++ b/test/integration/integration/tests/resources/tests/gql-api.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: false definitionFileRef: gql-sdl production: - - httpRouteRefs: + - routeRefs: - gql-api-gqlroute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/interceptors-api-level.yaml b/test/integration/integration/tests/resources/tests/interceptors-api-level.yaml index 5ca0c2725..af5d12643 100644 --- a/test/integration/integration/tests/resources/tests/interceptors-api-level.yaml +++ b/test/integration/integration/tests/resources/tests/interceptors-api-level.yaml @@ -26,7 +26,7 @@ spec: basePath: /interceptor/2.0.0 #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - api-interceptor-route organization: wso2-org apiProperties: diff --git a/test/integration/integration/tests/resources/tests/interceptors-resource-level.yaml b/test/integration/integration/tests/resources/tests/interceptors-resource-level.yaml index c4f790e83..06882d25a 100644 --- a/test/integration/integration/tests/resources/tests/interceptors-resource-level.yaml +++ b/test/integration/integration/tests/resources/tests/interceptors-resource-level.yaml @@ -26,7 +26,7 @@ spec: basePath: /interceptor/1.0.0 #definitionFileRef: swagger-definition-webhook production: - - httpRouteRefs: + - routeRefs: - resource-interceptor-route organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml b/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml index e1ffa437f..8dd1e4b96 100644 --- a/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml +++ b/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml @@ -26,7 +26,7 @@ spec: basePath: /test-api-with-multiple-port-listener/v1.0.0 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - test-api-with-multiple-port-listener-httproute organization: wso2-org --- @@ -42,7 +42,7 @@ spec: basePath: /test-api-with-multiple-port-listener-1/v1.0.0 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - test-api-with-multiple-port-listener-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/prod-and-sand-apis.yaml b/test/integration/integration/tests/resources/tests/prod-and-sand-apis.yaml index 16327affa..82c99b640 100644 --- a/test/integration/integration/tests/resources/tests/prod-and-sand-apis.yaml +++ b/test/integration/integration/tests/resources/tests/prod-and-sand-apis.yaml @@ -26,7 +26,7 @@ spec: basePath: /prod-sand-test-api/v1 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - prod-httproute organization: wso2-org --- @@ -128,9 +128,9 @@ spec: basePath: /prod-sand-test-api/v1 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - prod-httproute sandbox: - - httpRouteRefs: + - routeRefs: - sand-httproute organization: wso2-org diff --git a/test/integration/integration/tests/resources/tests/ratelimit-priority.yaml b/test/integration/integration/tests/resources/tests/ratelimit-priority.yaml index b2a59aeb3..819f5eac1 100644 --- a/test/integration/integration/tests/resources/tests/ratelimit-priority.yaml +++ b/test/integration/integration/tests/resources/tests/ratelimit-priority.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - ratelimit-priority-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/resource-scopes.yaml b/test/integration/integration/tests/resources/tests/resource-scopes.yaml index 7cb94975e..d0b5908ab 100644 --- a/test/integration/integration/tests/resources/tests/resource-scopes.yaml +++ b/test/integration/integration/tests/resources/tests/resource-scopes.yaml @@ -26,7 +26,7 @@ spec: basePath: /resource-scopes/v1 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - resource-scopes-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/trailing-slash.yaml b/test/integration/integration/tests/resources/tests/trailing-slash.yaml index c79dbed85..1b9bcd09d 100644 --- a/test/integration/integration/tests/resources/tests/trailing-slash.yaml +++ b/test/integration/integration/tests/resources/tests/trailing-slash.yaml @@ -26,7 +26,7 @@ spec: basePath: /trailing-slash/v1.0.0 #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - trailing-slash-httproute organization: wso2-org --- From 2a3b0683a48e66bc608f7371ebcf62772dffc32a Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Wed, 24 Jan 2024 15:16:32 +0530 Subject: [PATCH 04/10] remove debug logs --- .../data-plane/config-deployer/config-ds-configmap.yaml | 2 +- .../data-plane/gateway-components/common-log-conf.yaml | 2 +- helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 6a4e56650..bf2f22fb3 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -23,7 +23,7 @@ metadata: data: Config.toml: | [ballerina.log] - level = "DEBUG" + level = "INFO" [ballerina.http] traceLogConsole = false {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} diff --git a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml index b3dbf39e8..75407bd76 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml @@ -113,7 +113,7 @@ data: logLevel = "INFO" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" [[pkg]] name = "github.com/wso2/apk/common-controller/internal/utils" - logLevel = "DEBG" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" + logLevel = "INFO" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" [[pkg]] # The logging configuration for Router diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml index 62c690912..a97122786 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml @@ -23,7 +23,7 @@ metadata: data: Config.toml: | [ballerina.log] - level = "DEBUG" + level = "INFO" [ballerina.http] traceLogConsole = false {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} From 2587b2d28d58b3a3d514aa2410f2252e530cc12b Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Wed, 6 Mar 2024 10:46:08 +0530 Subject: [PATCH 05/10] fix lint issues --- adapter/internal/operator/synchronizer/synchronizer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/internal/operator/synchronizer/synchronizer.go b/adapter/internal/operator/synchronizer/synchronizer.go index 3305399d5..8f177d80c 100644 --- a/adapter/internal/operator/synchronizer/synchronizer.go +++ b/adapter/internal/operator/synchronizer/synchronizer.go @@ -224,7 +224,7 @@ func deployMultipleAPIsInGateway(event *APIEvent, successChannel *chan SuccessEv updated := xds.UpdateXdsCacheOnAPIChange(updatedLabelsMap) if updated { - loggers.LoggerAPKOperator.Info("XDS cache updated for apis: %+v", updatedAPIs) + loggers.LoggerAPKOperator.Infof("XDS cache updated for apis: %+v", updatedAPIs) *successChannel <- SuccessEvent{ APINamespacedName: updatedAPIs, State: event.EventType, @@ -234,7 +234,7 @@ func deployMultipleAPIsInGateway(event *APIEvent, successChannel *chan SuccessEv paritionCh <- event } } else { - loggers.LoggerAPKOperator.Info("XDS cache not updated for APIs : %+v", updatedAPIs) + loggers.LoggerAPKOperator.Infof("XDS cache not updated for APIs : %+v", updatedAPIs) } } From cecc7f2d0c1124cca6b91ed2e0892f30c439cd93 Mon Sep 17 00:00:00 2001 From: tharindu1st Date: Wed, 6 Mar 2024 16:37:30 +0530 Subject: [PATCH 06/10] httpRouteRef=>routeRef change --- developer/tryout/samples/sample-api.yaml | 6 +++--- .../wso2-apk-config-deployer-api.yaml | 4 ++-- .../wso2-apk-config-generator-api.yaml | 4 ++-- .../gateway-runtime/jwks-domain-api.yaml | 4 ++-- .../idp/authenticationEndpoint-domain-api.yaml | 4 ++-- .../templates/idp/commonoauth-domain-api.yaml | 4 ++-- helm-charts/templates/idp/dcr-domain-api.yaml | 4 ++-- helm-charts/templates/idp/oauth-domain-api.yaml | 4 ++-- .../config-deployer-service/ballerina/APIClient.bal | 4 ++-- .../ballerina/Dependencies.toml | 2 +- .../ballerina/Dependencies.toml.template | 13 +++++++------ .../ballerina/modules/model/API.bal | 2 +- .../api-interceptor/sample-api.yaml | 4 ++-- .../gateway-interceptor/sample-api.yaml | 4 ++-- .../resource-interceptor/sample-api.yaml | 4 ++-- .../k8s-resources/api.yaml | 4 ++-- test/performance/artifacts/api.yaml | 4 ++-- 17 files changed, 38 insertions(+), 37 deletions(-) diff --git a/developer/tryout/samples/sample-api.yaml b/developer/tryout/samples/sample-api.yaml index 50de733e3..a954bae37 100644 --- a/developer/tryout/samples/sample-api.yaml +++ b/developer/tryout/samples/sample-api.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -apiVersion: dp.wso2.com/v1alpha1 +apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: name: http-bin-api @@ -24,11 +24,11 @@ spec: basePath: /http-bin-api/1.0.8 definitionFileRef: swagger-definition-http-bin-api production: - - httpRouteRefs: + - routeRefs: - prod-http-route-http-bin-api-p1 - prod-http-route-http-bin-api-p2 sandbox: - - httpRouteRefs: + - routeRefs: - sand-http-route-http-bin-api organization: a3b58ccf-6ecc-4557-b5bb-0a35cce38256 --- diff --git a/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-deployer-api.yaml b/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-deployer-api.yaml index d1a39ea98..c51f866fd 100644 --- a/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-deployer-api.yaml +++ b/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-deployer-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.dp.configdeployer.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: "{{ template "apk-helm.resource.prefix" . }}-wso2-apk-config-deployer-api" namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: organization: "apk-system" definitionFileRef: "{{ template "apk-helm.resource.prefix" . }}-wso2-apk-config-deployer-api-definition" production: - - httpRouteRefs: + - routeRefs: - "{{ template "apk-helm.resource.prefix" . }}-config-deploy-api-route" systemAPI: true {{- end -}} diff --git a/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-generator-api.yaml b/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-generator-api.yaml index 61f4dc246..7a8dc6645 100644 --- a/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-generator-api.yaml +++ b/helm-charts/templates/data-plane/config-deployer/wso2-apk-config-generator-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.dp.configdeployer.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: "{{ template "apk-helm.resource.prefix" . }}-wso2-apk-config-generator-api" namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: organization: "apk-system" definitionFileRef: "{{ template "apk-helm.resource.prefix" . }}-wso2-apk-config-generator-api-definition" production: - - httpRouteRefs: + - routeRefs: - "{{ template "apk-helm.resource.prefix" . }}-config-api-route" systemAPI: true {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/jwks-domain-api.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/jwks-domain-api.yaml index 7e5eecc6a..85767ba67 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/jwks-domain-api.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/jwks-domain-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if .Values.idp.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: {{ template "apk-helm.resource.prefix" . }}-jwks-endpoint-ds-api namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: organization: "apk-system" isDefaultVersion: true production: - - httpRouteRefs: + - routeRefs: - {{ template "apk-helm.resource.prefix" . }}-jwks-endpoint-ds-httproute systemAPI: true {{- end -}} diff --git a/helm-charts/templates/idp/authenticationEndpoint-domain-api.yaml b/helm-charts/templates/idp/authenticationEndpoint-domain-api.yaml index 9b6b31268..c759c62f6 100644 --- a/helm-charts/templates/idp/authenticationEndpoint-domain-api.yaml +++ b/helm-charts/templates/idp/authenticationEndpoint-domain-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if .Values.idp.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: {{ template "apk-helm.resource.prefix" . }}-authentication-endpoint-ds-api namespace: {{ .Release.Namespace }} @@ -29,7 +29,7 @@ spec: basePath: "/authenticationEndpoint/1.0.0" organization: "apk-system" production: - - httpRouteRefs: + - routeRefs: - {{ template "apk-helm.resource.prefix" . }}-authentication-endpoint-ds-httproute systemAPI: true {{- end -}} diff --git a/helm-charts/templates/idp/commonoauth-domain-api.yaml b/helm-charts/templates/idp/commonoauth-domain-api.yaml index dfcc4602a..9e0d7a04d 100644 --- a/helm-charts/templates/idp/commonoauth-domain-api.yaml +++ b/helm-charts/templates/idp/commonoauth-domain-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if .Values.idp.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: {{ template "apk-helm.resource.prefix" . }}-commonoauth-api namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: isDefaultVersion: true organization: "apk-system" production: - - httpRouteRefs: + - routeRefs: - {{ template "apk-helm.resource.prefix" . }}-commonoauth-ds-httproute systemAPI: true {{- end -}} diff --git a/helm-charts/templates/idp/dcr-domain-api.yaml b/helm-charts/templates/idp/dcr-domain-api.yaml index f4280ac65..d5a22a14b 100644 --- a/helm-charts/templates/idp/dcr-domain-api.yaml +++ b/helm-charts/templates/idp/dcr-domain-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if .Values.idp.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: {{ template "apk-helm.resource.prefix" . }}-dcr-api namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: isDefaultVersion: true organization: "apk-system" production: - - httpRouteRefs: + - routeRefs: - {{ template "apk-helm.resource.prefix" . }}-dcr-ds-httproute systemAPI: true {{- end -}} diff --git a/helm-charts/templates/idp/oauth-domain-api.yaml b/helm-charts/templates/idp/oauth-domain-api.yaml index 6f13f7976..179ef80c8 100644 --- a/helm-charts/templates/idp/oauth-domain-api.yaml +++ b/helm-charts/templates/idp/oauth-domain-api.yaml @@ -15,7 +15,7 @@ # under the License. {{- if .Values.idp.enabled }} kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: {{ template "apk-helm.resource.prefix" . }}-oauth-api namespace: {{ .Release.Namespace }} @@ -30,7 +30,7 @@ spec: isDefaultVersion: true organization: "apk-system" production: - - httpRouteRefs: + - routeRefs: - {{ template "apk-helm.resource.prefix" . }}-oauth-ds-httproute systemAPI: true {{- end -}} diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index c0009d947..fba8b6790 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -449,10 +449,10 @@ public class APIClient { } if productionRoutes.length() > 0 { - k8sAPI.spec.production = [{httpRouteRefs: productionRoutes}]; + k8sAPI.spec.production = [{routeRef: productionRoutes}]; } if sandboxRoutes.length() > 0 { - k8sAPI.spec.sandbox = [{httpRouteRefs: sandboxRoutes}]; + k8sAPI.spec.sandbox = [{routeRef: sandboxRoutes}]; } if apkConf.id != () { k8sAPI.metadata["annotations"] = {[API_UUID_ANNOTATION] : apkConf.id}; diff --git a/runtime/config-deployer-service/ballerina/Dependencies.toml b/runtime/config-deployer-service/ballerina/Dependencies.toml index bb1ba5577..45dbaad3f 100644 --- a/runtime/config-deployer-service/ballerina/Dependencies.toml +++ b/runtime/config-deployer-service/ballerina/Dependencies.toml @@ -70,7 +70,7 @@ modules = [ [[package]] org = "ballerina" name = "http" -version = "2.10.7" +version = "2.10.8" dependencies = [ {org = "ballerina", name = "auth"}, {org = "ballerina", name = "cache"}, diff --git a/runtime/config-deployer-service/ballerina/Dependencies.toml.template b/runtime/config-deployer-service/ballerina/Dependencies.toml.template index 526506973..c07db3430 100644 --- a/runtime/config-deployer-service/ballerina/Dependencies.toml.template +++ b/runtime/config-deployer-service/ballerina/Dependencies.toml.template @@ -5,7 +5,7 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.8.0" +distribution-version = "2201.8.5" [[package]] org = "ballerina" @@ -33,7 +33,7 @@ dependencies = [ [[package]] org = "ballerina" name = "constraint" -version = "1.4.0" +version = "1.5.0" dependencies = [ {org = "ballerina", name = "jballerina.java"} ] @@ -44,7 +44,7 @@ modules = [ [[package]] org = "ballerina" name = "crypto" -version = "2.5.0" +version = "2.6.2" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "time"} @@ -70,7 +70,7 @@ modules = [ [[package]] org = "ballerina" name = "http" -version = "2.10.7" +version = "2.10.8" dependencies = [ {org = "ballerina", name = "auth"}, {org = "ballerina", name = "cache"}, @@ -368,7 +368,7 @@ modules = [ [[package]] org = "ballerinax" name = "postgresql" -version = "1.10.0" +version = "1.11.0" dependencies = [ {org = "ballerina", name = "crypto"}, {org = "ballerina", name = "io"}, @@ -397,7 +397,8 @@ dependencies = [ modules = [ {org = "wso2", packageName = "apk_common_lib", moduleName = "apk_common_lib"}, {org = "wso2", packageName = "apk_common_lib", moduleName = "apk_common_lib.java.io"}, - {org = "wso2", packageName = "apk_common_lib", moduleName = "apk_common_lib.java.lang"} + {org = "wso2", packageName = "apk_common_lib", moduleName = "apk_common_lib.java.lang"}, + {org = "wso2", packageName = "apk_common_lib", moduleName = "apk_common_lib.org.wso2.apk.common"} ] [[package]] diff --git a/runtime/config-deployer-service/ballerina/modules/model/API.bal b/runtime/config-deployer-service/ballerina/modules/model/API.bal index b0510e540..7cb81ea24 100644 --- a/runtime/config-deployer-service/ballerina/modules/model/API.bal +++ b/runtime/config-deployer-service/ballerina/modules/model/API.bal @@ -58,7 +58,7 @@ public type APIStatus record { }; public type EnvConfig record { - string[] httpRouteRefs; + string[] routeRef; }; public type APIList record { diff --git a/samples/custom-ratelimit-interceptor-service/api-interceptor/sample-api.yaml b/samples/custom-ratelimit-interceptor-service/api-interceptor/sample-api.yaml index d78610b24..39a150ff5 100644 --- a/samples/custom-ratelimit-interceptor-service/api-interceptor/sample-api.yaml +++ b/samples/custom-ratelimit-interceptor-service/api-interceptor/sample-api.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -apiVersion: dp.wso2.com/v1alpha1 +apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: name: http-bin-api-basic @@ -24,7 +24,7 @@ spec: basePath: /http-bin-api/1.0.8 definitionFileRef: swagger-definition-http-bin-api production: - - httpRouteRefs: + - routeRefs: - prod-http-route-http-bin-api-basic organization: a3b58ccf-6ecc-4557-b5bb-0a35cce38256 --- diff --git a/samples/custom-ratelimit-interceptor-service/gateway-interceptor/sample-api.yaml b/samples/custom-ratelimit-interceptor-service/gateway-interceptor/sample-api.yaml index d78610b24..39a150ff5 100644 --- a/samples/custom-ratelimit-interceptor-service/gateway-interceptor/sample-api.yaml +++ b/samples/custom-ratelimit-interceptor-service/gateway-interceptor/sample-api.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -apiVersion: dp.wso2.com/v1alpha1 +apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: name: http-bin-api-basic @@ -24,7 +24,7 @@ spec: basePath: /http-bin-api/1.0.8 definitionFileRef: swagger-definition-http-bin-api production: - - httpRouteRefs: + - routeRefs: - prod-http-route-http-bin-api-basic organization: a3b58ccf-6ecc-4557-b5bb-0a35cce38256 --- diff --git a/samples/custom-ratelimit-interceptor-service/resource-interceptor/sample-api.yaml b/samples/custom-ratelimit-interceptor-service/resource-interceptor/sample-api.yaml index 85adc1264..87daab988 100644 --- a/samples/custom-ratelimit-interceptor-service/resource-interceptor/sample-api.yaml +++ b/samples/custom-ratelimit-interceptor-service/resource-interceptor/sample-api.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -apiVersion: dp.wso2.com/v1alpha1 +apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: name: http-bin-api-basic @@ -24,7 +24,7 @@ spec: basePath: /http-bin-api/1.0.8 definitionFileRef: swagger-definition-http-bin-api production: - - httpRouteRefs: + - routeRefs: - prod-http-route-http-bin-api-basic organization: a3b58ccf-6ecc-4557-b5bb-0a35cce38256 --- diff --git a/samples/request-response-mediation-interceptors/k8s-resources/api.yaml b/samples/request-response-mediation-interceptors/k8s-resources/api.yaml index 472e8305b..14acdb6a1 100644 --- a/samples/request-response-mediation-interceptors/k8s-resources/api.yaml +++ b/samples/request-response-mediation-interceptors/k8s-resources/api.yaml @@ -14,7 +14,7 @@ # limitations under the License. # ----------------------------------------------------------------------- -apiVersion: dp.wso2.com/v1alpha1 +apiVersion: dp.wso2.com/v1alpha2 kind: API metadata: name: interceptor-api @@ -31,7 +31,7 @@ spec: - name: propertyName2 value: "propertyValue2" production: - - httpRouteRefs: + - routeRefs: - interceptor-route --- apiVersion: dp.wso2.com/v1alpha1 diff --git a/test/performance/artifacts/api.yaml b/test/performance/artifacts/api.yaml index 23aa76b99..0343f8f85 100644 --- a/test/performance/artifacts/api.yaml +++ b/test/performance/artifacts/api.yaml @@ -1,6 +1,6 @@ --- kind: "API" -apiVersion: "dp.wso2.com/v1alpha1" +apiVersion: "dp.wso2.com/v1alpha2" metadata: name: "default-api-definition-endpoint-test" labels: @@ -18,7 +18,7 @@ spec: organization: "default" definitionFileRef: "default-api-definition-endpoint-test-definition" production: - - httpRouteRefs: + - routeRefs: - "01ee3d7a-36d0-176e-aee2-6b601948eb64" sandbox: null apiProperties: [] From 94c48c40ca793547eef338d58540df0cab3dcd29 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Sun, 25 Feb 2024 09:20:51 +0530 Subject: [PATCH 07/10] Add Prometheus support to common controller --- adapter/go.sum | 5 +- .../commoncontroller/common_controller.go | 6 + common-controller/go.mod | 3 +- common-controller/go.sum | 1 + common-controller/pkg/metrics/metrics.go | 191 ++++++++++++++++-- .../common-controller-service.yaml | 5 + 6 files changed, 196 insertions(+), 15 deletions(-) diff --git a/adapter/go.sum b/adapter/go.sum index b59fd5b9a..97362525d 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -117,8 +117,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -214,6 +214,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index d270085b2..3be8bbb1a 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -214,6 +214,12 @@ func InitCommonControllerServer(conf *config.Config) { loggers.LoggerAPKOperator.Info("Starting common controller ....") + // Start the metrics server + if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { + loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port, conf.CommonController.Metrics.CollectionInterval) + } + rateLimiterCache := xds.GetRateLimiterCache() rlsSrv := xdsv3.NewServer(ctx, rateLimiterCache, &xds.Callbacks{}) diff --git a/common-controller/go.mod b/common-controller/go.mod index 568002fa9..c1bdd0728 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -7,7 +7,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/procfs v0.12.0 github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -23,6 +23,7 @@ require ( github.com/redis/go-redis/v9 v9.2.1 github.com/wso2/apk/adapter v0.0.0-20231214082511-af2c8b8a19f1 github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 + github.com/shirou/gopsutil/v3 v3.24.1 google.golang.org/grpc v1.62.0 ) diff --git a/common-controller/go.sum b/common-controller/go.sum index 22c976b82..f5a353ede 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -314,6 +314,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go index c2da1fa33..ef7824ebb 100644 --- a/common-controller/pkg/metrics/metrics.go +++ b/common-controller/pkg/metrics/metrics.go @@ -16,33 +16,200 @@ * under the License. */ -// Package metrics holds the implementation for exposing adapter metrics to prometheus +// Package metrics holds the implementation for exposing common controller metrics to prometheus package metrics import ( "fmt" "net/http" + "os" + "runtime" "strconv" + "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/procfs" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" "github.com/wso2/apk/adapter/pkg/logging" logger "github.com/wso2/apk/common-controller/internal/loggers" - metrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32) { +var ( + prometheusMetricRegistry = prometheus.NewRegistry() + + hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "host_info", + Help: "Host Info", + }, []string{"os"}) + + availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_available_cpu_total", + Help: "Number of available CPUs.", + }) + + freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_free_physical_memory_bytes", + Help: "Amount of free physical memory.", + }) + + totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_total_virtual_memory_bytes", + Help: "Amount of total virtual memory.", + }) + usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_used_virtual_memory_bytes", + Help: "Amount of used virtual memory.", + }) + + systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_system_cpu_load_percentage", + Help: "System-wide CPU usage as a percentage.", + }) + + loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "os_system_load_average", + Help: "Current load of CPU in the host system for the last {x} minutes", + }, []string{"duration"}) + + processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_start_time_seconds", + Help: "Start time of the process since unix epoch in seconds.", + }) + + processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_open_fds", + Help: "Number of open file descriptors.", + }) +) + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) + + // Register other metrics + prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, + systemCPULoad, loadAvg, processStartTime, processOpenFDs) +} + +// recordMetrics record custom golang metrics +var recordMetrics = func(collectionInterval int32) { + for { + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfo.WithLabelValues(host.OS).Set(1) + availableCPUs.Set(float64(runtime.NumCPU())) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory.Set(float64(v.Free)) + usedVirtualMemory.Set(float64(v.Used)) + totalVirtualMemory.Set(float64(v.Total)) - collector := metrics.CustomMetricsCollector() - prometheus.MustRegister(collector) - http.Handle("/metrics", promhttp.Handler()) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad.Set(averagePercentage) + + pid := os.Getpid() + p, err := procfs.NewProc(pid) + if handleError(err, "Failed to get current process") { + return + } + stat, err := p.Stat() + if handleError(err, "Failed to get process stats") { + return + } + t, err := stat.StartTime() + if handleError(err, "Failed to read process start time") { + return + } + processStartTime.Set(t) + fds, err := p.FileDescriptorsLen() + if handleError(err, "Failed to read file descriptor count") { + return + } + processOpenFDs.Set(float64(fds)) + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + loadAvg.WithLabelValues("1m").Set(avg.Load1) + loadAvg.WithLabelValues("5m").Set(avg.Load5) + loadAvg.WithLabelValues("15m").Set(avg.Load15) + + // Sleep before the next measurement + time.Sleep(time.Duration(collectionInterval) * time.Second) + } + +} + +func handleError(err error, message string) bool { if err != nil { - logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, }) + return true + } + return false +} + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32, collectionInterval int32) { + done := make(chan struct{}) // Channel to indicate recordMetrics routine exit + + // Start the Prometheus metrics server + go func() { + http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + if err != nil { + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } + }() + + for { + // Start the recordMetrics goroutine + go func() { + defer func() { + if r := recover(); r != nil { + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), + Severity: logging.MAJOR, + ErrorCode: 1111, + }) + } + }() + recordMetrics(collectionInterval) + done <- struct{}{} // Signal that the goroutine has completed + }() + + // Wait for the previous recordMetrics goroutine to complete + <-done + + // Log and restart the goroutine + logger.LoggerAPKOperator.Info("Restarting recordMetrics goroutine...") + time.Sleep(3 * time.Second) } } diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml index e5adf645f..debcb579f 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml @@ -38,4 +38,9 @@ spec: - name: https-internal-api protocol: TCP port: 18003 + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-common-controller-port + protocol: TCP + port: 18009 + {{ end }} {{- end -}} From c951c1bd7848898f5a890154660065b1e14e52f0 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 28 Feb 2024 09:55:11 +0530 Subject: [PATCH 08/10] Add Prometheus support to Config DS --- .../data-plane/config-deployer/config-ds-deployment.yaml | 7 +++++-- .../data-plane/config-deployer/config-ds-service.yaml | 8 ++++++++ helm-charts/values.yaml | 1 - 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index 27d71680b..f5cae8cd6 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -77,6 +77,9 @@ spec: {{- else }} subPath: tls.key {{- end }} + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml + subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.pem {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} @@ -114,8 +117,8 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-config-ds-configmap - - name: prometheus-jmx-config-volume - configMap: + - name: prometheus-jmx-config-volume + configMap: name: prometheus-jmx-config-configds - name: config-ds-tls-volume secret: diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml index b19708a7f..ffd488cb4 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml @@ -28,4 +28,12 @@ spec: - name: servlet-https protocol: TCP port: 9443 + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-config-ds-port-lang + protocol: TCP + port: 18008 + - name: prometheus-config-ds-port-system + protocol: TCP + port: 18010 + {{ end }} {{- end -}} diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index bbc801126..91d7b1ca0 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -265,7 +265,6 @@ wso2: # requests: # cpu: 0.1 # memory: 64Mi - idp: enabled: true listener: From a5da5bf38590aa539fb20da56dfac524405f576b Mon Sep 17 00:00:00 2001 From: CrowleyRajapakse Date: Thu, 7 Mar 2024 17:33:46 +0530 Subject: [PATCH 09/10] fixing impl issues and test issues --- runtime/config-deployer-service/ballerina/APIClient.bal | 4 ++-- .../config-deployer-service/ballerina/modules/model/API.bal | 2 +- .../tests/resources/tests/jwt-api-level-security.yaml | 2 +- .../tests/resources/tests/jwt-resource-level-security.yaml | 2 +- .../integration/tests/resources/tests/owner-reference.yaml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index fba8b6790..da9bc516d 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -449,10 +449,10 @@ public class APIClient { } if productionRoutes.length() > 0 { - k8sAPI.spec.production = [{routeRef: productionRoutes}]; + k8sAPI.spec.production = [{routeRefs: productionRoutes}]; } if sandboxRoutes.length() > 0 { - k8sAPI.spec.sandbox = [{routeRef: sandboxRoutes}]; + k8sAPI.spec.sandbox = [{routeRefs: sandboxRoutes}]; } if apkConf.id != () { k8sAPI.metadata["annotations"] = {[API_UUID_ANNOTATION] : apkConf.id}; diff --git a/runtime/config-deployer-service/ballerina/modules/model/API.bal b/runtime/config-deployer-service/ballerina/modules/model/API.bal index 7cb81ea24..25d7bd40c 100644 --- a/runtime/config-deployer-service/ballerina/modules/model/API.bal +++ b/runtime/config-deployer-service/ballerina/modules/model/API.bal @@ -58,7 +58,7 @@ public type APIStatus record { }; public type EnvConfig record { - string[] routeRef; + string[] routeRefs; }; public type APIList record { diff --git a/test/integration/integration/tests/resources/tests/jwt-api-level-security.yaml b/test/integration/integration/tests/resources/tests/jwt-api-level-security.yaml index 7bfb7e9df..d290af4fe 100644 --- a/test/integration/integration/tests/resources/tests/jwt-api-level-security.yaml +++ b/test/integration/integration/tests/resources/tests/jwt-api-level-security.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - api-level-jwt-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/jwt-resource-level-security.yaml b/test/integration/integration/tests/resources/tests/jwt-resource-level-security.yaml index 332351577..5f2cb2da5 100644 --- a/test/integration/integration/tests/resources/tests/jwt-resource-level-security.yaml +++ b/test/integration/integration/tests/resources/tests/jwt-resource-level-security.yaml @@ -27,7 +27,7 @@ spec: isDefaultVersion: true #definitionFileRef: definition-file production: - - httpRouteRefs: + - routeRefs: - resource-level-jwt-httproute organization: wso2-org --- diff --git a/test/integration/integration/tests/resources/tests/owner-reference.yaml b/test/integration/integration/tests/resources/tests/owner-reference.yaml index 2c75e16ab..dad1bb4ee 100644 --- a/test/integration/integration/tests/resources/tests/owner-reference.yaml +++ b/test/integration/integration/tests/resources/tests/owner-reference.yaml @@ -132,7 +132,7 @@ spec: organization: "default" isDefaultVersion: true production: - - httpRouteRefs: + - routeRefs: - "hr1" - "hr2" apiProperties: [] @@ -151,7 +151,7 @@ spec: organization: "default" isDefaultVersion: true production: - - httpRouteRefs: + - routeRefs: - "hr2" apiProperties: [] status: null From a4331edb7479cfe1bc81bebea7721a92f590f778 Mon Sep 17 00:00:00 2001 From: CrowleyRajapakse Date: Thu, 7 Mar 2024 17:58:49 +0530 Subject: [PATCH 10/10] fixing rebase issues --- adapter/go.sum | 5 +- .../commoncontroller/common_controller.go | 6 - common-controller/go.mod | 3 +- common-controller/go.sum | 1 - common-controller/pkg/metrics/metrics.go | 191 ++---------------- .../config-deployer/config-ds-deployment.yaml | 3 - .../config-deployer/config-ds-service.yaml | 8 - .../common-controller-service.yaml | 5 - 8 files changed, 15 insertions(+), 207 deletions(-) diff --git a/adapter/go.sum b/adapter/go.sum index 97362525d..b59fd5b9a 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -117,8 +117,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -214,7 +214,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index 3be8bbb1a..d270085b2 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -214,12 +214,6 @@ func InitCommonControllerServer(conf *config.Config) { loggers.LoggerAPKOperator.Info("Starting common controller ....") - // Start the metrics server - if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { - loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port, conf.CommonController.Metrics.CollectionInterval) - } - rateLimiterCache := xds.GetRateLimiterCache() rlsSrv := xdsv3.NewServer(ctx, rateLimiterCache, &xds.Callbacks{}) diff --git a/common-controller/go.mod b/common-controller/go.mod index c1bdd0728..568002fa9 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -7,7 +7,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 + github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -23,7 +23,6 @@ require ( github.com/redis/go-redis/v9 v9.2.1 github.com/wso2/apk/adapter v0.0.0-20231214082511-af2c8b8a19f1 github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 - github.com/shirou/gopsutil/v3 v3.24.1 google.golang.org/grpc v1.62.0 ) diff --git a/common-controller/go.sum b/common-controller/go.sum index f5a353ede..22c976b82 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -314,7 +314,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go index ef7824ebb..c2da1fa33 100644 --- a/common-controller/pkg/metrics/metrics.go +++ b/common-controller/pkg/metrics/metrics.go @@ -16,200 +16,33 @@ * under the License. */ -// Package metrics holds the implementation for exposing common controller metrics to prometheus +// Package metrics holds the implementation for exposing adapter metrics to prometheus package metrics import ( "fmt" "net/http" - "os" - "runtime" "strconv" - "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/procfs" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/load" - "github.com/shirou/gopsutil/v3/mem" "github.com/wso2/apk/adapter/pkg/logging" logger "github.com/wso2/apk/common-controller/internal/loggers" + metrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) -var ( - prometheusMetricRegistry = prometheus.NewRegistry() - - hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", - }, []string{"os"}) - - availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", - }) - - freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory.", - }) - - totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_total_virtual_memory_bytes", - Help: "Amount of total virtual memory.", - }) - usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_used_virtual_memory_bytes", - Help: "Amount of used virtual memory.", - }) - - systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", - }) - - loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", - }, []string{"duration"}) - - processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_start_time_seconds", - Help: "Start time of the process since unix epoch in seconds.", - }) - - processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_open_fds", - Help: "Number of open file descriptors.", - }) -) - -func init() { - // Register the Go collector with the registry - goCollector := collectors.NewGoCollector() - prometheusMetricRegistry.MustRegister(goCollector) - - // Register other metrics - prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs) -} - -// recordMetrics record custom golang metrics -var recordMetrics = func(collectionInterval int32) { - for { - host, err := host.Info() - if handleError(err, "Failed to get host info") { - return - } - hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) - - v, err := mem.VirtualMemory() - if handleError(err, "Failed to read virtual memory metrics") { - return - } - freePhysicalMemory.Set(float64(v.Free)) - usedVirtualMemory.Set(float64(v.Used)) - totalVirtualMemory.Set(float64(v.Total)) - - percentages, err := cpu.Percent(0, false) - if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { - return - } - totalPercentage := 0.0 - for _, p := range percentages { - totalPercentage += p - } - averagePercentage := totalPercentage / float64(len(percentages)) - systemCPULoad.Set(averagePercentage) - - pid := os.Getpid() - p, err := procfs.NewProc(pid) - if handleError(err, "Failed to get current process") { - return - } - stat, err := p.Stat() - if handleError(err, "Failed to get process stats") { - return - } - t, err := stat.StartTime() - if handleError(err, "Failed to read process start time") { - return - } - processStartTime.Set(t) - fds, err := p.FileDescriptorsLen() - if handleError(err, "Failed to read file descriptor count") { - return - } - processOpenFDs.Set(float64(fds)) - - avg, err := load.Avg() - if handleError(err, "Failed to read cpu load averages") { - return - } - loadAvg.WithLabelValues("1m").Set(avg.Load1) - loadAvg.WithLabelValues("5m").Set(avg.Load5) - loadAvg.WithLabelValues("15m").Set(avg.Load15) - - // Sleep before the next measurement - time.Sleep(time.Duration(collectionInterval) * time.Second) - } - -} +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { -func handleError(err error, message string) bool { + collector := metrics.CustomMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintf(message, err.Error()), - Severity: logging.MINOR, - ErrorCode: 1109, + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, }) - return true - } - return false -} - -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32, collectionInterval int32) { - done := make(chan struct{}) // Channel to indicate recordMetrics routine exit - - // Start the Prometheus metrics server - go func() { - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) - if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, - }) - } - }() - - for { - // Start the recordMetrics goroutine - go func() { - defer func() { - if r := recover(); r != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), - Severity: logging.MAJOR, - ErrorCode: 1111, - }) - } - }() - recordMetrics(collectionInterval) - done <- struct{}{} // Signal that the goroutine has completed - }() - - // Wait for the previous recordMetrics goroutine to complete - <-done - - // Log and restart the goroutine - logger.LoggerAPKOperator.Info("Restarting recordMetrics goroutine...") - time.Sleep(3 * time.Second) } } diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index f5cae8cd6..df93172d1 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -77,9 +77,6 @@ spec: {{- else }} subPath: tls.key {{- end }} - - name: prometheus-jmx-config-volume - mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml - subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.pem {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml index ffd488cb4..b19708a7f 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml @@ -28,12 +28,4 @@ spec: - name: servlet-https protocol: TCP port: 9443 - {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - name: prometheus-config-ds-port-lang - protocol: TCP - port: 18008 - - name: prometheus-config-ds-port-system - protocol: TCP - port: 18010 - {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml index debcb579f..e5adf645f 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml @@ -38,9 +38,4 @@ spec: - name: https-internal-api protocol: TCP port: 18003 - {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - name: prometheus-common-controller-port - protocol: TCP - port: 18009 - {{ end }} {{- end -}}