From 61e5121603a39fcd5453fe94be1f61fffcf9da44 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Mon, 7 Sep 2020 18:54:58 +0530 Subject: [PATCH] [Development][Add] Added Index template support (#13) * Updated lifecycle CRD Signed-off-by: iamabhishek-dubey * Updated CRD for index template Signed-off-by: iamabhishek-dubey * Updated CRD for index template Signed-off-by: iamabhishek-dubey * Updated CRD for index template Signed-off-by: iamabhishek-dubey * Updated Documentation Signed-off-by: iamabhishek-dubey * Updated Documentation Signed-off-by: iamabhishek-dubey * Added function to generate index template Signed-off-by: iamabhishek-dubey * Added elastic connection details Signed-off-by: iamabhishek-dubey * Added code for index template Signed-off-by: iamabhishek-dubey * Added controller for index template Signed-off-by: iamabhishek-dubey * Updated helm charts for latest code Signed-off-by: iamabhishek-dubey * Updated Documentation for latest code Signed-off-by: iamabhishek-dubey --- README.md | 74 ++-------- api/v1alpha1/indextemplate_types.go | 18 ++- api/v1alpha1/zz_generated.deepcopy.go | 41 +++++- ...ogging.opstreelabs.in_indexlifecycles.yaml | 12 +- ...logging.opstreelabs.in_indextemplates.yaml | 71 +++++++--- config/samples/index-template-example.yaml | 17 +++ config/samples/lifecycle-example.yaml | 1 + .../logging_v1alpha1_indextemplate.yaml | 7 - controllers/indextemplate_controller.go | 28 +++- .../templates/index-lifecycle.yaml | 20 +++ .../templates/index-template.yaml | 21 +++ helm-charts/logging-setup/values.yaml | 33 ++++- utils/elasticsearch/index_template.go | 130 ++++++++++++++++++ 13 files changed, 375 insertions(+), 98 deletions(-) create mode 100644 config/samples/index-template-example.yaml delete mode 100644 config/samples/logging_v1alpha1_indextemplate.yaml create mode 100644 helm-charts/logging-setup/templates/index-lifecycle.yaml create mode 100644 helm-charts/logging-setup/templates/index-template.yaml create mode 100644 utils/elasticsearch/index_template.go diff --git a/README.md b/README.md index 68c1968..ff53a0d 100644 --- a/README.md +++ b/README.md @@ -34,24 +34,28 @@ A golang based CRD operator to setup and manage logging stack (Elasticsearch, Fl > The K8s API name is "logging.opstreelabs.in/v1alpha1" -Our roadmap is present in [ROADMAP](ROADMAP.md) +### Documentation + +[Documentation](https://docs.opstreelabs.in/logging-operator) ### Supported Features The "Logging Operator" includes these features:- - Elasticsearch different node types, like:- - - Master Node - - Data Node - - Ingestion Node - - Client/Coordinator Node -- Elasticsearch setup with/without TLS -- Customizable elasticsearch configuration and Heap size -- Fluentd as a log-shipper which already has JSON logs support + - **Master Node** => A node that has the master role (default), which makes it eligible to be elected as the master node, which controls the cluster. + - **Data Node** => A node that has the data role (default). Data nodes hold data and perform data related operations such as CRUD, search, and aggregations. + - **Ingestion Node** => A node that has the ingest role (default). Ingest nodes are able to apply an ingest pipeline to a document in order to transform and enrich the document before indexing. With a heavy ingest load, it makes sense to use dedicated ingest nodes and to not include the ingest role from nodes that have the master or data roles. + - **Client or Coordinator Node** => Requests like search requests or bulk-indexing requests may involve data held on different data nodes. A search request, for example, is executed in two phases which are coordinated by the node which receives the client request — the coordinating node. +- Elasticsearch setup with or without TLS on Transport and HTTP Layer +- Customizable elasticsearch configuration and configurable heap size +- Fluentd as a lightweight log-shipper and JSON field seperation support - Kibana integration with elasticsearch for logs visualization -- Seamless upgrade for Elasticsearch, Fluentd, and Kibana +- Seamless upgrade for Elasticsearch, Fluentd, and Kibana stack - Inculcated best practices for Kubernetes setup like `SecurityContext` and `Privilege Control` -- Loosely coupled setup, i.e. Elasticsearch, Fluentd, and Kibana can be setup individually as well. +- Loosely coupled setup, i.e. Elasticsearch, Fluentd, and Kibana setup can be done individually as well +- Index Lifecycle support to manage rollover and cleanup of indexes +- Index template support for configuring index settings like:- policy, replicas, shards etc. ### Architecture @@ -80,55 +84,7 @@ For the "Logging Operator" installation, we have categorized the steps in 3 part - RBAC setup for an operator to create resources in Kubernetes - Operator deployment and validation -#### Namespace setup - -Since we are going to use pre-baked manifests of Kubernetes in that case we need to setup the namespace with a specific name called "logging-operator". - -```shell -kubectl create ns logging-operator -``` - -#### CRD Setup - -So we have already pre-configured CRD in [config/crd](./config/crd) directory. We just have to run a magical `kubectl` commands. - -```shell -kubectl apply -f config/crd/ -``` - -#### RBAC setup - -Similar like CRD, we have pre-baked RBAC config files as well inside [config/crd](./config/rbac) which can be installed and configured by `kubectl` - -```shell -kubectl apply -f config/rbac/ -``` - -#### Operator Deployment and Validation - -Once all the initial steps are done, we can create the deployment for "Logging Operator". The deployment manifests for operator is present inside [config/manager/manager.yaml](./config/manager/manager.yaml) file. - -```shell -kubectl apply -f config/manager/manager.yaml -``` - -### Deployment Using Helm - -For quick deployment, we have pre-baked helm charts for logging operator deployment and logging stack setup. In case you don't want to customize the manifests file and want to deploy the cluster with some minimal configuration change, in that case, "Helm" can be used. - -```shell -helm upgrade logging-operator ./helm-charts/logging-operator/ \ - -f ./helm-charts/logging-operator/values.yaml --namespace logging-operator --install -``` - -Once the logging operator setup is completed, we can create the logging stack for our requirement. - -```shell -helm upgrade logging-stack ./helm-charts/logging-setup/ \ - -f ./helm-charts/logging-setup/values.yaml --set elasticsearch.master.replicas=3 \ - --set elasticsearch.data.replicas=3 --set elasticsearch.ingestion.replicas=1 \ - --set elasticsearch.client.replicas=1 --namespace logging-operator --install -``` +The detailed installation steps are present in [Documentation Guide](https://docs.opstreelabs.in/logging-operator) ### Examples diff --git a/api/v1alpha1/indextemplate_types.go b/api/v1alpha1/indextemplate_types.go index 037fc53..417cb8d 100644 --- a/api/v1alpha1/indextemplate_types.go +++ b/api/v1alpha1/indextemplate_types.go @@ -22,14 +22,24 @@ import ( // IndexTemplateSpec defines the desired state of IndexTemplate type IndexTemplateSpec struct { - Rollover Rollover `json:"rollover,omitempty"` - Delete Delete `json:"delete,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + IndexPatterns []string `json:"indexPatterns,omitempty"` + IndexTemplateSettings IndexTemplateSettings `json:"settings,omitempty"` + Elasticsearch ManagementElasticsearch `json:"elasticsearch,omitempty"` +} + +// IndexTemplateSettings defines the desired state for settings of index +type IndexTemplateSettings struct { + Shards int32 `json:"shards,omitempty"` + Replicas int32 `json:"replicas,omitempty"` + IndexLifecycleName string `json:"indexLifeCycleName,omitempty"` + RollOverAlias string `json:"rolloverAlias,omitempty"` } // IndexTemplateStatus defines the observed state of IndexTemplate type IndexTemplateStatus struct { - Rollover Rollover `json:"rollover,omitempty"` - Delete Delete `json:"delete,omitempty"` + IndexPatterns []string `json:"indexPatterns,omitempty"` + IndexTemplateSettings IndexTemplateSettings `json:"settings,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 673001c..b8e1374 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -406,8 +406,8 @@ func (in *IndexTemplate) DeepCopyInto(out *IndexTemplate) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexTemplate. @@ -460,11 +460,36 @@ func (in *IndexTemplateList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IndexTemplateSettings) DeepCopyInto(out *IndexTemplateSettings) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexTemplateSettings. +func (in *IndexTemplateSettings) DeepCopy() *IndexTemplateSettings { + if in == nil { + return nil + } + out := new(IndexTemplateSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IndexTemplateSpec) DeepCopyInto(out *IndexTemplateSpec) { *out = *in - out.Rollover = in.Rollover - out.Delete = in.Delete + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.IndexPatterns != nil { + in, out := &in.IndexPatterns, &out.IndexPatterns + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.IndexTemplateSettings = in.IndexTemplateSettings + in.Elasticsearch.DeepCopyInto(&out.Elasticsearch) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexTemplateSpec. @@ -480,8 +505,12 @@ func (in *IndexTemplateSpec) DeepCopy() *IndexTemplateSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IndexTemplateStatus) DeepCopyInto(out *IndexTemplateStatus) { *out = *in - out.Rollover = in.Rollover - out.Delete = in.Delete + if in.IndexPatterns != nil { + in, out := &in.IndexPatterns, &out.IndexPatterns + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.IndexTemplateSettings = in.IndexTemplateSettings } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexTemplateStatus. diff --git a/config/crd/bases/logging.opstreelabs.in_indexlifecycles.yaml b/config/crd/bases/logging.opstreelabs.in_indexlifecycles.yaml index 0b34750..049de32 100644 --- a/config/crd/bases/logging.opstreelabs.in_indexlifecycles.yaml +++ b/config/crd/bases/logging.opstreelabs.in_indexlifecycles.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 + controller-gen.kubebuilder.io/version: v0.3.0 creationTimestamp: null name: indexlifecycles.logging.opstreelabs.in spec: @@ -83,6 +83,16 @@ spec: type: object type: object type: object + additionalPrinterColumns: + - jsonPath: .spec.rollover.maxSize + name: Rollover Size + type: string + - jsonPath: .spec.rollover.maxAge + name: Rollover Age + type: string + - jsonPath: .spec.delete.minAge + name: Deletion Age + type: string served: true storage: true subresources: diff --git a/config/crd/bases/logging.opstreelabs.in_indextemplates.yaml b/config/crd/bases/logging.opstreelabs.in_indextemplates.yaml index 9bbe791..a147f8b 100644 --- a/config/crd/bases/logging.opstreelabs.in_indextemplates.yaml +++ b/config/crd/bases/logging.opstreelabs.in_indextemplates.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 + controller-gen.kubebuilder.io/version: v0.3.0 creationTimestamp: null name: indextemplates.logging.opstreelabs.in spec: @@ -36,40 +36,75 @@ spec: spec: description: IndexTemplateSpec defines the desired state of IndexTemplate properties: - delete: - description: Delete is the struct for index deletion + elasticsearch: + description: ManagementElasticsearch is the struct for elasticsearch + configuration for fluentd properties: - minAge: + host: + type: string + password: + type: string + username: type: string type: object - rollover: - description: Rollover is the struct for index roll overing + enabled: + type: boolean + indexPatterns: + items: + type: string + type: array + settings: + description: IndexTemplateSettings defines the desired state for settings + of index properties: - maxAge: + indexLifeCycleName: type: string - maxSize: + replicas: + format: int32 + type: integer + rolloverAlias: type: string + shards: + format: int32 + type: integer type: object type: object status: description: IndexTemplateStatus defines the observed state of IndexTemplate properties: - delete: - description: Delete is the struct for index deletion - properties: - minAge: - type: string - type: object - rollover: - description: Rollover is the struct for index roll overing + indexPatterns: + items: + type: string + type: array + settings: + description: IndexTemplateSettings defines the desired state for settings + of index properties: - maxAge: + indexLifeCycleName: type: string - maxSize: + replicas: + format: int32 + type: integer + rolloverAlias: type: string + shards: + format: int32 + type: integer type: object type: object type: object + additionalPrinterColumns: + - jsonPath: .spec.settings.shards + name: Shards + type: integer + format: int32 + - jsonPath: .spec.settings.replicas + name: Replicas + type: integer + format: int32 + - jsonPath: .spec.settings.indexLifeCycleName + name: Lifecycle Policy + type: string served: true storage: true subresources: diff --git a/config/samples/index-template-example.yaml b/config/samples/index-template-example.yaml new file mode 100644 index 0000000..f425e7e --- /dev/null +++ b/config/samples/index-template-example.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: logging.opstreelabs.in/v1alpha1 +kind: IndexTemplate +metadata: + name: indextemplate-sample +spec: + enabled: true + indexPatterns: ["kubernetes-default-*", "kubernetes-istio-system-*"] + settings: + shards: 1 + replicas: 1 + indexLifeCycleName: indexlifecycle-example + rolloverAlias: "kubernetes" + elasticsearch: + host: "https://elasticsearch-cluster-ingestion:9200" + username: elastic + password: "Opstree@1234" diff --git a/config/samples/lifecycle-example.yaml b/config/samples/lifecycle-example.yaml index 8d01db0..979c8e3 100644 --- a/config/samples/lifecycle-example.yaml +++ b/config/samples/lifecycle-example.yaml @@ -1,3 +1,4 @@ +--- apiVersion: logging.opstreelabs.in/v1alpha1 kind: IndexLifecycle metadata: diff --git a/config/samples/logging_v1alpha1_indextemplate.yaml b/config/samples/logging_v1alpha1_indextemplate.yaml deleted file mode 100644 index e85019b..0000000 --- a/config/samples/logging_v1alpha1_indextemplate.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: logging.opstreelabs.in/v1alpha1 -kind: IndexTemplate -metadata: - name: indextemplate-sample -spec: - # Add fields here - foo: bar diff --git a/controllers/indextemplate_controller.go b/controllers/indextemplate_controller.go index b45d1b7..032c285 100644 --- a/controllers/indextemplate_controller.go +++ b/controllers/indextemplate_controller.go @@ -18,9 +18,14 @@ package controllers import ( "context" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" + + elasticutils "logging-operator/utils/elasticsearch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,9 +44,28 @@ type IndexTemplateReconciler struct { func (r *IndexTemplateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { _ = context.Background() - _ = r.Log.WithValues("indextemplate", req.NamespacedName) + reqLogger := r.Log.WithValues("index-template", req.NamespacedName) + + instance := &loggingv1alpha1.IndexTemplate{} + err := r.Get(context.TODO(), req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } + return ctrl.Result{RequeueAfter: time.Second * 10}, err + } + + if err := controllerutil.SetControllerReference(instance, instance, r.Scheme); err != nil { + return ctrl.Result{RequeueAfter: time.Second * 10}, err + } + + if instance.Spec.Enabled != nil && *instance.Spec.Enabled != false { + elasticutils.CompareandUpdateIndexTemplate(instance) + } else { + elasticutils.DeleteIndexTemplate(instance) + } - // your logic here + reqLogger.Info("Will reconcile after 10 seconds", "IndexTemplate.Namespace", instance.Namespace, "IndexTemplate.Name", instance.Name) return ctrl.Result{}, nil } diff --git a/helm-charts/logging-setup/templates/index-lifecycle.yaml b/helm-charts/logging-setup/templates/index-lifecycle.yaml new file mode 100644 index 0000000..5b14860 --- /dev/null +++ b/helm-charts/logging-setup/templates/index-lifecycle.yaml @@ -0,0 +1,20 @@ +{{- if .Values.lifecycle.enabled }} +{{- range .Values.lifecycle.policies }} +--- +apiVersion: logging.opstreelabs.in/v1alpha1 +kind: IndexLifecycle +metadata: + name: {{ .name }} +spec: + enabled: {{ .status }} + rollover: + maxSize: {{ .maxSize }} + maxAge: {{ .maxAge }} + delete: + minAge: {{ .deletionAge }} + elasticsearch: + host: {{ $.Values.lifecycle.elasticsearch.host }} + username: {{ $.Values.lifecycle.elasticsearch.username }} + password: {{ $.Values.lifecycle.elasticsearch.password }} +{{- end }} +{{- end }} diff --git a/helm-charts/logging-setup/templates/index-template.yaml b/helm-charts/logging-setup/templates/index-template.yaml new file mode 100644 index 0000000..74e3d6c --- /dev/null +++ b/helm-charts/logging-setup/templates/index-template.yaml @@ -0,0 +1,21 @@ +{{- if .Values.indexTemplate.enabled }} +{{- range .Values.indexTemplate.templates }} +--- +apiVersion: logging.opstreelabs.in/v1alpha1 +kind: IndexTemplate +metadata: + name: {{ .name }} +spec: + enabled: {{ .status }} + indexPatterns: {{ .indexPatterns }} + settings: + shards: {{ .shards }} + replicas: {{ .replicas }} + indexLifeCycleName: {{ .indexLifeCycleName }} + rolloverAlias: {{ .rolloverAlias }} + elasticsearch: + host: {{ $.Values.indexTemplate.elasticsearch.host }} + username: {{ $.Values.indexTemplate.elasticsearch.username }} + password: {{ $.Values.indexTemplate.elasticsearch.password }} +{{- end }} +{{- end }} diff --git a/helm-charts/logging-setup/values.yaml b/helm-charts/logging-setup/values.yaml index 612816d..b93f449 100644 --- a/helm-charts/logging-setup/values.yaml +++ b/helm-charts/logging-setup/values.yaml @@ -197,7 +197,7 @@ kibana: imagePullPolicy: IfNotPresent elasticsearch: - host: prod-cluster-client + host: https://prod-cluster-client:9200 username: elastic password: Opstree@1234 tlsEnabled: true @@ -220,3 +220,34 @@ kibana: values: - "kibana" topologyKey: kubernetes.io/hostname + +lifecycle: + enabled: false + policies: + - name: prod-lifecycle + status: true + maxSize: 2GB + maxAge: 30d + deletionAge: 15d + elasticsearch: + host: https://prod-cluster-ingestion:9200 + username: elastic + password: Opstree@1234 + +indexTemplate: + enabled: false + templates: + - name: prod-template + status: true + shards: 1 + replicas: 1 + indexLifeCycleName: prod-lifecycle + rolloverAlias: kubernetes + indexPatterns: + - "kubernetes-default-*" + - "kubernetes-istio-system-*" + elasticsearch: + host: https://prod-cluster-ingestion:9200 + username: elastic + password: Opstree@1234 + diff --git a/utils/elasticsearch/index_template.go b/utils/elasticsearch/index_template.go new file mode 100644 index 0000000..34d6401 --- /dev/null +++ b/utils/elasticsearch/index_template.go @@ -0,0 +1,130 @@ +package elasticutils + +import ( + "bytes" + "crypto/tls" + "encoding/json" + loggingv1alpha1 "logging-operator/api/v1alpha1" + "net/http" + "reflect" +) + +type IndexTemplateType struct { + IndexPatterns []string `json:"index_patterns"` + Settings struct { + NumberOfShards int32 `json:"number_of_shards"` + NumberOfReplicas int32 `json:"number_of_replicas"` + IndexLifecycleName string `json:"index.lifecycle.name"` + IndexLifecycleRolloverAlias string `json:"index.lifecycle.rollover_alias"` + } `json:"settings"` +} + +// CreateUpdateIndexTemplate will create and update the index template +func CreateUpdateIndexTemplate(cr *loggingv1alpha1.IndexTemplate) { + reqLogger := log.WithValues("Namespace", cr.Namespace, "Template.Name", cr.ObjectMeta.Name, "Resource.Type", "Index Template") + indexTemplateData, err := json.Marshal(generateIndexTemplate(cr)) + + if err != nil { + reqLogger.Error(err, "Error while generating index template data") + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + client := &http.Client{Transport: tr} + requestBody := bytes.NewReader(indexTemplateData) + + req, err := http.NewRequest("PUT", *cr.Spec.Elasticsearch.Host+"/_template/"+cr.ObjectMeta.Name+"?pretty", requestBody) + if err != nil { + reqLogger.Error(err, "Error while generating request information") + } + + if cr.Spec.Elasticsearch.Username != nil && cr.Spec.Elasticsearch.Password != nil { + req.SetBasicAuth(*cr.Spec.Elasticsearch.Username, *cr.Spec.Elasticsearch.Password) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + reqLogger.Error(err, "Request failed while creating index template") + } + defer resp.Body.Close() + reqLogger.Info("Successfully created the index template") +} + +// DeleteIndexTemplate will delete the index template +func DeleteIndexTemplate(cr *loggingv1alpha1.IndexTemplate) { + reqLogger := log.WithValues("Namespace", cr.Namespace, "Template.Name", cr.ObjectMeta.Name, "Resource.Type", "Index Template") + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + req, err := http.NewRequest("DELETE", *cr.Spec.Elasticsearch.Host+"/_template/"+cr.ObjectMeta.Name, nil) + if err != nil { + reqLogger.Error(err, "Error while generating request information") + } + + if cr.Spec.Elasticsearch.Username != nil && cr.Spec.Elasticsearch.Password != nil { + req.SetBasicAuth(*cr.Spec.Elasticsearch.Username, *cr.Spec.Elasticsearch.Password) + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + reqLogger.Error(err, "Request failed while deleting index template") + } + defer resp.Body.Close() + reqLogger.Info("Successfully deleted the index template") +} + +// CompareandUpdateIndexTemplate will compare and create the index template +func CompareandUpdateIndexTemplate(cr *loggingv1alpha1.IndexTemplate) { + reqLogger := log.WithValues("Namespace", cr.Namespace, "Template.Name", cr.ObjectMeta.Name, "Resource.Type", "Index Template") + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + req, err := http.NewRequest("GET", *cr.Spec.Elasticsearch.Host+"/_template/"+cr.ObjectMeta.Name, nil) + if err != nil { + reqLogger.Error(err, "Error while generating request information") + } + + if cr.Spec.Elasticsearch.Username != nil && cr.Spec.Elasticsearch.Password != nil { + req.SetBasicAuth(*cr.Spec.Elasticsearch.Username, *cr.Spec.Elasticsearch.Password) + } + + resp, err := client.Do(req) + if err != nil { + reqLogger.Error(err, "Request failed while getting index template") + CreateUpdateIndexTemplate(cr) + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + + var existingData IndexTemplateType + err = decoder.Decode(&existingData) + + compareResult := reflect.DeepEqual(existingData, generateIndexTemplate(cr)) + if compareResult != true { + CreateUpdateIndexTemplate(cr) + } + return +} + +func generateIndexTemplate(cr *loggingv1alpha1.IndexTemplate) IndexTemplateType { + + indexTemplate := IndexTemplateType{} + + indexTemplate.IndexPatterns = cr.Spec.IndexPatterns + indexTemplate.Settings.NumberOfShards = cr.Spec.IndexTemplateSettings.Shards + indexTemplate.Settings.NumberOfReplicas = cr.Spec.IndexTemplateSettings.Replicas + indexTemplate.Settings.IndexLifecycleName = cr.Spec.IndexTemplateSettings.IndexLifecycleName + indexTemplate.Settings.IndexLifecycleRolloverAlias = cr.Spec.IndexTemplateSettings.RollOverAlias + + return indexTemplate +}