diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml index 659dd1ec979..00db250e275 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml @@ -1,14 +1,21 @@ inputs: - - name: udp-cef - id: udp-cef-${kubernetes.hints.container_id} - type: udp + - name: filestream-cef + id: filestream-cef-${kubernetes.hints.container_id} + type: filestream use_output: default streams: - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true data_stream: dataset: cef.log type: logs - host: localhost:9003 + exclude_files: + - .gz$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.cef.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log processors: - rename: fields: @@ -16,20 +23,23 @@ inputs: to: event.original - decode_cef: field: event.original + prospector: + scanner: + symlinks: true tags: - cef - forwarded data_stream.namespace: default - - name: tcp-cef - id: tcp-cef-${kubernetes.hints.container_id} - type: tcp + - name: udp-cef + id: udp-cef-${kubernetes.hints.container_id} + type: udp use_output: default streams: - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true data_stream: dataset: cef.log type: logs - host: localhost:9004 + host: localhost:9003 processors: - rename: fields: @@ -41,23 +51,16 @@ inputs: - cef - forwarded data_stream.namespace: default - - name: filestream-cef - id: filestream-cef-${kubernetes.hints.container_id} - type: filestream + - name: tcp-cef + id: tcp-cef-${kubernetes.hints.container_id} + type: tcp use_output: default streams: - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true data_stream: dataset: cef.log type: logs - exclude_files: - - .gz$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.cef.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log + host: localhost:9004 processors: - rename: fields: @@ -65,9 +68,6 @@ inputs: to: event.original - decode_cef: field: event.original - prospector: - scanner: - symlinks: true tags: - cef - forwarded diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml index 760582f2305..f3ecc3651a3 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/crowdstrike.yml @@ -1,4 +1,35 @@ inputs: + - name: aws-s3-crowdstrike + id: aws-s3-crowdstrike-${kubernetes.hints.container_id} + type: aws-s3 + use_output: default + streams: + - condition: ${kubernetes.hints.crowdstrike.fdr.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true + data_stream: + dataset: crowdstrike.fdr + type: logs + queue_url: null + sqs.notification_parsing_script.source: | + function parse(n) { + var m = JSON.parse(n); + var evts = []; + var files = m.files; + var bucket = m.bucket; + if (!Array.isArray(files) || (files.length == 0) || bucket == null || bucket == "") { + return evts; + } + files.forEach(function(f){ + var evt = new S3EventV2(); + evt.SetS3BucketName(bucket); + evt.SetS3ObjectKey(f.path); + evts.push(evt); + }); + return evts; + } + tags: + - forwarded + - crowdstrike-fdr + data_stream.namespace: default - name: filestream-crowdstrike id: filestream-crowdstrike-${kubernetes.hints.container_id} type: filestream @@ -48,34 +79,3 @@ inputs: - forwarded - crowdstrike-fdr data_stream.namespace: default - - name: aws-s3-crowdstrike - id: aws-s3-crowdstrike-${kubernetes.hints.container_id} - type: aws-s3 - use_output: default - streams: - - condition: ${kubernetes.hints.crowdstrike.fdr.enabled} == true or ${kubernetes.hints.crowdstrike.enabled} == true - data_stream: - dataset: crowdstrike.fdr - type: logs - queue_url: null - sqs.notification_parsing_script.source: | - function parse(n) { - var m = JSON.parse(n); - var evts = []; - var files = m.files; - var bucket = m.bucket; - if (!Array.isArray(files) || (files.length == 0) || bucket == null || bucket == "") { - return evts; - } - files.forEach(function(f){ - var evt = new S3EventV2(); - evt.SetS3BucketName(bucket); - evt.SetS3ObjectKey(f.path); - evts.push(evt); - }); - return evts; - } - tags: - - forwarded - - crowdstrike-fdr - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml index a5d43104711..efeb43a8b3d 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/elasticsearch.yml @@ -1,129 +1,4 @@ inputs: - - name: filestream-elasticsearch - id: filestream-elasticsearch-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.elasticsearch.audit.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true - data_stream: - dataset: elasticsearch.audit - type: logs - exclude_files: - - .gz$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.elasticsearch.audit.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - - add_fields: - fields: - ecs.version: 1.10.0 - target: "" - - decode_json_fields: - fields: - - message - target: _json - - rename: - fields: - - from: _json.request.body - to: _request - ignore_missing: true - - drop_fields: - fields: - - _json - - detect_mime_type: - field: _request - target: http.request.mime_type - - drop_fields: - fields: - - _request - ignore_missing: true - prospector: - scanner: - symlinks: true - - condition: ${kubernetes.hints.elasticsearch.deprecation.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true - data_stream: - dataset: elasticsearch.deprecation - type: logs - exclude_files: - - .gz$ - - _slowlog.log$ - - _access.log$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.elasticsearch.deprecation.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - - condition: ${kubernetes.hints.elasticsearch.gc.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true - data_stream: - dataset: elasticsearch.gc - type: logs - exclude_files: - - .gz$ - exclude_lines: - - '^(OpenJDK|Java HotSpot).* Server VM ' - - '^CommandLine flags: ' - - '^Memory: ' - - ^{ - multiline: - match: after - negate: true - pattern: ^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{) - parsers: - - container: - format: auto - stream: ${kubernetes.hints.elasticsearch.gc.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_fields: - fields: - ecs.version: 1.10.0 - target: "" - prospector: - scanner: - symlinks: true - - condition: ${kubernetes.hints.elasticsearch.server.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true - data_stream: - dataset: elasticsearch.server - type: logs - exclude_files: - - .gz$ - - _slowlog.log$ - - _access.log$ - - _deprecation.log$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.elasticsearch.server.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - - condition: ${kubernetes.hints.elasticsearch.slowlog.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true - data_stream: - dataset: elasticsearch.slowlog - type: logs - exclude_files: - - .gz$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.elasticsearch.slowlog.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - data_stream.namespace: default - name: elasticsearch/metrics-elasticsearch id: elasticsearch/metrics-elasticsearch-${kubernetes.hints.container_id} type: elasticsearch/metrics @@ -276,3 +151,128 @@ inputs: scope: node username: ${kubernetes.hints.elasticsearch.shard.username|kubernetes.hints.elasticsearch.username|''} data_stream.namespace: default + - name: filestream-elasticsearch + id: filestream-elasticsearch-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.elasticsearch.audit.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true + data_stream: + dataset: elasticsearch.audit + type: logs + exclude_files: + - .gz$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.elasticsearch.audit.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + - add_fields: + fields: + ecs.version: 1.10.0 + target: "" + - decode_json_fields: + fields: + - message + target: _json + - rename: + fields: + - from: _json.request.body + to: _request + ignore_missing: true + - drop_fields: + fields: + - _json + - detect_mime_type: + field: _request + target: http.request.mime_type + - drop_fields: + fields: + - _request + ignore_missing: true + prospector: + scanner: + symlinks: true + - condition: ${kubernetes.hints.elasticsearch.deprecation.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true + data_stream: + dataset: elasticsearch.deprecation + type: logs + exclude_files: + - .gz$ + - _slowlog.log$ + - _access.log$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.elasticsearch.deprecation.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + - condition: ${kubernetes.hints.elasticsearch.gc.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true + data_stream: + dataset: elasticsearch.gc + type: logs + exclude_files: + - .gz$ + exclude_lines: + - '^(OpenJDK|Java HotSpot).* Server VM ' + - '^CommandLine flags: ' + - '^Memory: ' + - ^{ + multiline: + match: after + negate: true + pattern: ^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{) + parsers: + - container: + format: auto + stream: ${kubernetes.hints.elasticsearch.gc.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_fields: + fields: + ecs.version: 1.10.0 + target: "" + prospector: + scanner: + symlinks: true + - condition: ${kubernetes.hints.elasticsearch.server.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true + data_stream: + dataset: elasticsearch.server + type: logs + exclude_files: + - .gz$ + - _slowlog.log$ + - _access.log$ + - _deprecation.log$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.elasticsearch.server.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + - condition: ${kubernetes.hints.elasticsearch.slowlog.enabled} == true or ${kubernetes.hints.elasticsearch.enabled} == true + data_stream: + dataset: elasticsearch.slowlog + type: logs + exclude_files: + - .gz$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.elasticsearch.slowlog.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml index 92907934bce..8f578662878 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/hid_bravura_monitor.yml @@ -1,4 +1,16 @@ inputs: + - name: winlog-hid_bravura_monitor + id: winlog-hid_bravura_monitor-${kubernetes.hints.container_id} + type: winlog + use_output: default + streams: + - condition: ${kubernetes.hints.hid_bravura_monitor.winlog.enabled} == true or ${kubernetes.hints.hid_bravura_monitor.enabled} == true + data_stream: + dataset: hid_bravura_monitor.winlog + type: logs + name: Hitachi-Hitachi ID Systems-Hitachi ID Suite/Operational + tags: null + data_stream.namespace: default - name: filestream-hid_bravura_monitor id: filestream-hid_bravura_monitor-${kubernetes.hints.container_id} type: filestream @@ -30,15 +42,3 @@ inputs: - .gz$ tags: null data_stream.namespace: default - - name: winlog-hid_bravura_monitor - id: winlog-hid_bravura_monitor-${kubernetes.hints.container_id} - type: winlog - use_output: default - streams: - - condition: ${kubernetes.hints.hid_bravura_monitor.winlog.enabled} == true or ${kubernetes.hints.hid_bravura_monitor.enabled} == true - data_stream: - dataset: hid_bravura_monitor.winlog - type: logs - name: Hitachi-Hitachi ID Systems-Hitachi ID Suite/Operational - tags: null - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml index 8f35f1980e2..34d6d24b582 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml @@ -1,4 +1,31 @@ inputs: + - name: iis/metrics-iis + id: iis/metrics-iis-${kubernetes.hints.container_id} + type: iis/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.iis.application_pool.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.application_pool + type: metrics + metricsets: + - application_pool + period: ${kubernetes.hints.iis.application_pool.period|kubernetes.hints.iis.period|'10s'} + - condition: ${kubernetes.hints.iis.webserver.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.webserver + type: metrics + metricsets: + - webserver + period: ${kubernetes.hints.iis.webserver.period|kubernetes.hints.iis.period|'10s'} + - condition: ${kubernetes.hints.iis.website.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.website + type: metrics + metricsets: + - website + period: ${kubernetes.hints.iis.website.period|kubernetes.hints.iis.period|'10s'} + data_stream.namespace: default - name: filestream-iis id: filestream-iis-${kubernetes.hints.container_id} type: filestream @@ -45,30 +72,3 @@ inputs: tags: - iis-error data_stream.namespace: default - - name: iis/metrics-iis - id: iis/metrics-iis-${kubernetes.hints.container_id} - type: iis/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.iis.application_pool.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.application_pool - type: metrics - metricsets: - - application_pool - period: ${kubernetes.hints.iis.application_pool.period|kubernetes.hints.iis.period|'10s'} - - condition: ${kubernetes.hints.iis.webserver.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.webserver - type: metrics - metricsets: - - webserver - period: ${kubernetes.hints.iis.webserver.period|kubernetes.hints.iis.period|'10s'} - - condition: ${kubernetes.hints.iis.website.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.website - type: metrics - metricsets: - - website - period: ${kubernetes.hints.iis.website.period|kubernetes.hints.iis.period|'10s'} - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml index 9ccbf653368..789d088b443 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/iptables.yml @@ -1,4 +1,20 @@ inputs: + - name: udp-iptables + id: udp-iptables-${kubernetes.hints.container_id} + type: udp + use_output: default + streams: + - condition: ${kubernetes.hints.iptables.log.enabled} == true or ${kubernetes.hints.iptables.enabled} == true + data_stream: + dataset: iptables.log + type: logs + host: localhost:9001 + processors: + - add_locale: null + tags: + - iptables-log + - forwarded + data_stream.namespace: default - name: filestream-iptables id: filestream-iptables-${kubernetes.hints.container_id} type: filestream @@ -41,19 +57,3 @@ inputs: tags: - iptables-log data_stream.namespace: default - - name: udp-iptables - id: udp-iptables-${kubernetes.hints.container_id} - type: udp - use_output: default - streams: - - condition: ${kubernetes.hints.iptables.log.enabled} == true or ${kubernetes.hints.iptables.enabled} == true - data_stream: - dataset: iptables.log - type: logs - host: localhost:9001 - processors: - - add_locale: null - tags: - - iptables-log - - forwarded - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml index a167b6e182f..de31c9c19ac 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/kafka.yml @@ -1,4 +1,33 @@ inputs: + - name: filestream-kafka + id: filestream-kafka-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.kafka.log.enabled} == true or ${kubernetes.hints.kafka.enabled} == true + data_stream: + dataset: kafka.log + type: logs + exclude_files: + - .gz$ + multiline: + match: after + negate: true + pattern: ^\[ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.kafka.log.stream|'all'} + paths: + - /opt/kafka*/var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + prospector: + scanner: + symlinks: true + tags: + - kafka-log + data_stream.namespace: default - name: kafka/metrics-kafka id: kafka/metrics-kafka-${kubernetes.hints.container_id} type: kafka/metrics @@ -36,32 +65,3 @@ inputs: period: ${kubernetes.hints.kafka.partition.period|kubernetes.hints.kafka.period|'10s'} username: ${kubernetes.hints.kafka.partition.username|kubernetes.hints.kafka.username|''} data_stream.namespace: default - - name: filestream-kafka - id: filestream-kafka-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.kafka.log.enabled} == true or ${kubernetes.hints.kafka.enabled} == true - data_stream: - dataset: kafka.log - type: logs - exclude_files: - - .gz$ - multiline: - match: after - negate: true - pattern: ^\[ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.kafka.log.stream|'all'} - paths: - - /opt/kafka*/var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - prospector: - scanner: - symlinks: true - tags: - - kafka-log - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml index 499a6e9d659..8038816ab50 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/kibana.yml @@ -61,6 +61,27 @@ inputs: to: kibana.background_task_utilization ignore_missing: true username: ${kubernetes.hints.kibana.background_task_utilization.username|kubernetes.hints.kibana.username|''} + - condition: ${kubernetes.hints.kibana.task_manager_metrics.enabled} == true or ${kubernetes.hints.kibana.enabled} == true + data_stream: + dataset: kibana.task_manager_metrics + type: metrics + hosts: + - ${kubernetes.hints.kibana.task_manager_metrics.host|kubernetes.hints.kibana.host|'http://localhost:5601'} + method: GET + metricsets: + - json + namespace: task_manager_metrics + password: ${kubernetes.hints.kibana.task_manager_metrics.password|kubernetes.hints.kibana.password|''} + path: /api/task_manager/metrics + period: ${kubernetes.hints.kibana.task_manager_metrics.period|kubernetes.hints.kibana.period|'10s'} + processors: + - rename: + fail_on_error: false + fields: + - from: http.task_manager_metrics + to: kibana.task_manager_metrics + ignore_missing: true + username: ${kubernetes.hints.kibana.task_manager_metrics.username|kubernetes.hints.kibana.username|''} data_stream.namespace: default - name: kibana/metrics-kibana id: kibana/metrics-kibana-${kubernetes.hints.container_id} diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml index 89c3aed53ca..96118a58ecc 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml @@ -1,32 +1,4 @@ inputs: - - name: logstash/metrics-logstash - id: logstash/metrics-logstash-${kubernetes.hints.container_id} - type: logstash/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.logstash.node.enabled} == true or ${kubernetes.hints.logstash.enabled} == true - data_stream: - dataset: logstash.stack_monitoring.node - type: metrics - hosts: - - ${kubernetes.hints.logstash.node.host|kubernetes.hints.logstash.host|'http://localhost:9600'} - metricsets: - - node - password: ${kubernetes.hints.logstash.node.password|kubernetes.hints.logstash.password|''} - period: ${kubernetes.hints.logstash.node.period|kubernetes.hints.logstash.period|'10s'} - username: ${kubernetes.hints.logstash.node.username|kubernetes.hints.logstash.username|''} - - condition: ${kubernetes.hints.logstash.node_stats.enabled} == true or ${kubernetes.hints.logstash.enabled} == true - data_stream: - dataset: logstash.stack_monitoring.node_stats - type: metrics - hosts: - - ${kubernetes.hints.logstash.node_stats.host|kubernetes.hints.logstash.host|'http://localhost:9600'} - metricsets: - - node_stats - password: ${kubernetes.hints.logstash.node_stats.password|kubernetes.hints.logstash.password|''} - period: ${kubernetes.hints.logstash.node_stats.period|kubernetes.hints.logstash.period|'10s'} - username: ${kubernetes.hints.logstash.node_stats.username|kubernetes.hints.logstash.username|''} - data_stream.namespace: default - name: filestream-logstash id: filestream-logstash-${kubernetes.hints.container_id} type: filestream @@ -79,3 +51,280 @@ inputs: scanner: symlinks: true data_stream.namespace: default + - name: logstash/metrics-logstash + id: logstash/metrics-logstash-${kubernetes.hints.container_id} + type: logstash/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.logstash.node.enabled} == true or ${kubernetes.hints.logstash.enabled} == true + data_stream: + dataset: logstash.stack_monitoring.node + type: metrics + hosts: + - ${kubernetes.hints.logstash.node.host|kubernetes.hints.logstash.host|'http://localhost:9600'} + metricsets: + - node + password: ${kubernetes.hints.logstash.node.password|kubernetes.hints.logstash.password|''} + period: ${kubernetes.hints.logstash.node.period|kubernetes.hints.logstash.period|'10s'} + username: ${kubernetes.hints.logstash.node.username|kubernetes.hints.logstash.username|''} + - condition: ${kubernetes.hints.logstash.node_stats.enabled} == true or ${kubernetes.hints.logstash.enabled} == true + data_stream: + dataset: logstash.stack_monitoring.node_stats + type: metrics + hosts: + - ${kubernetes.hints.logstash.node_stats.host|kubernetes.hints.logstash.host|'http://localhost:9600'} + metricsets: + - node_stats + password: ${kubernetes.hints.logstash.node_stats.password|kubernetes.hints.logstash.password|''} + period: ${kubernetes.hints.logstash.node_stats.period|kubernetes.hints.logstash.period|'10s'} + username: ${kubernetes.hints.logstash.node_stats.username|kubernetes.hints.logstash.username|''} + data_stream.namespace: default + - name: cel-logstash + id: cel-logstash-${kubernetes.hints.container_id} + type: cel + use_output: default + streams: + - auth.basic.password: null + auth.basic.user: null + condition: ${kubernetes.hints.logstash.node_cel.enabled} == true and ${kubernetes.hints.logstash.enabled} == true + config_version: "2" + data_stream: + dataset: logstash.node + type: metrics + interval: ${kubernetes.hints.logstash.node_cel.period|kubernetes.hints.logstash.period|'30s'} + program: |- + get(state.url) + .as(resp, bytes(resp.Body) + .decode_json().as(body, + {"logstash":{"node":{"stats":{ + "events":body.events, + "jvm":{ + "uptime_in_millis":body.jvm.uptime_in_millis, + "mem":body.jvm['mem'].drop("pools"), + "threads":body.jvm.threads + }, + "queue":body.queue, + "reloads":body.reloads, + "process":body.process, + "os":{ + "cpu":body.process.cpu, + "cgroup":has(body.os.group) ? body.os.cgroup : {}, + }, + "logstash":{ + "ephemeral_id":body.ephemeral_id, + "host":body.host, + "http_address":body.http_address, + "name":body.name, + "pipeline":body.pipeline, + "pipelines":body.pipelines.map(pipeline, pipeline != '.monitoring-logstash', [pipeline]).flatten(), + "snapshot":body.snapshot, + "status":body.status, + "uuid":body.id, + "version":body.version, + } + }} + }}) + ) + .as(eve, { + "events":[eve] + }) + redact: + fields: null + resource.url: http://localhost:9600/_node/stats?graph=true + - auth.basic.password: null + auth.basic.user: null + condition: ${kubernetes.hints.logstash.pipeline.enabled} == true and ${kubernetes.hints.logstash.enabled} == true + config_version: "2" + data_stream: + dataset: logstash.pipeline + type: metrics + interval: ${kubernetes.hints.logstash.pipeline.period|kubernetes.hints.logstash.period|'30s'} + program: |- + get(state.url) + .as(resp, bytes(resp.Body) + .decode_json().as(body, + body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name} + .with({ + "elasticsearch.cluster.id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))), + "host":{ + "name":body.name, + "address":body.http_address, + }, + "total":{ + "flow":body.pipelines[pipeline_name].flow, + "time":{ + "queue_push_duration":{ + "ms":body.pipelines[pipeline_name].events.queue_push_duration_in_millis, + }, + "duration":{ + "ms":body.pipelines[pipeline_name].events.duration_in_millis, + }, + }, + "reloads":{ + "successes":body.pipelines[pipeline_name].reloads.successes, + "failures":body.pipelines[pipeline_name].reloads.failures + }, + "events":{ + "out":body.pipelines[pipeline_name].events.out, + "in":body.pipelines[pipeline_name].events["in"], + "filtered":body.pipelines[pipeline_name].events.filtered, + }, + "queues":{ + "type":body.pipelines[pipeline_name].queue.type, + "events":body.pipelines[pipeline_name].queue.events_count, + "current_size":{ + "bytes":body.pipelines[pipeline_name].queue.queue_size_in_bytes, + }, + "max_size":{ + "bytes":body.pipelines[pipeline_name].queue.max_queue_size_in_bytes, + } + } + } + }) + ) + )) + .as(pipelines, { + "events":pipelines.map(pipeline, {"logstash":{"pipeline":pipeline}})}) + redact: + fields: null + resource.url: http://localhost:9600/_node/stats?graph=true&vertices=true + - auth.basic.password: null + auth.basic.user: null + condition: ${kubernetes.hints.logstash.plugins.enabled} == true and ${kubernetes.hints.logstash.enabled} == true + config_version: "2" + data_stream: + dataset: logstash.plugins + type: metrics + interval: ${kubernetes.hints.logstash.plugins.period|kubernetes.hints.logstash.period|'1m'} + program: |- + get(state.url) + .as(resp, bytes(resp.Body) + .decode_json().as(body, + body.pipelines.map(pipeline_name, pipeline_name != ".monitoring-logstash", {"name":pipeline_name}.with(body.pipelines[pipeline_name]) + .with({ + "es_cluster_id":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), each.cluster_uuid))), + "es_cluster_id_map":((body.pipelines[pipeline_name].vertices).as(vertices, vertices.map(each, has(each.cluster_uuid), {"plugin_id":each.id, "cluster_id":each.cluster_uuid}))), + "outputs":body.pipelines[pipeline_name].plugins.outputs, + "inputs":body.pipelines[pipeline_name].plugins.inputs, + "filters":body.pipelines[pipeline_name].plugins.filters, + "codecs":body.pipelines[pipeline_name].plugins.codecs, + "host":{ + "name":body.name, + "address":body.http_address, + } + }) + ) + )).as(events, events.map(event, + { + "inputs":event.inputs.map(input, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event.es_cluster_id, + "plugin":{ + "type":"input", + "input":{ + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == input.id), tuple.cluster_id), + "name":input.name, + "id":input.id, + "flow": has(input.flow) ? input.flow : {}, + "events":{ + "out":input.events.out, + }, + "time":{ + "queue_push_duration":{ + "ms":input.events.queue_push_duration_in_millis + } + } + } + } + }.drop_empty() + ), + "codecs":event.codecs.map(codec, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event.es_cluster_id, + "plugin":{ + "type":"codec", + "codec":{ + "id":codec.id, + "name":codec.name, + "flow": has(codec.flow) ? codec.flow : {}, + "decode":{ + "duration":{ + "ms":codec.decode.duration_in_millis + }, + "in":codec.decode.writes_in, + "out":codec.decode.out, + }, + "encode":{ + "in":codec.encode.writes_in, + "duration":{ + "ms":codec.encode.duration_in_millis + } + } + } + } + }.drop_empty() + ), + "filters":event.filters.map(filter, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event.es_cluster_id, + "plugin":{ + "type":"filter", + "filter":{ + "id":filter.id, + "name":filter.name, + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == filter.id), tuple.cluster_id), + "flow": has(filter.flow) ? filter.flow : {}, + "events":{ + "in":filter.events['in'], + "out":filter.events.out, + }, + "time":{ + "duration":{ + "ms":filter.events.duration_in_millis + } + } + } + } + }.drop_empty() + ), + "outputs":event.outputs.map(output, + { + "name":event.name, + "id":event.hash, + "host":event.host, + "elasticsearch.cluster.id":event.es_cluster_id, + "plugin":{ + "type":"output", + "output":{ + "id":output.id, + "name":output.name, + "elasticsearch.cluster.id":event.es_cluster_id_map.map(tuple, (tuple.plugin_id == output.id), tuple.cluster_id), + "flow": has(output.flow) ? output.flow : {}, + "events":{ + "in":output.events['in'], + "out":output.events.out, + }, + "time":{ + "duration":{ + "ms":output.events.duration_in_millis + } + } + } + } + }.drop_empty() + ) + }).collate(["filters", "outputs", "inputs", "codecs"])).as(plugins, { + "events":plugins.map(plugin, {"logstash":{"pipeline":plugin}})}) + redact: + fields: null + resource.url: http://localhost:9600/_node/stats?graph=true&vertices=true + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml index 5c9eb7fddc2..443f7b9ca3d 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/microsoft_sqlserver.yml @@ -99,6 +99,7 @@ inputs: dataset: microsoft_sqlserver.transaction_log type: metrics driver: mssql + fetch_from_all_databases: false hosts: - sqlserver://${kubernetes.hints.microsoft_sqlserver.transaction_log.username|kubernetes.hints.microsoft_sqlserver.username|'domain\username'}:${kubernetes.hints.microsoft_sqlserver.transaction_log.password|kubernetes.hints.microsoft_sqlserver.password|'verysecurepassword'}@${kubernetes.hints.microsoft_sqlserver.transaction_log.host|kubernetes.hints.microsoft_sqlserver.host|'localhost'} metricsets: @@ -110,24 +111,24 @@ inputs: response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('master')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('master') ; response_format: table - - query: USE [master] ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('master') ; + - query: USE [master]; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('master') ; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='model'; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('model')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('model') ; response_format: table - - query: USE [model] ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('model') ; + - query: USE [model]; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('model') ; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='tempdb'; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('tempdb')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('tempdb') ; response_format: table - - query: USE [tempdb] ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('tempdb') ; + - query: USE [tempdb]; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('tempdb') ; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='msdb'; response_format: table - query: SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('msdb')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('msdb') ; response_format: table - - query: USE [msdb] ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('msdb') ; + - query: USE [msdb]; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('msdb') ; response_format: table data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml index 6af480629d2..a14f5e6af35 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml @@ -53,7 +53,7 @@ inputs: password: ${kubernetes.hints.mongodb.dbstats.password|kubernetes.hints.mongodb.password|''} period: ${kubernetes.hints.mongodb.dbstats.period|kubernetes.hints.mongodb.period|'10s'} ssl.certificate: null - ssl.enabled: false + ssl.enabled: null ssl.key: null ssl.verification_mode: null username: ${kubernetes.hints.mongodb.dbstats.username|kubernetes.hints.mongodb.username|''} diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml index df50544f5d9..71f7a70effd 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml @@ -1,43 +1,4 @@ inputs: - - name: mysql/metrics-mysql - id: mysql/metrics-mysql-${kubernetes.hints.container_id} - type: mysql/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.mysql.galera_status.enabled} == true and ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.galera_status - type: metrics - hosts: - - ${kubernetes.hints.mysql.galera_status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - galera_status - password: ${kubernetes.hints.mysql.galera_status.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.galera_status.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.galera_status.username|kubernetes.hints.mysql.username|'root'} - - condition: ${kubernetes.hints.mysql.performance.enabled} == true or ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.performance - type: metrics - hosts: - - ${kubernetes.hints.mysql.performance.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - performance - password: ${kubernetes.hints.mysql.performance.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.performance.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.performance.username|kubernetes.hints.mysql.username|'root'} - - condition: ${kubernetes.hints.mysql.status.enabled} == true or ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.status - type: metrics - hosts: - - ${kubernetes.hints.mysql.status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - status - password: ${kubernetes.hints.mysql.status.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.status.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.status.username|kubernetes.hints.mysql.username|'root'} - data_stream.namespace: default - name: filestream-mysql id: filestream-mysql-${kubernetes.hints.container_id} type: filestream @@ -87,3 +48,42 @@ inputs: scanner: symlinks: true data_stream.namespace: default + - name: mysql/metrics-mysql + id: mysql/metrics-mysql-${kubernetes.hints.container_id} + type: mysql/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.mysql.galera_status.enabled} == true and ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.galera_status + type: metrics + hosts: + - ${kubernetes.hints.mysql.galera_status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - galera_status + password: ${kubernetes.hints.mysql.galera_status.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.galera_status.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.galera_status.username|kubernetes.hints.mysql.username|'root'} + - condition: ${kubernetes.hints.mysql.performance.enabled} == true or ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.performance + type: metrics + hosts: + - ${kubernetes.hints.mysql.performance.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - performance + password: ${kubernetes.hints.mysql.performance.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.performance.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.performance.username|kubernetes.hints.mysql.username|'root'} + - condition: ${kubernetes.hints.mysql.status.enabled} == true or ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.status + type: metrics + hosts: + - ${kubernetes.hints.mysql.status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - status + password: ${kubernetes.hints.mysql.status.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.status.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.status.username|kubernetes.hints.mysql.username|'root'} + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml index 4630a5b5e9e..4cac1ee1875 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml @@ -1,4 +1,27 @@ inputs: + - name: filestream-nats + id: filestream-nats-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.nats.log.enabled} == true or ${kubernetes.hints.nats.enabled} == true + data_stream: + dataset: nats.log + type: logs + exclude_files: + - .gz$ + parsers: + - container: + format: auto + stream: ${kubernetes.hints.nats.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + tags: + - nats-log + data_stream.namespace: default - name: nats/metrics-nats id: nats/metrics-nats-${kubernetes.hints.container_id} type: nats/metrics @@ -59,26 +82,3 @@ inputs: - subscriptions period: ${kubernetes.hints.nats.subscriptions.period|kubernetes.hints.nats.period|'10s'} data_stream.namespace: default - - name: filestream-nats - id: filestream-nats-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.nats.log.enabled} == true or ${kubernetes.hints.nats.enabled} == true - data_stream: - dataset: nats.log - type: logs - exclude_files: - - .gz$ - parsers: - - container: - format: auto - stream: ${kubernetes.hints.nats.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - tags: - - nats-log - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/nginx.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/nginx.yml index 4e5879469a4..3251df395e1 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/nginx.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/nginx.yml @@ -1,4 +1,54 @@ inputs: + - name: filestream-nginx + id: filestream-nginx-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.nginx.access.enabled} == true or ${kubernetes.hints.nginx.enabled} == true + data_stream: + dataset: nginx.access + type: logs + exclude_files: + - .gz$ + ignore_older: 72h + parsers: + - container: + format: auto + stream: ${kubernetes.hints.nginx.access.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + prospector: + scanner: + symlinks: true + tags: + - nginx-access + - condition: ${kubernetes.hints.nginx.error.enabled} == true or ${kubernetes.hints.nginx.enabled} == true + data_stream: + dataset: nginx.error + type: logs + exclude_files: + - .gz$ + ignore_older: 72h + multiline: + match: after + negate: true + pattern: '^\d{4}\/\d{2}\/\d{2} ' + parsers: + - container: + format: auto + stream: ${kubernetes.hints.nginx.error.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + prospector: + scanner: + symlinks: true + tags: + - nginx-error + data_stream.namespace: default - name: httpjson-nginx id: httpjson-nginx-${kubernetes.hints.container_id} type: httpjson @@ -97,53 +147,3 @@ inputs: period: ${kubernetes.hints.nginx.stubstatus.period|kubernetes.hints.nginx.period|'10s'} server_status_path: /nginx_status data_stream.namespace: default - - name: filestream-nginx - id: filestream-nginx-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.nginx.access.enabled} == true or ${kubernetes.hints.nginx.enabled} == true - data_stream: - dataset: nginx.access - type: logs - exclude_files: - - .gz$ - ignore_older: 72h - parsers: - - container: - format: auto - stream: ${kubernetes.hints.nginx.access.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - prospector: - scanner: - symlinks: true - tags: - - nginx-access - - condition: ${kubernetes.hints.nginx.error.enabled} == true or ${kubernetes.hints.nginx.enabled} == true - data_stream: - dataset: nginx.error - type: logs - exclude_files: - - .gz$ - ignore_older: 72h - multiline: - match: after - negate: true - pattern: '^\d{4}\/\d{2}\/\d{2} ' - parsers: - - container: - format: auto - stream: ${kubernetes.hints.nginx.error.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - prospector: - scanner: - symlinks: true - tags: - - nginx-error - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/oracle.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/oracle.yml index 29e0c8f1699..e5dac21fdf8 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/oracle.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/oracle.yml @@ -1,4 +1,31 @@ inputs: + - name: filestream-oracle + id: filestream-oracle-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.oracle.database_audit.enabled} == true or ${kubernetes.hints.oracle.enabled} == true + data_stream: + dataset: oracle.database_audit + type: logs + exclude_files: + - .gz$ + exclude_lines: + - ^Audit file + parsers: + - multiline: + match: after + negate: true + pattern: ^[A-Za-z]{3}\s+[A-Za-z]{3}\s+[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}\s[0-9]{4}\s\S[0-9]{2}:[0-9]{2} + timeout: 10 + type: pattern + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + tags: + - oracle-database_audit + data_stream.namespace: default - name: sql/metrics-oracle id: sql/metrics-oracle-${kubernetes.hints.container_id} type: sql/metrics @@ -92,30 +119,3 @@ inputs: - query: WITH data_files AS (SELECT file_name, file_id, tablespace_name, bytes, status, maxbytes, user_bytes, online_status FROM sys.dba_data_files UNION SELECT file_name, file_id, tablespace_name, bytes, status, maxbytes, user_bytes, status AS ONLINE_STATUS FROM sys.dba_temp_files), spaces AS (SELECT b.tablespace_name TB_NAME, tbs_size TB_SIZE_USED, a.free_space TB_SIZE_FREE FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+) = b.tablespace_name AND a.tablespace_name != 'TEMP'), temp_spaces AS (SELECT tablespace_name, tablespace_size, allocated_space, free_space FROM dba_temp_free_space WHERE tablespace_name = 'TEMP'), details AS (SELECT df.file_name, df.file_id, df.tablespace_name, df.bytes, df.status, df.maxbytes, df.user_bytes, df.online_status, sp.tb_size_used, sp.tb_size_free FROM data_files df, spaces sp WHERE df.tablespace_name = sp.tb_name UNION SELECT df.file_name, df.file_id, df.tablespace_name, df.bytes, df.status, df.maxbytes, df.user_bytes, df.online_status, tsp.tablespace_size - tsp.free_space AS TB_SIZE_USED, tsp.free_space AS TB_SIZE_FREE FROM data_files df, temp_spaces tsp WHERE df.tablespace_name = tsp.tablespace_name) SELECT file_name, file_id, tablespace_name, bytes, status, maxbytes, user_bytes, online_status, tb_size_used, tb_size_free, SUM(bytes) over() AS TOTAL_BYTES FROM details response_format: table data_stream.namespace: default - - name: filestream-oracle - id: filestream-oracle-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.oracle.database_audit.enabled} == true or ${kubernetes.hints.oracle.enabled} == true - data_stream: - dataset: oracle.database_audit - type: logs - exclude_files: - - .gz$ - exclude_lines: - - ^Audit file - parsers: - - multiline: - match: after - negate: true - pattern: ^[A-Za-z]{3}\s+[A-Za-z]{3}\s+[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}\s[0-9]{4}\s\S[0-9]{2}:[0-9]{2} - timeout: 10 - type: pattern - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - tags: - - oracle-database_audit - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml index f21316d7581..96708c77e38 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml @@ -1,4 +1,24 @@ inputs: + - name: tcp-pfsense + id: tcp-pfsense-${kubernetes.hints.container_id} + type: tcp + use_output: default + streams: + - condition: ${kubernetes.hints.pfsense.log.enabled} == true and ${kubernetes.hints.pfsense.enabled} == true + data_stream: + dataset: pfsense.log + type: logs + host: localhost:9001 + processors: + - add_locale: null + - add_fields: + fields: + tz_offset: local + target: _tmp + tags: + - pfsense + - forwarded + data_stream.namespace: default - name: filestream-pfsense id: filestream-pfsense-${kubernetes.hints.container_id} type: filestream @@ -43,23 +63,3 @@ inputs: - pfsense - forwarded data_stream.namespace: default - - name: tcp-pfsense - id: tcp-pfsense-${kubernetes.hints.container_id} - type: tcp - use_output: default - streams: - - condition: ${kubernetes.hints.pfsense.log.enabled} == true and ${kubernetes.hints.pfsense.enabled} == true - data_stream: - dataset: pfsense.log - type: logs - host: localhost:9001 - processors: - - add_locale: null - - add_fields: - fields: - tz_offset: local - target: _tmp - tags: - - pfsense - - forwarded - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml index e5b613a4804..3fd3cb5bb5c 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml @@ -59,8 +59,8 @@ inputs: - remote_write port: 9201 rate_counters: true - ssl.certificate: /etc/pki/server/cert.pem - ssl.enabled: null + ssl.certificate: null + ssl.enabled: false ssl.key: null types_patterns.exclude: null types_patterns.include: null diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml index 8557717a5db..7e7c6e3de88 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml @@ -1,4 +1,26 @@ inputs: + - name: filestream-sentinel_one + id: filestream-sentinel_one-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.sentinel_one.container_logs.enabled} == true + data_stream: + dataset: sentinel_one.container_logs + type: logs + exclude_files: [] + exclude_lines: [] + parsers: + - container: + format: auto + stream: all + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + tags: [] + data_stream.namespace: default - name: httpjson-sentinel_one id: httpjson-sentinel_one-${kubernetes.hints.container_id} type: httpjson @@ -195,25 +217,3 @@ inputs: - forwarded - sentinel_one-threat data_stream.namespace: default - - name: filestream-sentinel_one - id: filestream-sentinel_one-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.sentinel_one.container_logs.enabled} == true - data_stream: - dataset: sentinel_one.container_logs - type: logs - exclude_files: [] - exclude_lines: [] - parsers: - - container: - format: auto - stream: all - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - tags: [] - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/snyk.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/snyk.yml index 990c39b9f14..e59e7eff7c9 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/snyk.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/snyk.yml @@ -1,4 +1,26 @@ inputs: + - name: filestream-snyk + id: filestream-snyk-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.snyk.container_logs.enabled} == true + data_stream: + dataset: snyk.container_logs + type: logs + exclude_files: [] + exclude_lines: [] + parsers: + - container: + format: auto + stream: all + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + tags: [] + data_stream.namespace: default - name: httpjson-snyk id: httpjson-snyk-${kubernetes.hints.container_id} type: httpjson @@ -117,25 +139,3 @@ inputs: - forwarded - snyk-vulnerabilities data_stream.namespace: default - - name: filestream-snyk - id: filestream-snyk-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.snyk.container_logs.enabled} == true - data_stream: - dataset: snyk.container_logs - type: logs - exclude_files: [] - exclude_lines: [] - parsers: - - container: - format: auto - stream: all - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - tags: [] - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml index 96a643f41ea..5127a4ba11d 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml @@ -1,4 +1,32 @@ inputs: + - name: synthetics/http-synthetics + id: synthetics/http-synthetics-${kubernetes.hints.container_id} + type: synthetics/http + use_output: default + streams: + - __ui: null + check.request.method: null + condition: ${kubernetes.hints.synthetics.http.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true + data_stream: + dataset: http + type: synthetics + enabled: true + ipv4: true + ipv6: true + max_attempts: 2 + max_redirects: null + name: null + password: ${kubernetes.hints.synthetics.http.password|kubernetes.hints.synthetics.password|''} + response.include_body: null + response.include_headers: null + run_from.geo.name: Fleet managed + run_from.id: fleet_managed + schedule: '@every 3m' + timeout: ${kubernetes.hints.synthetics.http.timeout|kubernetes.hints.synthetics.timeout|''} + type: http + urls: null + username: ${kubernetes.hints.synthetics.http.username|kubernetes.hints.synthetics.username|''} + data_stream.namespace: default - name: synthetics/tcp-synthetics id: synthetics/tcp-synthetics-${kubernetes.hints.container_id} type: synthetics/tcp @@ -13,12 +41,8 @@ inputs: hosts: ${kubernetes.hints.synthetics.tcp.host|kubernetes.hints.synthetics.host|''} ipv4: true ipv6: true + max_attempts: 2 name: null - processors: - - add_fields: - fields: - monitor.fleet_managed: true - target: "" proxy_use_local_resolver: false run_from.geo.name: Fleet managed run_from.id: fleet_managed @@ -40,12 +64,8 @@ inputs: hosts: ${kubernetes.hints.synthetics.icmp.host|kubernetes.hints.synthetics.host|''} ipv4: true ipv6: true + max_attempts: 2 name: null - processors: - - add_fields: - fields: - monitor.fleet_managed: true - target: "" run_from.geo.name: Fleet managed run_from.id: fleet_managed schedule: '@every 3m' @@ -64,12 +84,8 @@ inputs: dataset: browser type: synthetics enabled: true + max_attempts: 2 name: null - processors: - - add_fields: - fields: - monitor.fleet_managed: true - target: "" run_from.geo.name: Fleet managed run_from.id: fleet_managed schedule: '@every 3m' @@ -117,35 +133,3 @@ inputs: symlinks: true tags: [] data_stream.namespace: default - - name: synthetics/http-synthetics - id: synthetics/http-synthetics-${kubernetes.hints.container_id} - type: synthetics/http - use_output: default - streams: - - __ui: null - check.request.method: null - condition: ${kubernetes.hints.synthetics.http.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true - data_stream: - dataset: http - type: synthetics - enabled: true - ipv4: true - ipv6: true - max_redirects: null - name: null - password: ${kubernetes.hints.synthetics.http.password|kubernetes.hints.synthetics.password|''} - processors: - - add_fields: - fields: - monitor.fleet_managed: true - target: "" - response.include_body: null - response.include_headers: null - run_from.geo.name: Fleet managed - run_from.id: fleet_managed - schedule: '@every 3m' - timeout: ${kubernetes.hints.synthetics.http.timeout|kubernetes.hints.synthetics.timeout|''} - type: http - urls: null - username: ${kubernetes.hints.synthetics.http.username|kubernetes.hints.synthetics.username|''} - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/tomcat.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/tomcat.yml index e88d1490bc4..f0af9f8ad74 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/tomcat.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/tomcat.yml @@ -1,27 +1,20 @@ inputs: - - name: filestream-tomcat - id: filestream-tomcat-${kubernetes.hints.container_id} - type: filestream + - name: udp-tomcat + id: udp-tomcat-${kubernetes.hints.container_id} + type: udp use_output: default streams: - - condition: ${kubernetes.hints.tomcat.log.enabled} == true and ${kubernetes.hints.tomcat.enabled} == true + - condition: ${kubernetes.hints.tomcat.log.enabled} == true or ${kubernetes.hints.tomcat.enabled} == true data_stream: dataset: tomcat.log type: logs - exclude_files: - - .gz$ fields: observer: product: TomCat type: Web vendor: Apache fields_under_root: true - parsers: - - container: - format: auto - stream: ${kubernetes.hints.tomcat.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log + host: localhost:9523 processors: - script: lang: javascript @@ -2764,16 +2757,14 @@ inputs: target_field: url.registered_domain target_subdomain_field: url.subdomain - add_locale: null - prospector: - scanner: - symlinks: true tags: - tomcat-log - forwarded + udp: null data_stream.namespace: default - - name: udp-tomcat - id: udp-tomcat-${kubernetes.hints.container_id} - type: udp + - name: tcp-tomcat + id: tcp-tomcat-${kubernetes.hints.container_id} + type: tcp use_output: default streams: - condition: ${kubernetes.hints.tomcat.log.enabled} == true or ${kubernetes.hints.tomcat.enabled} == true @@ -5532,24 +5523,31 @@ inputs: tags: - tomcat-log - forwarded - udp: null + tcp: null data_stream.namespace: default - - name: tcp-tomcat - id: tcp-tomcat-${kubernetes.hints.container_id} - type: tcp + - name: filestream-tomcat + id: filestream-tomcat-${kubernetes.hints.container_id} + type: filestream use_output: default streams: - - condition: ${kubernetes.hints.tomcat.log.enabled} == true or ${kubernetes.hints.tomcat.enabled} == true + - condition: ${kubernetes.hints.tomcat.log.enabled} == true and ${kubernetes.hints.tomcat.enabled} == true data_stream: dataset: tomcat.log type: logs + exclude_files: + - .gz$ fields: observer: product: TomCat type: Web vendor: Apache fields_under_root: true - host: localhost:9523 + parsers: + - container: + format: auto + stream: ${kubernetes.hints.tomcat.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log processors: - script: lang: javascript @@ -8292,8 +8290,10 @@ inputs: target_field: url.registered_domain target_subdomain_field: url.subdomain - add_locale: null + prospector: + scanner: + symlinks: true tags: - tomcat-log - forwarded - tcp: null data_stream.namespace: default