Skip to content

Commit

Permalink
Merge branch 'dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
divinity666 committed Dec 18, 2021
2 parents ec77809 + 56f8889 commit 356a30d
Show file tree
Hide file tree
Showing 16 changed files with 288 additions and 86 deletions.
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
dist: focal
language: ruby
rvm:
- 2.5.5
Expand Down
45 changes: 21 additions & 24 deletions FUNCTION_CALLS.md

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions lib/VERSION.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# frozen_string_literal: true

# Version information
GRAFANA_REPORTER_VERSION = [0, 5, 0].freeze
GRAFANA_REPORTER_VERSION = [0, 5, 1].freeze
# Release date
GRAFANA_REPORTER_RELEASE_DATE = '2021-11-05'
GRAFANA_REPORTER_RELEASE_DATE = '2021-12-18'
8 changes: 6 additions & 2 deletions lib/grafana/influxdb_datasource.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@ def request(query_description)
# replace $timeFilter variable
query = query.gsub(/\$timeFilter(?=\W|$)/, "time >= #{query_description[:from]}ms and time <= #{query_description[:to]}ms")

interval = query_description[:variables].delete('interval') || ((query_description[:to].to_i - query_description[:from].to_i) / 1000).to_i
interval = interval.raw_value if interval.is_a?(Variable)

# replace grafana variables $__interval and $__interval_ms in query
query = query.gsub(/\$(?:__)?interval(?=\W|$)/, "#{((query_description[:to].to_i - query_description[:from].to_i) / 1000 / 1000).to_i}s")
query = query.gsub(/\$(?:__)?interval_ms(?=\W|$)/, "#{((query_description[:to].to_i - query_description[:from].to_i) / 1000).to_i}")
# TODO: check where calculation and replacement of interval variable should take place
query = query.gsub(/\$(?:__)?interval(?=\W|$)/, "#{interval.is_a?(String) ? interval : "#{(interval / 1000).to_i}s"}")
query = query.gsub(/\$(?:__)?interval_ms(?=\W|$)/, "#{interval}")

url = "/api/datasources/proxy/#{id}/query?db=#{@model['database']}&q=#{ERB::Util.url_encode(query)}&epoch=ms"

Expand Down
53 changes: 42 additions & 11 deletions lib/grafana/prometheus_datasource.rb
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,25 @@ def self.handles?(model)
def request(query_description)
raise MissingSqlQueryError if query_description[:raw_query].nil?

# TODO: properly allow endpoint to be set - also check raw_query method
end_point = @endpoint ? @endpoint : "query_range"
query_hash = query_description[:raw_query].is_a?(Hash) ? query_description[:raw_query] : {}

# TODO: set query option 'step' on request
url = "/api/datasources/proxy/#{id}/api/v1/#{end_point}?"\
"start=#{query_description[:from]}&end=#{query_description[:to]}"\
"&query=#{replace_variables(query_description[:raw_query], query_description[:variables])}"
# read instant value and convert instant value to boolean value
instant = query_description[:variables].delete('instant') || query_hash[:instant] || false
instant = instant.raw_value if instant.is_a?(Variable)
instant = instant.to_s.downcase == 'true'
interval = query_description[:variables].delete('interval') || query_hash[:interval] || 15
interval = interval.raw_value if interval.is_a?(Variable)
query = query_hash[:query] || query_description[:raw_query]

url = if instant
"/api/datasources/proxy/#{id}/api/v1/query?time=#{query_description[:to]}&query="\
"#{CGI.escape(replace_variables(query, query_description[:variables]))}"
else
"/api/datasources/proxy/#{id}/api/v1/query_range?start=#{query_description[:from]}"\
"&end=#{query_description[:to]}"\
"&query=#{CGI.escape(replace_variables(query, query_description[:variables]))}"\
"&step=#{interval}"
end

webrequest = query_description[:prepared_request]
webrequest.relative_url = url
Expand All @@ -32,8 +44,8 @@ def request(query_description)

# @see AbstractDatasource#raw_query_from_panel_model
def raw_query_from_panel_model(panel_query_target)
@endpoint = panel_query_target['format'] == 'time_series' && (panel_query_target['instant'] == false || !panel_query_target['instant']) ? 'query_range' : 'query'
panel_query_target['expr']
{ query: panel_query_target['expr'], instant: panel_query_target['instant'],
interval: panel_query_target['step'] }
end

# @see AbstractDatasource#default_variable_format
Expand All @@ -45,16 +57,35 @@ def default_variable_format

# @see AbstractDatasource#preformat_response
def preformat_response(response_body)
json = JSON.parse(response_body)['data']['result']
json = JSON.parse(response_body)

# handle response with error result
unless json['error'].nil?
return { header: ['error'], content: [[ json['error'] ]] }
end

result_type = json['data']['resultType']
json = json['data']['result']

headers = ['time']
content = {}

# handle vector queries
if result_type == 'vector'
return {
header: (headers << 'value') + json.first['metric'].keys,
content: [ [json.first['value'][0], json.first['value'][1]] + json.first['metric'].values ]
}
end

# handle scalar queries
if result_type =~ /^(?:scalar|string)$/
return { header: headers << result_type, content: [[json[0], json[1]]] }
end

# keep sorting, if json has only one target item, otherwise merge results and return
# as a time sorted array
# TODO properly set headlines
if json.length == 1
return { header: headers << json.first['metric'].to_s, content: [[json.first['value'][1], json.first['value'][0]]] } if json.first.has_key?('value') # this happens for the special case of calls to '/query' endpoint
return { header: headers << json.first['metric']['mode'], content: json.first['values'] }
end

Expand Down
33 changes: 30 additions & 3 deletions lib/grafana_reporter/asciidoctor/help.rb
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@ def github(headline_level = 2)
private

def github_options
{ headline_separator: '#', code_begin: '`', code_end: '`', table_begin: "\n", head_postfix_col: '| -- ' }
{ headline_separator: '#', code_begin: '`', code_end: '`', table_begin: "\n", head_postfix_col: '| -- ',
table_linebreak: "<br />"}
end

def asciidoctor_options
{ headline_separator: '=', code_begin: '`+', code_end: '+`', table_begin: "\n[%autowidth.stretch, "\
"options=\"header\"]\n|===\n", table_end: "\n|===" }
"options=\"header\"]\n|===\n", table_end: "\n|===", table_linebreak: "\n\n" }
end

def help_text(opts)
Expand Down Expand Up @@ -82,7 +83,7 @@ def functions_as_text(opts = {})
#{v[:description]}#{"\n\nSee also: #{v[:see]}" if v[:see]}#{unless v[:options].empty?
%(
#{opts[:table_begin]}| Option | Description#{"\n#{opts[:head_postfix_col] * 2}" if opts[:head_postfix_col]}
#{v[:options].sort.map { |_opt_k, opt_v| "| #{opts[:code_begin]}#{opt_v[:call]}#{opts[:code_end]} | #{opt_v[:description].gsub('|', '\|')}#{"\nSee also: #{opt_v[:see]}" if opt_v[:see]}" }.join("\n") }#{opts[:table_end]})
#{v[:options].sort.map { |_opt_k, opt_v| "| #{opts[:code_begin]}#{opt_v[:call]}#{opts[:code_end]} | #{opt_v[:description].gsub('|', '\|')}#{"#{opts[:table_linebreak]}See also: #{opt_v[:see]}" if opt_v[:see]}" }.join("\n") }#{opts[:table_end]})
end}
)
end
Expand Down Expand Up @@ -256,6 +257,20 @@ def raw_help_yaml
Set a timeout for the current query. If not overridden with `grafana_default_timeout` in the report template,
this defaults to 60 seconds.
interval:
call: interval="<intervaL>"
description: >-
Used to set the interval size for timescale datasources, whereas the value is used without further
conversion directly in the datasource specific interval parameter.
Prometheus default: 15 (passed as `step` parameter)
Influx default: similar to grafana default, i.e. `(to_time - from_time) / 1000`
(replaces `interval_ms` and `interval` variables in query)
instant:
call: instant="true"
description: >-
Optional parameter for Prometheus `instant` queries. Ignored for other datasources than Prometheus.
# ----------------------------------
# FUNCTION DOCUMENTATION STARTS HERE
# ----------------------------------
Expand Down Expand Up @@ -403,6 +418,8 @@ def raw_help_yaml
transpose:
from_timezone:
to_timezone:
instant:
interval:
grafana_panel_query_value:
call: 'grafana_panel_query_value:<panel_id>[query="<query_letter>",options]'
Expand All @@ -425,6 +442,8 @@ def raw_help_yaml
to:
from_timezone:
to_timezone:
instant:
interval:
grafana_sql_table:
call: 'include::grafana_sql_table:<datasource_id>[sql="<sql_query>",options]'
Expand All @@ -446,12 +465,18 @@ def raw_help_yaml
transpose:
from_timezone:
to_timezone:
instant:
interval:
grafana_sql_value:
call: 'grafana_sql_value:<datasource_id>[sql="<sql_query>",options]'
description: >-
Returns the value in the first column and the first row of the given query.
Grafana variables will be replaced in the SQL statement.
Please note that asciidoctor might fail, if you use square brackets in your
sql statement. To overcome this issue, you'll need to escape the closing
square brackets, i.e. +]+ needs to be replaced with +\\]+.
see: https://grafana.com/docs/grafana/latest/variables/syntax/
standard_options:
filter_columns:
Expand All @@ -463,6 +488,8 @@ def raw_help_yaml
to:
from_timezone:
to_timezone:
instant:
interval:
YAML_HELP
end
end
Expand Down
2 changes: 1 addition & 1 deletion lib/grafana_reporter/asciidoctor/processor_mixin.rb
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def build_attribute_hash(document_hash, item_hash)
k =~ /^(?:timeout|from|to)$/ ||
k =~ /filter_columns|format|replace_values_.*|transpose|from_timezone|
to_timezone|result_type|query|table_formatter|include_headline|
column_divider|row_divider/x
column_divider|row_divider|instant|interval/x
end)

result
Expand Down
13 changes: 11 additions & 2 deletions lib/grafana_reporter/asciidoctor/sql_value_inline_macro.rb
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,24 @@ def process(parent, target, attrs)
@report.next_step
instance = attrs['instance'] || parent.document.attr('grafana_default_instance') || 'default'
attrs['result_type'] = 'sql_value'
sql = attrs['sql']
@report.logger.debug("Processing SqlValueInlineMacro (instance: #{instance}, datasource: #{target},"\
" sql: #{attrs['sql']})")
" sql: #{sql})")

# translate sql statement to fix asciidoctor issue
# refer https://github.com/asciidoctor/asciidoctor/issues/4072#issuecomment-991305715
sql_translated = CGI::unescapeHTML(sql) if sql
if sql != sql_translated
@report.logger.debug("Translating SQL query to fix asciidoctor issue: #{sql_translated}")
sql = sql_translated
end

begin
# catch properly if datasource could not be identified
query = QueryValueQuery.new(@report.grafana(instance),
variables: build_attribute_hash(parent.document.attributes, attrs))
query.datasource = @report.grafana(instance).datasource_by_id(target)
query.raw_query = attrs['sql']
query.raw_query = sql

create_inline(parent, :quoted, query.execute)
rescue Grafana::GrafanaError => e
Expand Down
50 changes: 27 additions & 23 deletions lib/grafana_reporter/configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -263,13 +263,13 @@ def get_config(path)
cur_pos
end

def validate_schema(schema, subject)
def validate_schema(schema, subject, pattern = nil)
return nil if subject.nil?

schema.each do |key, config|
type, min_occurence, next_level = config
type, min_occurence, pattern, next_level = config

validate_schema(next_level, subject[key]) if next_level
validate_schema(next_level, subject[key], pattern) if next_level

if key.nil?
# apply to all on this level
Expand All @@ -289,9 +289,13 @@ def validate_schema(schema, subject)
elsif subject.is_a?(Hash)
if !subject.key?(key) && min_occurence.positive?
raise ConfigurationDoesNotMatchSchemaError.new(key, 'occur', min_occurence, 0)
end
if !subject[key].is_a?(type) && subject.key?(key)
elsif !subject[key].is_a?(type) && subject.key?(key)
raise ConfigurationDoesNotMatchSchemaError.new(key, 'be a', type, subject[key].class)
elsif pattern
# validate for regex
unless subject[key].to_s =~ pattern
raise ConfigurationDoesNotMatchSchemaError.new(key, 'match pattern', pattern.inspect, subject[key].to_s)
end
end

else
Expand All @@ -312,35 +316,35 @@ def schema(explicit)
{
'grafana' =>
[
Hash, 1,
Hash, 1, nil,
{
nil =>
[
Hash, 1,
Hash, 1, nil,
{
'host' => [String, 1],
'api_key' => [String, 0]
'host' => [String, 1, %r{^http(s)?://.+}],
'api_key' => [String, 0, %r{^(?:[\w]+[=]*)?$}]
}
]
}
],
'default-document-attributes' => [Hash, explicit ? 1 : 0],
'to_file' => [String, 0],
'default-document-attributes' => [Hash, explicit ? 1 : 0, nil],
'to_file' => [String, 0, nil],
'grafana-reporter' =>
[
Hash, 1,
Hash, 1, nil,
{
'check-for-updates' => [Integer, 0],
'debug-level' => [String, 0],
'run-mode' => [String, 0],
'test-instance' => [String, 0],
'templates-folder' => [String, explicit ? 1 : 0],
'report-class' => [String, 1],
'reports-folder' => [String, explicit ? 1 : 0],
'report-retention' => [Integer, explicit ? 1 : 0],
'ssl-cert' => [String, 0],
'webservice-port' => [Integer, explicit ? 1 : 0],
'callbacks' => [Hash, 0, { nil => [String, 1] }]
'check-for-updates' => [Integer, 0, /^[0-9]*$/],
'debug-level' => [String, 0, /^(?:DEBUG|INFO|WARN|ERROR|FATAL|UNKNOWN)?$/],
'run-mode' => [String, 0, /^(?:test|single-render|webservice)?$/],
'test-instance' => [String, 0, nil],
'templates-folder' => [String, explicit ? 1 : 0, nil],
'report-class' => [String, 1, nil],
'reports-folder' => [String, explicit ? 1 : 0, nil],
'report-retention' => [Integer, explicit ? 1 : 0, nil],
'ssl-cert' => [String, 0, nil],
'webservice-port' => [Integer, explicit ? 1 : 0, nil],
'callbacks' => [Hash, 0, nil, { nil => [String, 1, nil] }]
}
]
}
Expand Down
4 changes: 2 additions & 2 deletions lib/grafana_reporter/errors.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def initialize(datasource, message)
# Raised if the return value of a datasource request does not match the expected return hash.
class DatasourceRequestInvalidReturnValueError < GrafanaReporterError
def initialize(datasource, message)
super("The datasource request to '#{datasource.name}' (#{datasource.class})"\
super("The datasource request to '#{datasource.name}' (#{datasource.class}) "\
"returned an invalid value: '#{message}'")
end
end
Expand Down Expand Up @@ -65,7 +65,7 @@ def initialize(folder, config_item)
# Details about how to fix that are provided in the message.
class ConfigurationDoesNotMatchSchemaError < ConfigurationError
def initialize(item, verb, expected, currently)
super("Configuration file does not match schema definition. Expected '#{item}' to #{verb} '#{expected}',"\
super("Configuration file does not match schema definition. Expected '#{item}' to #{verb} '#{expected}', "\
"but was '#{currently}'.")
end
end
Expand Down
6 changes: 3 additions & 3 deletions ruby-grafana-reporter.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ Gem::Specification.new do |s|
s.version = GRAFANA_REPORTER_VERSION.join('.')
s.date = GRAFANA_REPORTER_RELEASE_DATE
s.summary = 'Reporter Service for Grafana'
s.description = 'Build reports based on grafana dashboards in asciidoctor syntax. Runs '\
'as webservice for easy integration with grafana, or as a standalone, '\
s.description = 'Build reports based on grafana dashboards in asciidoctor or ERB syntax. '\
'Runs as webservice for easy integration with grafana, or as a standalone, '\
'command line utility.'
''\
'By default the reports will be converted to PDF documents, whereas other '\
'target formats can be used as well.'
'target formats are supported as well.'
s.author = 'Christian Kohlmeyer'
s.email = 'kohly@gmx.de'
s.files = folders.collect { |folder| Dir[File.join(__dir__, 'lib', *folder, '*.rb')].sort }.flatten << 'LICENSE' << 'README.md'
Expand Down
3 changes: 3 additions & 0 deletions spec/integration/panel_query_table_include_processor_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ class MyReader; def unshift_line(*args); end; end

context 'prometheus' do
it 'can handle prometheus requests' do
@report.logger.level = ::Logger::Severity::DEBUG
allow(@report.logger).to receive(:debug)
expect(@report.logger).to receive(:debug).with(/^Requesting .*&step/)
expect(@report.logger).not_to receive(:error)
expect(Asciidoctor.convert("include::grafana_panel_query_table:#{STUBS[:panel_prometheus][:id]}[query=\"#{STUBS[:panel_prometheus][:letter]}\",dashboard=\"#{STUBS[:dashboard]}\",from=\"0\",to=\"0\"]", to_file: false)).to match(/<p>\| 1617728730 \| \| \| \| 0.011986814503580401 \| 0.6412945761450544\n\|/)
end
Expand Down
Loading

0 comments on commit 356a30d

Please sign in to comment.