From 57e8ac99374967dc30aad096afea53f5bd1d0ec8 Mon Sep 17 00:00:00 2001 From: Michal Vesely Date: Fri, 11 Aug 2017 11:10:19 +0200 Subject: [PATCH 01/12] parameterize initial django db creation --- manifests/config.pp | 3 ++- manifests/init.pp | 2 ++ manifests/params.pp | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/manifests/config.pp b/manifests/config.pp index e37c363e..f23d6684 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -98,7 +98,8 @@ # first init of user db for graphite exec { 'Initial django db creation': - command => "${::graphite::gr_python_binary} manage.py syncdb --noinput", + command => $::graphite::gr_django_init_command, + provider => $::graphite::gr_django_init_provider, cwd => $graphite_web_managepy_location, refreshonly => true, require => $syncdb_require, diff --git a/manifests/init.pp b/manifests/init.pp index 2870dc8d..f7990e9c 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -732,6 +732,8 @@ $wsgi_processes = 5, $wsgi_threads = 5, $wsgi_inactivity_timeout = 120, + $gr_django_init_provider = $::graphite::params::gr_django_init_provider, + $gr_django_init_command = $::graphite::params::gr_django_init_command, $gr_django_tagging_pkg = $::graphite::params::django_tagging_pkg, $gr_django_tagging_ver = $::graphite::params::django_tagging_ver, $gr_django_tagging_source = $::graphite::params::django_tagging_source, diff --git a/manifests/params.pp b/manifests/params.pp index fbdc6000..2afd003b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -37,6 +37,10 @@ $install_prefix = '/opt/' + # variables for django db initialization + $gr_django_init_provider = 'posix' + $gr_django_init_command = "${::graphite::gr_python_binary} manage.py syncdb --noinput" + # variables to workaround unusual graphite install target: # https://github.com/graphite-project/carbon/issues/86 $pyver = $::osfamily ? { From 8800d66389cd5396e3c6baeb1b9caae7ccc0557d Mon Sep 17 00:00:00 2001 From: Michal Vesely Date: Tue, 29 Aug 2017 11:41:01 +0200 Subject: [PATCH 02/12] move django init command to init class --- manifests/init.pp | 10 ++++++++-- manifests/params.pp | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index f7990e9c..1af3bed5 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -437,6 +437,12 @@ # [*wsgi_inactivity-timeout*] # WSGI inactivity-timeout in seconds. # Default is 120 +# [*gr_django_init_provider*] +# Provider for the Django DB initialization exec. +# Default: 'posix' +# [*gr_django_init_command] +# Command to use for the Django DB initialization exec. +# default: "${::graphite::gr_python_binary} manage.py syncdb --noinput" # [*gr_django_tagging_pkg*] # String. The name of the django tagging package to install # Default: django-tagging @@ -732,8 +738,8 @@ $wsgi_processes = 5, $wsgi_threads = 5, $wsgi_inactivity_timeout = 120, - $gr_django_init_provider = $::graphite::params::gr_django_init_provider, - $gr_django_init_command = $::graphite::params::gr_django_init_command, + $gr_django_init_provider = "${::graphite::params::gr_python_binary} manage.py syncdb --noinput", + $gr_django_init_command = $::graphite::params::django_init_command, $gr_django_tagging_pkg = $::graphite::params::django_tagging_pkg, $gr_django_tagging_ver = $::graphite::params::django_tagging_ver, $gr_django_tagging_source = $::graphite::params::django_tagging_source, diff --git a/manifests/params.pp b/manifests/params.pp index 2afd003b..5a22f86d 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -38,8 +38,8 @@ $install_prefix = '/opt/' # variables for django db initialization - $gr_django_init_provider = 'posix' - $gr_django_init_command = "${::graphite::gr_python_binary} manage.py syncdb --noinput" + $django_init_provider = 'posix' +# $django_init_command = "${::graphite::gr_python_binary} manage.py syncdb --noinput" # variables to workaround unusual graphite install target: # https://github.com/graphite-project/carbon/issues/86 From 1115f60ec71b76b91c20682844113d65901f3443 Mon Sep 17 00:00:00 2001 From: Michal Vesely Date: Tue, 29 Aug 2017 12:01:43 +0200 Subject: [PATCH 03/12] fix switched around django_init_command/provider --- manifests/init.pp | 6 +++--- manifests/params.pp | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index 1af3bed5..6bc1fba6 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -442,7 +442,7 @@ # Default: 'posix' # [*gr_django_init_command] # Command to use for the Django DB initialization exec. -# default: "${::graphite::gr_python_binary} manage.py syncdb --noinput" +# default: "${::graphite::params::python_binary} manage.py syncdb --noinput" # [*gr_django_tagging_pkg*] # String. The name of the django tagging package to install # Default: django-tagging @@ -738,8 +738,8 @@ $wsgi_processes = 5, $wsgi_threads = 5, $wsgi_inactivity_timeout = 120, - $gr_django_init_provider = "${::graphite::params::gr_python_binary} manage.py syncdb --noinput", - $gr_django_init_command = $::graphite::params::django_init_command, + $gr_django_init_provider = $::graphite::params::django_init_provider, + $gr_django_init_command = "${::graphite::params::python_binary} manage.py syncdb --noinput", $gr_django_tagging_pkg = $::graphite::params::django_tagging_pkg, $gr_django_tagging_ver = $::graphite::params::django_tagging_ver, $gr_django_tagging_source = $::graphite::params::django_tagging_source, diff --git a/manifests/params.pp b/manifests/params.pp index 5a22f86d..677e5ff3 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -39,7 +39,6 @@ # variables for django db initialization $django_init_provider = 'posix' -# $django_init_command = "${::graphite::gr_python_binary} manage.py syncdb --noinput" # variables to workaround unusual graphite install target: # https://github.com/graphite-project/carbon/issues/86 From e6aa73cf34de0385848414ca8ce79254c1139b2b Mon Sep 17 00:00:00 2001 From: Filip Chabik Date: Tue, 29 Aug 2017 10:23:26 +0200 Subject: [PATCH 04/12] Add a new parameter: gr_carbonlink_hashing_type (graphite-web 0.9.16+). --- README.md | 4 ++++ manifests/init.pp | 4 ++++ .../webapp/graphite/local_settings.py.erb | 15 +++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/README.md b/README.md index 03ff3f13..eb1af2aa 100644 --- a/README.md +++ b/README.md @@ -737,6 +737,10 @@ Default is undef (array). Array of webbapp hosts. eg.: ['10.0.2.2:80', '10.0.2.3 Default is undef (array). Array of carbonlink hosts. eg.: ['10.0.2.2:80', '10.0.2.3:80'] +##### `gr_carbonlink_hashing_type` + +Default is 'undef' (string). Defines consistent-hashing type for 0.9.16+, e.g.: 'carbon_ch' + ##### `gr_cluster_fetch_timeout` Default is 6. Timeout to fetch series data. diff --git a/manifests/init.pp b/manifests/init.pp index 2870dc8d..0ba7860d 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -322,6 +322,9 @@ # [*gr_carbonlink_query_bulk*] # Boolean. 0.9.13 function. Using 'query-bulk' queries for carbon. # Default: false +# [*gr_carbonlink_hashing_type*] +# String. 0.9.16 function. Defining 'consistent-hashing' type. +# Default: carbon_ch # [*gr_cluster_fetch_timeout*] # Timeout to fetch series data. Default = 6 # [*gr_cluster_find_timeout*] @@ -761,6 +764,7 @@ $gr_disable_webapp_cache = false, $gr_enable_logrotation = true, $gr_carbonlink_query_bulk = undef, + $gr_carbonlink_hashing_type = undef, $gr_carbonlink_hosts_timeout = '1.0', $gr_rendering_hosts = undef, $gr_rendering_hosts_timeout = '1.0', diff --git a/templates/opt/graphite/webapp/graphite/local_settings.py.erb b/templates/opt/graphite/webapp/graphite/local_settings.py.erb index 3476ff8b..999380bc 100644 --- a/templates/opt/graphite/webapp/graphite/local_settings.py.erb +++ b/templates/opt/graphite/webapp/graphite/local_settings.py.erb @@ -286,6 +286,21 @@ CARBONLINK_TIMEOUT = <%= scope.lookupvar('graphite::gr_carbonlink_hosts_timeout' CARBONLINK_QUERY_BULK = <%= scope.lookupvar('graphite::gr_carbonlink_query_bulk') %> <% end -%> +<%- if scope.lookupvar('graphite::gr_graphite_ver') >= '0.9.16' -%> +# Type of metric hashing function. +# The default `carbon_ch` is Graphite's traditional consistent-hashing implementation. +# Alternatively, you can use `fnv1a_ch`, which supports the Fowler-Noll-Vo hash +# function (FNV-1a) hash implementation offered by the carbon-c-relay project +# https://github.com/grobian/carbon-c-relay +# +# Supported values: carbon_ch, fnv1a_ch +# +#CARBONLINK_HASHING_TYPE = 'carbon_ch' +<% unless [:undef, nil].include? scope.lookupvar('graphite::gr_carbonlink_hashing_type') -%> +CARBONLINK_HASHING_TYPE = '<%= scope.lookupvar('graphite::gr_carbonlink_hashing_type') %>' +<% end -%> +<% end -%> + ##################################### # Additional Django Settings # ##################################### From 8c43d379c6feb867281b8638ed3396b20326c0aa Mon Sep 17 00:00:00 2001 From: Daniel Werdermann Date: Wed, 25 Oct 2017 11:03:13 +0200 Subject: [PATCH 05/12] just cosmetic --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb1af2aa..65bdbcb0 100644 --- a/README.md +++ b/README.md @@ -739,7 +739,7 @@ Default is undef (array). Array of carbonlink hosts. eg.: ['10.0.2.2:80', '10.0. ##### `gr_carbonlink_hashing_type` -Default is 'undef' (string). Defines consistent-hashing type for 0.9.16+, e.g.: 'carbon_ch' +Default is undef (string). Defines consistent-hashing type for 0.9.16+, e.g.: 'carbon_ch' ##### `gr_cluster_fetch_timeout` From ff0c611276e5f9d1d0b4a101fea13c3611a76d9f Mon Sep 17 00:00:00 2001 From: Daniel Werdermann Date: Mon, 6 Nov 2017 19:25:16 +0100 Subject: [PATCH 06/12] graphite.metrics.search removed from graphite 1.0 See https://github.com/graphite-project/graphite-web/commit/8187904ee03655390ab2561cea21d573a371d12b#diff-46c6eecf33ea420f578e510fc1a5a170 This fixes #349 Closes #350 --- templates/opt/graphite/conf/graphite.wsgi.erb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/templates/opt/graphite/conf/graphite.wsgi.erb b/templates/opt/graphite/conf/graphite.wsgi.erb index 840dbf5d..45c3d5f7 100644 --- a/templates/opt/graphite/conf/graphite.wsgi.erb +++ b/templates/opt/graphite/conf/graphite.wsgi.erb @@ -41,9 +41,3 @@ if whitenoise: directory = os.path.join(os.path.dirname(module.__file__), 'static') if os.path.isdir(directory): application.add_files(directory, prefix=prefix) - -# Initializing the search index can be very expensive. The import below -# ensures the index is preloaded before any requests are handed to the -# process. -log.info("graphite.wsgi - pid %d - reloading search index" % os.getpid()) -import graphite.metrics.search # noqa From 57ac34977286c121d4025369890642f903b1ae4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kolbj=C3=B8rn=20Barmen?= <30319643+kbarmen@users.noreply.github.com> Date: Wed, 6 Dec 2017 15:41:14 +0100 Subject: [PATCH 07/12] Update params.pp Added support for Debian 9 aka stretch Pip needs python-setuptools to install the required pip modules. --- manifests/params.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/params.pp b/manifests/params.pp index 677e5ff3..6ce99df3 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -78,6 +78,7 @@ 'python-psycopg2', 'python-simplejson', 'python-sqlite', + 'python-setuptools', ] if $::operatingsystem == 'Ubuntu' { @@ -102,7 +103,7 @@ $extra_pip_install_options = undef } - /jessie|trusty|utopic|vivid|wily/: { + /stretch|jessie|trusty|utopic|vivid|wily/: { $apache_24 = true $graphitepkgs = union($common_os_pkgs, ['python-cairo',]) $libpath = "/usr/lib/python${pyver}/dist-packages" From 816f59eb66f1bdf53509e3d0d5493a97f02ec6cc Mon Sep 17 00:00:00 2001 From: Daniel Werdermann Date: Mon, 11 Dec 2017 14:40:28 +0100 Subject: [PATCH 08/12] rename contributing file --- DEVELOP.md => CONTRIBUTING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename DEVELOP.md => CONTRIBUTING.md (100%) diff --git a/DEVELOP.md b/CONTRIBUTING.md similarity index 100% rename from DEVELOP.md rename to CONTRIBUTING.md From 1c4d5524ab9e3cb352bae299117f605fc97e5c72 Mon Sep 17 00:00:00 2001 From: Daniel Werdermann Date: Fri, 15 Dec 2017 08:29:01 +0100 Subject: [PATCH 09/12] Update LICENSE --- LICENSE | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 2cc091aa..932b3409 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,4 @@ -Version: Apache 2.0 - -echocat puppet-graphite, Copyright (c) 2011-2014 echocat +Copyright 2011 echocat Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,4 +10,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. From e58d63b3ac7beef3e4b2bbf9f77652b44a1a1cb3 Mon Sep 17 00:00:00 2001 From: Johan van den Dorpe Date: Tue, 16 Jan 2018 11:23:46 +0000 Subject: [PATCH 10/12] Add support and tests for graphite-web 1.0 cluster config values --- manifests/init.pp | 3 ++ spec/classes/graphite_spec.rb | 37 +++++++++++++++++++ .../webapp/graphite/local_settings.py.erb | 8 ++++ 3 files changed, 48 insertions(+) diff --git a/manifests/init.pp b/manifests/init.pp index e002f1c7..08aba63b 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -818,6 +818,9 @@ $graphiteweb_storage_dir_REAL = $gr_graphiteweb_storage_dir $graphiteweb_install_lib_dir_REAL = pick($gr_graphiteweb_install_lib_dir, "${graphiteweb_webapp_dir_REAL}/graphite") + # Check for Graphite version 1 and above + $version_1 = versioncmp($gr_graphite_ver, '1.0') + # The anchor resources allow the end user to establish relationships # to the "main" class and preserve the relationship to the # implementation classes through a transitive relationship to diff --git a/spec/classes/graphite_spec.rb b/spec/classes/graphite_spec.rb index bd8bee93..6c585581 100644 --- a/spec/classes/graphite_spec.rb +++ b/spec/classes/graphite_spec.rb @@ -7,6 +7,40 @@ it { is_expected.to raise_error(Puppet::Error,/unsupported os,.+\./ )} end + shared_context 'Graphite 0.9 cluster settings' do + let(:params) {{ + :gr_graphite_ver => '0.9.16', + :gr_cluster_servers => [ '10.0.0.1', '10.0.0.2' ], + :gr_cluster_fetch_timeout => 6, + :gr_cluster_find_timeout => 2, + :gr_cluster_retry_delay => 10, + :gr_cluster_cache_duration => 120, + }} + + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_FETCH_TIMEOUT = 6/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_FIND_TIMEOUT = 2/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_RETRY_DELAY = 10/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FIND_CACHE_DURATION = 120/) } + + end + + shared_context 'Graphite 1.0 cluster settings' do + let(:params) {{ + :gr_graphite_ver => '1.1.1', + :gr_cluster_servers => [ '10.0.0.1', '10.0.0.2' ], + :gr_cluster_fetch_timeout => 6, + :gr_cluster_find_timeout => 2, + :gr_cluster_retry_delay => 10, + :gr_cluster_cache_duration => 120, + }} + + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FETCH_TIMEOUT = 6/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FIND_TIMEOUT = 2/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_RETRY_DELAY = 10/) } + it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/FIND_CACHE_DURATION = 120/) } + + end + on_supported_os.each do |os, facts| context "on #{os}" do let(:facts) do @@ -20,6 +54,9 @@ it { is_expected.to contain_class('graphite::config').that_comes_before('Anchor[graphite::end]') } it { is_expected.to contain_anchor('graphite::end') } + it_behaves_like 'Graphite 0.9 cluster settings' + it_behaves_like 'Graphite 1.0 cluster settings' + end end diff --git a/templates/opt/graphite/webapp/graphite/local_settings.py.erb b/templates/opt/graphite/webapp/graphite/local_settings.py.erb index 999380bc..ebe79658 100644 --- a/templates/opt/graphite/webapp/graphite/local_settings.py.erb +++ b/templates/opt/graphite/webapp/graphite/local_settings.py.erb @@ -233,12 +233,20 @@ DATABASES = { <% else %> CLUSTER_SERVERS = ['<%= scope.lookupvar('graphite::gr_cluster_servers').join("','") %>'] +<% if scope.lookupvar('graphite::version_1') >= 0 %> +REMOTE_FETCH_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_fetch_timeout') %> +REMOTE_FIND_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_find_timeout') %> +REMOTE_RETRY_DELAY = <%= scope.lookupvar('graphite::gr_cluster_retry_delay') %> +FIND_CACHE_DURATION = <%= scope.lookupvar('graphite::gr_cluster_cache_duration') %> +<% else %> REMOTE_STORE_FETCH_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_fetch_timeout') %> REMOTE_STORE_FIND_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_find_timeout') %> REMOTE_STORE_RETRY_DELAY = <%= scope.lookupvar('graphite::gr_cluster_retry_delay') %> REMOTE_FIND_CACHE_DURATION = <%= scope.lookupvar('graphite::gr_cluster_cache_duration') %> <% end %> +<% end %> + ## Prefetch cache # set to True to fetch all metrics using a single http request per remote server # instead of one http request per target, per remote server. From ea86b373e06cebc4db052b101f49989d2fc41f7c Mon Sep 17 00:00:00 2001 From: Johan van den Dorpe Date: Thu, 18 Jan 2018 13:36:32 +0000 Subject: [PATCH 11/12] Add comment in local_settings.py explaining config name changes --- templates/opt/graphite/webapp/graphite/local_settings.py.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/opt/graphite/webapp/graphite/local_settings.py.erb b/templates/opt/graphite/webapp/graphite/local_settings.py.erb index ebe79658..ba14ecd0 100644 --- a/templates/opt/graphite/webapp/graphite/local_settings.py.erb +++ b/templates/opt/graphite/webapp/graphite/local_settings.py.erb @@ -230,9 +230,12 @@ DATABASES = { #REMOTE_STORE_FIND_TIMEOUT = 2.5 # Timeout for metric find requests #REMOTE_STORE_RETRY_DELAY = 60 # Time before retrying a failed remote webapp #REMOTE_FIND_CACHE_DURATION = 300 # Time to cache remote metric find results + <% else %> CLUSTER_SERVERS = ['<%= scope.lookupvar('graphite::gr_cluster_servers').join("','") %>'] +# Remote cluster settings config names changed in Graphite 1.0. +# See https://github.com/echocat/puppet-graphite/pull/356 for more info <% if scope.lookupvar('graphite::version_1') >= 0 %> REMOTE_FETCH_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_fetch_timeout') %> REMOTE_FIND_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_find_timeout') %> From 3938459ea29aed82cdf8ba5e107edb771ebf354c Mon Sep 17 00:00:00 2001 From: Daniel Werdermann Date: Thu, 1 Feb 2018 09:16:14 +0100 Subject: [PATCH 12/12] prepare release 7.3.0 --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17207294..7711fe4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 2018-02-01 - 7.3.0 (Feature release) + +#### Features: + +- Add support for Debian 9 (stretch) +- Add support for graphite 1.x +- Add `gr_carbonlink_hashing_type` (since graphite 0.9.16+) +- Add `gr_django_init_provider` +- Add `gr_django_init_command` + +#### Bugfixes: + +- Support renamed parameters in local_settings.py for graphite 1.x+ + ## 2017-08-01 - 7.2.0 (Feature release) #### Features: