Merge pull request #358 from echocat/develop

backmerge release 7.3.0

CHANGELOG.md

@@ -1,3 +1,17 @@
+## 2018-02-01 - 7.3.0 (Feature release)
+
+#### Features:
+
+- Add support for Debian 9 (stretch)
+- Add support for graphite 1.x
+- Add `gr_carbonlink_hashing_type` (since graphite 0.9.16+)
+- Add `gr_django_init_provider`
+- Add `gr_django_init_command`
+
+#### Bugfixes:
+
+- Support renamed parameters in local_settings.py for graphite 1.x+
+
 ## 2017-08-01 - 7.2.0 (Feature release)
 
 #### Features:

LICENSE

@@ -1,6 +1,4 @@
-Version: Apache 2.0
-
-echocat puppet-graphite, Copyright (c) 2011-2014 echocat
+Copyright 2011 echocat
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,4 +10,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

README.md

@@ -737,6 +737,10 @@ Default is undef (array). Array of webbapp hosts. eg.: ['10.0.2.2:80', '10.0.2.3
 
 Default is undef (array). Array of carbonlink hosts. eg.: ['10.0.2.2:80', '10.0.2.3:80']
 
+##### `gr_carbonlink_hashing_type`
+
+Default is undef (string). Defines consistent-hashing type for 0.9.16+, e.g.: 'carbon_ch'
+
 ##### `gr_cluster_fetch_timeout`
 
 Default is 6. Timeout to fetch series data.

manifests/config.pp

@@ -98,7 +98,8 @@
 
   # first init of user db for graphite
   exec { 'Initial django db creation':
-    command     => "${::graphite::gr_python_binary} manage.py syncdb --noinput",
+    command     => $::graphite::gr_django_init_command,
+    provider    => $::graphite::gr_django_init_provider,
     cwd         => $graphite_web_managepy_location,
     refreshonly => true,
     require     => $syncdb_require,

manifests/init.pp

@@ -322,6 +322,9 @@
 # [*gr_carbonlink_query_bulk*]
 #   Boolean. 0.9.13 function. Using 'query-bulk' queries for carbon.
 #   Default: false
+# [*gr_carbonlink_hashing_type*]
+#   String. 0.9.16 function. Defining 'consistent-hashing' type.
+#   Default: carbon_ch
 # [*gr_cluster_fetch_timeout*]
 #   Timeout to fetch series data.   Default = 6
 # [*gr_cluster_find_timeout*]
@@ -437,6 +440,12 @@
 # [*wsgi_inactivity-timeout*]
 #   WSGI inactivity-timeout in seconds.
 #   Default is 120
+# [*gr_django_init_provider*]
+#   Provider for the Django DB initialization exec.
+#   Default: 'posix'
+# [*gr_django_init_command]
+#   Command to use for the Django DB initialization exec.
+#   default: "${::graphite::params::python_binary} manage.py syncdb --noinput"
 # [*gr_django_tagging_pkg*]
 #   String. The name of the django tagging package to install
 #   Default: django-tagging
@@ -732,6 +741,8 @@
   $wsgi_processes                         = 5,
   $wsgi_threads                           = 5,
   $wsgi_inactivity_timeout                = 120,
+  $gr_django_init_provider                = $::graphite::params::django_init_provider,
+  $gr_django_init_command                 = "${::graphite::params::python_binary} manage.py syncdb --noinput",
   $gr_django_tagging_pkg                  = $::graphite::params::django_tagging_pkg,
   $gr_django_tagging_ver                  = $::graphite::params::django_tagging_ver,
   $gr_django_tagging_source               = $::graphite::params::django_tagging_source,
@@ -761,6 +772,7 @@
   $gr_disable_webapp_cache                = false,
   $gr_enable_logrotation                  = true,
   $gr_carbonlink_query_bulk               = undef,
+  $gr_carbonlink_hashing_type             = undef,
   $gr_carbonlink_hosts_timeout            = '1.0',
   $gr_rendering_hosts                     = undef,
   $gr_rendering_hosts_timeout             = '1.0',
@@ -806,6 +818,9 @@
   $graphiteweb_storage_dir_REAL     = $gr_graphiteweb_storage_dir
   $graphiteweb_install_lib_dir_REAL = pick($gr_graphiteweb_install_lib_dir, "${graphiteweb_webapp_dir_REAL}/graphite")
 
+  # Check for Graphite version 1 and above
+  $version_1 = versioncmp($gr_graphite_ver, '1.0')
+
   # The anchor resources allow the end user to establish relationships
   # to the "main" class and preserve the relationship to the
   # implementation classes through a transitive relationship to

manifests/params.pp

@@ -37,6 +37,9 @@
 
   $install_prefix     = '/opt/'
 
+  # variables for django db initialization
+  $django_init_provider = 'posix'
+
   # variables to workaround unusual graphite install target:
   # https://github.com/graphite-project/carbon/issues/86
   $pyver              = $::osfamily ? {
@@ -75,6 +78,7 @@
         'python-psycopg2',
         'python-simplejson',
         'python-sqlite',
+        'python-setuptools',
       ]
 
       if $::operatingsystem == 'Ubuntu' {
@@ -99,7 +103,7 @@
           $extra_pip_install_options = undef
         }
 
-        /jessie|trusty|utopic|vivid|wily/: {
+        /stretch|jessie|trusty|utopic|vivid|wily/: {
           $apache_24                 = true
           $graphitepkgs              = union($common_os_pkgs, ['python-cairo',])
           $libpath                   = "/usr/lib/python${pyver}/dist-packages"

spec/classes/graphite_spec.rb

@@ -7,6 +7,40 @@
     it { is_expected.to raise_error(Puppet::Error,/unsupported os,.+\./ )}
   end
 
+  shared_context 'Graphite 0.9 cluster settings' do
+    let(:params) {{
+      :gr_graphite_ver => '0.9.16',
+      :gr_cluster_servers => [ '10.0.0.1', '10.0.0.2' ],
+      :gr_cluster_fetch_timeout => 6,
+      :gr_cluster_find_timeout => 2,
+      :gr_cluster_retry_delay => 10,
+      :gr_cluster_cache_duration => 120,
+    }}
+
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_FETCH_TIMEOUT = 6/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_FIND_TIMEOUT = 2/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_STORE_RETRY_DELAY = 10/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FIND_CACHE_DURATION = 120/) }
+
+  end
+
+  shared_context 'Graphite 1.0 cluster settings' do
+    let(:params) {{
+      :gr_graphite_ver => '1.1.1',
+      :gr_cluster_servers => [ '10.0.0.1', '10.0.0.2' ],
+      :gr_cluster_fetch_timeout => 6,
+      :gr_cluster_find_timeout => 2,
+      :gr_cluster_retry_delay => 10,
+      :gr_cluster_cache_duration => 120,
+    }}
+
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FETCH_TIMEOUT = 6/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_FIND_TIMEOUT = 2/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/REMOTE_RETRY_DELAY = 10/) }
+    it { is_expected.to contain_file('/opt/graphite/webapp/graphite/local_settings.py').with_content(/FIND_CACHE_DURATION = 120/) }
+
+  end
+
   on_supported_os.each do |os, facts|
     context "on #{os}" do
       let(:facts) do
@@ -20,6 +54,9 @@
       it { is_expected.to contain_class('graphite::config').that_comes_before('Anchor[graphite::end]') }
       it { is_expected.to contain_anchor('graphite::end') }
 
+      it_behaves_like 'Graphite 0.9 cluster settings'
+      it_behaves_like 'Graphite 1.0 cluster settings'
+
     end
   end
 

templates/opt/graphite/conf/graphite.wsgi.erb

@@ -41,9 +41,3 @@ if whitenoise:
         directory = os.path.join(os.path.dirname(module.__file__), 'static')
         if os.path.isdir(directory):
             application.add_files(directory, prefix=prefix)
-
-# Initializing the search index can be very expensive. The import below
-# ensures the index is preloaded before any requests are handed to the
-# process.
-log.info("graphite.wsgi - pid %d - reloading search index" % os.getpid())
-import graphite.metrics.search  # noqa

templates/opt/graphite/webapp/graphite/local_settings.py.erb

@@ -230,15 +230,26 @@ DATABASES = {
 #REMOTE_STORE_FIND_TIMEOUT = 2.5  # Timeout for metric find requests
 #REMOTE_STORE_RETRY_DELAY = 60    # Time before retrying a failed remote webapp
 #REMOTE_FIND_CACHE_DURATION = 300 # Time to cache remote metric find results
+
 <% else %>
 CLUSTER_SERVERS = ['<%= scope.lookupvar('graphite::gr_cluster_servers').join("','") %>']
 
+# Remote cluster settings config names changed in Graphite 1.0.
+# See https://github.com/echocat/puppet-graphite/pull/356 for more info
+<% if scope.lookupvar('graphite::version_1') >= 0 %>
+REMOTE_FETCH_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_fetch_timeout') %>
+REMOTE_FIND_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_find_timeout') %>
+REMOTE_RETRY_DELAY = <%= scope.lookupvar('graphite::gr_cluster_retry_delay') %>
+FIND_CACHE_DURATION = <%= scope.lookupvar('graphite::gr_cluster_cache_duration') %>
+<% else %>
 REMOTE_STORE_FETCH_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_fetch_timeout') %>
 REMOTE_STORE_FIND_TIMEOUT = <%= scope.lookupvar('graphite::gr_cluster_find_timeout') %>
 REMOTE_STORE_RETRY_DELAY = <%= scope.lookupvar('graphite::gr_cluster_retry_delay') %>
 REMOTE_FIND_CACHE_DURATION = <%= scope.lookupvar('graphite::gr_cluster_cache_duration') %>
 <% end %>
 
+<% end %>
+
 ## Prefetch cache
 # set to True to fetch all metrics using a single http request per remote server
 # instead of one http request per target, per remote server.
@@ -286,6 +297,21 @@ CARBONLINK_TIMEOUT = <%= scope.lookupvar('graphite::gr_carbonlink_hosts_timeout'
 CARBONLINK_QUERY_BULK = <%= scope.lookupvar('graphite::gr_carbonlink_query_bulk') %>
 <% end -%>
 
+<%- if scope.lookupvar('graphite::gr_graphite_ver') >= '0.9.16' -%>
+# Type of metric hashing function.
+# The default `carbon_ch` is Graphite's traditional consistent-hashing implementation.
+# Alternatively, you can use `fnv1a_ch`, which supports the Fowler-Noll-Vo hash
+# function (FNV-1a) hash implementation offered by the carbon-c-relay project
+# https://github.com/grobian/carbon-c-relay
+#
+# Supported values: carbon_ch, fnv1a_ch
+#
+#CARBONLINK_HASHING_TYPE = 'carbon_ch'
+<% unless [:undef, nil].include? scope.lookupvar('graphite::gr_carbonlink_hashing_type') -%>
+CARBONLINK_HASHING_TYPE = '<%= scope.lookupvar('graphite::gr_carbonlink_hashing_type') %>'
+<% end -%>
+<% end -%>
+
 #####################################
 # Additional Django Settings #
 #####################################