From a9f4fe0639c2d501e3f1d429cc0978d8f752ab0b Mon Sep 17 00:00:00 2001 From: Nick Walker Date: Tue, 29 Dec 2015 14:30:58 -0800 Subject: [PATCH 1/4] Add autovacuum settings to pe-postgresql In the past I've seen the console and puppetdb databases grow much larger than expected and I have a hypothesis that this is because the default 20% setting for autovacuum is too high for our workload. So, there's a potential that changing this to 1% might be too low and might reduce performance but it should reduce disk usage and it can always be tuned upward if the performance hit is too large, however, it is much more difficult to reduce the size of a database after it has grown too large. Basically, I'm erroring on the side of using less disk space and hopefully less outages due to running out of disk space and we can open a conversation on performance later if one needs to be had. --- .../manifests/pe_postgresql_management.pp | 29 +++++++++++++++++++ site/role/manifests/all_in_one_pe.pp | 1 + 2 files changed, 30 insertions(+) create mode 100644 site/profile/manifests/pe_postgresql_management.pp diff --git a/site/profile/manifests/pe_postgresql_management.pp b/site/profile/manifests/pe_postgresql_management.pp new file mode 100644 index 0000000..c4b81c3 --- /dev/null +++ b/site/profile/manifests/pe_postgresql_management.pp @@ -0,0 +1,29 @@ +class profile::pe_postgresql_management ( + $autovacuum_scale_factor = '.01', + $manage_postgresql_service = true, +) { + + $postgresql_service_resource_name = 'postgresqld' + $postgresql_service_name = 'pe-postgresql' + $notify_postgresql_service = $manage_postgresql_service ? { + true => Service[$postgresql_service_resource_name], + default => undef, + } + + if $manage_postgresql_service { + service { $postgresql_service_resource_name : + name => $postgresql_service_name, + ensure => running, + enable => true, + } + } + + #http://www.postgresql.org/docs/9.4/static/runtime-config-autovacuum.html + postgresql_conf { ['autovacuum_vacuum_scale_factor', 'autovacuum_analyze_scale_factor'] : + ensure => present, + target => '/opt/puppetlabs/server/data/postgresql/9.4/data/postgresql.conf', + value => $autovacuum_scale_factor, + notify => $notify_postgresql_service, + } + +} diff --git a/site/role/manifests/all_in_one_pe.pp b/site/role/manifests/all_in_one_pe.pp index 9e93155..054ef28 100644 --- a/site/role/manifests/all_in_one_pe.pp +++ b/site/role/manifests/all_in_one_pe.pp @@ -2,5 +2,6 @@ class role::all_in_one_pe { include profile::puppetmaster include profile::git_webhook + include profile::pe_postgresql_management } From d5854041f81c1feee78775ce211672396db90ae9 Mon Sep 17 00:00:00 2001 From: Nick Walker Date: Tue, 29 Dec 2015 14:30:58 -0800 Subject: [PATCH 2/4] Notify console-services when restarting postgresql Due to an issue where console-services fails to complete about 2 requests after a restart of postgresql, let's go ahead and restart the console-services service if we restart the postgresql service. If the console-services is not on the same node as postgresql then set all_in_one_pe_install to false and console-services will not be restarted. --- site/profile/manifests/pe_postgresql_management.pp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/site/profile/manifests/pe_postgresql_management.pp b/site/profile/manifests/pe_postgresql_management.pp index c4b81c3..7cca5da 100644 --- a/site/profile/manifests/pe_postgresql_management.pp +++ b/site/profile/manifests/pe_postgresql_management.pp @@ -1,6 +1,7 @@ class profile::pe_postgresql_management ( $autovacuum_scale_factor = '.01', $manage_postgresql_service = true, + $all_in_one_pe_install = true, ) { $postgresql_service_resource_name = 'postgresqld' @@ -9,12 +10,17 @@ class profile::pe_postgresql_management ( true => Service[$postgresql_service_resource_name], default => undef, } + $notify_console_services = $all_in_one_pe_install ? { + true => Service['pe-console-services'], + default => undef, + } if $manage_postgresql_service { service { $postgresql_service_resource_name : name => $postgresql_service_name, ensure => running, enable => true, + notify => $notify_console_services } } From edba37efb1d0a5b8410fbd3164914f5deae7e4eb Mon Sep 17 00:00:00 2001 From: Nick Walker Date: Mon, 11 Jan 2016 09:40:46 -0800 Subject: [PATCH 3/4] Add pe_databases::maintenance to Puppetfile and profile Default to inlcuding a cron job for running vacuumdb --analyze and reindxdx for all databases. --- Puppetfile | 4 ++++ site/profile/manifests/pe_postgresql_management.pp | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Puppetfile b/Puppetfile index 6bc31f8..88dc8de 100644 --- a/Puppetfile +++ b/Puppetfile @@ -38,3 +38,7 @@ mod 'gms', mod 'pltraining-rbac', :git => 'https://github.com/puppetlabs/pltraining-rbac', :ref => '2f60e1789a721ce83f8df061e13f8bf81cd4e4ce' + +mod 'pe_databases', + :git => 'https://github.com/npwalker/pe_databases', + :ref => 'ce51abffa029d910f84b44160791e7406f2da864' diff --git a/site/profile/manifests/pe_postgresql_management.pp b/site/profile/manifests/pe_postgresql_management.pp index 7cca5da..08b6cb6 100644 --- a/site/profile/manifests/pe_postgresql_management.pp +++ b/site/profile/manifests/pe_postgresql_management.pp @@ -1,7 +1,8 @@ class profile::pe_postgresql_management ( - $autovacuum_scale_factor = '.01', - $manage_postgresql_service = true, - $all_in_one_pe_install = true, + $autovacuum_scale_factor = '.01', + $manage_postgresql_service = true, + $all_in_one_pe_install = true, + Boolean $include_pe_databases_maintenance = true, ) { $postgresql_service_resource_name = 'postgresqld' @@ -32,4 +33,9 @@ class profile::pe_postgresql_management ( notify => $notify_postgresql_service, } + #https://github.com/npwalker/pe_databases + if $include_pe_databases_maintenance { + include pe_databases::maintenance + } + } From acf1cf7ffd101e7d4ef3827c5d357c0f995f042f Mon Sep 17 00:00:00 2001 From: Nick Walker Date: Mon, 11 Jan 2016 09:47:16 -0800 Subject: [PATCH 4/4] Add data type validation to pe_postgresql_management profile Prior to this commit 3 of the 4 parameters were not validated. After this commit, the parameters are correctly validated. Due to MODULES-2960 $autovacuum_scale_factor is validated as a float but then is converted to a string to pass it to the value attribute of postgresql_conf. --- site/profile/manifests/pe_postgresql_management.pp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/site/profile/manifests/pe_postgresql_management.pp b/site/profile/manifests/pe_postgresql_management.pp index 08b6cb6..fd61376 100644 --- a/site/profile/manifests/pe_postgresql_management.pp +++ b/site/profile/manifests/pe_postgresql_management.pp @@ -1,8 +1,8 @@ class profile::pe_postgresql_management ( - $autovacuum_scale_factor = '.01', - $manage_postgresql_service = true, - $all_in_one_pe_install = true, - Boolean $include_pe_databases_maintenance = true, + Float[0,1] $autovacuum_scale_factor = 0.01, + Boolean $manage_postgresql_service = true, + Boolean $all_in_one_pe_install = true, + Boolean $include_pe_databases_maintenance = true, ) { $postgresql_service_resource_name = 'postgresqld' @@ -25,11 +25,13 @@ class profile::pe_postgresql_management ( } } + #The value attribute of postgresql_conf requires a string despite validating a float above + #https://tickets.puppetlabs.com/browse/MODULES-2960 #http://www.postgresql.org/docs/9.4/static/runtime-config-autovacuum.html postgresql_conf { ['autovacuum_vacuum_scale_factor', 'autovacuum_analyze_scale_factor'] : ensure => present, target => '/opt/puppetlabs/server/data/postgresql/9.4/data/postgresql.conf', - value => $autovacuum_scale_factor, + value => "${autovacuum_scale_factor}", notify => $notify_postgresql_service, }