Merge branch 'stable-3.1' into 'stable-3.1'

Till Brehm · Till Brehm · commit ba65ba3bf0cc · 2017-04-10T14:31:10.000+02:00
Make the webserver's rentention time for log files configurable

This feature ditches the current static 30 days approach of logfile purging and introduces a configurable retention time for each website.  
The retention time configuration can be found in the options tab of the website settings.  
  
  
  
Default retention time is the current 30 days value.  

However, by this approach it's possible to have retention times like 0 (forever), or very short retention times like 7 days which are often demanded by public/government data security officials which we have in germany.

See merge request !576
diff --git a/install/sql/incremental/upd_dev_collection.sql b/install/sql/incremental/upd_dev_collection.sql
@@ -1 +1,2 @@
-ALTER TABLE `web_domain` CHANGE `folder_directive_snippets` `folder_directive_snippets` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NULL;
+ALTER TABLE `web_domain` CHANGE `folder_directive_snippets` `folder_directive_snippets` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NULL;
+ALTER TABLE `web_domain` ADD `log_retention` INT NOT NULL DEFAULT '30' AFTER `https_port`;
diff --git a/install/sql/ispconfig3.sql b/install/sql/ispconfig3.sql
@@ -1986,6 +1986,7 @@ CREATE TABLE `web_domain` (
   `http_port` int(11) unsigned NOT NULL DEFAULT '80',
   `https_port` int(11) unsigned NOT NULL DEFAULT '443',
   `folder_directive_snippets` text,
+  `log_retention` int(11) NOT NULL DEFAULT '30',
   PRIMARY KEY  (`domain_id`),
   UNIQUE KEY `serverdomain` (  `server_id` , `ip_address`,  `domain` )
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ;
diff --git a/interface/web/sites/form/web_vhost_domain.tform.php b/interface/web/sites/form/web_vhost_domain.tform.php
@@ -917,7 +917,19 @@
 				'value'  => '',
 				'width'  => '3',
 				'maxlength' => '6'
-			)
+			),
+                        'log_retention' => array (
+                                'datatype' => 'INTEGER',
+                                'formtype' => 'TEXT',
+                                'validators' => array (  0 => array ( 'type' => 'REGEX',
+                                                'regex' => '/^([0-9]{1,4})$/',
+                                                'errmsg'=> 'log_retention_error_regex'),
+                                ),
+                                'default' => '30',
+                                'value' => '',
+                                'width' => '4',
+                                'maxlength' => '4'
+                        )
 			//#################################
 			// ENDE Datatable fields
 			//#################################
diff --git a/interface/web/sites/lib/lang/ar_web_vhost_domain.lng b/interface/web/sites/lib/lang/ar_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/bg_web_vhost_domain.lng b/interface/web/sites/lib/lang/bg_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/br_web_vhost_domain.lng b/interface/web/sites/lib/lang/br_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'Porta HTTPS';
 $wb['http_port_error_regex'] = 'Porta HTTP inválida.';
 $wb['https_port_error_regex'] = 'Porta HTTPS inválida.';
 $wb['enable_pagespeed_txt'] = 'Habilitar PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/ca_web_vhost_domain.lng b/interface/web/sites/lib/lang/ca_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/cz_web_vhost_domain.lng b/interface/web/sites/lib/lang/cz_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/de_web_vhost_domain.lng b/interface/web/sites/lib/lang/de_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Log-Dateien Aufbewahrungszeit';
+$wb['log_retention_error_regex'] = "Aufbewahrungszeit in Tagen (Erlaubte Werte: min. 0 - max. 9999)";
 ?>
diff --git a/interface/web/sites/lib/lang/dk_web_vhost_domain.lng b/interface/web/sites/lib/lang/dk_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/el_web_vhost_domain.lng b/interface/web/sites/lib/lang/el_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/en_web_vhost_domain.lng b/interface/web/sites/lib/lang/en_web_vhost_domain.lng
@@ -153,4 +153,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/es_web_vhost_domain.lng b/interface/web/sites/lib/lang/es_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/fi_web_vhost_domain.lng b/interface/web/sites/lib/lang/fi_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/fr_web_vhost_domain.lng b/interface/web/sites/lib/lang/fr_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/hr_web_vhost_domain.lng b/interface/web/sites/lib/lang/hr_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/hu_web_vhost_domain.lng b/interface/web/sites/lib/lang/hu_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/id_web_vhost_domain.lng b/interface/web/sites/lib/lang/id_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/it_web_vhost_domain.lng b/interface/web/sites/lib/lang/it_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/ja_web_vhost_domain.lng b/interface/web/sites/lib/lang/ja_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/nl_web_vhost_domain.lng b/interface/web/sites/lib/lang/nl_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Poort';
 $wb['http_port_error_regex'] = 'HTTP Poort niet correct.';
 $wb['https_port_error_regex'] = 'HTTPS Poort niet correct.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/pl_web_vhost_domain.lng b/interface/web/sites/lib/lang/pl_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/pt_web_vhost_domain.lng b/interface/web/sites/lib/lang/pt_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/ro_web_vhost_domain.lng b/interface/web/sites/lib/lang/ro_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/se_web_vhost_domain.lng b/interface/web/sites/lib/lang/se_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/sk_web_vhost_domain.lng b/interface/web/sites/lib/lang/sk_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/lib/lang/tr_web_vhost_domain.lng b/interface/web/sites/lib/lang/tr_web_vhost_domain.lng
@@ -148,4 +148,6 @@ $wb['https_port_txt'] = 'HTTPS Port';
 $wb['http_port_error_regex'] = 'HTTP Port invalid.';
 $wb['https_port_error_regex'] = 'HTTPS Port invalid.';
 $wb['enable_pagespeed_txt'] = 'Enable PageSpeed';
+$wb['log_retention_txt'] = 'Logfiles retention time';
+$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
 ?>
diff --git a/interface/web/sites/templates/web_vhost_domain_advanced.htm b/interface/web/sites/templates/web_vhost_domain_advanced.htm
@@ -35,6 +35,10 @@ <h1><tmpl_var name="list_head_txt"></h1>
                 </div>
                 <input name="system_group" id="system_group" value="{tmpl_var name='system_group'}" type="hidden" />
             </div>
+            <div class="form-group">
+                <label for="log_retention" class="col-sm-3 control-label">{tmpl_var name='log_retention_txt'}</label>
+                <div class="col-sm-9"><input name="log_retention" id="log_retention" value="{tmpl_var name='log_retention'}" type="text" class="form-control" /></div>
+            </div>
             <div class="form-group apache">
                 <label for="allow_override" class="col-sm-3 control-label">{tmpl_var name='allow_override_txt'}</label>
                 <div class="col-sm-9"><input type="text" name="allow_override" id="allow_override" value="{tmpl_var name='allow_override'}" class="form-control" /></div></div>
diff --git a/server/lib/classes/cron.d/200-logfiles.inc.php b/server/lib/classes/cron.d/200-logfiles.inc.php
@@ -60,7 +60,7 @@ public function onRunJob() {
 		// Manage and compress web logfiles and create traffic statistics
 		//######################################################################################################
 
-		$sql = "SELECT domain_id, domain, type, document_root, web_folder, parent_domain_id FROM web_domain WHERE (type = 'vhost' or type = 'vhostsubdomain' or type = 'vhostalias') AND server_id = ?";
+		$sql = "SELECT domain_id, domain, type, document_root, web_folder, parent_domain_id, log_retention FROM web_domain WHERE (type = 'vhost' or type = 'vhostsubdomain' or type = 'vhostalias') AND server_id = ?";
 		$records = $app->db->queryAllRecords($sql, $conf['server_id']);
 		foreach($records as $rec) {
 
@@ -76,6 +76,8 @@ public function onRunJob() {
 				unset($tmp);
 			}
 
+                        $log_retention = $rec['log_retention'];
+
 			$logfile = $rec['document_root'].'/' . $log_folder . '/'.$yesterday.'-access.log';
 			$total_bytes = 0;
 
@@ -142,21 +144,17 @@ public function onRunJob() {
 				exec("cat /dev/null > $logfile");
 			}
 
-			// delete logfiles after 30 days
-			$month_ago = date('Ymd', time() - 86400 * 30);
-			$logfile = escapeshellcmd($rec['document_root'].'/' . $log_folder . '/'.$month_ago.'-access.log.gz');
-			if(@is_file($logfile)) {
-				unlink($logfile);
-			}
-
-			//* Delete older Log files, in case that we missed them before due to serverdowntimes.
-			$datepart = date('Ym', time() - 86400 * 31 * 2);
+			// delete logfiles after x days (default 30)
+                        if($log_retention > 0) {
+                        foreach (glob($rec['document_root'].'/' . $log_folder . '/'."*.log*") as $logfile) {
+                        $now   = time();
+                        if (is_file($logfile))
+                                if ($now - filemtime($logfile) >= 60 * 60 * 24 * $log_retention)
+                                        unlink($logfile);
+                        }
 
-			$logfile = escapeshellcmd($rec['document_root']).'/' . $log_folder . '/'.$datepart.'*-access.log.gz';
-			exec('rm -f '.$logfile);
+                        }
 
-			$logfile = escapeshellcmd($rec['document_root']).'/' . $log_folder . '/'.$datepart.'*-access.log';
-			exec('rm -f '.$logfile);
 		}
 
 		//* Delete old logfiles in /var/log/ispconfig/httpd/ that were created by vlogger for the hostname of the server

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		-ALTER TABLE `web_domain` CHANGE `folder_directive_snippets` `folder_directive_snippets` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NULL;
	`1`	+ALTER TABLE `web_domain` CHANGE `folder_directive_snippets` `folder_directive_snippets` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NULL;
	`2`	+ALTER TABLE `web_domain` ADD `log_retention` INT NOT NULL DEFAULT '30' AFTER `https_port`;