Commit 08e07650 authored by Paul Cammish's avatar Paul Cammish
Browse files

Improve CI performance, fixes #323, other small fixes

parent 8f3dfa94
stages:
- Package
- Publish
- Install
- Upgrade
- Test
# This will run through the packages one at a time, building them into artifacts
# This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with
......@@ -97,8 +96,9 @@ Build Repo:
Clean Install:
tags:
- vbox:bullseye
stage: Install
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
- autotest/enable_backports
......@@ -120,18 +120,21 @@ Clean Install:
- repo/*
expire_in: 1 hour
Upgrade Testing:
Upgrade Stable:
tags:
- vbox:bullseye
stage: Upgrade
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- git describe --all
- autotest/install_then_upgrade bullseye-testing
- autotest/install_then_upgrade bullseye
- run-parts --verbose --exit-on-error autotest/test.d
- hostname
- dpkg -l 'sympl*'
allow_failure: false
when: delayed
start_in: 30 seconds
variables:
CI_DEBUG_TRACE: "false"
artifacts:
......@@ -139,23 +142,25 @@ Upgrade Testing:
- repo/*
expire_in: 1 hour
Upgrade Stable:
Upgrade Testing:
tags:
- vbox:bullseye
stage: Upgrade
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- git describe --all
- autotest/install_then_upgrade bullseye
- autotest/install_then_upgrade bullseye-testing
- run-parts --verbose --exit-on-error autotest/test.d
- hostname
- dpkg -l 'sympl*'
allow_failure: false
when: delayed
start_in: 5 minutes
start_in: 1 minute
variables:
CI_DEBUG_TRACE: "false"
artifacts:
paths:
- repo/*
expire_in: 1 hour
......@@ -7,8 +7,8 @@
## Install
* [ ] Run Install script as per https://wiki.sympl.host/Installation_Instructions without dpkg prompts.
* [ ] User is pointed to https://wiki.sympl.host for docs, and https://forum.sympl.host for issues.
* [ ] Run Install script as per https://wiki.sympl.io/Installation_Instructions without dpkg prompts.
* [ ] User is pointed to https://wiki.sympl.io for docs, and https://forum.sympl.io for issues.
* [ ] User has to set a new password for `sympl`, and is suggested to use an SSH key.
* [ ] User can log in as the `sympl` user.
......
CHANGELOG
---------
2022-04-26
sympl-core
* Fix control logic in sympl-filesystem-security
2022-04-06
sympl-web
* Use AWFFull names for stats files rather than Webalizer
2022-03-23
sympl-core
* Replace tempfile with mktemp in sympl-generate-dhparams
sympl-mail
* Correctly regenerate dhparams for dovecot
2022-03-22
sympl-mail
* Clean up logging warning in exim main.log
* Update references from sympl.host to sympl.io
sympl-webmail
* Fix log warnings from Roundcube in user.log
sympl-core, sympl-web, sympl-cron
* Update references from sympl.host to sympl.io
2021-12-13
sympl-core
* Updated workaround for Let's Encrypt cross-signed intermediate
......
......@@ -21,51 +21,52 @@ touch /srv/$(hostname -f)/config/antivirus
systemctl unmask clamav-freshclam
service clamav-freshclam stop
rm -rf /run/clamav
service clamav-freshclam start
sleep 3
echo -n "I: Waiting for clamav to download databases."
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo -n ' 1/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
echo -n ' 2/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
echo " 3/3 done."
invoke-rc.d clamav-daemon restart
break
fi
echo -n "."
sleep 1
done
#service clamav-freshclam start
#
#sleep 3
#
#echo -n "I: Waiting for clamav to download databases."
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
# echo -n ' 1/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
# echo -n ' 2/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
# echo " 3/3 done."
# invoke-rc.d clamav-daemon restart
# break
# fi
# echo -n "."
# sleep 1
#done
# horrible hack
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo "We have what look to be valid definitions..."
else
echo "Failed to download clamAV definitions, going to fallback."
cd /var/lib/clamav/
wget -qO clamav.tar.gz http://sympl.host/clamav.tar.gz
tar -xvf clamav.tar.gz
fi
#if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
# echo "We have what look to be valid definitions..."
#else
# echo "Failed to download clamAV definitions, going to fallback."
cd /var/lib/clamav/
wget -qO clamav.tar.gz http://sympl.io/clamav.tar.gz
tar -xvf clamav.tar.gz
#fi
# Start clamav, whatever state it's currently in
systemctl unmask clamav-daemon
service clamav-daemon stop
service clamav-daemon start
service clamav-freshclam start
# Enable antispam and antivirus as default, otherwise monit would stop them.
echo "I: Enabling Antivirus and Antispam config"
......
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
echo 'Running sympl-test...'
/usr/bin/sympl-test > /dev/null 2>&1
# ...so, we'll ignore that and run it again.
/usr/bin/sympl-test
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
#echo 'Running sympl-test...'
#/usr/bin/sympl-test > /dev/null 2>&1
# fairly sure this was a race conditon where apache wasn't reloading quick enough
# so run it once, and retry if that fails,
# as theres still some uncommon race conditions
if ! /usr/bin/sympl-test ; then echo Trying again ; /usr/bin/sympl-test ; fi
......@@ -2,7 +2,7 @@
#
# Sympl command line interface.
#
# Copyright 2019-2020 the Sympl Project - https://sympl.host
# Copyright 2019-2022 the Sympl Project - https://sympl.io
#
# Licenced under GPL3+
#
......@@ -154,7 +154,7 @@ Examples:
sympl update
Note: Not all functions are implimented yet.
Please report bugs via http://bugs.sympl.host.
Please report bugs via http://bugs.sympl.io.
"
}
......
sympl-core (11.20220426.0) stable; urgency=medium
* Fix control logic in sympl-filesystem-security
-- Paul Cammish <sympl@kelduum.net> Tue, 26 Apr 2022 10:00:00 +0100
sympl-core (11.20220323.0) stable; urgency=medium
* Replace tempfile with mktemp in sympl-generate-dhparams
-- Paul Cammish <sympl@kelduum.net> Tue, 22 Mar 2022 14:00:00 +0000
sympl-core (11.20220322.1) stable; urgency=low
* Update references from sympl.host to sympl.io
-- Paul Cammish <sympl@kelduum.net> Tue, 22 Mar 2022 14:00:00 +0000
sympl-core (11.20211213.2) stable; urgency=medium
* Fix for MOTD installation
......
......@@ -167,7 +167,7 @@ if [ -d /etc/sympl ]; then
fi
for domain in $( find /srv -maxdepth 1 -mindepth 1 ! -type l -type d -print | grep -v '^/srv/\.' | grep '\.' ); do
if [ ! -f ${domain}/config/do-not-secure ] || [ -f /etc/sympl/disable-filesystem-security ] ; then
if [ ! -f ${domain}/config/disable-filesystem-security ] && [ ! -f ${domain}/config/do-not-secure ] ; then
secure_domain_dir ${domain}
fi
done
......
......@@ -41,7 +41,7 @@ dir=$(dirname "$fileName")
mkdir -m 750 -p "$dir"
length=2048
tmpfile=$(tempfile -m 0600 -d "$dir" -p .dh)
tmpfile=$(TMPDIR="$dir" mktemp XXXXXXXXXXX.dh )
if [ ! -f "$tmpfile" ] ; then
echo "temporary file '$tmpfile' doesn't exist, stopping."
......
......@@ -253,7 +253,7 @@ Need Help?
For advice on securing your machine please consult the documentation
available upon the sympl website:
http://sympl.host/
http://sympl.io/
EOF
end
......
sympl-cron (11.20220322.1) stable; urgency=low
* Update reference to sympl.host to sympl.io
-- Paul Cammish <sympl@kelduum.net> Tue, 22 Mar 2022 14:00:00 +0000
sympl-cron (11.20210818.1) stable; urgency=medium
* Debian Bullseye Release
......
......@@ -39,7 +39,7 @@ USAGE
The --test option can be useful for determining when the commands
will next get executed.
Full documentation is available at http://sympl.host
Full documentation is available at http://sympl.io
BUGS
......
......@@ -19,14 +19,14 @@ class TestFTP < Test::Unit::TestCase
@domain.create()
# Hack to disable TLS enforcement, as the ruby lib doesnt support it
system 'echo 1 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3'
system 'echo 1 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall -w -s 9 -q "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
end
def teardown
# Re-enable TLS enforcement
system 'echo 2 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3'
system 'echo 2 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall -w -s 9 -q "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
#
# Delete the temporary domain
......
sympl-mail (11.20220323.0) stable; urgency=medium
* Correctly regenerate dhparams for dovecot
-- Paul Cammish <sympl@kelduum.net> Wed, 23 Mar 2022 11:47:00 +0000
sympl-mail (11.20220322.2) stable; urgency=low
* Update references from sympl.host to sympl.io
-- Paul Cammish <sympl@kelduum.net> Tue, 22 Mar 2022 14:00:00 +0000
sympl-mail (11.20220322.1) stable; urgency=medium
* Clean up logging warning in exim main.log
-- Paul Cammish <sympl@kelduum.net> Tue, 22 Mar 2022 13:26:00 +0000
sympl-mail (11.20210921.1) stable; urgency=medium
* Deal with mail correctly when no mailboxes directory exists, fixes #317
......
......@@ -6,4 +6,5 @@ if [ "$*" = "--verbose" ] || [ "$*" = "-v" ]; then
fi
sympl-generate-dhparams $verbose/etc/ssl/private/exim4-dhparams.pem Debian-exim
sympl-generate-dhparams $verbose/etc/ssl/private/dovecot-dhparams.pem dovecot
......@@ -12,7 +12,7 @@ tls_advertise_hosts = *
# Allow localhost to authenticate without TLS, or any TLS connection.
auth_advertise_hosts = localhost : ${if eq{$tls_cipher}{}{no_matching_hosts}{*}}
auth_advertise_hosts = localhost : ${if eq{$tls_cipher}{}{localhost}{*}}
# Specify the location of the Exim server's TLS certificate and private key.
# The private key must not be encrypted (password protected). You can put
......
......@@ -26,7 +26,7 @@ local_ip: 192.168.0.1
local_domains:
- local.domain
- other.domain
- sympl.host
- sympl.io
rewrite_domains:
symlinked.domain: local.domain
remote_ip: 192.168.4.1
......
......@@ -406,6 +406,10 @@ class Exim4ConfigTest < Test::Unit::TestCase
do_acl_script('exim4_acl_tests/antivirus_accept')
FileUtils.touch(File.join(config_dir, "antivirus"))
# Fairly hacky way to ensure clamAV is up and running - stopping it, starting it and waiting for it to say it's loaded sigs
# deals with an annoying race condition under load testing, where it hasnt finished reading sigs before the tests get to it
system('sync ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl reload clamav-daemon.service ; timeout 120 journalctl -u clamav-daemon.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "[0-9]* signatures" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1')
# OK the file is there now, so reject (as per default)
do_acl_script('exim4_acl_tests/antivirus_reject')
......
......@@ -2,7 +2,7 @@
##
#
# This file is automatically generated from the template located at
# /etc/sympl/apache.d/webalizer.conf.erb
# /etc/sympl/apache.d/awffull.conf.erb
#
# Feel free to make changes to this file. If changes are made, then this file
# will not be updated automatically when the template changes.
......@@ -17,7 +17,7 @@
#
# LogFile
# LogType defines the log type being processed. Normally, the Webalizer
# LogType defines the log type being processed. Normally, the awffull
# expects a CLF or Combined web server log as input. Using this option,
# you can process ftp logs as well (xferlog as produced by wu-ftp and
# others), or Squid native logs. Values can be 'clf', 'ftp' or 'squid',
......@@ -33,9 +33,9 @@ OutputDir <%= domain.stats_dir %>
# HistoryName allows you to specify the name of the history file produced
# by the Webalizer. The history file keeps the data for up to 12 months
# by the awffull. The history file keeps the data for up to 12 months
# worth of logs, used for generating the main HTML page (index.html).
# The default is a file named "webalizer.hist", stored in the specified
# The default is a file named "awffull.hist", stored in the specified
# output directory. If you specify just the filename (without a path),
# it will be kept in the specified output directory. Otherwise, the path
# is relative to the output directory, unless absolute (leading /).
......@@ -45,13 +45,13 @@ HistoryName <%= history_name %>
# Incremental processing allows multiple partial log files to be used
# instead of one huge one. Useful for large sites that have to rotate
# their log files more than once a month. The Webalizer will save its
# their log files more than once a month. AWFFull will save its
# internal state before exiting, and restore it the next time run, in
# order to continue processing where it left off. This mode also causes
# The Webalizer to scan for and ignore duplicate records (records already
# AWFFull to scan for and ignore duplicate records (records already
# processed by a previous run). See the README file for additional
# information. The value may be 'yes' or 'no', with a default of 'no'.
# The file 'webalizer.current' is used to store the current state data,
# The file 'awffull.current' is used to store the current state data,
# and is located in the output directory of the program (unless changed
# with the IncrementalName option below). Please read at least the section
# on Incremental processing in the README file before you enable this option.
......@@ -61,7 +61,7 @@ Incremental yes
# IncrementalName allows you to specify the filename for saving the
# incremental data in. It is similar to the HistoryName option where the
# name is relative to the specified output directory, unless an absolute
# filename is specified. The default is a file named "webalizer.current"
# filename is specified. The default is a file named "awffull.current"
# kept in the normal output directory. If you don't specify "Incremental"
# as 'yes' then this option has no meaning.
......@@ -80,7 +80,7 @@ ReportTitle Usage Statistics for
# clicking on URL's in the report to go to the proper location in
# the event you are running the report on a 'virtual' web server,
# or for a server different than the one the report resides on.
# If not specified here, or on the command line, webalizer will
# If not specified here, or on the command line, awffull will
# try to get the hostname via a uname system call. If that fails,
# it will default to "localhost".
......@@ -93,7 +93,7 @@ HostName <%= domain %>
#HTMLExtension html
# PageType lets you tell the Webalizer what types of URL's you
# PageType lets you tell AWFFull what types of URL's you
# consider a 'page'. Most people consider html and cgi documents
# as pages, while not images and audio files. If no types are
# specified, defaults will be used ('htm*', 'cgi' and HTMLExtension
......@@ -132,7 +132,7 @@ PageType php
# DNSChildren allows you to specify how many "children" processes are
# run to perform DNS lookups to create or update the DNS cache file.
# If a number is specified, the DNS cache file will be created/updated
# each time the Webalizer is run, immediately prior to normal processing,
# each time AWFFull is run, immediately prior to normal processing,
# by running the specified number of "children" processes to perform
# DNS lookups. If used, the DNS cache filename MUST be specified as
# well. The default value is zero (0), which disables DNS cache file
......@@ -153,7 +153,7 @@ PageType php
# block, immediately after the <TITLE> line. Maximum line length
# is 80 characters, so use multiple lines if needed.
#HTMLHead <META NAME="author" CONTENT="The Webalizer">
#HTMLHead <META NAME="author" CONTENT="AWFFull">
# HTMLBody defined the HTML code to be inserted, starting with the
# <BODY> tag. If not specified, the default is shown below. If
......@@ -220,7 +220,7 @@ PageType php
#GMTTime no
# Debug prints additional information for error messages. This
# will cause webalizer to dump bad records/fields instead of just
# will cause awffull to dump bad records/fields instead of just
# telling you it found a bad one. As usual, the value can be
# either "yes" or "no". The default is "no". It shouldn't be
# needed unless you start getting a lot of Warning or Error
......@@ -229,7 +229,7 @@ PageType php
#Debug no
# FoldSeqErr forces the Webalizer to ignore sequence errors.
# FoldSeqErr forces AWFFull to ignore sequence errors.
# This is useful for Netscape and other web servers that cache
# the writing of log records and do not guarentee that they
# will be in chronological order. The use of the FoldSeqErr
......@@ -336,12 +336,12 @@ PageType php
#AllSearchStr no
#AllUsers no
# The Webalizer normally strips the string 'index.' off the end of
# AWFFull normally strips the string 'index.' off the end of
# URL's in order to consolidate URL totals. For example, the URL
# /somedir/index.html is turned into /somedir/ which is really the
# same URL. This option allows you to specify additional strings
# to treat in the same way. You don't need to specify 'index.' as
# it is always scanned for by The Webalizer, this option is just to
# it is always scanned for by AWFFull, this option is just to
# specify _additional_ strings if needed. If you don't need any,
# don't specify any as each string will be scanned for in EVERY
# log record... A bunch of them will degrade performance. Also,
......@@ -354,7 +354,7 @@ PageType php
# The Hide*, Group* and Ignore* and Include* keywords allow you to
# change the way Sites, URL's, Referrers, User Agents and Usernames
# are manipulated. The Ignore* keywords will cause The Webalizer to
# are manipulated. The Ignore* keywords will cause AWFFull to
# completely ignore records as if they didn't exist (and thus not
# counted in the main site totals). The Hide* keywords will prevent
# things from being displayed in the 'Top' tables, but will still be
......@@ -471,7 +471,7 @@ GroupURL /cgi-bin/* CGI Scripts
# The Ignore* keywords allow you to completely ignore log records based
# on hostname, URL, user agent, referrer or username. I hessitated in
# adding these, since the Webalizer was designed to generate _accurate_
# adding these, since AWFFull was designed to generate _accurate_
# statistics about a web servers performance. By choosing to ignore
# records, the accuracy of reports become skewed, negating why I wrote
# this program in the first place. However, due to popular demand, here
......@@ -501,7 +501,7 @@ IgnoreReferrer localhost
#IgnoreUser *
#IncludeUser someuser
# The MangleAgents allows you to specify how much, if any, The Webalizer
# The MangleAgents allows you to specify how much, if any, AWFFull
# should mangle user agent names. This allows several levels of detail
# to be produced when reporting user agent statistics. There are six
# levels that can be specified, which define different levels of detail
......@@ -582,7 +582,7 @@ SearchEngine google.be q=
#DumpUsers no
#DumpSearchStr no
# If you compiled Webalizer with GeoIP library, it becomes enabled
# If you compiled Awffull with GeoIP library, it becomes enabled
# by default. But if you wish to disable it, just set GeoIP to 'no'.
# You may also want to specify database file path manually, if you
# don't have one installed on system (in case of static build).
......@@ -592,7 +592,7 @@ SearchEngine google.be q=
# The custom bar graph Colors are defined here. Declare them
# in the standard hexadecimal way (as HTML, without the '#')
# If none are given, you will get the standard webalizer colors.
# If none are given, you will get the standard awffull colors.
#ColorHit 00805c
#ColorFile 0000ff
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment