Commit 3c42a803 authored by Paul Cammish's avatar Paul Cammish
Browse files

Improve CI performance, Fixes #323

parent bad21db0
stages:
- Package
- Publish
- Install
- Upgrade
- Test
# This will run through the packages one at a time, building them into artifacts
# This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with
......@@ -97,8 +96,9 @@ Build Repo:
Clean Install:
tags:
- vbox:stretch
stage: Install
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
- git describe --all
......@@ -120,18 +120,21 @@ Clean Install:
- repo/*
expire_in: 1 hour
Upgrade Testing:
Upgrade Stable:
tags:
- vbox:stretch
stage: Upgrade
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- git describe --all
- autotest/install_then_upgrade stretch-testing
- autotest/install_then_upgrade stretch
- run-parts --verbose --exit-on-error autotest/test.d
- hostname
- dpkg -l 'sympl*'
allow_failure: false
when: delayed
start_in: 30 seconds
variables:
CI_DEBUG_TRACE: "false"
artifacts:
......@@ -139,20 +142,21 @@ Upgrade Testing:
- repo/*
expire_in: 1 hour
Upgrade Stable:
Upgrade Testing:
tags:
- vbox:stretch
stage: Upgrade
stage: Test
script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner )
- git describe --all
- autotest/install_then_upgrade stretch
- autotest/install_then_upgrade stretch-testing
- run-parts --verbose --exit-on-error autotest/test.d
- hostname
- dpkg -l 'sympl*'
allow_failure: false
when: delayed
start_in: 5 minutes
start_in: 1 minute
variables:
CI_DEBUG_TRACE: "false"
artifacts:
......
CHANGELOG
---------
2022-04-26
sympl-core
* Fix control logic in sympl-filesystem-security
2021-12-13
sympl-core
* Updated workaround for Let's Encrypt cross-signed intermediate
......
......@@ -21,52 +21,53 @@ touch /srv/$(hostname -f)/config/antivirus
systemctl unmask clamav-freshclam
service clamav-freshclam stop
rm -rf /run/clamav
service clamav-freshclam start
sleep 3
echo -n "I: Waiting for clamav to download/update databases."
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo -n ' 1/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
echo -n ' 2/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
echo " 3/3 done."
invoke-rc.d clamav-daemon restart
break
fi
echo -n "."
sleep 1
done
#service clamav-freshclam start
#sleep 3
#echo -n "I: Waiting for clamav to download/update databases."
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
# echo -n ' 1/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
# echo -n ' 2/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
# echo " 3/3 done."
# invoke-rc.d clamav-daemon restart
# break
# fi
# echo -n "."
# sleep 1
#done
#
# horrible hack
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo "We have what look to be valid definitions..."
else
echo "Failed to download clamAV definitions, going to fallback."
cd /var/lib/clamav/
wget -qO clamav.tar.gz http://sympl.host/clamav.tar.gz
tar -xvf clamav.tar.gz
fi
#if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
# echo "We have what look to be valid definitions..."
#else
#echo "Failed to download clamAV definitions, going to fallback."
cd /var/lib/clamav/
wget -qO clamav.tar.gz http://sympl.host/clamav.tar.gz
tar -xvf clamav.tar.gz
#fi
# Start clamav, whatever state it's currently in
systemctl unmask clamav-daemon
service clamav-daemon stop
service clamav-daemon start
service clamav-freshclam start
# Enable antispam and antivirus as default, otherwise monit would stop them.
echo "I: Enabling Antivirus and Antispam config"
......
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
echo 'Running sympl-test...'
/usr/bin/sympl-test > /dev/null 2>&1
# ...so, we'll ignore that and run it again.
/usr/bin/sympl-test
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
#echo 'Running sympl-test...'
#/usr/bin/sympl-test > /dev/null 2>&1
# fairly sure this was a race conditon where apache wasn't reloading quick enough
# so run it once, and retry if that fails,
# as theres still some uncommon race conditions
if ! /usr/bin/sympl-test ; then echo Trying again ; /usr/bin/sympl-test ; fi
sympl-core (9.20220426.0) stable; urgency=medium
* Fix control logic in sympl-filesystem-security
-- Paul Cammish <sympl@kelduum.net> Tue, 26 Apr 2022 10:00:00 +0100
sympl-core (9.20211213.0) stable; urgency=medium
* Updated workaround for Let's Encrypt cross-signed intermediate
......
......@@ -167,7 +167,7 @@ if [ -d /etc/sympl ]; then
fi
for domain in $( find /srv -maxdepth 1 -mindepth 1 ! -type l -type d -print | grep -v '^/srv/\.' | grep '\.' ); do
if [ ! -f ${domain}/config/do-not-secure ] || [ -f /etc/sympl/disable-filesystem-security ] ; then
if [ ! -f ${domain}/config/disable-filesystem-security ] && [ ! -f ${domain}/config/do-not-secure ] ; then
secure_domain_dir ${domain}
fi
done
......
......@@ -19,14 +19,14 @@ class TestFTP < Test::Unit::TestCase
@domain.create()
# Hack to disable TLS enforcement, as the ruby lib doesnt support it
system 'echo 1 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3'
system 'echo 1 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
end
def teardown
# Re-enable TLS enforcement
system 'echo 2 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3'
system 'echo 2 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
#
# Delete the temporary domain
......
......@@ -406,6 +406,10 @@ class Exim4ConfigTest < Test::Unit::TestCase
do_acl_script('exim4_acl_tests/antivirus_accept')
FileUtils.touch(File.join(config_dir, "antivirus"))
# Fairly hacky way to ensure clamAV is up and running - stopping it, starting it and waiting for it to say it's loaded sigs
# deals with an annoying race condition under load testing, where it hasnt finished reading sigs before the tests get to it
system('sync ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl reload clamav-daemon.service ; timeout 120 journalctl -u clamav-daemon.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "[0-9]* signatures" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1')
# OK the file is there now, so reject (as per default)
do_acl_script('exim4_acl_tests/antivirus_reject')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment