Commit 3c42a803 authored by Paul Cammish's avatar Paul Cammish
Browse files

Improve CI performance, Fixes #323

parent bad21db0
stages: stages:
- Package - Package
- Publish - Publish
- Install - Test
- Upgrade
# This will run through the packages one at a time, building them into artifacts # This will run through the packages one at a time, building them into artifacts
# This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with # This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with
...@@ -97,8 +96,9 @@ Build Repo: ...@@ -97,8 +96,9 @@ Build Repo:
Clean Install: Clean Install:
tags: tags:
- vbox:stretch - vbox:stretch
stage: Install stage: Test
script: script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner ) - chmod -x $( which gitlab-runner )
- echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
- git describe --all - git describe --all
...@@ -120,18 +120,21 @@ Clean Install: ...@@ -120,18 +120,21 @@ Clean Install:
- repo/* - repo/*
expire_in: 1 hour expire_in: 1 hour
Upgrade Testing: Upgrade Stable:
tags: tags:
- vbox:stretch - vbox:stretch
stage: Upgrade stage: Test
script: script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner ) - chmod -x $( which gitlab-runner )
- git describe --all - git describe --all
- autotest/install_then_upgrade stretch-testing - autotest/install_then_upgrade stretch
- run-parts --verbose --exit-on-error autotest/test.d - run-parts --verbose --exit-on-error autotest/test.d
- hostname - hostname
- dpkg -l 'sympl*' - dpkg -l 'sympl*'
allow_failure: false allow_failure: false
when: delayed
start_in: 30 seconds
variables: variables:
CI_DEBUG_TRACE: "false" CI_DEBUG_TRACE: "false"
artifacts: artifacts:
...@@ -139,20 +142,21 @@ Upgrade Testing: ...@@ -139,20 +142,21 @@ Upgrade Testing:
- repo/* - repo/*
expire_in: 1 hour expire_in: 1 hour
Upgrade Stable: Upgrade Testing:
tags: tags:
- vbox:stretch - vbox:stretch
stage: Upgrade stage: Test
script: script:
- ntpdate -s europe.pool.ntp.org
- chmod -x $( which gitlab-runner ) - chmod -x $( which gitlab-runner )
- git describe --all - git describe --all
- autotest/install_then_upgrade stretch - autotest/install_then_upgrade stretch-testing
- run-parts --verbose --exit-on-error autotest/test.d - run-parts --verbose --exit-on-error autotest/test.d
- hostname - hostname
- dpkg -l 'sympl*' - dpkg -l 'sympl*'
allow_failure: false allow_failure: false
when: delayed when: delayed
start_in: 5 minutes start_in: 1 minute
variables: variables:
CI_DEBUG_TRACE: "false" CI_DEBUG_TRACE: "false"
artifacts: artifacts:
......
CHANGELOG CHANGELOG
--------- ---------
2022-04-26
sympl-core
* Fix control logic in sympl-filesystem-security
2021-12-13 2021-12-13
sympl-core sympl-core
* Updated workaround for Let's Encrypt cross-signed intermediate * Updated workaround for Let's Encrypt cross-signed intermediate
......
...@@ -21,52 +21,53 @@ touch /srv/$(hostname -f)/config/antivirus ...@@ -21,52 +21,53 @@ touch /srv/$(hostname -f)/config/antivirus
systemctl unmask clamav-freshclam systemctl unmask clamav-freshclam
service clamav-freshclam stop service clamav-freshclam stop
rm -rf /run/clamav rm -rf /run/clamav
service clamav-freshclam start
sleep 3
echo -n "I: Waiting for clamav to download/update databases."
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo -n ' 1/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
echo -n ' 2/3 '
break
fi
echo -n "."
sleep 1
done
for i in $(seq 1 100) ; do
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
echo " 3/3 done."
invoke-rc.d clamav-daemon restart
break
fi
echo -n "."
sleep 1
done
#service clamav-freshclam start
#sleep 3
#echo -n "I: Waiting for clamav to download/update databases."
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
# echo -n ' 1/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] ; then
# echo -n ' 2/3 '
# break
# fi
# echo -n "."
# sleep 1
#done
#for i in $(seq 1 100) ; do
# if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] ; then
# echo " 3/3 done."
# invoke-rc.d clamav-daemon restart
# break
# fi
# echo -n "."
# sleep 1
#done
#
# horrible hack # horrible hack
if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then #if [ -f "/var/lib/clamav/bytecode.cvd" -o -f "/var/lib/clamav/bytecode.cld" ] && [ -f "/var/lib/clamav/daily.cvd" -o -f "/var/lib/clamav/daily.cld" ] && [ -f "/var/lib/clamav/main.cvd" -o -f "/var/lib/clamav/main.cld" ] ; then
echo "We have what look to be valid definitions..." # echo "We have what look to be valid definitions..."
else #else
echo "Failed to download clamAV definitions, going to fallback." #echo "Failed to download clamAV definitions, going to fallback."
cd /var/lib/clamav/ cd /var/lib/clamav/
wget -qO clamav.tar.gz http://sympl.host/clamav.tar.gz wget -qO clamav.tar.gz http://sympl.host/clamav.tar.gz
tar -xvf clamav.tar.gz tar -xvf clamav.tar.gz
fi #fi
# Start clamav, whatever state it's currently in # Start clamav, whatever state it's currently in
systemctl unmask clamav-daemon systemctl unmask clamav-daemon
service clamav-daemon stop service clamav-daemon stop
service clamav-daemon start service clamav-daemon start
service clamav-freshclam start
# Enable antispam and antivirus as default, otherwise monit would stop them. # Enable antispam and antivirus as default, otherwise monit would stop them.
echo "I: Enabling Antivirus and Antispam config" echo "I: Enabling Antivirus and Antispam config"
......
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
echo 'Running sympl-test...'
/usr/bin/sympl-test > /dev/null 2>&1
# ...so, we'll ignore that and run it again.
/usr/bin/sympl-test
#!/bin/bash
# It's unclear why, but the first time this runs, one of the AV tests fails.
#echo 'Running sympl-test...'
#/usr/bin/sympl-test > /dev/null 2>&1
# fairly sure this was a race conditon where apache wasn't reloading quick enough
# so run it once, and retry if that fails,
# as theres still some uncommon race conditions
if ! /usr/bin/sympl-test ; then echo Trying again ; /usr/bin/sympl-test ; fi
sympl-core (9.20220426.0) stable; urgency=medium
* Fix control logic in sympl-filesystem-security
-- Paul Cammish <sympl@kelduum.net> Tue, 26 Apr 2022 10:00:00 +0100
sympl-core (9.20211213.0) stable; urgency=medium sympl-core (9.20211213.0) stable; urgency=medium
* Updated workaround for Let's Encrypt cross-signed intermediate * Updated workaround for Let's Encrypt cross-signed intermediate
......
...@@ -167,7 +167,7 @@ if [ -d /etc/sympl ]; then ...@@ -167,7 +167,7 @@ if [ -d /etc/sympl ]; then
fi fi
for domain in $( find /srv -maxdepth 1 -mindepth 1 ! -type l -type d -print | grep -v '^/srv/\.' | grep '\.' ); do for domain in $( find /srv -maxdepth 1 -mindepth 1 ! -type l -type d -print | grep -v '^/srv/\.' | grep '\.' ); do
if [ ! -f ${domain}/config/do-not-secure ] || [ -f /etc/sympl/disable-filesystem-security ] ; then if [ ! -f ${domain}/config/disable-filesystem-security ] && [ ! -f ${domain}/config/do-not-secure ] ; then
secure_domain_dir ${domain} secure_domain_dir ${domain}
fi fi
done done
......
...@@ -19,14 +19,14 @@ class TestFTP < Test::Unit::TestCase ...@@ -19,14 +19,14 @@ class TestFTP < Test::Unit::TestCase
@domain.create() @domain.create()
# Hack to disable TLS enforcement, as the ruby lib doesnt support it # Hack to disable TLS enforcement, as the ruby lib doesnt support it
system 'echo 1 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3' system 'echo 1 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
end end
def teardown def teardown
# Re-enable TLS enforcement # Re-enable TLS enforcement
system 'echo 2 > /etc/pure-ftpd/conf/TLS ; service pure-ftpd restart ; sleep 3' system 'echo 2 > /etc/pure-ftpd/conf/TLS ; sync ; systemctl stop pure-ftpd.service ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl start pure-ftpd.service ; timeout 120 journalctl -u pure-ftpd.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "Started pure-ftpd.service" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1'
# #
# Delete the temporary domain # Delete the temporary domain
......
...@@ -406,6 +406,10 @@ class Exim4ConfigTest < Test::Unit::TestCase ...@@ -406,6 +406,10 @@ class Exim4ConfigTest < Test::Unit::TestCase
do_acl_script('exim4_acl_tests/antivirus_accept') do_acl_script('exim4_acl_tests/antivirus_accept')
FileUtils.touch(File.join(config_dir, "antivirus")) FileUtils.touch(File.join(config_dir, "antivirus"))
# Fairly hacky way to ensure clamAV is up and running - stopping it, starting it and waiting for it to say it's loaded sigs
# deals with an annoying race condition under load testing, where it hasnt finished reading sigs before the tests get to it
system('sync ; date=$(date "+%Y-%m-%d %H:%M:%S") ; systemctl reload clamav-daemon.service ; timeout 120 journalctl -u clamav-daemon.service --since="$date" --follow | while read line ; do if [ $( echo $line | grep -c "[0-9]* signatures" ) -eq 1 ]; then killall "journalctl" 2>&1 >/dev/null ; fi ; done ; sleep 1')
# OK the file is there now, so reject (as per default) # OK the file is there now, so reject (as per default)
do_acl_script('exim4_acl_tests/antivirus_reject') do_acl_script('exim4_acl_tests/antivirus_reject')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment