Significantly Improved CI

.gitlab-ci.yml

 stages:
-  - package
-  - publish
-  - install
+  - Package
+  - Publish
+  - Install
+  - Upgrade
 
 # This will run through the packages one at a time, building them into artifacts
 # This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with
@@ -11,13 +12,13 @@ stages:
   tags:
     - docker
   image: kelduum/sympl-build-buster:v1
-  stage: package
+  stage: Package
   script:
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - apt-get update && apt-get install -q -yy gitlab-runner rake devscripts rdoc graphviz ruby-dev gem2deb txt2man dh-systemd apache2-dev libcap-dev php7.3-common dh-golang golang-go
-    - export PACKAGE=${CI_BUILD_NAME##package:}
+    - export PACKAGE=${CI_BUILD_NAME#sympl-}
     - git describe --all
-    - echo "$CI_BUILD_NAME" -  "$PACKAGE" - "${CI_BUILD_NAME##package:}"
+    - echo "Building '$CI_BUILD_NAME' in directory '$PACKAGE'"
     - cd $PACKAGE
     - dpkg-buildpackage --no-sign --build=full
     - cd ..
@@ -26,46 +27,46 @@ stages:
   artifacts:
     paths:
       - deb/
-    expire_in: 4 days
+    expire_in: 1 day
 
 # List of packages to build
 
-package:core:
+sympl-core:
   <<: *package
-package:backup:
+sympl-backup:
   <<: *package
-package:cron:
+sympl-cron:
   <<: *package
-package:dns:
+sympl-dns:
   <<: *package
-package:mail:
+sympl-mail:
   <<: *package
-package:firewall:
+sympl-firewall:
   <<: *package
-package:ftp:
+sympl-ftp:
   <<: *package
-package:web:
+sympl-web:
   <<: *package
-package:monit:
+sympl-monit:
   <<: *package
-package:mysql:
+sympl-mysql:
   <<: *package
-package:phpmyadmin:
+#sympl-phpmyadmin:
+#  <<: *package
+sympl-updater:
   <<: *package
-package:updater:
+sympl-webmail:
   <<: *package
-package:webmail:
-  <<: *package
-  
+
 # This builds the debian packages and drops them in a very simple local repo
 #   so they can be installed during the next (install) step.
 # Again, it uses the kelduum/sympl-build docker image with the relevant tools pre-installed
 
-publish:
+Build Repo:
   tags:
     - docker
   image: kelduum/sympl-build-buster:v1
-  stage: publish
+  stage: Publish
   script:
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - apt-get update && apt-get install -q -yy dpkg-dev git
@@ -73,116 +74,85 @@ publish:
     - dpkg-scanpackages deb > deb/Packages
     - gzip -k deb/Packages
     - echo '## Building repo...'
-    - mkdir -p repo/pool/main/s repo/pool/main/l repo/dists/buster/main/binary-amd64
+    - mkdir -p repo/pool/main/s repo/pool/main/l repo/dists/local/main/binary-amd64
     - cp deb/s*.deb repo/pool/main/s
     - cp deb/l*.deb repo/pool/main/l
     - cd repo
-    - dpkg-scanpackages -m pool | gzip > dists/buster/main/binary-amd64/Packages.gz
-    - dpkg-scanpackages -m pool > dists/buster/main/binary-amd64/Packages
+    - dpkg-scanpackages -m pool | gzip > dists/local/main/binary-amd64/Packages.gz
+    - dpkg-scanpackages -m pool > dists/local/main/binary-amd64/Packages
   artifacts:
     paths:
       - deb/*
       - repo/*
-    expire_in: 1 month
+    expire_in: 1 day
 
 
-# This stage runs on a VM running VirtualBox 5, which the gitlab-runner controls.
-# The vbox:buster image is a pre-installed minimal copy of Debian Stretch.
-#
-# Clean is a minimal install, where quick has all of the pre-reqs installed,
-# other than unattended-upgrades as it tries to run at reboot.
+# The next two stages runs on a VM running VirtualBox 5, which the gitlab-runner controls.
+# The image is a pre-installed minimal copy of Debian, ideally with the dependencies pre
+# downloaded.
 
-install:quick:
+Clean Install:
   tags:
-    - vbox:buster-quick
-  stage: install
-  allow_failure: false
+    - vbox:buster
+  stage: Install
   script:
+    - chmod -x $( which gitlab-runner )
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - git describe --all
-    - chmod -x $( which gitlab-runner )
-    - hostname -b quick.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
+    - apt-get -qq update; apt-get -qq -y upgrade
     - cp -r repo/ /
     - chmod -R 664 /repo ; chmod -R +X /repo
-    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
+    - echo "deb [trusted=yes] file:/repo local main" > /etc/apt/sources.list.d/local.list
     - apt-get -qq update
     - apt-get -y --allow-unauthenticated install --install-recommends sympl-core
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
-#  allow_failure: true
-#  when: manual
   variables:
     CI_DEBUG_TRACE: "false"
+  allow_failure: false
+  when: always
   artifacts:
     paths:
       - repo/*
-    expire_in: 4 days
+    expire_in: 1 hour
 
-
-install:clean:
+Upgrade Testing:
   tags:
     - vbox:buster
-  stage: install
-  allow_failure: false
+  stage: Upgrade
   script:
-    - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-    - git describe --all
     - chmod -x $( which gitlab-runner )
-    - hostname -b clean.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
-    - apt-get -qq update; apt-get -qq -y upgrade
-    - cp -r repo/ /
-    - chmod -R 664 /repo ; chmod -R +X /repo
-    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
-    - apt-get -qq update || true
-    - apt-get -y --allow-unauthenticated install --install-recommends sympl-core
+    - git describe --all
+    - autotest/install_then_upgrade buster-testing
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
+  allow_failure: false
   variables:
     CI_DEBUG_TRACE: "false"
-#  allow_failure: true
-#  when: manual
   artifacts:
     paths:
       - repo/*
-    expire_in: 4 days
-
+    expire_in: 1 hour
 
-install:upgrade:
+Upgrade Stable:
   tags:
     - vbox:buster
-  stage: install
-  allow_failure: false
+  stage: Upgrade
   script:
-    - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-    - git describe --all
     - chmod -x $( which gitlab-runner )
-    - hostname -b upgrade.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
-    - apt-get -qq update; apt-get -qq -y upgrade
-    - wget -qO- http://mirror.mythic-beasts.com/mythic/support@mythic-beasts.com.gpg.key | apt-key add -
-    - echo deb http://packages.mythic-beasts.com/mythic/ buster-testing main > /etc/apt/sources.list.d/sympl_mythic-beasts.list
-    - apt-get -qq update
-    - apt-get -qq install $( autotest/get_previous_versions buster-testing )
-    - dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }' | sort > pre-upgrade
-    - cp -r repo/ /
-    - chmod -R 664 /repo ; chmod -R +X /repo
-    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
-    - apt-get -qq update || true
-    - apt-get -y --allow-unauthenticated --with-new-pkgs upgrade
-    - apt-get -y autoremove
-    - dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }' | sort > post-upgrade
-    - autotest/checkupgrade
+    - git describe --all
+    - autotest/install_then_upgrade buster
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
-  allow_failure: true
- # when: manual
+  allow_failure: false
+  when: delayed
+  start_in: 5 minutes
   variables:
     CI_DEBUG_TRACE: "false"
   artifacts:
     paths:
       - repo/*
-    expire_in: 4 days
-
-
+    expire_in: 1 hour

autotest/change-hostname

+#!/bin/bash
+
+hostname -b $1
+hostname > /etc/hostname
+export HOSTNAME=$(hostname)
+sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts
+echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
+echo "Hostname set to $(hostname --fqdn)"
+

autotest/get_previous_versions

-#!/bin/bash
-
-# If we're not on master, then install the already public versions
-if [[ ! $( git describe --all ) == *master ]] ; then
-
-  echo "--install-recommends sympl-core"
-
-else
-
-  if [ "$1" == "" ]; then
-    echo "Error: need a distro name, such as 'stretch-testing'" > /dev/stderr
-    exit 1
-  fi
-
-  # Get the current versions
-  head -n 1 ./*/debian/changelog | grep '^sympl' | sed -e 's|(||' -e 's|).*||' > .built
-  touch .install-ver
-
-  # While we don't have a list
-  while [ "$( cat .install-ver | wc -l )" != "$( cat .built | wc -l )" ]; do 
-
-    # Sleep for a bit if it's not the first time
-    if [ -s .install-ver ]; then
-      sleep 10
-    fi
-
-    # Pull the list of packages
-    # Using £ is untidy, but means you don't have to use GNU sed
-    wget -q -O - "http://packages.mythic-beasts.com/mythic/dists/$1/main/source/Sources.gz" \
-    | gunzip \
-    | grep -A 2 '^Package: sympl' \
-    | grep -v ^Binary \
-    | sed -e 's|^Package: ||' -e 's|^Version: ||' \
-    | tr '\n' ' ' \
-    | sed -e 's| -- |£|g' \
-    | tr '£' '\n' \
-    > .packages
-
-    # Get the newest version of each package, other than duplicates
-    cat .packages .built \
-    | sort -r \
-    | uniq -u \
-    | awk '!seen[$1] { print $1 " " $2 } {++seen[$1]}' \
-    > .install
-
-    # Filter for only packages in the repo
-    rm .install-ver
-    for package in $( find . -type f -name 'changelog' | sed -e 's|^\./||' -e 's|/.*||' | tr '\n' ' ' ); do
-      grep "^sympl-${package} " .install >> .install-ver
-    done
-
-    # Check for any versions to be installed which match the ones we built, force a retry if so
-    retry=false
-    cat .built | while read line; do
-      if [ $( grep -c "$line" .install-ver ) != 0 ]; then retry=true; fi
-    done
-    if [ $retry == true ]; then echo invalid > .install-ver ; fi
-
-  done
-
-cat .install-ver | tr ' ' '=' | tr '\n' ' '
-
-rm .install-ver .install .packages .built
-
-fi

autotest/install_then_upgrade

+#!/bin/bash
+
+set -e
+
+if [ "x$1" == "x" ]; then
+  echo "Error: I need a repo name, such as 'stretch-testing'" > /dev/stderr
+  exit 1
+fi
+
+REPO=$1
+
+head -n 1 ./*/debian/changelog | grep '^sympl' | sed -e 's|(||' -e 's|).*||' | sort > /tmp/local_versions
+
+wget -q -O - "http://packages.mythic-beasts.com/mythic/dists/$REPO/main/source/Sources.gz" \
+  | gunzip \
+  | grep -A 2 '^Package: sympl' \
+  | grep -v ^Binary \
+  | sed -e 's|^Package: ||' -e 's|^Version: ||' \
+  | tr '\n' ' ' \
+  | sed -e 's| -- |\n|g' \
+  | sort -r \
+  > /tmp/packages
+
+#echo local_versons
+#cat /tmp/local_versions
+#echo ---
+#echo packages
+#cat /tmp/packages
+
+cp /tmp/local_versions /tmp/from_repo
+
+cat /tmp/local_versions | while read PACKAGE LOCAL_VER ; do
+#  echo Checking for previous version of $PACKAGE $LOCAL_VER
+  if grep "^$PACKAGE " /tmp/packages > /tmp/check-package; then
+    LOCAL_VER_NUM="$( echo $LOCAL_VER | tr -d '.' )"
+    cat /tmp/check-package | while read skip REPO_VER ; do
+      REPO_VER_NUM="$( echo $REPO_VER | tr -d '.' )"
+      if [ $REPO_VER_NUM -lt $LOCAL_VER_NUM ]; then
+        sed -i "s|^$PACKAGE .*|$PACKAGE $REPO_VER|" /tmp/from_repo
+        break
+    #  else
+    #    echo skipping $REPO_VER
+      fi
+    done
+  else
+    sed -i "s|^$PACKAGE .*||" /tmp/from_repo
+ #   echo new package: $PACKAGE
+  fi
+done
+
+rm /tmp/check-package /tmp/packages
+
+#diff /tmp/from_repo /tmp/local_versions
+echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+apt-get -qq update
+apt-get -qq -y upgrade
+
+wget -qO- http://mirror.mythic-beasts.com/mythic/support@mythic-beasts.com.gpg.key | apt-key add -
+
+echo deb http://packages.mythic-beasts.com/mythic/ $REPO main > /etc/apt/sources.list.d/sympl_mythic-beasts.list
+
+cp -r repo/ /
+chmod -R 664 /repo ; chmod -R +X /repo
+echo "deb [trusted=yes] file:/repo local main" > /etc/apt/sources.list.d/local.list
+
+apt-get -qq update
+
+echo ---------- Installing -----------
+cat /tmp/from_repo | tr ' ' '\t'
+echo ---------------------------------
+
+apt-get -q -y install --allow-unauthenticated --install-recommends $( cat /tmp/from_repo | tr ' ' '=' | tr '\n' ' ' )
+
+rm /etc/apt/sources.list.d/sympl_mythic-beasts.list
+apt-get -qq update
+
+echo ----------- Upgrading -----------
+diff /tmp/from_repo /tmp/local_versions | grep '^> ' | sed -e 's|> ||' | tr ' ' '\t'
+echo ---------------------------------
+
+apt-get -q -y install --allow-unauthenticated --install-recommends $( diff /tmp/from_repo /tmp/local_versions | grep '^>' | sed -e 's|> ||' | tr ' ' '=' | tr '\n' ' ' )
+
+apt-get -y autoremove
+
+dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }'
+
+exit 0
+

autotest/test.d/01-setup

@@ -84,3 +84,4 @@ while [ `ps -ef | grep 'spamd child' | grep -v -c grep` -lt 2 ]; do
 done
 echo ' done.'
 
+

autotest/test.d/91-manual

@@ -4,6 +4,6 @@
 
 errors=0
 
-for command in $(find /bin /sbin /usr/bin /usr/sbin -name 'symbiosis-*' | grep -v 'symbiosis-crontab$\|symbiosis-dns-generate$\|symbiosis-generate-dhparams$\|symbiosis-httpd-logger$'); do echo -ne "\nTesting '$command --manual'" ; $command --manual > /tmp/autotest_out 2>&1 ; if [ $? -ne 0 ]; then errors=$((errors+1)); echo ' FAIL'; cat /tmp/autotest_out; else echo ' OK'; fi ; done
+for command in $(find /bin /sbin /usr/bin /usr/sbin -name 'symbiosis-*' | grep -v 'symbiosis-crontab$\|symbiosis-dns-generate$\|symbiosis-generate-dhparams$\|symbiosis-httpd-logger$'); do echo -ne "Testing '$command --manual'" ; $command --manual > /tmp/autotest_out 2>&1 ; if [ $? -ne 0 ]; then errors=$((errors+1)); echo ' FAIL'; cat /tmp/autotest_out; else echo ' OK'; fi ; done
 
 exit $errors