Revert "Merge branch 'buster-testing' of gitlab.mythic-beasts.com:sympl/sympl into buster-testing"

This reverts commit 563366d9

.gitlab-ci.yml

 stages:
-  - Package
-  - Publish
-  - Install
-  - Upgrade
+  - package
+  - publish
+  - install
 
 # This will run through the packages one at a time, building them into artifacts
 # This uses the kelduum/sympl-build:v1 docker image, which is Debian Stretch with
@@ -12,13 +11,13 @@ stages:
   tags:
     - docker
   image: kelduum/sympl-build-buster:v1
-  stage: Package
+  stage: package
   script:
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - apt-get update && apt-get install -q -yy gitlab-runner rake devscripts rdoc graphviz ruby-dev gem2deb txt2man dh-systemd apache2-dev libcap-dev php7.3-common dh-golang golang-go
-    - export PACKAGE=${CI_BUILD_NAME#sympl-}
+    - export PACKAGE=${CI_BUILD_NAME##package:}
     - git describe --all
-    - echo "Building '$CI_BUILD_NAME' in directory '$PACKAGE'"
+    - echo "$CI_BUILD_NAME" -  "$PACKAGE" - "${CI_BUILD_NAME##package:}"
     - cd $PACKAGE
     - dpkg-buildpackage --no-sign --build=full
     - cd ..
@@ -27,46 +26,46 @@ stages:
   artifacts:
     paths:
       - deb/
-    expire_in: 1 day
+    expire_in: 4 days
 
 # List of packages to build
 
-sympl-core:
+package:core:
   <<: *package
-sympl-backup:
+package:backup:
   <<: *package
-sympl-cron:
+package:cron:
   <<: *package
-sympl-dns:
+package:dns:
   <<: *package
-sympl-mail:
+package:mail:
   <<: *package
-sympl-firewall:
+package:firewall:
   <<: *package
-sympl-ftp:
+package:ftp:
   <<: *package
-sympl-web:
+package:web:
   <<: *package
-sympl-monit:
+package:monit:
   <<: *package
-sympl-mysql:
+package:mysql:
   <<: *package
-#sympl-phpmyadmin:
-#  <<: *package
-sympl-updater:
+package:phpmyadmin:
   <<: *package
-sympl-webmail:
+package:updater:
   <<: *package
-
+package:webmail:
+  <<: *package
+  
 # This builds the debian packages and drops them in a very simple local repo
 #   so they can be installed during the next (install) step.
 # Again, it uses the kelduum/sympl-build docker image with the relevant tools pre-installed
 
-Build Repo:
+publish:
   tags:
     - docker
   image: kelduum/sympl-build-buster:v1
-  stage: Publish
+  stage: publish
   script:
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - apt-get update && apt-get install -q -yy dpkg-dev git
@@ -74,85 +73,116 @@ Build Repo:
     - dpkg-scanpackages deb > deb/Packages
     - gzip -k deb/Packages
     - echo '## Building repo...'
-    - mkdir -p repo/pool/main/s repo/pool/main/l repo/dists/local/main/binary-amd64
+    - mkdir -p repo/pool/main/s repo/pool/main/l repo/dists/buster/main/binary-amd64
     - cp deb/s*.deb repo/pool/main/s
     - cp deb/l*.deb repo/pool/main/l
     - cd repo
-    - dpkg-scanpackages -m pool | gzip > dists/local/main/binary-amd64/Packages.gz
-    - dpkg-scanpackages -m pool > dists/local/main/binary-amd64/Packages
+    - dpkg-scanpackages -m pool | gzip > dists/buster/main/binary-amd64/Packages.gz
+    - dpkg-scanpackages -m pool > dists/buster/main/binary-amd64/Packages
   artifacts:
     paths:
       - deb/*
       - repo/*
-    expire_in: 1 day
+    expire_in: 1 month
 
 
-# The next two stages runs on a VM running VirtualBox 5, which the gitlab-runner controls.
-# The image is a pre-installed minimal copy of Debian, ideally with the dependencies pre
-# downloaded.
+# This stage runs on a VM running VirtualBox 5, which the gitlab-runner controls.
+# The vbox:buster image is a pre-installed minimal copy of Debian Stretch.
+#
+# Clean is a minimal install, where quick has all of the pre-reqs installed,
+# other than unattended-upgrades as it tries to run at reboot.
 
-Clean Install:
+install:quick:
   tags:
-    - vbox:buster
-  stage: Install
+    - vbox:buster-quick
+  stage: install
+  allow_failure: false
   script:
-    - chmod -x $( which gitlab-runner )
     - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - git describe --all
-    - apt-get -qq update; apt-get -qq -y upgrade
+    - chmod -x $( which gitlab-runner )
+    - hostname -b quick.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
     - cp -r repo/ /
     - chmod -R 664 /repo ; chmod -R +X /repo
-    - echo "deb [trusted=yes] file:/repo local main" > /etc/apt/sources.list.d/local.list
+    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
     - apt-get -qq update
     - apt-get -y --allow-unauthenticated install --install-recommends sympl-core
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
+#  allow_failure: true
+#  when: manual
   variables:
     CI_DEBUG_TRACE: "false"
-  allow_failure: false
-  when: always
   artifacts:
     paths:
       - repo/*
-    expire_in: 1 hour
+    expire_in: 4 days
 
-Upgrade Testing:
+
+install:clean:
   tags:
     - vbox:buster
-  stage: Upgrade
+  stage: install
+  allow_failure: false
   script:
-    - chmod -x $( which gitlab-runner )
+    - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - git describe --all
-    - autotest/install_then_upgrade buster-testing
+    - chmod -x $( which gitlab-runner )
+    - hostname -b clean.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
+    - apt-get -qq update; apt-get -qq -y upgrade
+    - cp -r repo/ /
+    - chmod -R 664 /repo ; chmod -R +X /repo
+    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
+    - apt-get -qq update || true
+    - apt-get -y --allow-unauthenticated install --install-recommends sympl-core
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
-  allow_failure: false
   variables:
     CI_DEBUG_TRACE: "false"
+#  allow_failure: true
+#  when: manual
   artifacts:
     paths:
       - repo/*
-    expire_in: 1 hour
+    expire_in: 4 days
+
 
-Upgrade Stable:
+install:upgrade:
   tags:
     - vbox:buster
-  stage: Upgrade
+  stage: install
+  allow_failure: false
   script:
-    - chmod -x $( which gitlab-runner )
+    - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
     - git describe --all
-    - autotest/install_then_upgrade buster
+    - chmod -x $( which gitlab-runner )
+    - hostname -b upgrade.sympl.test ; hostname > /etc/hostname ; HOSTNAME=$(hostname) ; sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts ; echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
+    - apt-get -qq update; apt-get -qq -y upgrade
+    - wget -qO- http://mirror.mythic-beasts.com/mythic/support@mythic-beasts.com.gpg.key | apt-key add -
+    - echo deb http://packages.mythic-beasts.com/mythic/ buster-testing main > /etc/apt/sources.list.d/sympl_mythic-beasts.list
+    - apt-get -qq update
+    - apt-get -qq install $( autotest/get_previous_versions buster-testing )
+    - dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }' | sort > pre-upgrade
+    - cp -r repo/ /
+    - chmod -R 664 /repo ; chmod -R +X /repo
+    - echo "deb [trusted=yes] file:/repo buster main" > /etc/apt/sources.list.d/local.list
+    - apt-get -qq update || true
+    - apt-get -y --allow-unauthenticated --with-new-pkgs upgrade
+    - apt-get -y autoremove
+    - dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }' | sort > post-upgrade
+    - autotest/checkupgrade
     - run-parts --verbose --exit-on-error autotest/test.d
     - hostname
     - dpkg -l 'sympl*'
-  allow_failure: false
-  when: delayed
-  start_in: 5 minutes
+  allow_failure: true
+ # when: manual
   variables:
     CI_DEBUG_TRACE: "false"
   artifacts:
     paths:
       - repo/*
-    expire_in: 1 hour
+    expire_in: 4 days
+
+

autotest/change-hostname

-#!/bin/bash
-
-hostname -b $1
-hostname > /etc/hostname
-export HOSTNAME=$(hostname)
-sed -i -n '/^127\.0\.0\.1/!p' /etc/hosts
-echo -e "127.0.0.1\t$(hostname) $(hostname -s) localhost" >> /etc/hosts
-echo "Hostname set to $(hostname --fqdn)"
-

autotest/get_previous_versions

+#!/bin/bash
+
+# If we're not on master, then install the already public versions
+if [[ ! $( git describe --all ) == *master ]] ; then
+
+  echo "--install-recommends sympl-core"
+
+else
+
+  if [ "$1" == "" ]; then
+    echo "Error: need a distro name, such as 'stretch-testing'" > /dev/stderr
+    exit 1
+  fi
+
+  # Get the current versions
+  head -n 1 ./*/debian/changelog | grep '^sympl' | sed -e 's|(||' -e 's|).*||' > .built
+  touch .install-ver
+
+  # While we don't have a list
+  while [ "$( cat .install-ver | wc -l )" != "$( cat .built | wc -l )" ]; do 
+
+    # Sleep for a bit if it's not the first time
+    if [ -s .install-ver ]; then
+      sleep 10
+    fi
+
+    # Pull the list of packages
+    # Using £ is untidy, but means you don't have to use GNU sed
+    wget -q -O - "http://packages.mythic-beasts.com/mythic/dists/$1/main/source/Sources.gz" \
+    | gunzip \
+    | grep -A 2 '^Package: sympl' \
+    | grep -v ^Binary \
+    | sed -e 's|^Package: ||' -e 's|^Version: ||' \
+    | tr '\n' ' ' \
+    | sed -e 's| -- |£|g' \
+    | tr '£' '\n' \
+    > .packages
+
+    # Get the newest version of each package, other than duplicates
+    cat .packages .built \
+    | sort -r \
+    | uniq -u \
+    | awk '!seen[$1] { print $1 " " $2 } {++seen[$1]}' \
+    > .install
+
+    # Filter for only packages in the repo
+    rm .install-ver
+    for package in $( find . -type f -name 'changelog' | sed -e 's|^\./||' -e 's|/.*||' | tr '\n' ' ' ); do
+      grep "^sympl-${package} " .install >> .install-ver
+    done
+
+    # Check for any versions to be installed which match the ones we built, force a retry if so
+    retry=false
+    cat .built | while read line; do
+      if [ $( grep -c "$line" .install-ver ) != 0 ]; then retry=true; fi
+    done
+    if [ $retry == true ]; then echo invalid > .install-ver ; fi
+
+  done
+
+cat .install-ver | tr ' ' '=' | tr '\n' ' '
+
+rm .install-ver .install .packages .built
+
+fi

autotest/install_then_upgrade

-#!/bin/bash
-
-set -e
-
-if [ "x$1" == "x" ]; then
-  echo "Error: I need a repo name, such as 'stretch-testing'" > /dev/stderr
-  exit 1
-fi
-
-REPO=$1
-
-head -n 1 ./*/debian/changelog | grep '^sympl' | sed -e 's|(||' -e 's|).*||' | sort > /tmp/local_versions
-
-wget -q -O - "http://packages.mythic-beasts.com/mythic/dists/$REPO/main/source/Sources.gz" \
-  | gunzip \
-  | grep -A 2 '^Package: sympl' \
-  | grep -v ^Binary \
-  | sed -e 's|^Package: ||' -e 's|^Version: ||' \
-  | tr '\n' ' ' \
-  | sed -e 's| -- |\n|g' \
-  | sort -r \
-  > /tmp/packages
-
-#echo local_versons
-#cat /tmp/local_versions
-#echo ---
-#echo packages
-#cat /tmp/packages
-
-cp /tmp/local_versions /tmp/from_repo
-
-cat /tmp/local_versions | while read PACKAGE LOCAL_VER ; do
-#  echo Checking for previous version of $PACKAGE $LOCAL_VER
-  if grep "^$PACKAGE " /tmp/packages > /tmp/check-package; then
-    LOCAL_VER_NUM="$( echo $LOCAL_VER | tr -d '.' )"
-    cat /tmp/check-package | while read skip REPO_VER ; do
-      REPO_VER_NUM="$( echo $REPO_VER | tr -d '.' )"
-      if [ $REPO_VER_NUM -lt $LOCAL_VER_NUM ]; then
-        sed -i "s|^$PACKAGE .*|$PACKAGE $REPO_VER|" /tmp/from_repo
-        break
-    #  else
-    #    echo skipping $REPO_VER
-      fi
-    done
-  else
-    sed -i "s|^$PACKAGE .*||" /tmp/from_repo
- #   echo new package: $PACKAGE
-  fi
-done
-
-rm /tmp/check-package /tmp/packages
-
-#diff /tmp/from_repo /tmp/local_versions
-echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-apt-get -qq update
-apt-get -qq -y upgrade
-
-wget -qO- http://mirror.mythic-beasts.com/mythic/support@mythic-beasts.com.gpg.key | apt-key add -
-
-echo deb http://packages.mythic-beasts.com/mythic/ $REPO main > /etc/apt/sources.list.d/sympl_mythic-beasts.list
-
-cp -r repo/ /
-chmod -R 664 /repo ; chmod -R +X /repo
-echo "deb [trusted=yes] file:/repo local main" > /etc/apt/sources.list.d/local.list
-
-apt-get -qq update
-
-echo ---------- Installing -----------
-cat /tmp/from_repo | tr ' ' '\t'
-echo ---------------------------------
-
-apt-get -q -y install --allow-unauthenticated --install-recommends $( cat /tmp/from_repo | tr ' ' '=' | tr '\n' ' ' )
-
-rm /etc/apt/sources.list.d/sympl_mythic-beasts.list
-apt-get -qq update
-
-echo ----------- Upgrading -----------
-diff /tmp/from_repo /tmp/local_versions | grep '^> ' | sed -e 's|> ||' | tr ' ' '\t'
-echo ---------------------------------
-
-apt-get -q -y install --allow-unauthenticated --install-recommends $( diff /tmp/from_repo /tmp/local_versions | grep '^>' | sed -e 's|> ||' | tr ' ' '=' | tr '\n' ' ' )
-
-apt-get -y autoremove
-
-dpkg -l 'sympl-*' | grep '^ii' | awk '{ print $2 " " $3 }'
-
-exit 0
-

autotest/test.d/01-setup

@@ -84,4 +84,3 @@ while [ `ps -ef | grep 'spamd child' | grep -v -c grep` -lt 2 ]; do
 done
 echo ' done.'
 
-

autotest/test.d/91-manual

@@ -4,6 +4,6 @@
 
 errors=0
 
-for command in $(find /bin /sbin /usr/bin /usr/sbin -name 'symbiosis-*' | grep -v 'symbiosis-crontab$\|symbiosis-dns-generate$\|symbiosis-generate-dhparams$\|symbiosis-httpd-logger$'); do echo -ne "Testing '$command --manual'" ; $command --manual > /tmp/autotest_out 2>&1 ; if [ $? -ne 0 ]; then errors=$((errors+1)); echo ' FAIL'; cat /tmp/autotest_out; else echo ' OK'; fi ; done
+for command in $(find /bin /sbin /usr/bin /usr/sbin -name 'symbiosis-*' | grep -v 'symbiosis-crontab$\|symbiosis-dns-generate$\|symbiosis-generate-dhparams$\|symbiosis-httpd-logger$'); do echo -ne "\nTesting '$command --manual'" ; $command --manual > /tmp/autotest_out 2>&1 ; if [ $? -ne 0 ]; then errors=$((errors+1)); echo ' FAIL'; cat /tmp/autotest_out; else echo ' OK'; fi ; done
 
 exit $errors