Commit 6d5db9fb authored by Paul Cammish's avatar Paul Cammish
Browse files

Improved backup scripts, added sympl-sqldump

parent 9152b44d
CHANGELOG
---------
* 2019-06-13 - Improved SQL backup script
- New script with configurbility.
- Run sympl-sqldump --help for info.
* 2019-06-13 - Improved hostname & webmail installs
- Hostname misconfiguration will be repaired automatically
- If a FQDN isn't set, one will be created automatically.
......
#!/bin/bash
if [ $( diff pre-upgrade post-upgrade | wc -l ) -eq 0 ]; then
echo 'E: Versions not changed'
exit 1
if [ "x$CI_COMMIT_REF_PROTECTED" == "x" ]; then
echo 'E: Versions not changed'
exit 1
else
echo 'W: Versions not changed, but on we are on a protected branch so they may have been built already.'
fi
fi
diff pre-upgrade post-upgrade
......
#!/usr/bin/ruby
#
# Overview
# --------
#
# This script is designed to determine the name of any remote
# backup space associated with this machine and upload the local
# archive, produced by backup2l, to this location.
#
#
# Override
# --------
#
# If there is a file "/etc/symbiosis/dns.d/backup.name" it
# is assumed to contain the name and path of a remote location to
# use for the rsync upload.
#
#
# Disabling
# ---------
#
# To disable uploads completely create the empty file named
# "/etc/symbiosis/dns.d/backup.name".
#
# Steve
# --
#
require 'symbiosis/host'
#
# Local backup directory.
#
src = "/var/backups/"
#
# Automatically determine the primary backup space
#
name = Symbiosis::Host.primary_backup_space.to_s
#
# If we didn't get a name then exit.
#
if name.empty?
exit 0
end
#
# If we got a name of the form "foo.backup.bytemark.co.uk" truncate at the
# first part. Otherwise use the name untouched.
#
if name =~ /\.backup\.bytemark\.co\.uk$/
# If the name of the machine is example.vm.bytemark.co.uk we'll expect
# to upload to:
#
# example.backup.bytemark.co.uk::example/example.vm.bytemark.co.uk/
#
fqdn = `hostname --fqdn`.chomp
dest = name+"::"+name.split(".").first+"/"+fqdn
else
dest = name
end
#
# Now rsync.
#
# If the name of the machine is example.vm.bytemark.co.uk we'll expect
# to upload to:
#
# example.backup.bytemark.co.uk::example/example.vm.bytemark.co.uk/
#
puts "Sending backups to #{dest}...\n"
puts `rsync --bwlimit=768 --delete-before --delete-excluded --exclude 'localhost/TMP.*' --exclude 'localhost/*.lock' --quiet --archive --recursive --perms --no-owner --no-group --human-readable #{src} #{dest}`
#
# Exit with the exit status of the rsync command
#
exit $?.exitstatus
#!/usr/bin/ruby
#
# Overview
# --------
#
# This script is designed to determine the name of any remote
# backup space associated with this machine and download any files
# stored within that backup space, previously produced by backup2l,
# to the local machine.
#
# This is designed to ensure that no local backups are lost if the
# machine is re-imaged, or local backups are removed due to user-error
#
#
# Override
# --------
#
# If there is a file "/etc/symbiosis/dns.d/backup.name" it
# is assumed to contain the name and path of a remote location to
# use for the rsync upload.
#
#
# Disabling
# ---------
#
# To disable uploads/downloads completely create the empty file named
# "/etc/symbiosis/dns.d/backup.name".
#
# Steve
# --
#
require 'symbiosis/host'
require 'fileutils'
#
# Local backup directory.
#
src = "/var/backups"
#
# Automatically determine the primary backup space
#
name = Symbiosis::Host.primary_backup_space.to_s
#
# If we didn't get a name then exit.
#
if name.empty?
exit 0
end
#
# If we got a name of the form "foo.backup.bytemark.co.uk" truncate at the
# first part. Otherwise use the name untouched.
#
if name =~ /\.backup\.bytemark\.co\.uk$/
# Grab the FQDN
fqdn = `hostname --fqdn`.chomp
dest = name+"::"+name.split(".").first+"/"+fqdn
else
dest = name
end
FileUtils.mkdir_p("#{src}/localhost") unless File.directory?("#{src}/localhost")
#
# Now rsync.
#
# If the name of the machine is example.vm.bytemark.co.uk we'll expect
# to upload to:
#
# example.backup.bytemark.co.uk::example/example.vm.bytemark.co.uk/
#
puts "Ensuring directory structure is present..."
puts `rsync --quiet --perms --human-readable --dirs #{src}/localhost #{dest}/`
puts "\nSynchronising backups from #{dest}/localhost..."
puts `rsync --quiet --archive --recursive --no-perms --no-owner --no-group --human-readable #{dest}/localhost/ #{src}/localhost/`
#
# Exit with the exit status of the rsync command
#
exit $?.exitstatus
#!/usr/bin/ruby
#
# This script is designed to dump all the MySQL databases upon
# the local system.
#
require 'symbiosis/utils'
require 'uri'
backup_dir = "/var/backups/mysql"
encoding = "utf8"
database_cmd = "/usr/bin/mysql"
database_dump_cmd = "/usr/bin/mysqldump"
#
# If we don't have a backup directory then create it. Backup2l will complain
# if this isn't present.
#
Symbiosis::Utils.mkdir_p backup_dir unless File.exist?(backup_dir)
#
# If we don't have mysqld installed exit.
#
File.executable?("/usr/sbin/mysqld") or exit 0
File.executable?("/usr/bin/mysql") or exit 0
File.executable?("/usr/bin/mysqldump") or exit 0
File.readable?("/etc/mysql/debian.cnf") or exit 0
#
# Should we dump the events table? This is only possible with mysqldump >=
# 5.1.8. Default to "no".
#
dump_events_table = false
#
# Fetch mysqldump version
#
mysqldump_version_string = IO.popen("/usr/bin/mysqldump --version"){|io| io.readlines}.first
if mysqldump_version_string =~ /Distrib (\d+\.\d+\.\d+)/
mysqldump_version = $1.split(".").collect{|x| x.to_i }
min_mysqldump_version = [5,1,8]
dump_events_table = (mysqldump_version[0] > 5 or
(mysqldump_version[0] == 5 and mysqldump_version[1] > 1) or
(mysqldump_version[0] == 5 and mysqldump_version[1] == 1 and mysqldump_version[2] >= 8))
end
#
# Default to utf8.
#
cmd = %w(
/usr/bin/mysql
--defaults-file=/etc/mysql/debian.cnf
--skip-column-names
--batch
--default-character-set=UTF8
)
databases = IO.popen(cmd.join(" ")+" --execute 'SHOW DATABASES'"){|io| io.readlines}.collect{|l| l.chomp}
unless 0 == $?
puts "Failed to ascertain list of databases." if $VERBOSELOCAL
exit 1
end
#
# This allows us to specify a database on the command line (for testing).
#
databases = (databases & ARGV) unless ARGV.empty?
if databases.empty?
puts "No Mysql databases found" if $VERBOSELOCAL
exit 0
end
databases.each do |database|
dump = File.join(backup_dir,URI.escape(database,/[^a-zA-Z0-9._-]/)) + ".sql.gz"
cmd = %w(
/usr/bin/mysqldump
--defaults-file=/etc/mysql/debian.cnf
--opt
)
cmd << "--events" if "mysql" == database and dump_events_table
# debian-sys-maint doesn't have permission to lock these tables.
cmd << "--skip-lock-tables" if %w(performance_schema information_schema).include?(database)
cmd << "'#{database}' | gzip -9c"
Symbiosis::Utils.safe_open(dump, "a+") do |fh|
fh.truncate(0)
IO.popen(cmd.join(" ")) do |io|
fh.write(io.read(4096)) until io.eof?
end
unless 0 == $?
puts "mysqldump of #{database} failed." if $VERBOSELOCAL
end
end
warn "Dump of '#{database}' in #{dump} is zero in size." unless File.stat(dump).size?
end
#
# Exit sanely.
#
exit 0
sympl-backup (9.0.190613.0) stable; urgency=medium
* Removed deprecated backup scripts
-- Paul Cammish <sympl@kelduum.net> Thu, 13 Jun 2019 21:38:00 +0100
sympl-backup (9.0.190611.0) stable; urgency=medium
* Merged sympl-common into sympl-core
......
#!/bin/bash
#
# sympl-sqldump - a fairly simple utility to dump databases sanely
#
# Copyright 2019 Paul Cammish <paul.cammish@bytemark.co.uk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# Set basic environment
set -o pipefail
#set -e
if [ "$( echo "$@" | grep -c ' --debug ' )" != "0" ]; then DEBUG="TRUE"; fi
#############################################################################
# Base Debug/Error Functions #
#############################################################################
function _debug {
if [ $DEBUG ]; then
echo -e "\e[2mDEBUG: $@\e[0m"
fi
}
function _warn {
echo -e "\e[1m\e[33m WARN: $@\e[0m"
echo "WARNING: $@" >> "$DUMP_DIR/$LOGNAME"
echo "WARNING: $@" >> "$_TEMP-$LOGNAME"
}
function _error {
echo -e "\e[1m\e[31mERROR: $@\e[0m"
echo "ERROR: $@" >> "$DUMP_DIR/$LOGNAME"
echo "ERROR: $@" >> "$_TEMP-$LOGNAME"
_exit 1
}
function _verbose {
if [ $DEBUG ]; then
echo " INFO: $@"
elif [ $VERBOSE ]; then
echo "$@"
fi
if [ ! -d "$DUMP_DIR" ]; then mkdir -p $DUMP_DIR; fi
echo "$@" >> "$DUMP_DIR/$LOGNAME"
echo "$@" >> "$_TEMP-$LOGNAME"
}
function _check_stderr {
if [ $( cat "$STDERR" | wc -l ) -gt 0 ] || [ "$1" -ne "0" ]; then
_error "== errorcode $1 =="
cat "$STDERR" | while read line; do
_error "$line"
done
_exit 1
fi
rm "$STDERR"
}
function _exit {
_debug Exiting with errorcode: $1
if [ $DEBUG ]; then ls -la $DUMP_DIR; fi
rm $_TEMP-* 2> /dev/null
if [ -f "$DUMP_DIR/$LOGNAME" ]; then
tail -n 10000 $DUMP_DIR/$LOGNAME > $DUMP_DIR/$LOGNAME.new
mv $DUMP_DIR/$LOGNAME.new $DUMP_DIR/$LOGNAME
fi
exit $1
}
function _dump_failed {
_warn "Dump of database '$DATABASE' failed."
rm "$@"
}
function _bytesToHuman() {
# Taken from https://unix.stackexchange.com/a/259254
b=${1:-0}; d=''; s=0; S=(B {K,M,G,T,P,E,Z,Y});
while ((b > 1024)); do
d="$(printf ".%02d" $((b % 1024 * 100 / 1024)))"
b=$((b / 1024))
let s++
done
echo "$b$d${S[$s]}"
}
function _validate_dump {
TEST_FILENAME="$@"
_debug "Checking $TEST_FILENAME for valid dump"
if [ $( cat "$TEST_FILENAME" | gunzip | tail -n 3 | grep -c '^-- PostgreSQL database dump complete\|^-- Dump completed on\|^-- PostgreSQL database cluster dump complete' ) -eq 1 ]; then
_verbose " Completed $(_bytesToHuman $(gzip -l $TEST_FILENAME | tail -n 1 | awk '{ print $2 }' )) dump of '$DATABASE' at $( date ) ($(_bytesToHuman $(gzip -l $TEST_FILENAME | tail -n 1 | awk '{ print $1 }' )) gzip, $(gzip -l $TEST_FILENAME | tail -n 1 | awk '{ print $3 }'))"
else
_warn "Dump of $DATABASE appears to be truncated"
_dump_failed $TEST_FILENAME
fi
}
#############################################################################
# Set Defaults #
#############################################################################
# Note: To override thse defaults, copy and paste the modified entries to '/etc/sympl-sqldump.config',
# or if using another path, use the --config command line parameter
# Basic settings
DUMP_DIR=/var/backups/mysql
KEEP_MAX_COPIES="2"
# Complex settings - be careful!
LOGNAME=sympl-sqldump.log
_TEMP=/tmp/__sympl-sqldump
STDERR="$_TEMP-stderr"
SEPERATOR='--'
CONFIG_FILE=/etc/sympl-sqldump.config
# MySQL defaults
MYSQL=$( which mysql 2> /dev/null )
MYSQLDUMP=$( which mysqldump 2> /dev/null )
MYSQL_DEFAULTS="$HOME/.my.cnf"
MYSQL_SKIP_DB='information_schema performance_schema'
MYSQLDUMP_OPTIONS='--create-options --no-create-db --events --triggers --routines --dump-date --tz-utc'
# Output settings
#DEBUG=TRUE
#VERBOSE=TRUE
# Determine possible defaults
# Enable MySQL dumps if we can see both the client and the dump util in the path
if [ -f "$MYSQL" ] && [ -f "$MYSQLDUMP" ]; then
BACKUP_MYSQL=TRUE
else
unset BACKUP_MYSQL
fi
#############################################################################
# Import Default Overrides #
#############################################################################
function import_default_overrides {
if [ -f "$CONFIG_FILE" ]; then
if [ $( find "$CONFIG_FILE" -user root -perm 600 | wc -l ) == 1 ]; then
_debug Executing config file $CONFIG_FILE
. $CONFIG_FILE
else
_warn Config file found but not secured. Ignoring.
fi
fi
}
if [ "$( echo "$@" | grep -c ' --config ' )" == "0" ]; then
import_default_overrides
else
_debug "Config override detected in command line, will run once parsed"
fi
#############################################################################
# Read Command Line Parameters #
#############################################################################
_debug "Command line parameters: $@"
PARAMETERS=()
while [ $# -gt 0 ]; do
key="$1"
case $key in
--keep|-k)
KEEP_MAX_COPIES="$2"
shift; shift
;;
--dir|-d)
DUMP_DIR="$2"
shift; shift
;;
--mysql)
MYSQL="$2"
shift; shift
;;
--mysqldump)
MYSQLDUMP="$2"
shift; shift
;;
--mysql-defaults)
MYSQL_DEFAULTS="$2"
shift
;;
--debug)
DEBUG="TRUE"
shift
;;
--verbose|-v)
VERBOSE="TRUE"
shift
;;
--force)
FORCE="TRUE"
shift
;;
--config)
CONFIG_FILE="$2"
import_default_overrides
shift; shift
;;
# --restore) # Currently incomplete. Will be split to 'sympl-sqlrestore'
# RESTORE="$2"
# VERBOSE="TRUE"
# shift; shift
# ;;
*) # unhandled parameter
PARAMETERS+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
#set -- "${PARAMETERS[@]}"
# restore positional parameters
_debug "Runtime variables:
------------------------------------------
UNKNOWN |$PARAMETERS
CONFIG_FILE |$CONFIG_FILE
MYSQLDUMP_OPTIONS |$MYSQLDUMP_OPTIONS
KEEP_MAX_COPIES |$KEEP_MAX_COPIES
DUMP_DIR |$DUMP_DIR
MYSQL_SKIP_DB |$MYSQL_SKIP_DB
MYSQL |$MYSQL
MYSQLDUMP |$MYSQLDUMP
BACKUP_MYSQL |$BACKUP_MYSQL
------------------------------------------"
#############################################################################
# Default to help text if unhanded input #
#############################################################################
if [ "${#PARAMETERS}" -gt "0" ]; then
echo -e "Usage: \033[1msympl-sqldump <options>\033[0m
Common
--dir, -d <path> Override path to dump directory.
Will be created/chown'd/chmod'd to root user.
Defaults to $DUMP_DIR
--keep, -k <number> Number of dumps to keep before removing old copies.
Defaults to $KEEP_MAX_COPIES
--verbose, -v Output progress to both stdout and log file.
Executable paths
--mysql <file> Override path to 'mysql' executable.
--mysqldump <file> Override path to 'mysqldump' executable.
Authentication
--mysql-defaults Optional 'defaults-file' to use in debian.cnf
format. Defaults to $MYSQL_DEFAULTS,
debian.cnf and bytemark.cnf in that order.
Must be secured to root user only.
Other
--force Force a run even if currently locked.
--config <file> Specify a config file to override defaults.
Defaults to /etc/sympl-sqldump.conf if exists
and must be secured to root.
--help This text.
"
_exit 0
fi
#############################################################################
# Check for root priviledges #
#############################################################################
if [ $( id -u ) -ne 0 ]; then
_error This must be run as root.
_exit 256
fi
_verbose "Starting sympl-sqldump at $(date)"
#############################################################################
# Check for and Enable Lock #