Commit 93365268 authored by Patrick J Cherry's avatar Patrick J Cherry
Browse files

Rewrote pre-backup DB dumps in ruby. Now uses URL encoding to encode DB

dump names, as well as querying the DBs directly for info on which DBs actually
exist.  Closes #3396.
parent 0d61d44d
#!/usr/bin/ruby -w
#
# This script is designed to dump all the MySQL databases upon
# the local system.
#
require 'symbiosis/utils'
require 'uri'
backup_dir = "/var/backups/mysql"
encoding = "utf8"
database_cmd = "/usr/bin/mysql"
database_dump_cmd = "/usr/bin/mysqldump"
#
# If we don't have a backup directory then create it. Backup2l will complain
# if this isn't present.
#
Symbiosis::Utils.mkdir_p backup_dir unless File.exists?(backup_dir)
#
# If we don't have mysqld installed exit.
#
File.executable?("/usr/sbin/mysqld") or exit 0
File.executable?("/usr/bin/mysql") or exit 0
File.executable?("/usr/bin/mysqldump") or exit 0
File.readable?("/etc/mysql/debian.cnf") or exit 0
#
# Default to utf8.
#
cmd = %w(
/usr/bin/mysql
--defaults-file=/etc/mysql/debian.cnf
--skip-column-names
--batch
--default-character-set=UTF8
)
databases = IO.popen(cmd.join(" ")+" --execute 'SHOW DATABASES'"){|io| io.readlines}.collect{|l| l.chomp}
unless 0 == $?
puts "Failed to ascertain list of databases." if $VERBOSE
exit 1
end
#
# This allows us to specify a database on the command line (for testing).
#
databases = ARGV unless ARGV.empty?
databases.each do |database|
dump = File.join(backup_dir,URI.escape(database,/[^a-zA-Z0-9._-]/)) + ".sql.gz"
cmd = %w(
/usr/bin/mysqldump
--defaults-file=/etc/mysql/debian.cnf
--databases
--opt
)
cmd << "--skip-lock-tables" if "information_schema" == database
cmd << "'#{database}' | gzip -9c"
Symbiosis::Utils.safe_open(dump, "a+") do |fh|
fh.truncate(0)
IO.popen(cmd.join(" ")) do |io|
fh.write(io.read(4096)) until io.eof?
end
unless 0 == $?
puts "mysqldump of #{database} failed." if $VERBOSE
end
end
warn "Dump of '#{database}' in #{dump} is zero in size." unless File.stat(dump).size?
end
#
# Exit sanely.
#
exit 0
#!/usr/bin/ruby -w
#
# This script is designed to dump all the postgresql databases upon
# the local system.
#
#
require 'symbiosis/utils'
require 'etc'
require 'uri'
require 'pp'
backup_dir = "/var/backups/postgresql"
#
# If we don't have a backup directory then create it. Backup2l will complain
# if this isn't present.
#
Symbiosis::Utils.mkdir_p backup_dir unless File.exists?("/var/backups/mysql")
begin
user = Etc.getpwnam("postgres")
group = Etc.getgrnam("postgres")
#
# Use lchown to make sure that any symlink is not followed.
#
File.lchown(user.uid, group.gid, backup_dir)
rescue ArgumentError => err
#
# We've not found the postgres user -- postgres is not installed.
#
puts "Postgres user not found" if $VERBOSE
exit 0
end
#
# If we don't have postgres installed exit.
#
unless File.executable?("/usr/bin/psql") and File.executable?("/usr/bin/pg_dump")
puts "Neither /usr/bin/psql nor /usr/bin/pg_dump are executable." if $VERBOSE
exit 0
end
#
# Change user id to postgres
#
unless 0 == Process.uid
puts "Unable to drop privileges if not running as root." if $VERBOSE
exit 0
end
#
# Try to drop privs.
#
begin
Process::Sys.setgid(group.gid)
Process::Sys.setuid(user.uid)
rescue Errno::EPERM => err
puts "Unable to drop privileges from #{Process.uid}:#{Process.gid} to #{user.uid}:#{group.gid}" if $VERBOSE
exit 0
end
#
# Default to utf8.
#
cmd = %w(
/usr/bin/psql
--no-align
--tuples-only
--command
)
databases = IO.popen(cmd.join(" ")+" 'select datname from pg_database;'"){|io| io.readlines}.collect{|l| l.chomp}
unless 0 == $?
puts "Failed to ascertain list of postgres databases." if $VERBOSE
exit 0
end
databases = ARGV unless ARGV.empty?
if databases.empty?
puts "No Postgres databases found" if $VERBOSE
exit 0
end
databases.each do |database|
#
# Skip template0 as the "template0 database is normally marked datallowconn =
# false to prevent modification of it". This also prevents backing it up.
#
# See http://www.postgresql.org/docs/8.4/static/manage-ag-templatedbs.html
#
next if "template0" == database
dump = File.join(backup_dir, URI.escape(database,/[^a-zA-Z0-9._-]/))+".custom"
cmd = %w(
/usr/bin/pg_dump
--format=c
)
cmd << "'#{database}'"
#
# This dumps each database into the "custom" format, suitable for straight
# import back into postgres using pg_restore.
#
Symbiosis::Utils.safe_open(dump, "a+") do |fh|
fh.truncate(0)
IO.popen(cmd.join(" ")) do |io|
fh.write(io.read(4096)) until io.eof?
end
unless 0 == $?
puts "Failed to dump #{database}." if $VERBOSE
end
end
unless File.stat(dump).size?
warn "Failed #{database} dump #{dump} is zero in size."
next
end
end
#
# Exit sanely.
#
exit 0
symbiosis-backup (2012:0525) stable; urgency=low
* Updated postgres and mysql scripts to ruby, using URL encoding for dump
files. Closes #3396.
* Now with testing goodness, although this requires
libmysql-ruby/libpgsql-ruby to work.
-- Patrick Cherry <patrick@bytemark.co.uk> Fri, 25 May 2012 16:17:15 +0100
symbiosis-backup (2012:0416) stable; urgency=low
* Use "--quiet" when invoking rsync, rather than --verbose.
......@@ -11,7 +20,7 @@ symbiosis-backup (2012:0414) stable; urgency=low
easy restoration of dumped databases using pg_restore.
* MySQL dumps now use the --opt flag explicitly.
-- Patrick Cherry <patch@bitch.dominoid.net> Sat, 14 Apr 2012 12:30:19 +0100
-- Patrick Cherry <patrick@bytemark.co.uk> Sat, 14 Apr 2012 12:30:19 +0100
symbiosis-backup (2012:0222) stable; urgency=low
......
......@@ -11,7 +11,8 @@ Replaces: bytemark-vhost-simple-backup
Conflicts: bytemark-vhost-simple-backup
Provides: bytemark-vhost-simple-backup
Architecture: all
Depends: backup2l, ruby1.8, symbiosis-common (>>2011:1216), ${misc:Depends}
Depends: backup2l, ruby, ruby1.8, symbiosis-common (>>2011:1216), ${misc:Depends}
Recommends: libmysql-ruby1.8, libpgsql-ruby1.8
Description: Automatically backup your files
This package configures backup2l to backup your data.
.
......
bug.report usr/share/bug/symbiosis-backup/
etc/
backup.d etc/symbiosis/
test.d/* etc/symbiosis/test.d/
/etc/backup.d/
This directory is the root of the backup script used by the Bytemark Symbiosis
system.
The following directories are present:
pre-backup.d/
-> Every script in this directory is executed prior to a backup run.
post-backup.d/
-> Every script in this directory is executed after a backup run has completed.
conf.d/
-> The contents of this directory are concatenated together to form a
backup2l.conf file which is written to /etc/backup.d/backup2l.conf
#!/bin/bash
#
# This script is designed to dump all the MySQL databases upon
# the local system.
#
#
# If we don't have mysqld installed exit.
#
test -x /usr/sbin/mysqld || exit 0
test -x /etc/init.d/mysql || exit 0
test -e /etc/mysql/debian.cnf || exit 0
#
# If we don't have a backup directory then create it.
#
if [ ! -d /var/backups/mysql ]; then
mkdir -p /var/backups/mysql
fi
#
# Get the username and password
#
user=$(grep user /etc/mysql/debian.cnf | awk '{print $3}' | head -n 1)
pass=$(grep pass /etc/mysql/debian.cnf | awk '{print $3}' | head -n 1)
#
# For each database we find, dump it
#
for i in /var/lib/mysql/*/; do
# the name of the database
name=`basename $i`
# do the dump
mysqldump --opt --user="$user" --pass="$pass" $name | gzip > /var/backups/mysql/$name.gz
done
#
# Exit sanely.
#
exit 0
#!/bin/bash
#
# This script is designed to dump all the postgresql databases upon
# the local system.
#
#
# If we don't have postgresql installed exit.
#
test -x /etc/init.d/postgresql || exit 0
test -x /usr/bin/psql || exit 0
test -x /usr/bin/pg_dump || exit 0
id postgres >/dev/null 2>/dev/null || exit 0
#
# If we don't have a backup directory then create it.
#
if [ ! -d /var/backups/postgresql ]; then
mkdir -p /var/backups/postgresql
fi
#
# Make sure the postgres user owns the backup directory.
#
chown postgres.postgres /var/backups/postgresql
#
# Work out which databases we need to back up.
#
dbs=$(su - postgres -c "psql --no-align --tuples-only --command 'select datname from pg_database;'")
for db in $dbs ; do
#
# Skip template0 as the "template0 database is normally marked datallowconn =
# false to prevent modification of it". This also prevents backing it up.
#
# See http://www.postgresql.org/docs/8.4/static/manage-ag-templatedbs.html
#
if [ "$db" = "template0" ] ; then
continue
fi
#
# This dumps each database into the "custom" format, suitable for straight
# import back into postgres using pg_restore.
#
su - postgres -c "/usr/bin/pg_dump --format=c --file=/var/backups/postgresql/$db.custom $db"
done
#
# Exit sanely.
#
exit 0
#!/usr/bin/ruby
require 'iconv'
require 'uri'
require 'symbiosis/utils'
require 'test/unit'
begin
require 'mysql'
rescue LoadError
# Do nothing.
end
class TestMysqlDumps < Test::Unit::TestCase
def setup
@charsets = %w(UTF8 LATIN1)
@default_charset = "UTF8"
@backup_dir = "/var/backups/mysql"
@backup_script = File.expand_path(File.dirname(__FILE__) + "/../backup.d/pre-backup.d/10-dump-mysql")
@database = "symbiosis test "+Symbiosis::Utils.random_string(10)+" \303\242"
@table = "t\303\241ble"
@column = "c\303\266lumn"
@value = "v\303\245lue"
@defaults_file = "/etc/mysql/debian.cnf"
@username = @password = nil
parse_defaults_file(@defaults_file)
end
def teardown
@charsets.each do |charset|
database = @database + " #{charset}"
drop_db(database) if has_mysql?
dump_name = calculate_dump_name(database)
File.unlink(dump_name) if File.exists?(dump_name)
end
end
def has_mysql?
defined? Mysql and
@username and @password
end
def parse_defaults_file(fn)
File.open(fn,"r") do |fh|
found_client = false
until fh.eof?
line = fh.gets
case line.chomp
when /^\s*\[client\]/
found_client = true
when /^\s*\[/
break if found_client
when /\s*user\s*=\s*(\S+)/
@username = $1 if found_client
when /\s*password\s*=\s*(\S+)/
@password = $1 if found_client
end
end
end
rescue StandardError
@username = @password = nil
end
def calculate_dump_name(database, charset=@default_charset)
File.join(@backup_dir, URI.escape(Iconv.conv(@default_charset,charset,database),/[^a-zA-Z0-9._-]/)) + ".sql.gz"
end
def drop_db(database, charset=@default_charset)
dbh = Mysql.new(nil, @username, @password)
dbh.query("SET NAMES #{charset}")
dbh.query("SET CHARSET #{charset}")
dbh.query("DROP DATABASE IF EXISTS `#{database}`")
rescue Mysql::Error => err
warn err.to_s
ensure
dbh.close if dbh
end
#
# This test does the following:
# * creates a DB
# * populates it
# * checks it was populated as expected
# * takes a backup (using the script)
# * drops the DB
# * restores the DB
# * checks it was re-populated as expected
#
# It checks it for databases with funny charsets used in the DB name
#
def test_mysql_dump
unless has_mysql?
puts "Not running MySQL backup tests, since not all the requirements are in place."
return
end
@charsets.each do |charset|
database = Iconv.conv(charset, @default_charset, @database) + " #{charset}"
table = Iconv.conv(charset, @default_charset, @table)
column = Iconv.conv(charset, @default_charset, @column)
value = Iconv.conv(charset, @default_charset, @value)
res = nil
assert_nothing_raised("Failure when creating MySQL DB to test backups.") {
dbh = Mysql.new(nil, @username, @password)
dbh.query("SET CHARSET #{charset}")
dbh.query("SET NAMES #{charset}")
dbh.query("CREATE DATABASE `#{database}` CHARACTER SET #{charset};")
dbh.query "USE `#{database}`;"
dbh.query "CREATE TABLE `#{table}` (`#{column}` CHAR(20) CHARACTER SET #{charset});"
dbh.query "INSERT INTO `#{table}` (`#{column}`) VALUES (\"#{value}\");"
res = dbh.query "SELECT * FROM `#{table}`;"
dbh.close
}
#
# Make sure we've inserted the things properly
#
assert_equal(1, res.num_fields, "Mysql returned the wrong number of fields")
assert_equal(1, res.num_rows, "Mysql returned the wrong number of rows")
assert_equal(column, res.fetch_fields.first.name, "Mysql returned the wrong field name")
assert_equal(value, res.fetch_row.first, "Mysql returned the wrong value")
system(@backup_script, Iconv.conv(@default_charset,charset,database))
assert_equal(0, $?, "#{@backup_script} returned non-zero.")
drop_db(database, charset)
dump_name = calculate_dump_name(database, charset)
assert(File.exists?(dump_name),"Mysql dump file '#{dump_name}' does not exist.")
system("zcat #{dump_name} | /usr/bin/mysql --defaults-extra-file=/etc/mysql/debian.cnf")
assert_equal(0, $?, "Failed to restore MySQL database from dump.")
res = nil
assert_nothing_raised("Failure when reconnecting to MySQL DB to test backups.") {
dbh = Mysql.new(nil, @username, @password)
dbh.query("SET CHARSET #{charset};")
dbh.query("SET NAMES #{charset};")
dbh.query "USE `#{database}`;"
res = dbh.query "SELECT * FROM `#{table}`;"
dbh.close
}
#
# Make sure we've inserted the things properly
#
assert_equal(1, res.num_fields, "Mysql returned the wrong number of fields")
assert_equal(1, res.num_rows, "Mysql returned the wrong number of rows")
assert_equal(column, res.fetch_fields.first.name, "Mysql returned the wrong field name")
assert_equal(value, res.fetch_row.first, "Mysql returned the wrong value")
end
end
end
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment