Rewrote pre-backup DB dumps in ruby. Now uses URL encoding to encode DB

dump names, as well as querying the DBs directly for info on which DBs actually
exist.  Closes #3396.

backup/backup.d/pre-backup.d/10-dump-mysql

+#!/usr/bin/ruby -w
+#
+#  This script is designed to dump all the MySQL databases upon
+# the local system.
+#
+
+require 'symbiosis/utils'
+require 'uri'
+
+backup_dir = "/var/backups/mysql"
+encoding   = "utf8"
+
+database_cmd      = "/usr/bin/mysql"
+database_dump_cmd = "/usr/bin/mysqldump"
+
+#
+# If we don't have a backup directory then create it.  Backup2l will complain
+# if this isn't present.
+#
+Symbiosis::Utils.mkdir_p backup_dir unless File.exists?(backup_dir) 
+
+#
+#  If we don't have mysqld installed exit.
+#
+File.executable?("/usr/sbin/mysqld")    or exit 0
+File.executable?("/usr/bin/mysql")      or exit 0
+File.executable?("/usr/bin/mysqldump")  or exit 0
+File.readable?("/etc/mysql/debian.cnf") or exit 0
+
+#
+# Default to utf8.
+#
+cmd = %w(
+  /usr/bin/mysql
+  --defaults-file=/etc/mysql/debian.cnf
+  --skip-column-names 
+  --batch 
+  --default-character-set=UTF8
+)
+
+databases = IO.popen(cmd.join(" ")+" --execute 'SHOW DATABASES'"){|io| io.readlines}.collect{|l| l.chomp}
+
+unless 0 == $?
+  puts "Failed to ascertain list of databases." if $VERBOSE
+  exit 1
+end
+
+#
+# This allows us to specify a database on the command line (for testing).
+#
+databases = ARGV unless ARGV.empty?
+
+databases.each do |database|
+  dump = File.join(backup_dir,URI.escape(database,/[^a-zA-Z0-9._-]/)) + ".sql.gz"
+
+  cmd = %w(
+    /usr/bin/mysqldump 
+    --defaults-file=/etc/mysql/debian.cnf 
+    --databases
+    --opt
+  )
+
+  cmd << "--skip-lock-tables" if "information_schema" == database
+  cmd << "'#{database}' | gzip -9c"
+
+  Symbiosis::Utils.safe_open(dump, "a+") do |fh|
+    fh.truncate(0)
+    IO.popen(cmd.join(" ")) do |io|
+      fh.write(io.read(4096)) until io.eof?
+    end
+
+    unless 0 == $?
+      puts "mysqldump of #{database} failed." if $VERBOSE 
+    end
+  end
+
+  warn "Dump of '#{database}' in #{dump} is zero in size." unless File.stat(dump).size?
+end
+
+#
+# Exit sanely.
+#
+exit 0

backup/backup.d/pre-backup.d/20-dump-postgres

+#!/usr/bin/ruby -w
+#
+# This script is designed to dump all the postgresql databases upon
+# the local system.
+#
+#
+require 'symbiosis/utils'
+require 'etc'
+require 'uri'
+require 'pp'
+
+backup_dir = "/var/backups/postgresql"
+
+#
+# If we don't have a backup directory then create it.  Backup2l will complain
+# if this isn't present.
+#
+Symbiosis::Utils.mkdir_p backup_dir unless File.exists?("/var/backups/mysql") 
+
+begin 
+  user = Etc.getpwnam("postgres")
+  group = Etc.getgrnam("postgres")
+
+  #
+  # Use lchown to make sure that any symlink is not followed.
+  #
+  File.lchown(user.uid, group.gid, backup_dir)
+rescue ArgumentError => err
+  #
+  # We've not found the postgres user -- postgres is not installed.
+  #
+  puts "Postgres user not found" if $VERBOSE 
+  exit 0
+end
+
+
+#
+# If we don't have postgres installed exit.
+#
+unless File.executable?("/usr/bin/psql") and File.executable?("/usr/bin/pg_dump")
+  puts "Neither /usr/bin/psql nor /usr/bin/pg_dump are executable." if $VERBOSE
+  exit 0
+end 
+
+#
+# Change user id to postgres
+#
+unless 0 == Process.uid
+  puts "Unable to drop privileges if not running as root." if $VERBOSE
+  exit 0
+end
+
+#
+# Try to drop privs.
+#
+begin
+  Process::Sys.setgid(group.gid) 
+  Process::Sys.setuid(user.uid)
+rescue Errno::EPERM => err
+  puts "Unable to drop privileges from #{Process.uid}:#{Process.gid} to #{user.uid}:#{group.gid}" if $VERBOSE
+  exit 0
+end
+
+
+#
+# Default to utf8.
+#
+cmd = %w(
+  /usr/bin/psql
+  --no-align
+  --tuples-only
+  --command
+)
+
+databases = IO.popen(cmd.join(" ")+" 'select datname from pg_database;'"){|io| io.readlines}.collect{|l| l.chomp}
+
+unless 0 == $?
+  puts "Failed to ascertain list of postgres databases." if $VERBOSE
+  exit 0 
+end
+
+databases = ARGV unless ARGV.empty?
+
+if databases.empty?
+  puts "No Postgres databases found" if $VERBOSE
+  exit 0
+end
+
+databases.each do |database|
+  #
+  # Skip template0 as the "template0 database is normally marked datallowconn =
+  # false to prevent modification of it".  This also prevents backing it up.
+  #
+  # See http://www.postgresql.org/docs/8.4/static/manage-ag-templatedbs.html
+  #
+  next if "template0" == database 
+
+  dump = File.join(backup_dir, URI.escape(database,/[^a-zA-Z0-9._-]/))+".custom"
+
+  cmd = %w(
+    /usr/bin/pg_dump 
+    --format=c
+  )
+  cmd << "'#{database}'"
+
+  #
+  # This dumps each database into the "custom" format, suitable for straight
+  # import back into postgres using pg_restore.
+  #
+  Symbiosis::Utils.safe_open(dump, "a+") do |fh|
+    fh.truncate(0)
+    IO.popen(cmd.join(" ")) do |io|
+      fh.write(io.read(4096)) until io.eof?
+    end
+
+    unless 0 == $?
+      puts "Failed to dump #{database}." if $VERBOSE
+    end
+  end
+
+  unless File.stat(dump).size?
+    warn "Failed #{database} dump #{dump} is zero in size."
+    next
+  end
+end
+
+#
+# Exit sanely.
+#
+exit 0

backup/debian/changelog

+symbiosis-backup (2012:0525) stable; urgency=low
+
+  * Updated postgres and mysql scripts to ruby, using URL encoding for dump
+    files.  Closes #3396.
+  * Now with testing goodness, although this requires
+    libmysql-ruby/libpgsql-ruby to work.
+
+ -- Patrick Cherry <patrick@bytemark.co.uk>  Fri, 25 May 2012 16:17:15 +0100
+
 symbiosis-backup (2012:0416) stable; urgency=low
 
   * Use "--quiet" when invoking rsync, rather than --verbose.
@@ -11,7 +20,7 @@ symbiosis-backup (2012:0414) stable; urgency=low
     easy restoration of dumped databases using pg_restore.
   * MySQL dumps now use the --opt flag explicitly. 
 
- -- Patrick Cherry <patch@bitch.dominoid.net>  Sat, 14 Apr 2012 12:30:19 +0100
+ -- Patrick Cherry <patrick@bytemark.co.uk>  Sat, 14 Apr 2012 12:30:19 +0100
 
 symbiosis-backup (2012:0222) stable; urgency=low
 

backup/debian/control

@@ -11,7 +11,8 @@ Replaces: bytemark-vhost-simple-backup
 Conflicts: bytemark-vhost-simple-backup
 Provides: bytemark-vhost-simple-backup
 Architecture: all
-Depends: backup2l, ruby1.8, symbiosis-common (>>2011:1216), ${misc:Depends}
+Depends: backup2l, ruby, ruby1.8, symbiosis-common (>>2011:1216), ${misc:Depends}
+Recommends: libmysql-ruby1.8, libpgsql-ruby1.8
 Description: Automatically backup your files
  This package configures backup2l to backup your data.
  .

backup/debian/install

 bug.report           usr/share/bug/symbiosis-backup/
-etc/
+backup.d             etc/symbiosis/
+test.d/*             etc/symbiosis/test.d/

backup/etc/symbiosis/backup.d/README

-
-/etc/backup.d/
-
-  This directory is the root of the backup script used by the Bytemark Symbiosis
- system.
-
-  The following directories are present:
-
-    pre-backup.d/
-        ->  Every script in this directory is executed prior to a backup run.
-
-    post-backup.d/
-        -> Every script in this directory is executed after a backup run has completed.
-
-    conf.d/
-        -> The contents of this directory are concatenated together to form a
-           backup2l.conf file which is written to /etc/backup.d/backup2l.conf
-
-

backup/etc/symbiosis/backup.d/pre-backup.d/10-dump-mysql

-#!/bin/bash
-#
-#  This script is designed to dump all the MySQL databases upon
-# the local system.
-#
-
-
-#
-#  If we don't have mysqld installed exit.
-#
-test -x /usr/sbin/mysqld      || exit 0
-test -x /etc/init.d/mysql     || exit 0
-test -e /etc/mysql/debian.cnf || exit 0
-
-
-#
-#  If we don't have a backup directory then create it.
-#
-if [ ! -d /var/backups/mysql ]; then
-    mkdir -p /var/backups/mysql
-fi
-
-
-#
-#  Get the username and password
-#
-user=$(grep user /etc/mysql/debian.cnf | awk '{print $3}' | head -n 1)
-pass=$(grep pass /etc/mysql/debian.cnf | awk '{print $3}' | head -n 1)
-
-
-#
-#  For each database we find, dump it
-#
-for i in /var/lib/mysql/*/; do
-
-    # the name of the database
-    name=`basename $i`
-
-    # do the dump
-    mysqldump --opt --user="$user" --pass="$pass" $name | gzip > /var/backups/mysql/$name.gz
-
-done
-
-#
-# Exit sanely.
-#
-exit 0

backup/etc/symbiosis/backup.d/pre-backup.d/20-dump-postgres

-#!/bin/bash
-#
-#  This script is designed to dump all the postgresql databases upon
-# the local system.
-#
-
-#
-#  If we don't have postgresql installed exit.
-#
-test -x /etc/init.d/postgresql      || exit 0
-test -x /usr/bin/psql               || exit 0
-test -x /usr/bin/pg_dump            || exit 0
-id postgres  >/dev/null 2>/dev/null || exit 0
-
-#
-#  If we don't have a backup directory then create it.
-#
-if [ ! -d /var/backups/postgresql ]; then
-    mkdir -p /var/backups/postgresql
-fi
-
-#
-# Make sure the postgres user owns the backup directory.
-#
-chown postgres.postgres /var/backups/postgresql
-
-#
-# Work out which databases we need to back up.
-#
-dbs=$(su - postgres -c "psql --no-align --tuples-only --command 'select datname from pg_database;'")
-
-for db in $dbs ; do
-  #
-  # Skip template0 as the "template0 database is normally marked datallowconn =
-  # false to prevent modification of it".  This also prevents backing it up.
-  #
-  # See http://www.postgresql.org/docs/8.4/static/manage-ag-templatedbs.html
-  #
-  if [ "$db" = "template0" ] ; then
-    continue
-  fi
-
-  #
-  # This dumps each database into the "custom" format, suitable for straight
-  # import back into postgres using pg_restore.
-  #
-  su - postgres -c "/usr/bin/pg_dump --format=c --file=/var/backups/postgresql/$db.custom $db"
-done
-
-#
-# Exit sanely.
-#
-exit 0

backup/test.d/tc_backups_mysql.rb

+#!/usr/bin/ruby
+
+require 'iconv'
+require 'uri'
+require 'symbiosis/utils'
+require 'test/unit'
+
+begin
+  require 'mysql'
+rescue LoadError
+  # Do nothing.
+end
+
+class TestMysqlDumps < Test::Unit::TestCase
+
+  def setup
+    @charsets = %w(UTF8 LATIN1)
+    @default_charset = "UTF8"
+    @backup_dir = "/var/backups/mysql"
+    @backup_script = File.expand_path(File.dirname(__FILE__) + "/../backup.d/pre-backup.d/10-dump-mysql")
+
+    @database = "symbiosis test "+Symbiosis::Utils.random_string(10)+" \303\242"
+    @table    = "t\303\241ble"
+    @column   = "c\303\266lumn"
+    @value    = "v\303\245lue"
+
+    @defaults_file = "/etc/mysql/debian.cnf"
+
+    @username = @password = nil
+
+    parse_defaults_file(@defaults_file)
+
+  end
+
+  def teardown
+    @charsets.each do |charset|
+      database = @database + " #{charset}"
+      drop_db(database) if has_mysql?
+      dump_name = calculate_dump_name(database)
+      File.unlink(dump_name) if File.exists?(dump_name)
+    end
+  end
+
+  def has_mysql?
+    defined? Mysql and 
+      @username and @password 
+  end
+
+  def parse_defaults_file(fn)
+    File.open(fn,"r") do |fh|
+      found_client = false
+
+      until fh.eof? 
+	line = fh.gets
+        case line.chomp
+          when /^\s*\[client\]/
+            found_client = true
+          when /^\s*\[/
+	    break if found_client
+          when /\s*user\s*=\s*(\S+)/
+	    @username = $1 if found_client
+          when /\s*password\s*=\s*(\S+)/
+	    @password = $1 if found_client
+        end
+      end
+    end
+  rescue StandardError
+    @username = @password = nil
+  end
+
+  def calculate_dump_name(database, charset=@default_charset)
+    File.join(@backup_dir, URI.escape(Iconv.conv(@default_charset,charset,database),/[^a-zA-Z0-9._-]/)) + ".sql.gz"
+  end
+
+  def drop_db(database, charset=@default_charset)
+    dbh = Mysql.new(nil, @username, @password)
+    dbh.query("SET NAMES #{charset}")
+    dbh.query("SET CHARSET #{charset}")
+    dbh.query("DROP DATABASE IF EXISTS `#{database}`")
+  rescue Mysql::Error => err
+    warn err.to_s
+  ensure
+    dbh.close if dbh
+  end
+
+
+  #
+  # This test does the following:
+  #   * creates a DB
+  #   * populates it
+  #   * checks it was populated as expected
+  #   * takes a backup (using the script)
+  #   * drops the DB
+  #   * restores the DB
+  #   * checks it was re-populated as expected
+  #
+  # It checks it for databases with funny charsets used in the DB name
+  #
+  def test_mysql_dump
+    unless has_mysql?
+      puts "Not running MySQL backup tests, since not all the requirements are in place."
+      return
+    end
+
+    @charsets.each do |charset|
+      database = Iconv.conv(charset, @default_charset, @database) + " #{charset}"
+      table = Iconv.conv(charset, @default_charset, @table)
+      column = Iconv.conv(charset, @default_charset, @column)
+      value = Iconv.conv(charset, @default_charset, @value)
+      res = nil
+
+      assert_nothing_raised("Failure when creating MySQL DB to test backups.") {
+        dbh = Mysql.new(nil, @username, @password)
+        dbh.query("SET CHARSET #{charset}")
+        dbh.query("SET NAMES #{charset}")
+        dbh.query("CREATE DATABASE `#{database}` CHARACTER SET #{charset};")
+        dbh.query "USE `#{database}`;"
+        dbh.query "CREATE TABLE `#{table}` (`#{column}` CHAR(20) CHARACTER SET #{charset});"
+        dbh.query "INSERT INTO `#{table}` (`#{column}`) VALUES (\"#{value}\");"
+        res = dbh.query "SELECT * FROM `#{table}`;"
+        dbh.close
+      }
+
+      #
+      # Make sure we've inserted the things properly
+      #
+      assert_equal(1, res.num_fields, "Mysql returned the wrong number of fields")
+      assert_equal(1, res.num_rows, "Mysql returned the wrong number of rows")
+      assert_equal(column, res.fetch_fields.first.name, "Mysql returned the wrong field name")
+      assert_equal(value,  res.fetch_row.first, "Mysql returned the wrong value")
+
+      system(@backup_script, Iconv.conv(@default_charset,charset,database))
+      assert_equal(0, $?, "#{@backup_script} returned non-zero.")
+
+      drop_db(database, charset)
+ 
+      dump_name = calculate_dump_name(database, charset) 
+      assert(File.exists?(dump_name),"Mysql dump file '#{dump_name}' does not exist.")    
+
+      system("zcat #{dump_name} | /usr/bin/mysql --defaults-extra-file=/etc/mysql/debian.cnf")
+      assert_equal(0, $?, "Failed to restore MySQL database from dump.")
+
+      res = nil
+
+      assert_nothing_raised("Failure when reconnecting to MySQL DB to test backups.") {
+        dbh = Mysql.new(nil, @username, @password)
+        dbh.query("SET CHARSET #{charset};")
+        dbh.query("SET NAMES #{charset};")
+        dbh.query "USE `#{database}`;"
+        res = dbh.query "SELECT * FROM `#{table}`;"
+        dbh.close
+      }
+
+      #
+      # Make sure we've inserted the things properly
+      #
+      assert_equal(1, res.num_fields, "Mysql returned the wrong number of fields")
+      assert_equal(1, res.num_rows, "Mysql returned the wrong number of rows")
+      assert_equal(column, res.fetch_fields.first.name, "Mysql returned the wrong field name")
+      assert_equal(value,  res.fetch_row.first, "Mysql returned the wrong value")
+    
+    end
+
+  end
+  
+end