Commit bef94552 authored by Patrick J Cherry's avatar Patrick J Cherry
Browse files

Adjusted the logger to work better and not lose lines.

Adjustet the apache templates to use the new logger.
parent 0c10b257
......@@ -59,7 +59,7 @@ NameVirtualHost <%= ip %>:80
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog /var/log/apache2/<%= domain %>.error.log
CustomLog /var/log/apache2/<%= domain %>.access.log combined
ErrorLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/access.log" combined
</VirtualHost>
......@@ -86,8 +86,8 @@ NameVirtualHost <%= ip %>:80
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog /var/log/apache2/<%= domain %>.ssl.error.log
CustomLog /var/log/apache2/<%= domain %>.ssl.access.log combined
ErrorLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/ssl_error.log"
CustomLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/ssl_access.log" combined
</VirtualHost>
<VirtualHost <%= ips.collect{|ip| ip+":80"}.join(" ") %>>
......@@ -143,8 +143,9 @@ NameVirtualHost <%= ip %>:80
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog /var/log/apache2/<%= domain %>.error.log
CustomLog /var/log/apache2/<%= domain %>.access.log combined
ErrorLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-apache-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/access.log" combined
% end
</VirtualHost>
......@@ -96,6 +96,8 @@ NameVirtualHost <%= ip %>:443
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
CustomLog "| /usr/sbin/symbiosis-apache-logger" "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""
LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" zz_mass_hosting_combined
CustomLog "|| /usr/sbin/symbiosis-apache-logger -s -l ssl_access.log /var/log/apache2/zz-mass-hosting.ssl_access.log" zz_mass_hosting_combined
ErrorLog /var/log/apache2/zz-mass-hosting.error.log
</VirtualHost>
......@@ -80,7 +80,9 @@ NameVirtualHost <%= ip %>:80
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
CustomLog "| /usr/sbin/symbiosis-apache-logger" "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""
LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" zz_mass_hosting_combined
CustomLog "|| /usr/sbin/symbiosis-apache-logger -s -l access.log /var/log/apache2/zz-mass-hosting.access.log" zz_mass_hosting_combined
ErrorLog /var/log/apache2/zz-mass-hosting.error.log
</VirtualHost>
......@@ -4,23 +4,25 @@
# symbiosis-apache-logger - Log access requests on a per-domain basis.
#
# SYNOPSIS
# symbiosis-apache-logger [ --max-files | -f <n> ]
# [ --default-log | -d <filename> ] [ -h | --help ]
# [-m | --manual] [ -v | --verbose ] <filename>
#
#
# symbiosis-apache-logger [ --max-files | -f <n> ] [ -s | --sync ]
# [ --uid | -u <n> ] | [ --gid | -g <n> ]
# [ --log-name | -l <filename> ] [ -h | --help ]
# [-m | --manual] [ -v | --verbose ] <default_filename>
# OPTIONS
#
# -f, --max-files <n> Maxium number of log files to hold open. Defaults to
# 50.
#
# -d, --default-log <filename> The name of the logfile where hosts that are
# not Symbiosis hosts get their logs recorded.
# -l, --log-name <f> The name of the generated logs. Defaults to "access.log"
#
# -s, --sync Open the file in sync mode, i.e. all data are
# immediately flushed to the OS and not buffered by
# the script.
#
# -u, --uid <u> Set the UID -- privileges are dropped if this is set.
#
# -g, --gid <g> Set the GID
#
# -h, --help Show a help message, and exit.
#
# -m, --manual Show this manual, and exit.
......@@ -40,22 +42,26 @@ require 'getoptlong'
#
# The options set by the command line.
# The options set by the command line. These are all global variables.
#
help = false
manual = false
$VERBOSE = false
max_files = 50
default_log = "/var/log/apache2/zz-mass-hosting.log"
sync = false
access_log_filename = "access.log"
$help = false
$manual = false
$VERBOSE = false
$max_files = 50
$default_log = "/var/log/apache2/zz-mass-hosting.log"
$sync = false
$log_filename = "access.log"
$uid = nil
$gid = nil
opts = GetoptLong.new(
[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
[ '--manual', '-m', GetoptLong::NO_ARGUMENT ],
[ '--verbose', '-v', GetoptLong::NO_ARGUMENT ],
[ '--max-files', '-f', GetoptLong::REQUIRED_ARGUMENT ],
[ '--default-log','-d', GetoptLong::REQUIRED_ARGUMENT ],
[ '--log-name', '-l', GetoptLong::REQUIRED_ARGUMENT ],
[ '--uid', '-u', GetoptLong::REQUIRED_ARGUMENT ],
[ '--gid', '-g', GetoptLong::REQUIRED_ARGUMENT ],
[ '--sync' ,'-s', GetoptLong::NO_ARGUMENT ]
)
......@@ -63,30 +69,38 @@ begin
opts.each do |opt,arg|
case opt
when '--help'
help = true
$help = true
when '--manual'
manual = true
$manual = true
when '--verbose'
$VERBOSE = true
when "--sync"
sync = true
$sync = true
when "--max-files"
max_files = arg.to_i
when "--default-log"
default_log = File.expand_path(arg)
$max_files = arg.to_i
when "--log-filename"
$log_filename = arg
when "--uid"
$uid = arg.to_i
when "--gid"
$gid = arg.to_i
end
end
rescue
rescue => err
# any errors, show the help
help = true
warn err.to_s
$help = true
end
access_log_filename = ARGV.pop if ARGV.size > 0
#
# This is the default log name
#
$default_log = File.expand_path(ARGV.pop) if ARGV.size > 0
#
# CAUTION! Here be quality kode.
#
if manual or help
if $manual or $help
# Open the file, stripping the shebang line
lines = File.open(__FILE__){|fh| fh.readlines}[1..-1]
......@@ -98,14 +112,14 @@ if manual or help
line.chomp!
break if line.empty?
if help and !found_synopsis
if $help and !found_synopsis
found_synopsis = (line =~ /^#\s+SYNOPSIS\s*$/)
next
end
puts line[2..-1].to_s
break if help and found_synopsis and line =~ /^#\s*$/
break if $help and found_synopsis and line =~ /^#\s*$/
end
......@@ -117,47 +131,23 @@ require 'symbiosis/domains'
require 'symbiosis/utils'
#
# Our filehandle store is an array of File objects.
#
$filehandles = []
#
# Open a default file for all non-matching domains.
#
$default_filehandle = Symbiosis::Utils.safe_open(default_log,'a+')
$default_filehandle.sync = sync
#
# These two flags are supposed to help avoid race conditions -- we don't want
# to find a file handle to write to, and then find it closed when we want to
# use it.
#
$closing_logs = false
$write_lock = false
########################################################################
#
# Takes an array of filehandles and closes them all.
#
def do_close_all(fhs)
warn "Closing all files" if $VERBOSE
$closing_logs = true
#
# Sleep for 10 seconds if the write lock is true.
#
10.times do
break unless $write_lock
sleep 1
end
warn "Waited 10 seconds for the loop to finish. Closing anyway." if $VERBOSE
fhs.flatten.each do |fh|
#
# Don't try to close stuff that is already closed.
#
next if fh.closed?
begin
#
# Flush to disc!
#
warn "#{$0}: Flushing and closing #{fh.path}" if $VERBOSE
fh.flush
fh.close
rescue IOError
......@@ -166,157 +156,252 @@ def do_close_all(fhs)
end
end
########################################################################
#
# trap HUP -- reopen all files.
# Drop privs. Make sure either both UID/GID are set, or neither.
#
%w(HUP USR1).each do |sig|
trap(sig) do
do_close_all($filehandles + [$default_filehandle])
end
unless [$uid, $gid].all?{|x| x.nil?} or [$uid, $gid].all?{|x| x.is_a?(Integer)}
warn "#{$0}: Both UID and GID must be either unset or integers -- unsetting"
$uid = $gid = nil
end
#
# term INT, TERM -- close all files and exit.
#
%w(QUIT TERM INT).each do |sig|
trap(sig) do
do_close_all($filehandles + [$default_filehandle])
exit 0
unless 0 == Process.uid
warn "#{$0}: Unable to drop privileges if not running as root."
$uid = $gid = nil
end
if $uid and $gid
begin
Process::Sys.setgid($gid)
Process::Sys.setuid($uid)
rescue Errno::EPERM => err
warn "#{$0}: Unable to drop privileges from #{Process.uid}:#{Process.gid} to #{$uid}:#{$gid}"
$uid = $gid = nil
end
end
while (line = STDIN.gets) do
#
# gets returns nil if called at EOF, so we probably won't get his far, as
#
break if line.nil?
########################################################################
processing_thread = Thread.new do
#
# Here is where we sleep until told otherwise.
# Set up our finish-up and close-filehandles flags
#
10.times do
break unless $closing_logs
sleep 1
end
Thread.current['finish_now'] = false
Thread.current['close_filehandles'] = false
#
# This is our little flag to say that we've got a line and we're going to write to disc.
# This is our buffer. Allocate a thread-variable with the same name so the
# buffer can be added to outside the thread.
#
$write_lock = true
buffer = []
Thread.current['buffer'] = buffer
#
# Split the line into a domain name, and the rest of the line. The domain is
# always the first field. This is supplied by the REMOTE USER so suitable
# sanity checks have to be made.
# Our filehandle store is an array of File objects.
#
# This "split" splits the line into two at the first group of spaces.
filehandles = []
#
# irb(main):030:0> "a b c".split(" ",2)
# => ["a", " b c"]
# Set up some default file-handle options.
#
domain_name, line_without_domain_name = line.to_s.split(" ",2)
default_filehandle_opts = {:mode => 0644}
default_filehandle_opts[:uid] = $uid unless $uid.nil?
default_filehandle_opts[:gid] = $gid unless $gid.nil?
#
# Set up the filehandle as nil to force us to find it each time.
# Open a default file for all non-matching domains.
#
filehandle = nil
warn "#{$0}: Opening default log file #{$default_log}" if $VERBOSE
default_filehandle = Symbiosis::Utils.safe_open($default_log,'a+',default_filehandle_opts)
default_filehandle.sync = $sync
#
# Find our domain. This finds www and non-www prefixes, and returns nil
# unless the domain is sane.
# This is our buffer-processing loop.
#
if (domain = Symbiosis::Domains.find(domain_name))
#
# Change the domain name to the correct one.
#
domain_name = domain.name
loop do
if buffer.empty?
if Thread.current['close_filehandles'] or Thread.current['finish_now']
warn "#{$0}: Closing filehandles" if $VERBOSE
#
# Close all the filehandles.
#
do_close_all(filehandles + [default_filehandle])
#
# Reset our flag.
#
Thread.current['close_filehandles'] = false
#
# If the buffer is empty, we can break out of the loop, if needed.
#
break if Thread.current['finish_now']
end
#
# Sleep for a bit before checking the buffer again.
#
sleep 1
next
end
#
# Fetch the log filename
# Shift the first entry off the beginning of the buffer.
#
log_filename = File.expand_path(File.join(domain.log_dir, access_log_filename))
line = buffer.shift
#
# Split the line into a domain name, and the rest of the line. The domain is
# always the first field. This is supplied by the REMOTE USER so suitable
# sanity checks have to be made.
#
# This "split" splits the line into two at the first group of spaces.
#
# Fetch the file handle, or open the logfile, as needed.
# irb(main):030:0> "a b c".split(" ",2)
# => ["a", " b c"]
#
filehandle = $filehandles.find{|fh| fh.is_a?(File) and fh.path == log_filename}
domain_name, line_without_domain_name = line.to_s.split(" ",2)
#
# Remove the filehandle from the arry (we'll add it back later)
# Set up the filehandle as nil to force us to find it each time.
#
$filehandles.delete(filehandle)
filehandle = nil
#
# If no filehandle was found, or the filehandle we've found is duff,
# (re)-open it.
# Find our domain. This finds www and non-www prefixes, and returns nil
# unless the domain is sane. We can only do this if we're root.
#
unless filehandle.is_a?(File) and not filehandle.closed?
if 0 == Process.uid and (domain = Symbiosis::Domains.find(domain_name))
#
# Make sure we don't open more than 50 file handles.
# Change the domain name to the correct one.
#
domain_name = domain.name
#
if $filehandles.length >= max_files
other_filehandle = $filehandles.pop
other_filehandle.close
end
# Fetch the log filename
#
log_filename = File.expand_path(File.join(domain.log_dir, $log_filename))
#
# Fetch the file handle, or open the logfile, as needed.
#
filehandle = filehandles.find{|fh| fh.is_a?(File) and fh.path == log_filename}
#
# Remove the filehandle from the arry (we'll add it back later)
#
filehandles.delete(filehandle)
begin
#
# If no filehandle was found, or the filehandle we've found is duff,
# (re)-open it.
#
unless filehandle.is_a?(File) and not filehandle.closed?
#
# Set up a couple of things before we open the file. This will make
# sure the ownerships are correct.
# Make sure we don't open more than 50 file handles.
#
begin
Symbiosis::Utils.mkdir_p(File.dirname(log_filename), :uid => domain.uid, :gid => domain.gid, :mode => 0755)
rescue Errno::EEXIST
# ignore
if filehandles.length >= $max_files
other_filehandle = filehandles.pop
other_filehandle.close
end
filehandle = Symbiosis::Utils.safe_open(log_filename, "a+", :mode => 0644, :uid => domain.uid, :gid => domain.gid )
filehandle.sync = sync
begin
#
# Set up a couple of things before we open the file. This will make
# sure the ownerships are correct.
#
begin
warn "#{$0}: Creating directory #{File.dirname(log_filename)}" if $VERBOSE
Symbiosis::Utils.mkdir_p(File.dirname(log_filename), :uid => domain.uid, :gid => domain.gid, :mode => 0755)
rescue Errno::EEXIST
# ignore
end
warn "#{$0}: Opening log file #{log_filename}" if $VERBOSE
filehandle = Symbiosis::Utils.safe_open(log_filename, "a+", :mode => 0644, :uid => domain.uid, :gid => domain.gid )
filehandle.sync = $sync
rescue StandardError => err
filehandle = nil
warn "#{$0}: Caught #{err}" if $VERBOSE
end
rescue StandardError => err
filehandle = nil
warn "Caught #{err}" if $VERBOSE
end
end
end
if filehandle.nil?
warn "No file handle found -- logging to default file for #{domain_name.inspect}" if $VERBOSE
if filehandle.nil?
warn "#{$0}: No file handle found -- logging to default file for #{domain.inspect}" if $VERBOSE and domain.is_a?(Symbiosis::Domain)
#
# Make sure the default filehandle is open.
#
if $default_filehandle.nil? or $default_filehandle.closed?
$default_filehandle = Symbiosis::Utils.safe_open(default_log,'a+')
$default_filehandle.sync = sync
#
# Make sure the default filehandle is open.
#
if default_filehandle.nil? or default_filehandle.closed?
warn "#{$0}: Opening default log file #{$default_log}" if $VERBOSE
default_filehandle = Symbiosis::Utils.safe_open($default_log,'a+', default_filehandle_opts)
default_filehandle.sync = $sync
end
#
# Write the unadulterated line to the default log.
#
default_filehandle.write(line)
else
#
# Add the filehandle onto our array.
#
filehandles << filehandle
#
# Write the log, but without the domain on the front.
#
filehandle.write(line_without_domain_name)
end
#
# Write the unadulterated line to the default log.
#
$default_filehandle.write(line)
else
#
# Add the filehandle onto our array.
#
$filehandles << filehandle
#
# Write the log, but without the domain on the front.
#
filehandle.write(line_without_domain_name)
end
#
# This is our flag to say we've stopped writing.
#
$write_lock = false
end # End of the loop.
warn "#{$0}: Processing thread finished." if $VERBOSE
end
########################################################################
#
# trap HUP -- reopen all files.
#
%w(HUP USR1).each do |sig|
trap(sig) do
warn "#{$0}: Caught #{sig}" if $VERBOSE
processing_thread['close_filehandles'] = true
end
end
#
# If we get down here, then STDIN has been closed.
# term INT, TERM -- close all files and exit.
#
%w(QUIT TERM INT).each do |sig|
trap(sig) do
warn "#{$0}: Caught #{sig}" if $VERBOSE
processing_thread['finish_now'] = true
processing_thread.join
exit 0
end
end
#
# This will continue until STDIN is closed.
#
do_close_all($filehandles + [$default_filehandle])
while (line = STDIN.gets)
processing_thread['buffer'] << line
break unless processing_thread.alive?
end
#
# Finish off our thread.
#
processing_thread['finish_now'] = true
processing_thread.join
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment