Commit 691d43df authored by Patrick J Cherry's avatar Patrick J Cherry
Browse files

httpd: Moved the golang httpd-logger into the main httpd pacakge

This removes the old ruby logger, and the EM dependency.
parent fde6fe80
debian/files
debian/tmp
debian/symbiosis-httpd-logger
debian/debhelper-build-stamp
obj-x86_64-linux-gnu/
#
# Not sure this is the correct variable to use, but it appears to give the
# right name!
#
DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -q DEB_BUILD_GNU_TYPE)
all: obj-$(DEB_BUILD_GNU_TYPE)/bin/symbiosis-httpd-logger
clean:
$(RM) -r obj-$(DEB_BUILD_GNU_TYPE)
#
# Make sure the binary has built before running the test
#
test: obj-$(DEB_BUILD_GNU_TYPE)/bin/symbiosis-httpd-logger
ruby -I ../common/lib/ test/tc_apache_logger.rb
#
# The dh-golang stuff seems to put the binary in obj-$DEB_BUILD_GNU_TYPE/bin
#
obj-$(DEB_BUILD_GNU_TYPE)/bin/%: %.go
mkdir -p ${DEB_BUILD_GNU_TYPE}/bin
go build -o $@ $<
.PHONY: test clean all
symbiosis-httpd-logger (0.3.1) stable; urgency=medium
* Updated copyright and dh compatibility.
-- Patrick J Cherry <patrick@bytemark.co.uk> Mon, 24 Apr 2017 15:39:11 +0100
symbiosis-httpd-logger (0.3) stable; urgency=medium
* Mask out non-permissions bits from file mode inherited from directory
(thanks to @banburybill).
-- Patrick J Cherry <patrick@bytemark.co.uk> Wed, 19 Apr 2017 10:43:19 +0100
symbiosis-httpd-logger (0.2) stable; urgency=medium
* Non-maintainer upload.
* The sync flag is now obeyed (closes #44).
* Now creates log files with correct owner (closes #41).
* Fixed double logging to the default and per-host logs (closes #45).
* Verbose logging normalised.
* Debian packaging tidied simplified.
* Fixed up dpkg-divert to work properly.
* UID/GID for logs is always inherited from the log directory.
* Added stricter regex for log lines.
* Added prefix flag for testing.
-- Patrick J Cherry <patrick@bytemark.co.uk> Tue, 22 Nov 2016 12:06:58 +0000
symbiosis-httpd-logger (0.1) unstable; urgency=low
* Initial release.
-- Steve Kemp <steve@bytemark.co.uk> Wed, 06 Apr 2016 17:22:17 +0000
Source: symbiosis-httpd-logger
Section: web
Priority: extra
Maintainer: Steve Kemp <steve@bytemark.co.uk>
Build-Depends: debhelper (>=9), dh-golang, golang-go (>= 1.5.1)
Standards-Version: 3.9.4
Package: symbiosis-httpd-logger
Depends: ${misc:Depends}
Architecture: any
Description: golang based httpd logging process for Symbiosis.
This package replaces the original/default ruby-based logging process
which is executed by the http daemon, with the intention of saving RAM.
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: Bytemark Symbiosis
Source: https://github.com/BytemarkHosting/symbiosis
Files: *
Copyright: 2008-2017, Bytemark Ltd
License: GPL-2+
This program is free software; you can redistribute it
and/or modify it under the terms of the GNU General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later
version.
.
This program is distributed in the hope that it will be
useful, but WITHOUT ANY WARRANTY; without even the implied
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the GNU General Public License for more
details.
.
You should have received a copy of the GNU General Public
License along with this package; if not, write to the Free
Software Foundation, Inc., 51 Franklin St, Fifth Floor,
Boston, MA 02110-1301 USA
.
On Debian systems, the full text of the GNU General Public
License version 2 can be found in the file
`/usr/share/common-licenses/GPL-2'.
usr/bin/symbiosis-httpd-logger usr/sbin/symbiosis-httpd-logger
#!/bin/sh
set -e
#
# Skip, if we are upgrading
#
if [ "$1" = "upgrade" ]; then
exit 0
fi
package=symbiosis-httpd-logger
bin=/usr/sbin/symbiosis-httpd-logger
theirbin="$bin.dpkg-symbiosis.orig"
#
# Add the diversion if the file has not been generated by Symbiosis, and if the
# diversion doesn't exist.
#
if dpkg-divert --list "$package" | grep -xFq "diversion of $bin to $theirbin by $package"; then
if [ -e "$theirbin" ] ; then
# Remove and rename back again
dpkg-divert --remove --rename --package "$package" "$bin" || true
else
# Otherwise just remove the diversion
dpkg-divert --remove --package "$package" "$bin" || true
fi
fi
#DEBHELPER#
#!/bin/sh
set -e
#
# Skip, if we are not in "install" state
#
if [ "$1" != "install" ]; then
echo "I: Skipping $1"
exit 0
fi
package=symbiosis-httpd-logger
bin=/usr/sbin/symbiosis-httpd-logger
theirbin="$bin.dpkg-symbiosis.orig"
if [ -e $bin ] && ( ! dpkg-divert --list "$package" | grep -xFq "diversion of $bin to $theirbin by $package" ) ; then
dpkg-divert --divert "$theirbin" --rename --package "$package" --add "$bin"
fi
#DEBHELPER#
#!/usr/bin/make -f
export DH_GOPKG := symbiosis-httpd-logger
%:
dh $@ --buildsystem=golang --with=golang
#
# It would be nice to test as part of the build, but that required installing
# symbiosis-common.
#
# override_dh_auto_test:
# make test
#
#
require 'socket'
require 'test/unit'
require 'tmpdir'
require 'tempfile'
require 'symbiosis/domains'
class TestApacheLogger < Test::Unit::TestCase
def setup
@prefix = Dir.mktmpdir('srv')
File.chown(1000, 1000, @prefix) if Process.uid < 1000
@nonprefix = Dir.mktmpdir('other')
File.chown(1, 1, @nonprefix) if Process.uid < 1000
@default_log = Tempfile.new('tc_httpd_logger', @nonprefix)
@default_filename = @default_log.path
@domains = []
host_type = `dpkg-architecture -q DEB_BUILD_GNU_TYPE`.chomp || 'unknown'
#
# Go builds stuff in obj-..
#
@binary = File.expand_path(File.join(File.dirname(__FILE__), '..', 'obj-' + host_type, 'bin', 'symbiosis-httpd-logger'))
unless File.executable?(@binary)
@binary = '/usr/sbin/symbiosis-httpd-logger'
end
unless File.executable?(@binary)
omit('Missing binary for symbiosis-httpd-logger')
end
end
def teardown
#
# Close + remove our default logfile
#
if @default_log.is_a?(Tempfile)
FileUtils.mv(@default_log.path, @default_log.path + '.save') if $DEBUG
@default_log.close
end
return if $DEBUG
#
# Delete the temporary domain
#
@domains.each { |d| d.destroy unless d.nil? }
#
# Remove the @prefix directory
#
FileUtils.remove_entry_secure @prefix
FileUtils.remove_entry_secure @nonprefix
end
#
# This tests the options we can set.
#
def test_setup
flags = %w(-s -f 500 -u 100 -g 200 -l foo.log) + ['-p', @prefix, @default_filename]
flags.unshift('-v') if $DEBUG
IO.popen([@binary] + flags, 'r+') do |f|
f.close_write; f.close_read
end
assert_equal(0, $?.exitstatus, "#{@binary} exited non-zero when testing flags")
end
def test_logging
#
# Create the domain
#
10.times do |i|
@domains << Symbiosis::Domain.new("existent-#{i}.com", @prefix)
end
@domains.each(&:create)
#
# We'll delete this domain half-way
#
to_delete = Symbiosis::Domain.new('deleted.com', @prefix)
to_delete.create
@domains << to_delete
#
# Add a non-existent domain to the mis
#
non_existent = Symbiosis::Domain.new('non-existent.com', @prefix)
@domains << non_existent
test_lines = []
test_lines2 = []
10.times do |i|
test_lines += @domains.collect { |d| [d, "#{i} " + Symbiosis::Utils.random_string(40)] }
test_lines2 += @domains.collect { |d| [d, "#{10 + i} " + Symbiosis::Utils.random_string(40)] }
end
flags = %w(-s -f 4) + ['-p', @prefix, @default_filename]
flags.unshift('-v') if $DEBUG
IO.popen([@binary] + flags, 'r+') do |pi|
test_lines.each do |d, l|
pi.puts "#{d.name} #{l}"
end
to_delete.destroy
test_lines2.each do |d, l|
pi.puts "#{d.name} #{l}"
end
end
#
# If we get this far, open the domain's access logs and look for foo and bar. Mash the test lines into a hash.
#
log_line_hash = Hash.new { |h, k| h[k] = [] }
(test_lines + test_lines2).each do |d, l|
#
# We expect the last 10 lines of the deleted domain to appear, and all
# the non-existent ones in the shared access log.
#
unless [non_existent, to_delete].include?(d)
# next if to_delete == d and l =~ /^\d /
log_line_hash[d] << l
else
log_line_hash[non_existent] << "#{d.name} #{l}"
end
end
log_line_hash.each do |d, lines|
if File.directory?(d.directory)
# Make sure the file exists
access_log = File.join(d.log_dir, 'access.log')
else
# When we write to the default log, the domain is still attached.
access_log = @default_filename
end
assert(File.exist?(access_log), "Access log #{access_log} for #{d.name} not found")
# And that we can read it.
logged_lines = File.readlines(access_log).collect { |l| l.to_s.chomp }
assert_equal(lines, logged_lines, "Mismatch in logs for domain #{d} in file #{access_log}")
end
assert(!File.directory?(non_existent.directory), "Non-existent domain's directory created.")
assert(!File.directory?(to_delete.directory), "Non-existent domain's directory created.")
end
def test_symlinked_domain
domain = Symbiosis::Domain.new('example.com', @prefix)
domain.create
@domains << domain
FileUtils.ln_s(domain.directory, File.join(@prefix, 'example.org'))
FileUtils.ln_s(domain.name, File.join(@prefix, 'example.net'))
FileUtils.ln_s(@nonprefix, File.join(@prefix, 'example.info'))
FileUtils.ln_s('/var', File.join(@prefix, 'example.biz'))
test_lines = []
10.times do |i|
test_lines += %w(com net org info biz).collect { |d| ["example.#{d}", "#{i} " + Symbiosis::Utils.random_string(40)] }
end
flags = %w(-s -f 4) + ['-p', @prefix, @default_filename]
flags.unshift('-v') if $DEBUG
IO.popen([@binary] + flags, 'r+') do |pi|
test_lines.each do |d, l|
pi.puts "#{d} #{l}"
end
pi.close_write
end
assert(!File.exist?(File.join('/var', 'public', 'logs', 'access.log')), 'Log file outside /srv created.')
end
def test_as_root
omit('must be run as root') unless Process.uid == 0
domain = Symbiosis::Domain.new('example.com', @prefix)
domain.create
Symbiosis::Utils.mkdir_p(domain.log_dir, uid: domain.uid, gid: domain.gid)
# Add the set-group-id bit (to make sure it gets removed later)
File.chmod(02750, domain.log_dir)
log_dir_mode = (File.stat(domain.log_dir).mode & 0777)
@domains << domain
test_lines = []
10.times do |_i|
[domain.name, 'non-existent.horse'].each do |d|
test_lines << (d + ' ' + Symbiosis::Utils.random_string(40))
end
end
flags = %w(-s -f 4 ) + ['-u', domain.uid.to_s, '-g', domain.gid.to_s, '-p', @prefix, @default_filename]
flags.unshift('-v') if $DEBUG
IO.popen([@binary] + flags, 'r+') do |pi|
test_lines.each do |d, l|
pi.puts "#{d} #{l}"
end
pi.close_read
pi.close_write
end
assert_equal(0, $?.exitstatus, "#{@binary} exited non-zero when testing flags")
access_log = File.join(domain.log_dir, 'access.log')
assert(File.exist?(access_log), "Per-domain access log #{access_log} is missing!")
stat = File.stat(access_log)
assert_equal(domain.uid, stat.uid, "UID on the per-domain log #{access_log} does not match domain\'s UID")
assert_equal(domain.gid, stat.gid, "GID on the per-domain log #{access_log} does not match domain\'s GID")
# Test the mode twice, first make sure the file is not executable by anyone.
assert_equal(log_dir_mode & ~0111, stat.mode & 00777, "Mode on #{access_log} (0#{(stat.mode & 0777).to_s(8)}) is wrong")
# Now check that no sticky (etc) bits have been set.
assert_equal(0, stat.mode & 07000, "#{access_log} has non-permission mode bits set")
assert(File.exist?(@default_filename), "Default access log #{@default_filename} is missing!")
stat = File.stat(@default_filename)
dir_stat = File.stat(File.dirname(@default_filename))
assert_equal(stat.uid, dir_stat.uid, "UID on the default log #{@default_filename} does not match parent directory\'s UID")
assert_equal(stat.gid, dir_stat.gid, "GID on the default log #{@default_filename} does not match parent directory\'s GID")
end
end
obj-x86_64-linux-gnu/
sbin/symbiosis-httpd-logger
RUBYLIB := "${PWD}/lib:${PWD}/../common/lib"
nop:
@echo "Valid targets are (alphabetically) :"
@echo " "
@echo " clean - Remove bogus files."
@echo " docs - Create manpages for scripts in sbin."
@echo " "
#
# Not sure this is the correct variable to use, but it appears to give the
# right name!
#
DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -q DEB_BUILD_GNU_TYPE)
docs: man/symbiosis-httpd-configure.man man/symbiosis-httpd-logger.man man/symbiosis-httpd-generate-stats.man man/symbiosis-httpd-rotate-logs.man
all: sbin/symbiosis-httpd-logger docs
test:
docs: man/symbiosis-httpd-configure.man man/symbiosis-httpd-generate-stats.man man/symbiosis-httpd-rotate-logs.man man/symbiosis-httpd-logger.man
#
# The dh-golang stuff seems to put the binary in obj-$DEB_BUILD_GNU_TYPE/bin
#
obj-$(DEB_BUILD_GNU_TYPE)/bin/%: %.go
mkdir -p ${DEB_BUILD_GNU_TYPE}/bin
go build -o $@ $<
test: obj-$(DEB_BUILD_GNU_TYPE)/bin/symbiosis-httpd-logger
$(MAKE) -C vhost-alias test
RUBYLIB=${RUBYLIB} ruby test.d/t*.rb
.PHONY: test clean all docs
sbin/symbiosis-httpd-logger: obj-$(DEB_BUILD_GNU_TYPE)/bin/symbiosis-httpd-logger
cp -a $< $@
man/%.txt: ./txt/%.txt
cp -a $< $@
man/%.txt: ./sbin/%
[ -d man ] || mkdir man
RUBYLIB=${RUBYLIB} $< --manual > $@
......@@ -26,11 +39,13 @@ man/%.man: man/%.txt
test -s $@
clean:
$(RM) -r obj-$(DEB_BUILD_GNU_TYPE)
cd vhost-alias && make clean
@find . -name '.*~' -exec rm \{\} \;
@find . -name '.#*' -exec rm \{\} \;
@find . -name 'configure-stamp' -exec rm \{\} \;
@find . -name 'build-stamp' -exec rm \{\} \;
$(RM) -r man
$(RM) sbin/symbiosis-httpd-logger
.PHONY: clean nop symbiosis-httpd-man
.PHONY: test clean all docs
......@@ -82,7 +82,7 @@
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/access.log" combined
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/access.log" combined
</VirtualHost>
......@@ -120,8 +120,8 @@
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/ssl_error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/ssl_access.log" combined
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/ssl_error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/ssl_access.log" combined
</VirtualHost>
<VirtualHost <%= ips.collect{|ip| ip+":80"}.join(" ") %>>
......@@ -222,8 +222,8 @@
# made against, so that the cron-job in /etc/cron.daily may generate
# statistics for each domain.
#
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s -u <%= domain.uid %> -g <%= domain.gid %> <%= domain.log_dir %>/access.log" combined
ErrorLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/error.log"
CustomLog "|| /usr/sbin/symbiosis-httpd-logger -s <%= domain.log_dir %>/access.log" combined
% end
</VirtualHost>
......
symbiosis-httpd (2017:0904) stable; urgency=medium
* Replaced ruby httpd loggger with our golang version.
-- Patrick J Cherry <patrick@bytemark.co.uk> Mon, 04 Sep 2017 16:35:03 +0100
symbiosis-httpd (2017:0818) stable; urgency=medium
* Added a --diff-only option to symbiosis-httpd-configure.
......
......@@ -3,14 +3,17 @@ Section: web
Priority: optional
Maintainer: James Carter <jcarter@bytemark.co.uk>
Uploaders: Patrick J Cherry <patrick@bytemark.co.uk>, Steve Kemp <steve@bytemark.co.uk>
Build-Depends: debhelper (>= 7.0.0), apache2-dev, libcap-dev, php7.0-common, txt2man, gem2deb, ruby, symbiosis-common
Build-Depends: debhelper (>= 7.0.0), apache2-dev, libcap-dev, php7.0-common, txt2man, gem2deb, ruby, symbiosis-common, dh-golang, golang-go (>= 1.5.1)
Standards-Version: 3.9.6
XS-Ruby-Versions: all
Package: symbiosis-httpd
Architecture: all
Depends: apache2, libapache2-mod-php7.0, webalizer, libapache2-mod-vhost-bytemark (>= 2015:0101), ${misc:Depends}, symbiosis-common (>= 2017:0328), ruby | ruby-interpreter, ruby-eventmachine
Depends: apache2, libapache2-mod-php7.0, webalizer, libapache2-mod-vhost-bytemark (>= 2015:0101), ${misc:Depends}, symbiosis-common (>= 2017:0830), ruby | ruby-interpreter
Recommends: php7.0-mysql | php7.0-mysqli | php7.0-mysqlnd, php7.0-curl, php7.0-imagick, php7.0-mcrypt, php7.0-mhash, php7.0-xmlrpc, php7.0-gd, geoip-database
Replaces: symbiosis-httpd-logger
Conflicts: symbiosis-httpd-logger
Provides: symbiosis-httpd-logger
Description: Tools to manage Apache virtual hosting in Symbiosis
This package contains tools to manage the virtual hosting of websites
with Apache2.
......
#!/usr/bin/make -f
#
export DH_GOPKG := symbiosis-httpd-logger
%:
dh $@ --buildsystem=ruby --with ruby,apache2
dh $@ --buildsystem=ruby --with ruby,apache2,golang
override_dh_auto_build:
$(MAKE) docs
$(MAKE)
cd vhost-alias && $(MAKE) && $(MAKE) test
dh_auto_build
......
require 'symbiosis/domains'
require 'symbiosis/utils'
require 'eventmachine'
require 'pp'
module Symbiosis
class ApacheLogger < EventMachine::Connection
class DomainCache
def initialize(prefix, cache_time=10, clock=nil)
@prefix = prefix
@cache_time = cache_time
@cache = {}
@clock = clock
@clock ||= Proc.new { Time.now }
end
def [](k)
now = @clock.call
domain, cached_at = @cache[k]
if ( !domain.is_a?(Symbiosis::Domain) or !cached_at.is_a?(now.class) or ( cached_at + @cache_time ) <= now )
domain = Symbiosis::Domains.find(k, @prefix)
@cache[k] = [ domain, now ]
end
domain
end
end
def initialize(opts = {})
@sync_io = false
@max_filehandles = 50
@log_filename = "access.log"
@default_filehandle = nil
@default_filename = "/var/log/apache2/zz-mass-hosting.log"