symbiosis-apache-logger 9.9 KB
Newer Older
Patrick J Cherry's avatar
Patrick J Cherry committed
1
2
3
4
5
6
#!/usr/bin/ruby1.8 -w
#
# NAME
#   symbiosis-apache-logger - Log access requests on a per-domain basis.
#
# SYNOPSIS
7
8
9
10
#  symbiosis-apache-logger [ --max-files | -f <n> ] [ -s | --sync ]
#                          [ --uid | -u <n> ] | [ --gid | -g <n> ]
#                          [ --log-name | -l <filename> ] [ -h | --help ]
#                          [-m | --manual] [ -v | --verbose ] <default_filename>
11
#
Patrick J Cherry's avatar
Patrick J Cherry committed
12
13
14
15
16
# OPTIONS
#
#  -f, --max-files <n>     Maxium number of log files to hold open. Defaults to
#                          50.
#
17
#  -l, --log-name <f>      The name of the generated logs.  Defaults to "access.log"
Patrick J Cherry's avatar
Patrick J Cherry committed
18
19
20
21
22
#
#  -s, --sync              Open the file in sync mode, i.e. all data are
#                          immediately flushed to the OS and not buffered by
#                          the script.
#
23
24
25
26
#  -u, --uid <u>           Set the UID -- privileges are dropped if this is set.
#
#  -g, --gid <g>           Set the GID
#
Patrick J Cherry's avatar
Patrick J Cherry committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#  -h, --help              Show a help message, and exit.
#
#  -m, --manual            Show this manual, and exit.
#
#  -v, --verbose           Show verbose errors
#
# USAGE
#
#  In haste.
#
# AUTHOR
#
#  Patrick J Cherry <patrick@bytemark.co.uk> 
#

require 'getoptlong'


#
46
# The options set by the command line.  These are all global variables.
Patrick J Cherry's avatar
Patrick J Cherry committed
47
#
48
49
50
51
52
53
54
55
56
$help         = false
$manual       = false
$VERBOSE      = false
$max_files    = 50
$default_log  = "/var/log/apache2/zz-mass-hosting.log"
$sync         = false
$log_filename = "access.log"
$uid          = nil
$gid          = nil
Patrick J Cherry's avatar
Patrick J Cherry committed
57
58
59
60
61
62

opts = GetoptLong.new(
  [ '--help',       '-h', GetoptLong::NO_ARGUMENT ],
  [ '--manual',     '-m', GetoptLong::NO_ARGUMENT ],
  [ '--verbose',    '-v', GetoptLong::NO_ARGUMENT ],
  [ '--max-files',  '-f', GetoptLong::REQUIRED_ARGUMENT ],
63
64
65
  [ '--log-name',   '-l', GetoptLong::REQUIRED_ARGUMENT ],
  [ '--uid',        '-u', GetoptLong::REQUIRED_ARGUMENT ],
  [ '--gid',        '-g', GetoptLong::REQUIRED_ARGUMENT ],
Patrick J Cherry's avatar
Patrick J Cherry committed
66
67
68
69
70
71
72
  [ '--sync'       ,'-s', GetoptLong::NO_ARGUMENT ]
)

begin
  opts.each do |opt,arg|
    case opt
    when '--help'
73
      $help = true
Patrick J Cherry's avatar
Patrick J Cherry committed
74
    when '--manual'
75
      $manual = true
Patrick J Cherry's avatar
Patrick J Cherry committed
76
77
78
    when '--verbose'
      $VERBOSE = true
    when "--sync"
79
      $sync = true
Patrick J Cherry's avatar
Patrick J Cherry committed
80
    when "--max-files"
81
82
83
84
85
86
87
      $max_files = arg.to_i
    when "--log-filename"
      $log_filename = arg
    when "--uid"
      $uid = arg.to_i
    when "--gid"
      $gid = arg.to_i
Patrick J Cherry's avatar
Patrick J Cherry committed
88
89
    end
  end
90
rescue => err
Patrick J Cherry's avatar
Patrick J Cherry committed
91
  # any errors, show the help
92
93
  warn err.to_s
  $help = true
Patrick J Cherry's avatar
Patrick J Cherry committed
94
95
end

96
97
98
99
#
# This is the default log name
#
$default_log = File.expand_path(ARGV.pop) if ARGV.size > 0
Patrick J Cherry's avatar
Patrick J Cherry committed
100
101
102
103

#
# CAUTION! Here be quality kode.
#
104
if $manual or $help
Patrick J Cherry's avatar
Patrick J Cherry committed
105
106
107
108
109
110
111
112
113
114
115

  # Open the file, stripping the shebang line
  lines = File.open(__FILE__){|fh| fh.readlines}[1..-1]

  found_synopsis = false

  lines.each do |line|

    line.chomp!
    break if line.empty?

116
    if $help and !found_synopsis
Patrick J Cherry's avatar
Patrick J Cherry committed
117
118
119
120
121
122
      found_synopsis = (line =~ /^#\s+SYNOPSIS\s*$/)
      next
    end

    puts line[2..-1].to_s

123
    break if $help and found_synopsis and line =~ /^#\s*$/
Patrick J Cherry's avatar
Patrick J Cherry committed
124
125
126
127
128
129
130
131
132
133
134
135

  end

  exit 0
end


require 'symbiosis/domains'
require 'symbiosis/utils'



136
########################################################################
Patrick J Cherry's avatar
Patrick J Cherry committed
137
138
139
140
141
#
# Takes an array of filehandles and closes them all.
#
def do_close_all(fhs)
  fhs.flatten.each do |fh|
142
143
144
145
146
    #
    # Don't try to close stuff that is already closed.
    #
    next if fh.closed?

Patrick J Cherry's avatar
Patrick J Cherry committed
147
148
149
150
    begin
      #
      # Flush to disc!
      #
151
      warn "#{$0}: Flushing and closing #{fh.path}" if $VERBOSE
Patrick J Cherry's avatar
Patrick J Cherry committed
152
153
154
155
156
157
158
159
      fh.flush
      fh.close
    rescue IOError
      # ignore
    end
  end
end

160
161
162


########################################################################
Patrick J Cherry's avatar
Patrick J Cherry committed
163
#
164
# Drop privs.  Make sure either both UID/GID are set, or neither.
Patrick J Cherry's avatar
Patrick J Cherry committed
165
#
166
167
168
unless [$uid, $gid].all?{|x| x.nil?} or [$uid, $gid].all?{|x| x.is_a?(Integer)}
  warn "#{$0}: Both UID and GID must be either unset or integers -- unsetting"
  $uid = $gid = nil
Patrick J Cherry's avatar
Patrick J Cherry committed
169
170
end

171
172
173
174
175
176
177
178
179
180
181
182
unless 0 == Process.uid
  warn "#{$0}: Unable to drop privileges if not running as root."
  $uid = $gid = nil
end

if $uid and $gid
  begin
    Process::Sys.setgid($gid) 
    Process::Sys.setuid($uid)
  rescue Errno::EPERM => err
    warn "#{$0}: Unable to drop privileges from #{Process.uid}:#{Process.gid} to #{$uid}:#{$gid}"
    $uid = $gid = nil
Patrick J Cherry's avatar
Patrick J Cherry committed
183
184
185
  end
end

186
########################################################################
Patrick J Cherry's avatar
Patrick J Cherry committed
187

188
processing_thread = Thread.new do 
Patrick J Cherry's avatar
Patrick J Cherry committed
189
  #
190
  # Set up our finish-up and close-filehandles flags
Patrick J Cherry's avatar
Patrick J Cherry committed
191
  #
192
193
194
  Thread.current['finish_now']        = false
  Thread.current['close_filehandles'] = false

Patrick J Cherry's avatar
Patrick J Cherry committed
195
  #
196
197
  # This is our buffer.  Allocate a thread-variable with the same name so the
  # buffer can be added to outside the thread.
Patrick J Cherry's avatar
Patrick J Cherry committed
198
  #
199
200
  buffer = []
  Thread.current['buffer']     = buffer
Patrick J Cherry's avatar
Patrick J Cherry committed
201
202

  #
203
  # Our filehandle store is an array of File objects.
Patrick J Cherry's avatar
Patrick J Cherry committed
204
  #
205
206
  filehandles = []
  
Patrick J Cherry's avatar
Patrick J Cherry committed
207
  #
208
  # Set up some default file-handle options.
Patrick J Cherry's avatar
Patrick J Cherry committed
209
  #
210
211
212
213
  default_filehandle_opts = {:mode => 0644}
  default_filehandle_opts[:uid] = $uid unless $uid.nil? 
  default_filehandle_opts[:gid] = $gid unless $gid.nil? 
  
Patrick J Cherry's avatar
Patrick J Cherry committed
214
  #
215
  # Open a default file for all non-matching domains.
Patrick J Cherry's avatar
Patrick J Cherry committed
216
  #
217
218
219
  warn "#{$0}: Opening default log file #{$default_log}" if $VERBOSE 
  default_filehandle = Symbiosis::Utils.safe_open($default_log,'a+',default_filehandle_opts)
  default_filehandle.sync = $sync
Patrick J Cherry's avatar
Patrick J Cherry committed
220
221

  #
222
  # This is our buffer-processing loop.
Patrick J Cherry's avatar
Patrick J Cherry committed
223
  #
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
  loop do

    if buffer.empty?

      if Thread.current['close_filehandles'] or Thread.current['finish_now']
        warn "#{$0}: Closing filehandles" if $VERBOSE

        #
        # Close all the filehandles.
        #
        do_close_all(filehandles + [default_filehandle])

        #
        # Reset our flag.
        #
        Thread.current['close_filehandles'] = false

        #
        # If the buffer is empty, we can break out of the loop, if needed.
        #
        break if Thread.current['finish_now']
      end

      #
      # Sleep for a bit before checking the buffer again.
      #
      sleep 1
      next
    end
Patrick J Cherry's avatar
Patrick J Cherry committed
253
254

    #
255
    # Shift the first entry off the beginning of the buffer.
Patrick J Cherry's avatar
Patrick J Cherry committed
256
    #
257
258
259
260
261
262
263
264
    line = buffer.shift
 
    #
    # Split the line into a domain name, and the rest of the line.  The domain is
    # always the first field.  This is supplied by the REMOTE USER so suitable
    # sanity checks have to be made.
    #
    # This "split" splits the line into two at the first group of spaces.
Patrick J Cherry's avatar
Patrick J Cherry committed
265
    #
266
267
    # irb(main):030:0> "a  b c".split(" ",2)
    # => ["a", " b   c"]
Patrick J Cherry's avatar
Patrick J Cherry committed
268
    #
269
    domain_name, line_without_domain_name = line.to_s.split(" ",2)
Patrick J Cherry's avatar
Patrick J Cherry committed
270
271

    #
272
    # Set up the filehandle as nil to force us to find it each time.
Patrick J Cherry's avatar
Patrick J Cherry committed
273
    #
274
    filehandle = nil
Patrick J Cherry's avatar
Patrick J Cherry committed
275
276

    #
277
278
    # Find our domain.  This finds www and non-www prefixes, and returns nil
    # unless the domain is sane.  We can only do this if we're root.
Patrick J Cherry's avatar
Patrick J Cherry committed
279
    #
280
    if 0 == Process.uid and (domain = Symbiosis::Domains.find(domain_name))
Patrick J Cherry's avatar
Patrick J Cherry committed
281
      #
282
283
284
285
286
287
288
289
290
291
292
293
294
      # Fetch the log filename
      #
      log_filename = File.expand_path(File.join(domain.log_dir, $log_filename))

      #
      # Fetch the file handle, or open the logfile, as needed.
      #
      filehandle = filehandles.find{|fh| fh.is_a?(File) and fh.path == log_filename}

      #
      # Remove the filehandle from the arry (we'll add it back later)
      #
      filehandles.delete(filehandle) 
Patrick J Cherry's avatar
Patrick J Cherry committed
295

296
297
298
299
300
      #
      # If no filehandle was found, or the filehandle we've found is duff,
      # (re)-open it.
      #
      unless filehandle.is_a?(File) and not filehandle.closed?
Patrick J Cherry's avatar
Patrick J Cherry committed
301
        #
302
        # Make sure we don't open more than 50 file handles.
Patrick J Cherry's avatar
Patrick J Cherry committed
303
        #
304
305
306
        if filehandles.length >= $max_files
          other_filehandle = filehandles.pop
          other_filehandle.close
307
        end
Patrick J Cherry's avatar
Patrick J Cherry committed
308

309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
        begin
          #
          # Set up a couple of things before we open the file.  This will make
          # sure the ownerships are correct.
          #
          begin
            warn "#{$0}: Creating directory #{File.dirname(log_filename)}" if $VERBOSE 
            Symbiosis::Utils.mkdir_p(File.dirname(log_filename), :uid => domain.uid, :gid => domain.gid, :mode => 0755)
          rescue Errno::EEXIST
            # ignore
          end

          warn "#{$0}: Opening log file #{log_filename}" if $VERBOSE 
          filehandle = Symbiosis::Utils.safe_open(log_filename, "a+", :mode => 0644, :uid => domain.uid, :gid => domain.gid )
          filehandle.sync = $sync

        rescue StandardError => err
          filehandle = nil
          warn "#{$0}: Caught #{err}" if $VERBOSE
        end
Patrick J Cherry's avatar
Patrick J Cherry committed
329
330
331
332
333

      end

    end

334
335
    if filehandle.nil? 
      warn "#{$0}: No file handle found -- logging to default file for #{domain.inspect}" if $VERBOSE and domain.is_a?(Symbiosis::Domain)
Patrick J Cherry's avatar
Patrick J Cherry committed
336

337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
      #
      # Make sure the default filehandle is open.
      #
      if default_filehandle.nil? or default_filehandle.closed?
        warn "#{$0}: Opening default log file #{$default_log}" if $VERBOSE 
        default_filehandle = Symbiosis::Utils.safe_open($default_log,'a+', default_filehandle_opts)
        default_filehandle.sync = $sync
      end
      #
      # Write the unadulterated line to the default log.
      #
      default_filehandle.write(line)
    else
      #
      # Add the filehandle onto our array.
      #
      filehandles << filehandle
      #
      # Write the log, but without the domain on the front.
      #
      filehandle.write(line_without_domain_name)
Patrick J Cherry's avatar
Patrick J Cherry committed
358
359
    end

360
361
362
363
  end # End of the loop.

  warn "#{$0}: Processing thread finished." if $VERBOSE

Patrick J Cherry's avatar
Patrick J Cherry committed
364
365
end

366
367
368
369
370
371
372
373
374
375
########################################################################
#
# trap HUP -- reopen all files.
#
%w(HUP USR1).each do |sig|
  trap(sig) do
    warn "#{$0}: Caught #{sig}" if $VERBOSE 
    processing_thread['close_filehandles'] = true
  end
end
Patrick J Cherry's avatar
Patrick J Cherry committed
376
377

#
378
# term INT, TERM -- close all files and exit.
Patrick J Cherry's avatar
Patrick J Cherry committed
379
#
380
381
382
383
384
385
386
387
388
389
390
%w(QUIT TERM INT).each do |sig|
  trap(sig) do
    warn "#{$0}: Caught #{sig}" if $VERBOSE 
    processing_thread['finish_now'] = true
    processing_thread.join
    exit 0
  end
end

#
# This will continue until STDIN is closed.
Patrick J Cherry's avatar
Patrick J Cherry committed
391
#
392
393
394
395
396
397
398
399
400
401
402
while (line = STDIN.gets)
  processing_thread['buffer'] << line
  break unless processing_thread.alive?
end

#
# Finish off our thread.
#
processing_thread['finish_now'] = true
processing_thread.join

Patrick J Cherry's avatar
Patrick J Cherry committed
403