Added rubified blacklist generation

firewall/debian/symbiosis-firewall.cron.d

@@ -14,11 +14,11 @@
 #  Whitelist valid IP addresses every hour, but outside the scope of the
 # firewall test.
 #
-30   * * * * root [ -x /usr/bin/firewall-whitelist ] && /usr/bin/firewall-whitelist
+30   * * * * root [ -x /usr/sbin/symbiosis-firewall-whitelist ] && /usr/sbin/symbiosis-firewall-whitelist
 
 
 #
 # Check the firewall works every hour.
 #
-@hourly      root [ -x /usr/bin/firewall ] && /usr/bin/firewall --test
+# ourly      root [ -x /usr/sbin/symbiosis-firewall ] && /usr/sbin/
 

firewall/lib/symbiosis/firewall/blacklist.rb

+require 'symbiosis/firewall/pattern'
+require 'symbiosis/firewall/directory'
+require 'symbiosis/firewall/logtail'
+
+module Symbiosis
+  module Firewall
+    class Blacklist
+
+      attr_reader :attempts, :base_dir, :logtail_db
+
+      def initialize()
+        @attempts     = 20
+        @base_dir     = '/etc/symbiosis/firewall'
+        @logtail_db   = '/var/lib/symbiosis/firewall-logtail.db'
+        @patterns     = []
+      end
+
+      def base_dir=(b)
+        raise Errno::ENOENT, b unless File.directory?(b)
+
+        @base_dir = b
+      end
+
+      def logtail_db=(db)
+        @logtail_db = db
+      end
+      
+      def generate
+        results = do_read
+        do_parse_results(results)
+      end
+
+      private
+
+      def do_read
+        Dir.glob(File.join(@base_dir, "pattern.d", "*.patterns")) do |entry|
+          @patterns << Pattern.new("pattern.d/openssh.patterns")
+        end
+
+        logfiles = Hash.new
+        results = Hash.new{|h,k| h[k] = Hash.new{|i,l| i[l] = 0}}
+
+        @patterns.each do |pattern|
+          #
+          # Read the log file, if needed.
+          #
+          unless logfiles.has_key?(pattern.logfile)
+            loglines = []
+
+            begin
+              logtail = Logtail.new(pattern.logfile, @logtail_db)
+              loglines = logtail.readlines
+            rescue Errno::ENOENT => err
+              #
+              # Do nothing if the log file doesn't exist.
+              #
+            end
+            #
+            # Cache the log lines that we found
+            #
+            logfiles[pattern.logfile] = loglines
+          else
+            loglines = logfiles[pattern.logfile]
+          end
+
+
+          #
+          # Apply our pattern
+          #
+          new_results = pattern.apply(loglines)
+
+          #
+          # And add it on to our results.
+          #
+          new_results.each do |ip, ports|
+            ports.each do |port, hits|
+             results[ip][port] += hits
+            end
+          end
+        end
+
+        results
+      end
+
+      def do_parse(results = do_read)
+        #
+        # This is our result.  It is keyed on IP, and the values are an array
+        # of ports, or an array containing "all" for all ports.
+        #
+        blacklist = Hash.new{|h,k| h[k] = []}
+
+        results.each do |ip, ports|
+          #
+          # tot up on a per-ip basis
+          #
+          total_for_ip = 0
+
+          ports.each do |port, hits|
+            total_for_ip += hits
+
+            blacklist[ip] << port if hits > @attempts
+          end
+
+          #
+          # If an IP has exceeded the number of attempts on any port, block it from all ports.
+          #
+          if total_for_ip > @attempts
+            blacklist[ip] = %w(all)
+          end
+
+        end
+
+        blacklist
+      end
+
+    end
+
+  end
+
+end
+
+

firewall/lib/symbiosis/firewall/directory.rb

@@ -29,6 +29,10 @@ module Symbiosis
         @default = d
       end
 
+      def read
+        do_read
+      end
+
       #
       #
       def to_s
@@ -175,6 +179,10 @@ module Symbiosis
     #
     # 0 directories, 3 files
     #
+    # Each file can contain a list of ports/services/templates, or the word
+    # "all", or nothing at all.
+    #
+    #
     class IPListDirectory < Directory
 
       private
@@ -188,18 +196,17 @@ module Symbiosis
       #
       def do_read
 
-        template_path = do_find_template( self.default )
-        template = Template.new( template_path )
-        template.name = self.default
-        template.direction = self.direction
-        template.chain = self.chain unless self.chain.nil?
+        templates = []
+
+        #
+        # A hash of arrays
+        #
+        port_addresses = Hash.new{|i,j| i[j] = []}
 
-        addresses = []
         #
         #  Read the contents of the directory
         #
         Dir.entries( self.path ).each do |file|
-
           #
           #  Skip "dotfiles".
           #
@@ -208,15 +215,57 @@ module Symbiosis
           #
           #  Here we need to strip the optional ".auto" suffix.
           #
-          file = File.basename(file,".auto") 
+          ip = File.basename(file,".auto") 
+
+          #
+          # Now see if the file contains any lines for ports
+          #
+          ports = File.readlines(File.join(self.path, file))
 
           #
-          #  Save it away.
+          # Tidy port list, removing empty lines, and stripping out white
+          # space.
+          #
+          ports = ports.collect do |port|
+            port = port.chomp.strip
+            if port.empty?
+              nil
+            else
+              port
+            end
+          end.compact
+         
+          #
+          # Now we have our sanitised list, if the list is empty, assume all
+          # ports.  If nothing is specified, or one of the lines is "all", then
+          # the array can just contain "nil", which means all ports.
           #
-          addresses << file
+          if ports.empty? or ports.any?{|port| "all" == port}
+            ports = [nil] 
+          end
+
+          #
+          # Save each port/address combo.
+          #
+          ports.each do |port|
+            port_addresses[port] << ip
+          end
+        end
+
+        #
+        # Now translate our ports into templates.
+        #
+        port_addresses.each do |port, addresses|
+          template_path = do_find_template( self.default )
+          template = Template.new( template_path )
+          template.name = self.default
+          template.direction = self.direction
+          template.port = port unless port.nil? 
+          template.chain = self.chain unless self.chain.nil?
+          templates << [template, addresses]
         end
 
-        return [[template, addresses]]
+        return templates
       end
 
     end

firewall/lib/symbiosis/firewall/logtail.rb

+require 'digest/md5'
+require 'sqlite3'
+
+module Symbiosis
+  module Firewall
+    class Logtail
+
+      attr_reader :filename
+      #
+      # For testing.
+      attr_reader :dbh
+
+      def initialize(file, database = '/var/lib/symbiosis/firewall-logtail.db')
+        #
+        # hmm.. maybe we should deal with this a bit better?
+        #
+        raise Errno::ENOENT, file unless File.exists?(file)
+        @filename = file
+        
+        @identifier = nil
+        @lines = []
+
+        @dbh = SQLite3::Database.new(database)
+        @dbh.type_translation = true
+        @tbl_name = "logtail"
+        create_table
+      end
+      
+
+      #
+      # Returns the hash of the first line, or nil if no first line can be found. 
+      #
+      def identifier
+        return @identifier unless @identifier.nil?
+
+        line = nil
+        File.open(self.filename) do |fh|
+          line = fh.gets
+        end
+
+        if line.is_a?(String)
+          @identifier = Digest::MD5.new.hexdigest(line)
+        else
+          @identifier = nil
+        end
+
+        @identifier
+      end
+      
+      def pos=(new_pos)
+        @dbh.execute("INSERT OR REPLACE INTO #{@tbl_name}
+          VALUES (?, ?, ?)",
+          self.filename, self.identifier, new_pos
+        )
+
+        @pos = new_pos
+      end
+
+      def pos
+        return 0 if self.identifier.nil?
+        return @pos unless @pos.nil?
+
+        pos = @dbh.execute("SELECT pos FROM #{@tbl_name} WHERE filename = ? AND identifier = ? LIMIT 0,1", self.filename, self.identifier).flatten.first
+        pos = 0 if pos.nil?
+        @pos = pos
+      end
+      
+      def readlines
+        return @lines unless @lines.empty?
+        return @lines if self.identifier.nil?
+
+        File.open(self.filename) do |fh|
+          fh.pos = self.pos
+
+          while !fh.eof? 
+            @lines << fh.gets
+          end
+
+          #
+          # Record the position
+          #
+          self.pos=fh.pos
+        end
+
+        @lines
+      end
+
+      private
+      
+      #
+      # Creates the SQLite table.
+      #
+      def create_table
+        sql = "CREATE TABLE IF NOT EXISTS #{@tbl_name} 
+              (
+                filename   TEXT NOT NULL UNIQUE,
+                identifier TEXT NOT NULL,
+                pos        INTEGER NOT NULL
+              )"
+        @dbh.execute(sql)
+      end
+    end
+
+  end
+
+end
+

firewall/lib/symbiosis/firewall/pattern.rb

+
+require 'symbiosis/firewall/ipaddr'
+
+module Symbiosis
+  module Firewall
+    class Pattern
+
+      attr_reader :logfile
+
+      def initialize(filename)
+        @logfile  = nil
+        @ports    = nil
+        @patterns = []
+
+        File.readlines(filename).each do |line|
+          #
+          # Remove preceeding and trailing spaces, and newlines
+          #
+          line = line.strip.chomp
+
+          next if line.empty? or line =~ /^#/ 
+
+          #
+          #  Filename
+          #
+          if line =~ /^file\s*=\s*(.*)/  and @logfile.nil?
+            @logfile = $1
+
+          #
+          # Comma/space separated line of ports/services
+          #
+          elsif line =~ /^ports\s*=\s*(.*)/ and @ports.nil?
+            @ports = $1.split(/[^a-z0-9]+/i)
+
+          else
+            line = line.gsub("__IP__","(?:::ffff:)?([0-9a-fA-F:\.]+(?:/[0-9]+)?)")
+
+            @patterns << Regexp.new(line)
+          end
+
+        end
+
+      end
+
+
+      #
+      # Takes an array of log lines, and applies it patterns.  It returns a hash of hashes:
+      #
+      #  {
+      #    ip.ad.re.ss1 =>
+      #       { port1 => count1,
+      #         port2 => count2 },
+      #    ip.ad.re.ss2 =>
+      #       { port1 => count3,
+      #         port2 => count4 },
+      #  }
+      #
+      #
+      def apply(lines)
+        # This returns a has of IPs summed up.
+        results = Hash.new{|h,k| h[k] = Hash.new{|i,l| i[l] = 0 }}
+
+        lines.each do |line|
+          @patterns.each do |pattern|
+            next unless line =~ pattern
+            ip = $1
+
+            begin
+              ip = IPAddr.new(ip)
+            rescue ArgumentError => err
+              warn "Failed to parse IP #{ip.inspect}"
+            end
+
+            next unless ip.is_a?(IPAddr)
+
+            #
+            # Only apply /64 for ipv6 addresses.
+            #
+            ip = ip.mask( 64 ) if ip.ipv6?
+
+            @ports.each do |port|
+              results[ip.to_s][port] += 1
+            end
+          end
+        end
+
+        results
+      end
+
+    end
+
+  end
+
+end
+

firewall/lib/symbiosis/firewall/template.rb

@@ -93,6 +93,13 @@ module Symbiosis
       #  Set the source/dest
       #
       def address=( new_address )
+        #
+        # Cope with ranges.
+        #
+        if new_address.downcase =~ /^([0-9a-f\.:]+)-([0-9]+)$/
+          new_address = [$1, $2].join("/")
+        end
+
         @address = IPAddr.new(new_address)
       end
 

firewall/sbin/symbiosis-firewall-blacklist

+#! /usr/bin/ruby1.8
+#
+# NAME
+#
+#   symbiosis-firewall-blacklist -- Automatically blacklist IP addresses.
+#
+# SYNOPSIS
+#
+#  Options:
+#
+#   --prefix      The directory to operate upon.
+#
+#  Help Options:
+#
+#   --help        Show the help information for this script.
+#   --verbose     Show debugging information.
+#
+# This script is designed to automatically blacklist IP addresses which
+# have been used to successfully login via SSH.
+#
+# It does this by parsing the output of the "last" command, and creating
+# entries in /etc/symbiosis/firewall/blacklist.d/
+#
+# AUTHOR
+#
+#   Steve Kemp <steve@bytemark.co.uk>
+#
+
+#
+#  Modules we require
+#
+
+require 'getoptlong'
+require 'tempfile'
+require 'symbiosis/utmp'
+require 'symbiosis/firewall/directory'
+require 'symbiosis/firewall/template'
+require 'symbiosis/firewall/ipaddr'
+require 'symbiosis/firewall/logtail'
+require 'symbiosis/firewall/pattern'
+
+opts = GetoptLong.new(
+    [ '--help',    '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ],
+    [ '--prefix',  '-p', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--template-d', '-t', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+
+#
+#  The options set by the command line.
+#
+$HELP    = false
+$VERBOSE = false
+$PREFIX = "/etc/symbiosis/firewall/"
+delete   = false
+execute  = false
+template_dir = nil
+force = true
+
+opts.each do |opt,arg|
+  case opt
+    when '--help'
+      $HELP = true
+    when '--verbose'
+      $VERBOSE = true
+    when '--prefix'
+      $PREFIX = arg
+    when '--template-d'
+      template_dir = arg
+  end
+end
+
+
+#
+# CAUTION! Here be quality kode.
+#
+if $HELP
+  # Open the file, stripping the shebang line
+  lines = File.open(__FILE__){|fh| fh.readlines}[2..-1]
+
+  lines.each do |line|
+    line.chomp!
+    break if line.empty?
+    puts line[2..-1].to_s
+  end
+
+  exit 0
+end
+
+
+#
+#  Expire old entries first of all, then add new ones.
+#
+puts "Expiring old blacklist entries" if ( $VERBOSE )
+
+expired = 0
+blacklist_d = File.join($PREFIX, "blacklist.d")
+
+# ensure the directory exists.
+if ( ! File.directory?( "#{blacklist_d}" ) )
+  system( "mkdir -p #{blacklist_d}" )
+end
+
+if ( File.directory?( blacklist_d ) )
+
+  Dir.foreach( blacklist_d ) do |entry|
+    if ( ( entry =~ /\.auto$/i ) &&
+         (File.mtime( "#{blacklist_d}/#{entry}" ) <  ( Time.now - 8 * 24 * 60 * 60 ) ) )
+    then
+      puts "Removing #{blacklist_d}/#{entry}" if ( $VERBOSE )
+      File.unlink("#{blacklist_d}/#{entry}")
+      expired += 1
+    end
+  end
+
+end
+
+puts "Expiring done - removed #{expired} file(s)" if ( $VERBOSE )
+
+#
+# Fetch the IP addresses
+#
+blacklist = Symbiosis::Blacklist.new
+blacklist.attempts = attempts
+blacklist.base_dir = $PREFIX
+
+whitelist = Symbiosis::IPListDirectory.new(File.join($PREFIX, "whitelist.d"))
+
+
+#
+#  Did we update?
+#
+updated=false
+
+
+#
+#  Iterate over each IP
+#
+blacklist.generate.each do |ip, ports|
+
+  puts "Found IP address: #{ip}" if ( $VERBOSE )
+
+  fn = File.join(blacklist_d,ip.to_s.gsub("/","-")+".auto" )
+
+  if ( File.exists?(fn) )
+    puts "\tAlready blacklisted" if ( $VERBOSE )
+  else
+    # create the file
+    system( "touch #{fn}" )
+
+    updated=true
+    puts "\tAdding to blacklist" if ( $VERBOSE )