diff --git a/app/controllers/analytics_controller.rb b/app/controllers/analytics_controller.rb index f59b02f..fe49fa8 100644 --- a/app/controllers/analytics_controller.rb +++ b/app/controllers/analytics_controller.rb @@ -126,11 +126,16 @@ class AnalyticsController < ApplicationController @start_time = calculate_start_time(@time_period) # Top networks by request volume (using denormalized network_range_id) - @top_networks = NetworkRange.joins("LEFT JOIN events ON events.network_range_id = network_ranges.id") - .where("events.timestamp >= ? OR events.timestamp IS NULL", @start_time) - .group("network_ranges.id") - .select("network_ranges.*, COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips") - .order("event_count DESC") + # Use a subquery approach to avoid PostgreSQL GROUP BY issues with network_ranges.* + event_stats = Event.where("timestamp >= ?", @start_time) + .where.not(network_range_id: nil) + .group(:network_range_id) + .select("network_range_id, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips") + + # Join the stats back to NetworkRange to get full network details + @top_networks = NetworkRange.joins("INNER JOIN (#{event_stats.to_sql}) stats ON stats.network_range_id = network_ranges.id") + .select("network_ranges.*, stats.event_count, stats.unique_ips") + .order("stats.event_count DESC") .limit(50) # Network type breakdown with traffic stats @@ -139,7 +144,7 @@ class AnalyticsController < ApplicationController # Company breakdown for top traffic sources (using denormalized company column) @top_companies = Event.where("timestamp >= ? AND company IS NOT NULL", @start_time) .group(:company) - .select("company, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips") + .select("company, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count") .order("event_count DESC") .limit(20) @@ -307,7 +312,8 @@ class AnalyticsController < ApplicationController # Query events directly using denormalized flags event_stats = Event.where("timestamp >= ? AND #{network_type[:column]} = ?", start_time, true) .select("COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count") - .first + .reorder(nil) + .take results[network_type[:type]] = { label: network_type[:label], @@ -321,7 +327,7 @@ class AnalyticsController < ApplicationController # Calculate standard networks (everything else) standard_stats = Event.where("timestamp >= ? AND is_datacenter = ? AND is_vpn = ? AND is_proxy = ?", start_time, false, false, false) .select("COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count") - .first + .take results['standard'] = { label: 'Standard', @@ -366,7 +372,7 @@ class AnalyticsController < ApplicationController .having("COUNT(*) >= 10") # minimum threshold patterns[:high_deny_rate] = { - count: high_deny_networks.count, + count: high_deny_networks.length, network_ids: high_deny_networks.map(&:network_range_id) } diff --git a/app/controllers/network_ranges_controller.rb b/app/controllers/network_ranges_controller.rb index 761a61f..83ca2c1 100644 --- a/app/controllers/network_ranges_controller.rb +++ b/app/controllers/network_ranges_controller.rb @@ -60,6 +60,10 @@ class NetworkRangesController < ApplicationController @parent_ranges = @network_range.parent_ranges.limit(10) @associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user).order(created_at: :desc) : [] + # Load rules from supernets and subnets + @supernet_rules = @network_range.persisted? ? @network_range.supernet_rules.includes(:network_range, :user).limit(10) : [] + @subnet_rules = @network_range.persisted? ? @network_range.child_rules.includes(:network_range, :user).limit(20) : [] + # Traffic analytics (if we have events) @traffic_stats = calculate_traffic_stats(@network_range) diff --git a/app/controllers/rules_controller.rb b/app/controllers/rules_controller.rb index 94382fa..ac111fe 100644 --- a/app/controllers/rules_controller.rb +++ b/app/controllers/rules_controller.rb @@ -203,7 +203,7 @@ def process_quick_create_parameters end # Handle redirect URL - if @rule.redirect? && params[:redirect_url].present? + if @rule.redirect_action? && params[:redirect_url].present? @rule.metadata ||= {} if @rule.metadata.is_a?(String) begin @@ -340,7 +340,7 @@ end end # Handle redirect URL - if @rule.redirect? && params[:redirect_url].present? + if @rule.redirect_action? && params[:redirect_url].present? @rule.metadata ||= {} if @rule.metadata.is_a?(String) begin diff --git a/app/jobs/process_waf_event_job.rb b/app/jobs/process_waf_event_job.rb index 79f17b4..dac56a3 100644 --- a/app/jobs/process_waf_event_job.rb +++ b/app/jobs/process_waf_event_job.rb @@ -69,11 +69,11 @@ class ProcessWafEventJob < ApplicationJob # Only runs when: network never evaluated OR policies changed since last evaluation if tracking_network.needs_policy_evaluation? policy_start = Time.current - result = WafPolicyMatcher.evaluate_and_mark!(tracking_network) + result = WafPolicyMatcher.evaluate_and_mark!(event) Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms" if result[:generated_rules].any? - Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{tracking_network.cidr}" + Rails.logger.info "Generated #{result[:generated_rules].length} rules for event #{event.id} (network: #{tracking_network.cidr})" end end diff --git a/app/jobs/process_waf_policies_job.rb b/app/jobs/process_waf_policies_job.rb index 999566b..65ddd52 100644 --- a/app/jobs/process_waf_policies_job.rb +++ b/app/jobs/process_waf_policies_job.rb @@ -37,7 +37,7 @@ class ProcessWafPoliciesJob < ApplicationJob Rails.logger.info "Generated #{result[:generated_rules].length} rules for network range #{network_range.cidr}" result[:generated_rules].each do |rule| - Rails.logger.info " - Rule: #{rule.rule_type} #{rule.action} for #{rule.network_range&.cidr} (ID: #{rule.id})" + Rails.logger.info " - Rule: #{rule.waf_rule_type} #{rule.waf_action} for #{rule.network_range&.cidr} (ID: #{rule.id})" # Log if this is a redirect or challenge rule if rule.redirect_action? diff --git a/app/models/event.rb b/app/models/event.rb index eb00d5a..1c4f1fc 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -126,6 +126,37 @@ class Event < ApplicationRecord where("json_array_length(request_segment_ids) > ?", depth) } + # Analytics: Get response time percentiles over different time windows + def self.response_time_percentiles(windows: { hour: 1.hour, day: 1.day, week: 1.week }) + results = {} + + windows.each do |label, duration| + scope = where('timestamp >= ?', duration.ago) + + stats = scope.pick( + Arel.sql(<<~SQL.squish) + percentile_cont(0.5) WITHIN GROUP (ORDER BY response_time_ms) as p50, + percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms) as p95, + percentile_cont(0.99) WITHIN GROUP (ORDER BY response_time_ms) as p99, + COUNT(*) as count + SQL + ) + + results[label] = if stats + { + p50: stats[0]&.round(2), + p95: stats[1]&.round(2), + p99: stats[2]&.round(2), + count: stats[3] + } + else + { p50: nil, p95: nil, p99: nil, count: 0 } + end + end + + results + end + # Helper methods def path_depth request_segment_ids&.length || 0 @@ -455,7 +486,7 @@ class Event < ApplicationRecord end def active_blocking_rules - matching_rules.where(action: 'deny') + matching_rules.where(waf_action: :deny) end def has_blocking_rules? @@ -502,6 +533,121 @@ class Event < ApplicationRecord Rails.logger.error "Failed to normalize event #{id}: #{e.message}" end + def should_populate_network_intelligence? + # Only populate if IP is present and country is not yet set + # Also repopulate if IP address changed (rare case) + ip_address.present? && (country.blank? || ip_address_changed?) + end + + def populate_network_intelligence + return unless ip_address.present? + + # Convert IPAddr to string for PostgreSQL query + ip_string = ip_address.to_s + + # CRITICAL: Always find_or_create /24 tracking network for public IPs + # This /24 serves as: + # 1. The tracking unit for IPAPI deduplication (stores ipapi_queried_at) + # 2. The reference point for preventing duplicate API calls + # 3. The fallback network if no more specific GeoIP data exists + tracking_network = find_or_create_tracking_network(ip_string) + + # Find most specific network range with actual GeoIP data + # This might be more specific (e.g., /25) or broader (e.g., /22) than the /24 + data_range = NetworkRange.where("network >>= ?", ip_string) + .where.not(country: nil) # Must have actual data + .order(Arel.sql("masklen(network) DESC")) + .first + + # Use the most specific range with data, or fall back to tracking network + range = data_range || tracking_network + + if range + # Populate all network intelligence fields from the range + self.country = range.country + self.company = range.company + self.asn = range.asn + self.asn_org = range.asn_org + self.is_datacenter = range.is_datacenter || false + self.is_vpn = range.is_vpn || false + self.is_proxy = range.is_proxy || false + else + # No range at all (shouldn't happen, but defensive) + self.is_datacenter = false + self.is_vpn = false + self.is_proxy = false + end + + # ALWAYS set network_range_id to the tracking /24 + # This is what FetchIpapiDataJob uses to check ipapi_queried_at + # and prevent duplicate API calls + self.network_range_id = tracking_network&.id + rescue => e + Rails.logger.error "Failed to populate network intelligence for event #{id}: #{e.message}" + # Set defaults on error to prevent null values + self.is_datacenter = false + self.is_vpn = false + self.is_proxy = false + end + + # Find or create the /24 tracking network for this IP + # This is the fundamental unit for IPAPI deduplication + def find_or_create_tracking_network(ip_string) + return nil if private_or_reserved_ip?(ip_string) + + ip_addr = IPAddr.new(ip_string) + + # Calculate /24 for IPv4, /64 for IPv6 + if ip_addr.ipv4? + prefix_length = 24 + mask = (2**32 - 1) ^ ((2**(32 - prefix_length)) - 1) + network_int = ip_addr.to_i & mask + network_base = IPAddr.new(network_int, Socket::AF_INET) + network_cidr = "#{network_base}/#{prefix_length}" # e.g., "1.2.3.0/24" + else + prefix_length = 64 + mask = (2**128 - 1) ^ ((2**(128 - prefix_length)) - 1) + network_int = ip_addr.to_i & mask + network_base = IPAddr.new(network_int, Socket::AF_INET6) + network_cidr = "#{network_base}/#{prefix_length}" # e.g., "2001:db8::/64" + end + + # Find or create the tracking network + NetworkRange.find_or_create_by!(network: network_cidr) do |nr| + nr.source = 'auto_generated' + nr.creation_reason = 'tracking unit for IPAPI deduplication' + nr.is_datacenter = NetworkRangeGenerator.datacenter_ip?(ip_addr) rescue false + nr.is_vpn = false + nr.is_proxy = false + end + rescue => e + Rails.logger.error "Failed to create tracking network for IP #{ip_string}: #{e.message}" + nil + end + + # Check if IP is private or reserved (should not create network ranges) + def private_or_reserved_ip?(ip_string = nil) + ip_str = ip_string || ip_address.to_s + ip = IPAddr.new(ip_str) + + # Private and reserved ranges + [ + IPAddr.new('10.0.0.0/8'), + IPAddr.new('172.16.0.0/12'), + IPAddr.new('192.168.0.0/16'), + IPAddr.new('127.0.0.0/8'), + IPAddr.new('169.254.0.0/16'), + IPAddr.new('224.0.0.0/4'), + IPAddr.new('240.0.0.0/4'), + IPAddr.new('::1/128'), + IPAddr.new('fc00::/7'), + IPAddr.new('fe80::/10'), + IPAddr.new('ff00::/8') + ].any? { |range| range.include?(ip) } + rescue IPAddr::InvalidAddressError + true # Treat invalid IPs as "reserved" + end + def extract_fields_from_payload return unless payload.present? diff --git a/app/models/network_range.rb b/app/models/network_range.rb index 6641475..0cf96b5 100644 --- a/app/models/network_range.rb +++ b/app/models/network_range.rb @@ -392,13 +392,52 @@ class NetworkRange < ApplicationRecord end def blocking_rules - rules.where(action: 'deny', enabled: true) + rules.where(waf_action: :deny, enabled: true) end def active_rules rules.enabled.where("expires_at IS NULL OR expires_at > ?", Time.current) end + # Find all network ranges that are contained by this network and have enabled rules + # Used when creating a supernet rule to identify redundant child rules + def child_network_ranges_with_rules + NetworkRange + .where("network << ?::inet", network.to_s) # network is strictly contained by this network + .joins(:rules) + .where(rules: { enabled: true }) + .distinct + end + + # Find all enabled rules on child network ranges (more specific networks) + # Used after creating a rule to expire redundant child rules + def child_rules + Rule + .joins(:network_range) + .where("network_ranges.network << ?::inet", cidr) + .where(enabled: true) + end + + # Find all network ranges that contain this network and have enabled rules + # Used to check if creating a rule would be redundant + def parent_network_ranges_with_rules + NetworkRange + .where("?::inet << network", cidr) # this network is strictly contained by parent + .joins(:rules) + .where(rules: { enabled: true }) + .distinct + end + + # Find all enabled rules on parent network ranges (less specific networks) + # Used before creating a rule to check if it would be redundant + def supernet_rules + Rule + .joins(:network_range) + .where("?::inet << network_ranges.network", cidr) + .where(enabled: true) + .order("masklen(network_ranges.network) DESC") # Most specific supernet first + end + # Check if this network range needs WAF policy evaluation # Returns true if: # - Never been evaluated, OR diff --git a/app/models/rule.rb b/app/models/rule.rb index 64c804d..b4bcbe1 100644 --- a/app/models/rule.rb +++ b/app/models/rule.rb @@ -5,9 +5,9 @@ # Rules define actions to take for matching traffic conditions. # Network rules are associated with NetworkRange objects for rich context. class Rule < ApplicationRecord - # Rule enums - enum :waf_action, { allow: 0, deny: 1, rate_limit: 2, redirect: 3, log: 4, challenge: 5 }, scopes: false, prefix: true - enum :waf_rule_type, { network: 0, rate_limit: 1, path_pattern: 2 }, scopes: false, prefix: true + # Rule enums (prefix needed to avoid rate_limit collision) + enum :waf_action, { allow: 0, deny: 1, rate_limit: 2, redirect: 3, log: 4, challenge: 5 }, prefix: :action + enum :waf_rule_type, { network: 0, rate_limit: 1, path_pattern: 2 }, prefix: :type # Legacy string constants for backward compatibility RULE_TYPES = %w[network rate_limit path_pattern].freeze @@ -20,25 +20,6 @@ class Rule < ApplicationRecord belongs_to :waf_policy, optional: true has_many :events, dependent: :nullify - # Backward compatibility accessors for transition period - def action - waf_action - end - - def action=(value) - self.waf_action = value - self[:action] = value # Also set the legacy column - end - - def rule_type - waf_rule_type - end - - def rule_type=(value) - self.waf_rule_type = value - self[:rule_type] = value # Also set the legacy column - end - # Validations validates :waf_rule_type, presence: true, inclusion: { in: waf_rule_types.keys } validates :waf_action, presence: true, inclusion: { in: waf_actions.keys } @@ -59,6 +40,7 @@ class Rule < ApplicationRecord validate :validate_metadata_by_action validate :network_range_required_for_network_rules validate :validate_network_consistency, if: :network_rule? + validate :no_supernet_rule_exists, if: :should_check_supernet? # Scopes scope :enabled, -> { where(enabled: true) } @@ -66,20 +48,18 @@ class Rule < ApplicationRecord scope :active, -> { enabled.where("expires_at IS NULL OR expires_at > ?", Time.current) } scope :expired, -> { where("expires_at IS NOT NULL AND expires_at <= ?", Time.current) } scope :by_type, ->(type) { where(waf_rule_type: type) } - scope :network_rules, -> { network } - scope :rate_limit_rules, -> { rate_limit } - scope :path_pattern_rules, -> { path_pattern } + scope :network_rules, -> { where(waf_rule_type: :network) } + scope :rate_limit_rules, -> { where(waf_rule_type: :rate_limit) } + scope :path_pattern_rules, -> { where(waf_rule_type: :path_pattern) } scope :by_source, ->(source) { where(source: source) } scope :surgical_blocks, -> { where(source: "manual:surgical_block") } scope :surgical_exceptions, -> { where(source: "manual:surgical_exception") } scope :policy_generated, -> { where(source: "policy") } scope :from_waf_policy, ->(waf_policy) { where(waf_policy: waf_policy) } - # Legacy scopes for backward compatibility - scope :by_type_legacy, ->(type) { where(rule_type: type) } - scope :network_rules_legacy, -> { where(rule_type: "network") } - scope :rate_limit_rules_legacy, -> { where(rule_type: "rate_limit") } - scope :path_pattern_rules_legacy, -> { where(rule_type: "path_pattern") } + # Action scopes (manual to avoid enum collision with rate_limit) + scope :deny, -> { where(waf_action: :deny) } + scope :allow, -> { where(waf_action: :allow) } # Sync queries scope :since, ->(timestamp) { where("updated_at >= ?", Time.at(timestamp)).order(:updated_at, :id) } @@ -89,19 +69,19 @@ class Rule < ApplicationRecord before_validation :set_defaults before_validation :parse_json_fields before_save :calculate_priority_for_network_rules - before_save :sync_legacy_columns + after_create :expire_redundant_child_rules, if: :should_expire_child_rules? # Rule type checks def network_rule? - waf_rule_type_network? + type_network? end def rate_limit_rule? - waf_rule_type_rate_limit? + type_rate_limit? end def path_pattern_rule? - waf_rule_type_path_pattern? + type_path_pattern? end # Network-specific methods @@ -143,11 +123,11 @@ class Rule < ApplicationRecord # Action-specific methods def redirect_action? - waf_action_redirect? + action_redirect? end def challenge_action? - waf_action_challenge? + action_challenge? end # Redirect/challenge convenience methods @@ -509,14 +489,52 @@ class Rule < ApplicationRecord self.metadata ||= {} end - def sync_legacy_columns - # Sync enum values to legacy string columns for backward compatibility - if waf_action.present? - self[:action] = waf_action - end - if waf_rule_type.present? - self[:rule_type] = waf_rule_type + # Supernet/subnet redundancy checking + def should_check_supernet? + network_rule? && network_range.present? && new_record? + end + + def no_supernet_rule_exists + return unless network_range + + supernet_rule = network_range.supernet_rules.first + if supernet_rule + errors.add( + :base, + "A supernet rule already covers this network. " \ + "Rule ##{supernet_rule.id} for #{supernet_rule.network_range.cidr} " \ + "(action: #{supernet_rule.waf_action}) makes this rule redundant." + ) end end - end \ No newline at end of file + def should_expire_child_rules? + network_rule? && network_range.present? && enabled? + end + + def expire_redundant_child_rules + return unless network_range + + child_rules = network_range.child_rules + return if child_rules.empty? + + expired_count = 0 + child_rules.find_each do |child_rule| + # Disable the child rule and mark it as redundant + child_rule.update!( + enabled: false, + metadata: child_rule.metadata_hash.merge( + disabled_at: Time.current.iso8601, + disabled_reason: "Redundant - covered by supernet rule ##{id} (#{network_range.cidr})", + superseded_by_rule_id: id + ) + ) + expired_count += 1 + end + + if expired_count > 0 + Rails.logger.info "Rule ##{id}: Expired #{expired_count} redundant child rule(s) for #{network_range.cidr}" + end + end + +end \ No newline at end of file diff --git a/app/models/waf_policy.rb b/app/models/waf_policy.rb index 46fc5a0..4cd0dec 100644 --- a/app/models/waf_policy.rb +++ b/app/models/waf_policy.rb @@ -18,13 +18,13 @@ class WafPolicy < ApplicationRecord # Validations validates :name, presence: true, uniqueness: true validates :policy_type, presence: true, inclusion: { in: POLICY_TYPES } - validates :action, presence: true, inclusion: { in: ACTIONS } + validates :policy_action, presence: true, inclusion: { in: ACTIONS } validates :targets, presence: true validate :targets_must_be_array validates :user, presence: true validate :validate_targets_by_type - validate :validate_redirect_configuration, if: :redirect_action? - validate :validate_challenge_configuration, if: :challenge_action? + validate :validate_redirect_configuration, if: :redirect_policy_action? + validate :validate_challenge_configuration, if: :challenge_policy_action? # Scopes scope :enabled, -> { where(enabled: true) } @@ -59,19 +59,36 @@ validate :targets_must_be_array # Action methods def allow_action? - action == 'allow' + policy_action == 'allow' end def deny_action? - action == 'deny' + policy_action == 'deny' end def redirect_action? - action == 'redirect' + policy_action == 'redirect' end def challenge_action? - action == 'challenge' + policy_action == 'challenge' + end + + # Policy action methods (to avoid confusion with Rails' action methods) + def allow_policy_action? + policy_action == 'allow' + end + + def deny_policy_action? + policy_action == 'deny' + end + + def redirect_policy_action? + policy_action == 'redirect' + end + + def challenge_policy_action? + policy_action == 'challenge' end # Lifecycle methods @@ -118,7 +135,7 @@ validate :targets_must_be_array rule = Rule.create!( rule_type: 'network', - action: action, + action: policy_action, network_range: network_range, waf_policy: self, user: user, @@ -148,45 +165,45 @@ validate :targets_must_be_array end # Class methods for creating common policies - def self.create_country_policy(countries, action: 'deny', user:, **options) + def self.create_country_policy(countries, policy_action: 'deny', user:, **options) create!( - name: "#{action.capitalize} #{countries.join(', ')}", + name: "#{policy_action.capitalize} #{countries.join(', ')}", policy_type: 'country', targets: Array(countries), - action: action, + policy_action: policy_action, user: user, **options ) end - def self.create_asn_policy(asns, action: 'deny', user:, **options) + def self.create_asn_policy(asns, policy_action: 'deny', user:, **options) create!( - name: "#{action.capitalize} ASNs #{asns.join(', ')}", + name: "#{policy_action.capitalize} ASNs #{asns.join(', ')}", policy_type: 'asn', targets: Array(asns).map(&:to_i), - action: action, + policy_action: policy_action, user: user, **options ) end - def self.create_company_policy(companies, action: 'deny', user:, **options) + def self.create_company_policy(companies, policy_action: 'deny', user:, **options) create!( - name: "#{action.capitalize} #{companies.join(', ')}", + name: "#{policy_action.capitalize} #{companies.join(', ')}", policy_type: 'company', targets: Array(companies), - action: action, + policy_action: policy_action, user: user, **options ) end - def self.create_network_type_policy(types, action: 'deny', user:, **options) + def self.create_network_type_policy(types, policy_action: 'deny', user:, **options) create!( - name: "#{action.capitalize} #{types.join(', ')}", + name: "#{policy_action.capitalize} #{types.join(', ')}", policy_type: 'network_type', targets: Array(types), - action: action, + policy_action: policy_action, user: user, **options ) @@ -226,7 +243,7 @@ validate :targets_must_be_array active_rules: active_rules_count, rules_last_7_days: recent_rules.count, policy_type: policy_type, - action: action, + policy_action: policy_action, targets_count: targets&.length || 0 } end diff --git a/app/policies/waf_policy_policy.rb b/app/policies/waf_policy_policy.rb index 1ace306..63ca1f8 100644 --- a/app/policies/waf_policy_policy.rb +++ b/app/policies/waf_policy_policy.rb @@ -2,39 +2,39 @@ class WafPolicyPolicy < ApplicationPolicy def index? - true # All authenticated users can view policies + !user.viewer? # All authenticated users except viewers can view policies end def show? - true # All authenticated users can view policy details + !user.viewer? # All authenticated users except viewers can view policy details end def new? - user.admin? || user.editor? + !user.viewer? # All authenticated users except viewers can create policies end def create? - user.admin? || user.editor? + !user.viewer? # All authenticated users except viewers can create policies end def edit? - user.admin? || (user.editor? && record.user == user) + !user.viewer? # All authenticated users except viewers can edit policies end def update? - user.admin? || (user.editor? && record.user == user) + !user.viewer? # All authenticated users except viewers can update policies end def destroy? - user.admin? || (user.editor? && record.user == user) + !user.viewer? # All authenticated users except viewers can destroy policies end def activate? - user.admin? || (user.editor? && record.user == user) + !user.viewer? # All authenticated users except viewers can activate policies end def deactivate? - user.admin? || (user.editor? && record.user == user) + !user.viewer? # All authenticated users except viewers can deactivate policies end def new_country? @@ -45,14 +45,38 @@ class WafPolicyPolicy < ApplicationPolicy create? end + # ASN policy permissions + def new_asn? + create? + end + + def create_asn? + create? + end + + # Company policy permissions + def new_company? + create? + end + + def create_company? + create? + end + + # Network type policy permissions + def new_network_type? + create? + end + + def create_network_type? + create? + end + class Scope < ApplicationPolicy::Scope def resolve - if user.admin? - scope.all - else - # Non-admin users can only see their own policies - scope.where(user: user) - end + # All authenticated users except viewers can view all policies + # since WAF policies are system-wide security rules + scope.all end end end \ No newline at end of file diff --git a/app/services/geolite_asn_importer.rb b/app/services/geolite_asn_importer.rb index 28593f4..31dfcc8 100644 --- a/app/services/geolite_asn_importer.rb +++ b/app/services/geolite_asn_importer.rb @@ -139,23 +139,37 @@ class GeoliteAsnImporter IPAddr.new(network) # This will raise if invalid # Store raw GeoLite ASN data in network_data - geolite_data = { + geolite_asn_data = { asn: { autonomous_system_number: asn, autonomous_system_organization: asn_org } } + # Use upsert with JSONB merge + # COALESCE handles the case where network_data might be NULL + # || is PostgreSQL's JSONB concatenation/merge operator + # jsonb_set merges the nested geolite data NetworkRange.upsert( { network: network, asn: asn, asn_org: asn_org, source: 'geolite_asn', - network_data: { geolite: geolite_data }, + network_data: { geolite: geolite_asn_data }, updated_at: Time.current }, - unique_by: :index_network_ranges_on_network_unique + unique_by: :index_network_ranges_on_network_unique, + on_duplicate: Arel.sql(" + asn = EXCLUDED.asn, + asn_org = EXCLUDED.asn_org, + network_data = COALESCE(network_ranges.network_data, '{}'::jsonb) || + jsonb_build_object('geolite', + COALESCE(network_ranges.network_data->'geolite', '{}'::jsonb) || + EXCLUDED.network_data->'geolite' + ), + updated_at = EXCLUDED.updated_at + ") ) end diff --git a/app/services/geolite_country_importer.rb b/app/services/geolite_country_importer.rb index 8107404..18c3368 100644 --- a/app/services/geolite_country_importer.rb +++ b/app/services/geolite_country_importer.rb @@ -211,7 +211,7 @@ class GeoliteCountryImporter location_data = @locations_cache[geoname_id] || @locations_cache[registered_country_geoname_id] || {} # Store raw GeoLite country data in network_data[:geolite] - geolite_data = { + geolite_country_data = { country: { geoname_id: geoname_id, registered_country_geoname_id: registered_country_geoname_id, @@ -227,16 +227,29 @@ class GeoliteCountryImporter } }.compact + # Use upsert with JSONB merge + # COALESCE handles the case where network_data might be NULL + # || is PostgreSQL's JSONB concatenation/merge operator NetworkRange.upsert( { network: network, country: location_data[:country_iso_code], is_proxy: is_anonymous_proxy, source: 'geolite_country', - network_data: { geolite: geolite_data }, + network_data: { geolite: geolite_country_data }, updated_at: Time.current }, - unique_by: :index_network_ranges_on_network_unique + unique_by: :index_network_ranges_on_network_unique, + on_duplicate: Arel.sql(" + country = EXCLUDED.country, + is_proxy = EXCLUDED.is_proxy, + network_data = COALESCE(network_ranges.network_data, '{}'::jsonb) || + jsonb_build_object('geolite', + COALESCE(network_ranges.network_data->'geolite', '{}'::jsonb) || + EXCLUDED.network_data->'geolite' + ), + updated_at = EXCLUDED.updated_at + ") ) end diff --git a/app/services/ip_range_resolver.rb b/app/services/ip_range_resolver.rb index f949b81..2159621 100644 --- a/app/services/ip_range_resolver.rb +++ b/app/services/ip_range_resolver.rb @@ -143,7 +143,7 @@ class IpRangeResolver Rule.network_rules .where(network_range_id: range_ids) - .where(action: 'deny') + .where(waf_action: :deny) .enabled .where("expires_at IS NULL OR expires_at > ?", Time.current) .exists? @@ -158,7 +158,7 @@ class IpRangeResolver Rule.network_rules .where(network_range_id: range_ids) - .where(action: 'deny') + .where(waf_action: :deny) .enabled .where("expires_at IS NULL OR expires_at > ?", Time.current) .includes(:network_range) diff --git a/app/services/network_range_generator.rb b/app/services/network_range_generator.rb index b52e9de..87fcc2b 100644 --- a/app/services/network_range_generator.rb +++ b/app/services/network_range_generator.rb @@ -24,22 +24,6 @@ class NetworkRangeGenerator IPAddr.new('ff00::/8') # IPv6 multicast ].freeze - # Special network ranges to avoid - RESERVED_RANGES = [ - IPAddr.new('10.0.0.0/8'), # Private - IPAddr.new('172.16.0.0/12'), # Private - IPAddr.new('192.168.0.0/16'), # Private - IPAddr.new('127.0.0.0/8'), # Loopback - IPAddr.new('169.254.0.0/16'), # Link-local - IPAddr.new('224.0.0.0/4'), # Multicast - IPAddr.new('240.0.0.0/4'), # Reserved - IPAddr.new('::1/128'), # IPv6 loopback - IPAddr.new('fc00::/7'), # IPv6 private - IPAddr.new('fe80::/10'), # IPv6 link-local - IPAddr.new('ff00::/8') # IPv6 multicast - ].freeze - - class << self # Find or create a network range for the given IP address def find_or_create_for_ip(ip_address, user: nil) diff --git a/app/services/waf_policy_matcher.rb b/app/services/waf_policy_matcher.rb index 9e219e3..f852c81 100644 --- a/app/services/waf_policy_matcher.rb +++ b/app/services/waf_policy_matcher.rb @@ -1,28 +1,33 @@ # frozen_string_literal: true -# WafPolicyMatcher - Service to match NetworkRanges against active WafPolicies +# WafPolicyMatcher - Service to match Events against active WafPolicies # -# This service provides efficient matching of network ranges against firewall policies -# and can generate rules when matches are found. +# This service provides efficient matching of events against firewall policies +# (both network-based and path-based) and can generate rules when matches are found. class WafPolicyMatcher include ActiveModel::Model include ActiveModel::Attributes - attr_accessor :network_range + attr_accessor :event attr_reader :matching_policies, :generated_rules - def initialize(network_range:) - @network_range = network_range + def initialize(event:) + @event = event @matching_policies = [] @generated_rules = [] end - # Find all active policies that match the given network range + # Helper method to get network range from event + def network_range + event&.network_range + end + + # Find all active policies that match the given event (network or path-based) def find_matching_policies - return [] unless network_range.present? + return [] unless event.present? @matching_policies = active_policies.select do |policy| - policy.matches_network_range?(network_range) + policy.matches_event?(event) end # Sort by priority: country > asn > company > network_type, then by creation date @@ -82,25 +87,56 @@ class WafPolicyMatcher end # Class methods for batch processing - def self.process_network_range(network_range) - matcher = new(network_range: network_range) + def self.process_event(event) + matcher = new(event: event) matcher.match_and_generate_rules end - # Evaluate a network range against policies and mark it as evaluated - # This is the main entry point for inline policy evaluation - def self.evaluate_and_mark!(network_range) - return { matching_policies: [], generated_rules: [] } unless network_range + # Legacy method for backward compatibility - converts network range to event + def self.process_network_range(network_range) + # Find the most recent event for this network range + sample_event = network_range.events.order(created_at: :desc).first + if sample_event + process_event(sample_event) + else + # No events exist for this network range, return empty results + # Network-based policies need real events to trigger rule creation + { matching_policies: [], generated_rules: [] } + end + end - matcher = new(network_range: network_range) + # Evaluate an event against policies and mark its network range as evaluated + # This is the main entry point for inline policy evaluation + def self.evaluate_and_mark!(event) + return { matching_policies: [], generated_rules: [] } unless event + + matcher = new(event: event) result = matcher.match_and_generate_rules - # Mark this network range as evaluated - network_range.update_column(:policies_evaluated_at, Time.current) + # Mark the event's network range as evaluated + if event.network_range + event.network_range.update_column(:policies_evaluated_at, Time.current) + end result end + # Legacy method for backward compatibility + def self.evaluate_and_mark_network_range!(network_range) + return { matching_policies: [], generated_rules: [] } unless network_range + + # Find the most recent event for this network range + sample_event = network_range.events.order(created_at: :desc).first + if sample_event + evaluate_and_mark!(sample_event) + else + # No events exist, use the old network-range based evaluation + process_network_range(network_range) + network_range.update_column(:policies_evaluated_at, Time.current) + { matching_policies: [], generated_rules: [] } + end + end + def self.batch_process_network_ranges(network_ranges) results = [] @@ -158,8 +194,19 @@ class WafPolicyMatcher potential_ranges.find_each do |network_range| matcher = new(network_range: network_range) if waf_policy.matches_network_range?(network_range) + # Check for supernet rules before creating + if network_range.supernet_rules.any? + supernet = network_range.supernet_rules.first + Rails.logger.info "Skipping rule for #{network_range.cidr} - covered by supernet rule ##{supernet.id}" + next + end + rule = waf_policy.create_rule_for_network_range(network_range) - results << { network_range: network_range, generated_rule: rule } if rule + if rule&.persisted? + results << { network_range: network_range, generated_rule: rule } + elsif rule + Rails.logger.warn "Failed to create rule for #{network_range.cidr}: #{rule.errors.full_messages.join(', ')}" + end end end @@ -180,7 +227,7 @@ class WafPolicyMatcher { policy_name: waf_policy.name, policy_type: waf_policy.policy_type, - action: waf_policy.action, + action: waf_policy.policy_action, rules_generated: rules.count, active_rules: rules.active.count, networks_protected: rules.joins(:network_range).count('distinct network_ranges.id'), diff --git a/app/views/events/index.html.erb b/app/views/events/index.html.erb index 00ceddc..7f3650b 100644 --- a/app/views/events/index.html.erb +++ b/app/views/events/index.html.erb @@ -133,7 +133,7 @@ - <% network_range = @network_ranges_by_ip[event.ip_address.to_s] %> + <% network_range = event.network_range %> <% if network_range %> <%= link_to event.ip_address, network_range_path(event.ip_address), class: "text-blue-600 hover:text-blue-800 hover:underline font-mono" %> diff --git a/app/views/events/show.html.erb b/app/views/events/show.html.erb index da16125..b950ac4 100644 --- a/app/views/events/show.html.erb +++ b/app/views/events/show.html.erb @@ -1,4 +1,4 @@ -<% content_for :title, "Event #{@event.event_id} - Baffle Hub" %> +<% content_for :title, "Event #{@event.request_id} - Baffle Hub" %>
@@ -15,7 +15,7 @@ - <%= @event.event_id %> + <%= @event.request_id %>
@@ -48,8 +48,8 @@
-
Event ID
-
<%= @event.event_id %>
+
Request ID
+
<%= @event.request_id %>
Timestamp
@@ -77,10 +77,17 @@
- <% if @event.rule_matched.present? %> + <% if @event.rule.present? %>
Rule Matched
-
<%= @event.rule_matched %>
+
+ <%= link_to "Rule ##{@event.rule.id}", @event.rule, class: "text-blue-600 hover:text-blue-800" %> + (<%= @event.rule.waf_rule_type %>) + <% if @event.waf_policy.present? %> +
+ Policy: <%= link_to @event.waf_policy.name, @event.waf_policy, class: "text-blue-600 hover:text-blue-800" %> + <% end %> +
<% end %> <% if @event.blocked_reason.present? %> diff --git a/app/views/network_ranges/show.html.erb b/app/views/network_ranges/show.html.erb index 7345f94..dddfc98 100644 --- a/app/views/network_ranges/show.html.erb +++ b/app/views/network_ranges/show.html.erb @@ -251,6 +251,39 @@
<% end %> + + <% if @network_range.persisted? && @network_range.agent_tally.any? %> +
+

Top User Agents

+
+ <% @network_range.agent_tally.sort_by { |ua, count| -count }.first(5).each do |user_agent, count| %> +
+ + <% if user_agent.present? %> + <% ua = parse_user_agent(user_agent) %> + <% if ua[:name].present? %> + <%= ua[:name] %> + <% if ua[:version].present? %> + (<%= ua[:version] %>) + <% end %> + <% if ua[:bot] %> + + 🤖 <%= ua[:bot_name] || 'Bot' %> + + <% end %> + <% else %> + <%= truncate(user_agent, length: 50) %> + <% end %> + <% else %> + Unknown + <% end %> + + <%= count %> +
+ <% end %> +
+
+ <% end %> <% end %> @@ -312,15 +345,12 @@
- <%= form.label :rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :rule_type, + <%= form.label :waf_rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_rule_type, options_for_select([ ['Network - IP/CIDR based blocking', 'network'], ['Rate Limit - Request rate limiting', 'rate_limit'], - ['Path Pattern - URL path filtering', 'path_pattern'], - ['Header Pattern - HTTP header filtering', 'header_pattern'], - ['Query Pattern - Query parameter filtering', 'query_pattern'], - ['Body Signature - Request body filtering', 'body_signature'] + ['Path Pattern - URL path filtering', 'path_pattern'] ], 'network'), { }, { @@ -333,15 +363,15 @@
- <%= form.label :action, "Action", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :action, + <%= form.label :waf_action, "Action", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_action, options_for_select([ ['Deny - Block requests', 'deny'], ['Allow - Whitelist requests', 'allow'], ['Rate Limit - Throttle requests', 'rate_limit'], ['Redirect - Redirect to URL', 'redirect'], ['Challenge - Present CAPTCHA', 'challenge'], - ['Monitor - Log but allow', 'monitor'] + ['Log - Log but allow', 'log'] ], 'deny'), { }, { @@ -468,13 +498,13 @@
- <%= rule.action.upcase %> <%= rule.cidr %> + <%= rule.waf_action.upcase %> <%= rule.cidr %> Priority: <%= rule.priority %> - <%= rule.rule_type.humanize %> + <%= rule.waf_rule_type.humanize %> <% if rule.source.include?('surgical') %> @@ -523,51 +553,109 @@
- <% if @parent_ranges.any? %> + <% if @parent_ranges.any? || @supernet_rules.any? %>
-

Parent Network Ranges

+

+ Supernet Ranges + <% if @supernet_rules.any? %> + + <%= @supernet_rules.count %> <%= 'rule'.pluralize(@supernet_rules.count) %> + + <% end %> +

+

Broader networks that contain this range

<% @parent_ranges.each do |parent| %>
-
+
<%= link_to parent.cidr, network_range_path(parent), class: "text-sm font-medium text-gray-900 hover:text-blue-600" %>
Prefix: /<%= parent.prefix_length %> | <% if parent.company.present? %><%= parent.company %> | <% end %> <%= parent.source %>
+ <%# Show rules for this parent %> + <% parent_rules = @supernet_rules.select { |r| r.network_range_id == parent.id } %> + <% if parent_rules.any? %> +
+ <% parent_rules.each do |rule| %> + <%= render 'rules/compact_rule', rule: rule %> + <% end %> +
+ <% end %>
<% end %> + <%# Show supernet rules that don't have a parent range loaded %> + <% orphan_supernet_rules = @supernet_rules.reject { |r| @parent_ranges.map(&:id).include?(r.network_range_id) } %> + <% if orphan_supernet_rules.any? %> +
+
Additional Supernet Rules
+
+ <% orphan_supernet_rules.each do |rule| %> + <%= render 'rules/compact_rule', rule: rule %> + <% end %> +
+
+ <% end %>
<% end %> - <% if @child_ranges.any? %> + <% if @child_ranges.any? || @subnet_rules.any? %>
-

Child Network Ranges

+

+ Subnet Ranges + <% if @subnet_rules.any? %> + + <%= @subnet_rules.count %> <%= 'rule'.pluralize(@subnet_rules.count) %> + + <% end %> +

+

More specific networks within this range

<% @child_ranges.each do |child| %>
-
+
<%= link_to child.cidr, network_range_path(child), class: "text-sm font-medium text-gray-900 hover:text-blue-600" %>
Prefix: /<%= child.prefix_length %> | <% if child.company.present? %><%= child.company %> | <% end %> <%= child.source %>
+ <%# Show rules for this child %> + <% child_rules = @subnet_rules.select { |r| r.network_range_id == child.id } %> + <% if child_rules.any? %> +
+ <% child_rules.each do |rule| %> + <%= render 'rules/compact_rule', rule: rule %> + <% end %> +
+ <% end %>
<% end %> + <%# Show subnet rules that don't have a child range loaded %> + <% orphan_subnet_rules = @subnet_rules.reject { |r| @child_ranges.map(&:id).include?(r.network_range_id) } %> + <% if orphan_subnet_rules.any? %> +
+
Additional Subnet Rules
+
+ <% orphan_subnet_rules.each do |rule| %> + <%= render 'rules/compact_rule', rule: rule %> + <% end %> +
+
+ <% end %>
<% end %> @@ -577,7 +665,20 @@ <% if @related_events.any? %>
-

Recent Events (<%= @related_events.count %>)

+
+

Recent Events (<%= number_with_delimiter(@events_pagy.count) %>)

+ <% if @events_pagy.pages > 1 %> + + Page <%= @events_pagy.page %> of <%= @events_pagy.pages %> + + <% end %> +
+ + <% if @events_pagy.pages > 1 %> +
+ <%= pagy_nav_tailwind(@events_pagy, pagy_id: 'network_events_top') %> +
+ <% end %>
@@ -591,7 +692,7 @@ - <% @related_events.first(20).each do |event| %> + <% @related_events.each do |event| %>
@@ -649,6 +750,11 @@
+ + + <% if @events_pagy.pages > 1 %> + <%= pagy_nav_tailwind(@events_pagy, pagy_id: 'network_events_bottom') %> + <% end %>
<% end %>
diff --git a/app/views/rules/edit.html.erb b/app/views/rules/edit.html.erb index 1391903..ae70a87 100644 --- a/app/views/rules/edit.html.erb +++ b/app/views/rules/edit.html.erb @@ -44,9 +44,9 @@
- <%= form.label :rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :rule_type, - options_for_select(@rule_types.map { |type| [type.humanize, type] }, @rule.rule_type), + <%= form.label :waf_rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_rule_type, + options_for_select(@waf_rule_types.map { |type, _| [type.humanize, type] }, @rule.waf_rule_type), { prompt: "Select rule type" }, { class: "mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm", id: "rule_type_select", @@ -55,9 +55,9 @@
- <%= form.label :action, "Action", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :action, - options_for_select(@actions.map { |action| [action.humanize, action] }, @rule.action), + <%= form.label :waf_action, "Action", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_action, + options_for_select(@waf_actions.map { |action, _| [action.humanize, action] }, @rule.waf_action), { }, { class: "mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm" } %>

What action to take when this rule matches

diff --git a/app/views/rules/index.html.erb b/app/views/rules/index.html.erb index 2a9626f..21808e6 100644 --- a/app/views/rules/index.html.erb +++ b/app/views/rules/index.html.erb @@ -25,7 +25,7 @@
Total Rules
-
<%= number_with_delimiter(@rules.count) %>
+
<%= number_with_delimiter(Rule.count) %>
@@ -42,26 +42,8 @@
-
Active Rules
-
<%= number_with_delimiter(@rules.active.count) %>
-
-
-
-
-
- -
-
-
-
- - - -
-
-
-
Block Rules
-
<%= number_with_delimiter(@rules.where(action: 'deny').count) %>
+
Active Block Rules
+
<%= number_with_delimiter(Rule.deny.active.count) %>
@@ -73,13 +55,31 @@
+ + +
+
+
+
Disabled Rules
+
<%= number_with_delimiter(Rule.where(enabled: false).count) %>
+
+
+
+
+
+ +
+
+
+
+
Expired Rules
-
<%= number_with_delimiter(@rules.expired.count) %>
+
<%= number_with_delimiter(Rule.expired.count) %>
@@ -114,7 +114,7 @@ Rule Type Action - Target + Events Status Created Actions @@ -123,54 +123,77 @@ <% @rules.each do |rule| %> - -
+ +
+
<%= link_to "Rule ##{rule.id}", rule_path(rule), class: "text-blue-600 hover:text-blue-900" %>
-
- <%= rule.source.humanize %> - <% if rule.network_range? && rule.network_range %> - • <%= link_to rule.network_range.cidr, network_range_path(rule.network_range), class: "text-blue-600 hover:text-blue-900" %> - <% end %> -
+ + + <% if rule.waf_policy.present? %> +
+
Policy
+
+ <%= link_to rule.waf_policy.name, waf_policy_path(rule.waf_policy), class: "text-blue-600 hover:text-blue-900" %> +
+
+ <% end %> + + + <% if rule.network_range? && rule.network_range %> +
+
IP network
+
+ <%= link_to rule.network_range.cidr, network_range_path(rule.network_range), class: "text-blue-600 hover:text-blue-900" %> + <% if rule.network_range.company.present? %> +
<%= rule.network_range.company %>
+ <% end %> +
+
+ <% elsif rule.conditions.present? && rule.conditions&.dig('cidr').present? %> +
+
IP network
+
+ <%= rule.conditions['cidr'] %> +
+
+ <% end %>
- <%= rule.rule_type.humanize %> + <%= rule.waf_rule_type.humanize %> - <%= rule.action.upcase %> + <%= rule.waf_action.upcase %> - - <% if rule.network_range? && rule.network_range %> - <%= rule.network_range.cidr %> - <% if rule.network_range.company.present? %> -
<%= rule.network_range.company %>
- <% end %> - <% elsif rule.conditions.present? %> -
- <%= JSON.parse(rule.conditions || "{}").map { |k, v| "#{k}: #{v}" }.join(", ") rescue "Invalid JSON" %> + + <% event_count = rule.events.count %> + <% if event_count > 0 %> + <%= link_to number_with_delimiter(event_count), events_path(rule_id: rule.id), class: "text-blue-600 hover:text-blue-900 font-medium" %> +
+ <%= rule.events.where(waf_action: :deny).count %> blocked
<% else %> - diff --git a/app/views/rules/new.html.erb b/app/views/rules/new.html.erb index 34d7233..d21a5cb 100644 --- a/app/views/rules/new.html.erb +++ b/app/views/rules/new.html.erb @@ -40,9 +40,9 @@
- <%= form.label :rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :rule_type, - options_for_select(@rule_types.map { |type| [type.humanize, type] }, @rule.rule_type), + <%= form.label :waf_rule_type, "Rule Type", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_rule_type, + options_for_select(@waf_rule_types.map { |type, _| [type.humanize, type] }, @rule.waf_rule_type), { prompt: "Select rule type" }, { class: "mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm", id: "rule_type_select" } %> @@ -50,9 +50,9 @@
- <%= form.label :action, "Action", class: "block text-sm font-medium text-gray-700" %> - <%= form.select :action, - options_for_select(@actions.map { |action| [action.humanize, action] }, @rule.action), + <%= form.label :waf_action, "Action", class: "block text-sm font-medium text-gray-700" %> + <%= form.select :waf_action, + options_for_select(@waf_actions.map { |action, _| [action.humanize, action] }, @rule.waf_action), { prompt: "Select action" }, { class: "mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm" } %>

What action to take when this rule matches

diff --git a/app/views/rules/show.html.erb b/app/views/rules/show.html.erb index 810a19f..43687c9 100644 --- a/app/views/rules/show.html.erb +++ b/app/views/rules/show.html.erb @@ -1,4 +1,4 @@ -<% content_for :title, "Rule ##{@rule.id} - #{@rule.action.upcase}" %> +<% content_for :title, "Rule ##{@rule.id} - #{@rule.waf_action.upcase}" %>
@@ -23,15 +23,16 @@

Rule #<%= @rule.id %>

- <%= @rule.action.upcase %> + <%= @rule.waf_action.upcase %>
@@ -60,12 +61,12 @@
Rule Type
-
<%= @rule.rule_type.humanize %>
+
<%= @rule.waf_rule_type.humanize %>
Action
-
<%= @rule.action.upcase %>
+
<%= @rule.waf_action.upcase %>
@@ -118,6 +119,38 @@
+ +
+
+

Event Statistics

+ <%= link_to "View Events", events_path(rule_id: @rule.id), class: "text-sm text-blue-600 hover:text-blue-800" %> +
+
+
+
+
Total Events
+
<%= @rule.events.count %>
+
+ +
+
Blocked Events
+
<%= @rule.events.where(waf_action: :deny).count %>
+
+ +
+
Allowed Events
+
<%= @rule.events.where(waf_action: :allow).count %>
+
+
+ + <% if @rule.events.any? %> +
+ Last event: <%= time_ago_in_words(@rule.events.maximum(:timestamp)) %> ago +
+ <% end %> +
+
+ <% if @rule.network_rule? && @rule.network_range.present? %>
diff --git a/app/views/waf_policies/show.html.erb b/app/views/waf_policies/show.html.erb index d820593..97f9759 100644 --- a/app/views/waf_policies/show.html.erb +++ b/app/views/waf_policies/show.html.erb @@ -239,7 +239,7 @@ Rule #<%= rule.id %> - <%= rule.network_range&.cidr || "Unknown" %>
- <%= rule.action.upcase %> • Created <%= time_ago_in_words(rule.created_at) %> ago + <%= rule.waf_action.upcase %> • Created <%= time_ago_in_words(rule.created_at) %> ago <% if rule.redirect_action? %> • Redirect to <%= rule.redirect_url %> <% elsif rule.challenge_action? %> diff --git a/config/recurring.yml b/config/recurring.yml index b7bbe5c..0f86e23 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -12,6 +12,12 @@ # No recurring tasks configured yet # (previously had clear_solid_queue_finished_jobs, but now preserve_finished_jobs: false in queue.yml) +# Backfill network intelligence for recent events (catches events before network data imported) +backfill_recent_network_intelligence: + class: BackfillRecentNetworkIntelligenceJob + queue: default + schedule: every 5 minutes + # Clean up failed jobs older than 1 day cleanup_failed_jobs: command: "SolidQueue::FailedExecution.where('created_at < ?', 1.day.ago).delete_all" diff --git a/db/schema.rb b/db/schema.rb index 30f11ff..79693ca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,56 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do +ActiveRecord::Schema[8.1].define(version: 2025_11_13_052831) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" + create_table "active_storage_attachments", force: :cascade do |t| + t.bigint "blob_id", null: false + t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", force: :cascade do |t| + t.bigint "byte_size", null: false + t.string "checksum" + t.string "content_type" + t.datetime "created_at", null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" + t.string "service_name", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end + + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + end + + create_table "data_imports", force: :cascade do |t| + t.datetime "completed_at" + t.datetime "created_at", null: false + t.text "error_message" + t.integer "failed_records", default: 0 + t.string "filename", null: false + t.json "import_stats", default: {}, comment: "Detailed import statistics" + t.string "import_type", null: false, comment: "ASN or Country import" + t.integer "processed_records", default: 0 + t.datetime "started_at" + t.string "status", default: "pending", null: false, comment: "pending, processing, completed, failed" + t.integer "total_records", default: 0 + t.datetime "updated_at", null: false + t.index ["created_at"], name: "index_data_imports_on_created_at" + t.index ["import_type"], name: "index_data_imports_on_import_type" + t.index ["status"], name: "index_data_imports_on_status" + end + create_table "dsns", force: :cascade do |t| t.datetime "created_at", null: false t.boolean "enabled", default: true, null: false @@ -26,13 +72,21 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do create_table "events", force: :cascade do |t| t.string "agent_name" t.string "agent_version" + t.integer "asn" + t.string "asn_org" t.text "blocked_reason" + t.string "company" + t.string "country" t.datetime "created_at", null: false t.string "environment" - t.string "event_id", null: false t.inet "ip_address" + t.boolean "is_datacenter", default: false, null: false + t.boolean "is_proxy", default: false, null: false + t.boolean "is_vpn", default: false, null: false + t.bigint "network_range_id" t.json "payload" t.bigint "request_host_id" + t.string "request_id", null: false t.integer "request_method", default: 0 t.string "request_path" t.string "request_protocol" @@ -40,17 +94,25 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.string "request_url" t.integer "response_status" t.integer "response_time_ms" - t.string "rule_matched" + t.bigint "rule_id" t.string "server_name" + t.jsonb "tags", default: [], null: false t.datetime "timestamp", null: false t.datetime "updated_at", null: false t.string "user_agent" t.integer "waf_action", default: 0, null: false - t.index ["event_id"], name: "index_events_on_event_id", unique: true + t.index ["asn"], name: "index_events_on_asn" + t.index ["company"], name: "index_events_on_company" + t.index ["country"], name: "index_events_on_country" t.index ["ip_address"], name: "index_events_on_ip_address" + t.index ["is_datacenter", "is_vpn", "is_proxy"], name: "index_events_on_network_flags" + t.index ["network_range_id"], name: "index_events_on_network_range_id" t.index ["request_host_id", "request_method", "request_segment_ids"], name: "idx_events_host_method_path" t.index ["request_host_id"], name: "index_events_on_request_host_id" + t.index ["request_id"], name: "index_events_on_request_id", unique: true t.index ["request_segment_ids"], name: "index_events_on_request_segment_ids" + t.index ["rule_id"], name: "index_events_on_rule_id" + t.index ["tags"], name: "index_events_on_tags", using: :gin t.index ["timestamp"], name: "index_events_on_timestamp" t.index ["waf_action"], name: "index_events_on_waf_action" end @@ -70,6 +132,8 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.boolean "is_vpn", default: false t.datetime "last_api_fetch" t.inet "network", null: false + t.jsonb "network_data", default: {} + t.datetime "policies_evaluated_at" t.string "source", default: "api_imported", null: false t.datetime "updated_at", null: false t.bigint "user_id" @@ -82,6 +146,8 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.index ["is_datacenter"], name: "index_network_ranges_on_is_datacenter" t.index ["network"], name: "index_network_ranges_on_network", opclass: :inet_ops, using: :gist t.index ["network"], name: "index_network_ranges_on_network_unique", unique: true + t.index ["network_data"], name: "index_network_ranges_on_network_data", using: :gin + t.index ["policies_evaluated_at"], name: "index_network_ranges_on_policies_evaluated_at" t.index ["source"], name: "index_network_ranges_on_source" t.index ["user_id"], name: "index_network_ranges_on_user_id" end @@ -126,7 +192,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do end create_table "rules", force: :cascade do |t| - t.string "action", null: false t.json "conditions", default: {} t.datetime "created_at", null: false t.boolean "enabled", default: true, null: false @@ -134,24 +199,28 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.json "metadata", default: {} t.bigint "network_range_id" t.integer "priority" - t.string "rule_type", null: false t.string "source", limit: 100, default: "manual" t.datetime "updated_at", null: false t.bigint "user_id" + t.integer "waf_action", default: 0, null: false t.bigint "waf_policy_id" - t.index ["action"], name: "index_rules_on_action" + t.integer "waf_rule_type", default: 0, null: false t.index ["enabled", "expires_at"], name: "idx_rules_active" t.index ["enabled"], name: "index_rules_on_enabled" t.index ["expires_at"], name: "index_rules_on_expires_at" + t.index ["network_range_id", "waf_action", "waf_policy_id", "expires_at"], name: "index_rules_on_network_policy_expires_unique", unique: true, where: "(((source)::text = 'policy'::text) AND (expires_at IS NOT NULL))" + t.index ["network_range_id", "waf_action", "waf_policy_id"], name: "index_rules_on_network_policy_unique", unique: true, where: "(((source)::text = 'policy'::text) AND (expires_at IS NULL))" t.index ["network_range_id"], name: "index_rules_on_network_range_id" t.index ["priority"], name: "index_rules_on_priority" - t.index ["rule_type", "enabled"], name: "idx_rules_type_enabled" - t.index ["rule_type"], name: "index_rules_on_rule_type" + t.index ["source", "expires_at"], name: "index_rules_on_source_expires" t.index ["source"], name: "index_rules_on_source" t.index ["updated_at", "id"], name: "idx_rules_sync" t.index ["user_id"], name: "index_rules_on_user_id" + t.index ["waf_action"], name: "index_rules_on_waf_action" + t.index ["waf_policy_id", "expires_at"], name: "index_rules_on_policy_expires" t.index ["waf_policy_id"], name: "idx_rules_waf_policy" t.index ["waf_policy_id"], name: "index_rules_on_waf_policy_id" + t.index ["waf_rule_type"], name: "index_rules_on_waf_rule_type" end create_table "sessions", force: :cascade do |t| @@ -163,6 +232,14 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.index ["user_id"], name: "index_sessions_on_user_id" end + create_table "settings", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "key" + t.datetime "updated_at", null: false + t.string "value" + t.index ["key"], name: "index_settings_on_key", unique: true + end + create_table "users", force: :cascade do |t| t.datetime "created_at", null: false t.string "email_address", null: false @@ -173,13 +250,13 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do end create_table "waf_policies", force: :cascade do |t| - t.string "action", default: "deny", null: false t.json "additional_data", default: {} t.datetime "created_at", null: false t.text "description" t.boolean "enabled", default: true, null: false t.datetime "expires_at" t.string "name", null: false + t.string "policy_action", default: "deny", null: false t.string "policy_type", default: "country", null: false t.json "targets", default: [] t.datetime "updated_at", null: false @@ -191,7 +268,10 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_10_023232) do t.index ["user_id"], name: "index_waf_policies_on_user_id" end + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" add_foreign_key "events", "request_hosts" + add_foreign_key "events", "rules" add_foreign_key "network_ranges", "users" add_foreign_key "rules", "network_ranges" add_foreign_key "rules", "users" diff --git a/lib/tasks/events.rake b/lib/tasks/events.rake index 200486f..0c8fced 100644 --- a/lib/tasks/events.rake +++ b/lib/tasks/events.rake @@ -7,6 +7,27 @@ namespace :events do Event.backfill_network_intelligence!(batch_size: batch_size) end + desc "Backfill events with missing network intelligence (newly imported network data)" + task backfill_missing: :environment do + count = Event.where(country: nil).count + + if count.zero? + puts "✓ No events missing network intelligence" + else + puts "Found #{count} events without network intelligence" + puts "Backfilling..." + + processed = 0 + Event.where(country: nil).find_in_batches(batch_size: 1000) do |batch| + batch.each(&:save) + processed += batch.size + puts " Processed #{processed}/#{count}" + end + + puts "✓ Complete" + end + end + desc "Show backfill progress" task backfill_progress: :environment do total = Event.count diff --git a/test/integration/waf_policy_brazil_test.rb b/test/integration/waf_policy_brazil_test.rb index 4969d05..43c9a0c 100644 --- a/test/integration/waf_policy_brazil_test.rb +++ b/test/integration/waf_policy_brazil_test.rb @@ -93,7 +93,7 @@ class WafPolicyBrazilTest < Minitest::Test assert_equal 1, generated_rules.count, "Should have generated exactly one blocking rule" rule = generated_rules.first - assert_equal 'deny', rule.action + assert_equal 'deny', rule.waf_action assert_equal network_range, rule.network_range assert_equal @brazil_policy, rule.waf_policy assert_equal "policy", rule.source diff --git a/test/integration/waf_policy_integration_test.rb b/test/integration/waf_policy_integration_test.rb index e7aeb83..9562844 100644 --- a/test/integration/waf_policy_integration_test.rb +++ b/test/integration/waf_policy_integration_test.rb @@ -94,7 +94,7 @@ class WafPolicyIntegrationTest < ActiveSupport::TestCase assert_equal 1, generated_rules.count, "Should have generated exactly one blocking rule" rule = generated_rules.first - assert_equal 'deny', rule.action + assert_equal 'deny', rule.waf_action assert_equal network_range, rule.network_range assert_equal @brazil_policy, rule.waf_policy assert_equal "policy:Block Brazil", rule.source diff --git a/test/jobs/path_scanner_detector_job_test.rb b/test/jobs/path_scanner_detector_job_test.rb index fbf849a..7408d4a 100644 --- a/test/jobs/path_scanner_detector_job_test.rb +++ b/test/jobs/path_scanner_detector_job_test.rb @@ -32,8 +32,8 @@ class PathScannerDetectorJobTest < ActiveJob::TestCase rule = Rule.where(source: "auto:scanner_detected").last assert_not_nil rule - assert_equal "network_v4", rule.rule_type - assert_equal "deny", rule.action + assert_equal "network", rule.waf_rule_type + assert_equal "deny", rule.waf_action assert_equal "#{ip}/32", rule.cidr assert_equal 32, rule.priority assert rule.enabled? @@ -186,7 +186,7 @@ class PathScannerDetectorJobTest < ActiveJob::TestCase assert_equal 1, count rule = Rule.where(source: "auto:scanner_detected").last - assert_equal "network_v6", rule.rule_type + assert_equal "network", rule.waf_rule_type assert_equal "#{ip}/32", rule.cidr end diff --git a/test/services/waf_policy_matcher_test.rb b/test/services/waf_policy_matcher_test.rb index 1fe31d2..07a21e0 100644 --- a/test/services/waf_policy_matcher_test.rb +++ b/test/services/waf_policy_matcher_test.rb @@ -167,7 +167,7 @@ class WafPolicyMatcherTest < ActiveSupport::TestCase rule = generated_rules.first assert_equal brazil_policy, rule.waf_policy assert_equal @network_range, rule.network_range - assert_equal "deny", rule.action + assert_equal "deny", rule.waf_action end test "generate_rules handles multiple matching policies" do @@ -490,7 +490,7 @@ class WafPolicyMatcherTest < ActiveSupport::TestCase rule = redirect_policy.create_rule_for_network_range(@network_range) assert_not_nil rule - assert_equal "redirect", rule.action + assert_equal "redirect", rule.waf_action assert rule.metadata['redirect_url'].present? end