Many updates

app/controllers/analytics_controller.rb

@@ -56,11 +56,10 @@ class AnalyticsController < ApplicationController
       end
     end
 
-    # Top countries by event count - cached (this is the expensive JOIN query)
+    # Top countries by event count - cached (now uses denormalized country column)
     @top_countries = Rails.cache.fetch("#{cache_key_base}/top_countries", expires_in: cache_ttl) do
-      Event.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-            .where("timestamp >= ? AND network_ranges.country IS NOT NULL", @start_time)
-            .group("network_ranges.country")
+      Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
+            .group(:country)
             .count
             .sort_by { |_, count| -count }
             .first(10)
@@ -126,10 +125,10 @@ class AnalyticsController < ApplicationController
     @time_period = params[:period]&.to_sym || :day
     @start_time = calculate_start_time(@time_period)
 
-    # Top networks by request volume
-    @top_networks = NetworkRange.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
+    # Top networks by request volume (using denormalized network_range_id)
+    @top_networks = NetworkRange.joins("LEFT JOIN events ON events.network_range_id = network_ranges.id")
                                .where("events.timestamp >= ? OR events.timestamp IS NULL", @start_time)
-                               .group("network_ranges.id", "network_ranges.network", "network_ranges.company", "network_ranges.asn", "network_ranges.country", "network_ranges.is_datacenter", "network_ranges.is_vpn", "network_ranges.is_proxy")
+                               .group("network_ranges.id")
                                .select("network_ranges.*, COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips")
                                .order("event_count DESC")
                                .limit(50)
@@ -137,29 +136,26 @@ class AnalyticsController < ApplicationController
     # Network type breakdown with traffic stats
     @network_breakdown = calculate_network_type_stats(@start_time)
 
-    # Company breakdown for top traffic sources
-    @top_companies = NetworkRange.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                                 .where("events.timestamp >= ? AND network_ranges.company IS NOT NULL", @start_time)
-                                 .group("network_ranges.company")
-                                 .select("network_ranges.company, COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips, COUNT(DISTINCT network_ranges.id) as network_count")
-                                 .order("event_count DESC")
-                                 .limit(20)
+    # Company breakdown for top traffic sources (using denormalized company column)
+    @top_companies = Event.where("timestamp >= ? AND company IS NOT NULL", @start_time)
+                          .group(:company)
+                          .select("company, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
+                          .order("event_count DESC")
+                          .limit(20)
 
-    # ASN breakdown
-    @top_asns = NetworkRange.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                           .where("events.timestamp >= ? AND network_ranges.asn IS NOT NULL", @start_time)
-                           .group("network_ranges.asn", "network_ranges.asn_org")
-                           .select("network_ranges.asn, network_ranges.asn_org, COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips, COUNT(DISTINCT network_ranges.id) as network_count")
-                           .order("event_count DESC")
-                           .limit(15)
+    # ASN breakdown (using denormalized asn columns)
+    @top_asns = Event.where("timestamp >= ? AND asn IS NOT NULL", @start_time)
+                     .group(:asn, :asn_org)
+                     .select("asn, asn_org, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
+                     .order("event_count DESC")
+                     .limit(15)
 
-    # Geographic breakdown
-    @top_countries = NetworkRange.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                                 .where("events.timestamp >= ? AND network_ranges.country IS NOT NULL", @start_time)
-                                 .group("network_ranges.country")
-                                 .select("network_ranges.country, COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips, COUNT(DISTINCT network_ranges.id) as network_count")
-                                 .order("event_count DESC")
-                                 .limit(15)
+    # Geographic breakdown (using denormalized country column)
+    @top_countries = Event.where("timestamp >= ? AND country IS NOT NULL", @start_time)
+                          .group(:country)
+                          .select("country, COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips")
+                          .order("event_count DESC")
+                          .limit(15)
 
     # Suspicious network activity patterns
     @suspicious_patterns = calculate_suspicious_patterns(@start_time)
@@ -297,51 +293,41 @@ class AnalyticsController < ApplicationController
   end
 
   def calculate_network_type_stats(start_time)
-    # Get all network types with their traffic statistics
+    # Get all network types with their traffic statistics using denormalized columns
     network_types = [
-      { type: 'datacenter', label: 'Datacenter' },
-      { type: 'vpn', label: 'VPN' },
-      { type: 'proxy', label: 'Proxy' }
+      { type: 'datacenter', label: 'Datacenter', column: :is_datacenter },
+      { type: 'vpn', label: 'VPN', column: :is_vpn },
+      { type: 'proxy', label: 'Proxy', column: :is_proxy }
     ]
 
     results = {}
     total_events = Event.where("timestamp >= ?", start_time).count
 
     network_types.each do |network_type|
-      scope = case network_type[:type]
-              when 'datacenter' then NetworkRange.datacenter
-              when 'vpn' then NetworkRange.vpn
-              when 'proxy' then NetworkRange.proxy
-              end
+      # Query events directly using denormalized flags
+      event_stats = Event.where("timestamp >= ? AND #{network_type[:column]} = ?", start_time, true)
+                         .select("COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
+                         .first
 
-      if scope
-        network_stats = scope.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                            .where("events.timestamp >= ? OR events.timestamp IS NULL", start_time)
-                            .select("COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips, COUNT(DISTINCT network_ranges.id) as network_count")
-                            .first
-
-        results[network_type[:type]] = {
-          label: network_type[:label],
-          networks: network_stats.network_count,
-          events: network_stats.event_count,
-          unique_ips: network_stats.unique_ips,
-          percentage: total_events > 0 ? ((network_stats.event_count.to_f / total_events) * 100).round(1) : 0
-        }
-      end
+      results[network_type[:type]] = {
+        label: network_type[:label],
+        networks: event_stats.network_count || 0,
+        events: event_stats.event_count || 0,
+        unique_ips: event_stats.unique_ips || 0,
+        percentage: total_events > 0 ? ((event_stats.event_count.to_f / total_events) * 100).round(1) : 0
+      }
     end
 
     # Calculate standard networks (everything else)
-    standard_stats = NetworkRange.where(is_datacenter: false, is_vpn: false, is_proxy: false)
-                                .joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                                .where("events.timestamp >= ? OR events.timestamp IS NULL", start_time)
-                                .select("COUNT(events.id) as event_count, COUNT(DISTINCT events.ip_address) as unique_ips, COUNT(DISTINCT network_ranges.id) as network_count")
-                                .first
+    standard_stats = Event.where("timestamp >= ? AND is_datacenter = ? AND is_vpn = ? AND is_proxy = ?", start_time, false, false, false)
+                          .select("COUNT(*) as event_count, COUNT(DISTINCT ip_address) as unique_ips, COUNT(DISTINCT network_range_id) as network_count")
+                          .first
 
     results['standard'] = {
       label: 'Standard',
-      networks: standard_stats.network_count,
-      events: standard_stats.event_count,
-      unique_ips: standard_stats.unique_ips,
+      networks: standard_stats.network_count || 0,
+      events: standard_stats.event_count || 0,
+      unique_ips: standard_stats.unique_ips || 0,
       percentage: total_events > 0 ? ((standard_stats.event_count.to_f / total_events) * 100).round(1) : 0
     }
 
@@ -351,51 +337,51 @@ class AnalyticsController < ApplicationController
   def calculate_suspicious_patterns(start_time)
     patterns = {}
 
-    # High volume networks (top 1% by request count)
-    total_networks = NetworkRange.joins("LEFT JOIN events ON events.ip_address <<= network_ranges.network")
-                                .where("events.timestamp >= ?", start_time)
-                                .distinct.count
+    # High volume networks (top 1% by request count) - using denormalized network_range_id
+    total_networks = Event.where("timestamp >= ? AND network_range_id IS NOT NULL", start_time)
+                          .distinct.count(:network_range_id)
 
-    high_volume_threshold = [total_networks * 0.01, 1].max
-    high_volume_networks = NetworkRange.joins("INNER JOIN events ON events.ip_address <<= network_ranges.network")
-                                      .where("events.timestamp >= ?", start_time)
-                                      .group("network_ranges.id")
-                                      .having("COUNT(events.id) > ?", Event.where("timestamp >= ?", start_time).count / total_networks)
-                                      .count
+    if total_networks > 0
+      avg_events_per_network = Event.where("timestamp >= ?", start_time).count / total_networks
+      high_volume_networks = Event.where("timestamp >= ? AND network_range_id IS NOT NULL", start_time)
+                                  .group(:network_range_id)
+                                  .having("COUNT(*) > ?", avg_events_per_network * 5)
+                                  .count
 
-    patterns[:high_volume] = {
-      count: high_volume_networks.count,
-      networks: high_volume_networks.keys
-    }
+      patterns[:high_volume] = {
+        count: high_volume_networks.count,
+        networks: high_volume_networks.keys
+      }
+    else
+      patterns[:high_volume] = { count: 0, networks: [] }
+    end
 
-    # Networks with high deny rates (> 50% blocked requests)
-    high_deny_networks = NetworkRange.joins("INNER JOIN events ON events.ip_address <<= network_ranges.network")
-                                    .where("events.timestamp >= ?", start_time)
-                                    .group("network_ranges.id")
-                                    .select("network_ranges.id,
-                                           COUNT(CASE WHEN events.waf_action = 1 THEN 1 END) as denied_count,
-                                           COUNT(events.id) as total_count")
-                                    .having("COUNT(CASE WHEN events.waf_action = 1 THEN 1 END)::float / COUNT(events.id) > 0.5")
-                                    .having("COUNT(events.id) >= 10") # minimum threshold
+    # Networks with high deny rates (> 50% blocked requests) - using denormalized network_range_id
+    high_deny_networks = Event.where("timestamp >= ? AND network_range_id IS NOT NULL", start_time)
+                              .group(:network_range_id)
+                              .select("network_range_id,
+                                     COUNT(CASE WHEN waf_action = 1 THEN 1 END) as denied_count,
+                                     COUNT(*) as total_count")
+                              .having("COUNT(CASE WHEN waf_action = 1 THEN 1 END)::float / COUNT(*) > 0.5")
+                              .having("COUNT(*) >= 10") # minimum threshold
 
     patterns[:high_deny_rate] = {
       count: high_deny_networks.count,
-      network_ids: high_deny_networks.map(&:id)
+      network_ids: high_deny_networks.map(&:network_range_id)
     }
 
-    # Networks appearing as multiple subnets (potential botnets)
-    company_subnets = NetworkRange.where("company IS NOT NULL")
-                                 .where("timestamp >= ? OR timestamp IS NULL", start_time)
-                                 .group(:company)
-                                 .select(:company, "COUNT(DISTINCT network) as subnet_count")
-                                 .having("COUNT(DISTINCT network) > 5")
-                                 .order("subnet_count DESC")
-                                 .limit(10)
+    # Companies appearing with multiple IPs (potential botnets) - using denormalized company column
+    company_subnets = Event.where("timestamp >= ? AND company IS NOT NULL", start_time)
+                           .group(:company)
+                           .select("company, COUNT(DISTINCT ip_address) as ip_count")
+                           .having("COUNT(DISTINCT ip_address) > 5")
+                           .order("ip_count DESC")
+                           .limit(10)
 
-    patterns[:distributed_companies] = company_subnets.map do |company|
+    patterns[:distributed_companies] = company_subnets.map do |stat|
       {
-        company: company.company,
-        subnets: company.subnet_count
+        company: stat.company,
+        subnets: stat.ip_count
       }
     end
 

app/controllers/events_controller.rb

@@ -2,28 +2,35 @@
 
 class EventsController < ApplicationController
   def show
-    @event = Event.find(params[:id])
-    @network_range = NetworkRange.contains_ip(@event.ip_address.to_s).first
+    @event = Event.includes(:network_range).find(params[:id])
 
-    # Auto-generate network range if no match found
+    # Use denormalized network_range_id if available (much faster)
+    @network_range = @event.network_range
+
+    # Fallback to IP lookup if network_range_id is missing
     unless @network_range
-      @network_range = NetworkRangeGenerator.find_or_create_for_ip(@event.ip_address)
-      Rails.logger.debug "Auto-generated network range #{@network_range&.cidr} for IP #{@event.ip_address}" if @network_range
+      @network_range = NetworkRange.contains_ip(@event.ip_address.to_s).first
+
+      # Auto-generate network range if no match found
+      unless @network_range
+        @network_range = NetworkRangeGenerator.find_or_create_for_ip(@event.ip_address)
+        Rails.logger.debug "Auto-generated network range #{@network_range&.cidr} for IP #{@event.ip_address}" if @network_range
+      end
     end
   end
 
   def index
-    @events = Event.order(timestamp: :desc)
+    @events = Event.includes(:network_range, :rule).order(timestamp: :desc)
     Rails.logger.debug "Found #{@events.count} total events"
     Rails.logger.debug "Action: #{params[:waf_action]}"
 
     # Apply filters
     @events = @events.by_ip(params[:ip]) if params[:ip].present?
     @events = @events.by_waf_action(params[:waf_action]) if params[:waf_action].present?
-    @events = @events.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-                     .where("network_ranges.country = ?", params[:country]) if params[:country].present?
+    @events = @events.by_country(params[:country]) if params[:country].present?
+    @events = @events.where(rule_id: params[:rule_id]) if params[:rule_id].present?
 
-    # Network-based filters
+    # Network-based filters (now using denormalized columns)
     @events = @events.by_company(params[:company]) if params[:company].present?
     @events = @events.by_network_type(params[:network_type]) if params[:network_type].present?
     @events = @events.by_asn(params[:asn]) if params[:asn].present?
@@ -37,24 +44,10 @@ class EventsController < ApplicationController
     # Paginate
     @pagy, @events = pagy(@events, items: 50)
 
-    # Preload network ranges for all unique IPs to avoid N+1 queries
-    unique_ips = @events.pluck(:ip_address).uniq.compact
-    @network_ranges_by_ip = {}
-    unique_ips.each do |ip|
-      ip_string = ip.to_s  # IPAddr objects can be converted to string
-      range = NetworkRange.contains_ip(ip_string).first
-
-      # Auto-generate network range if no match found
-      unless range
-        range = NetworkRangeGenerator.find_or_create_for_ip(ip)
-        Rails.logger.debug "Auto-generated network range #{range&.cidr} for IP #{ip_string}" if range
-      end
-
-      @network_ranges_by_ip[ip_string] = range if range
-    end
+    # Network ranges are now preloaded via includes(:network_range)
+    # The denormalized network_range_id makes this much faster than IP containment lookups
 
     Rails.logger.debug "Events count after pagination: #{@events.count}"
     Rails.logger.debug "Pagy info: #{@pagy.count} total, #{@pagy.pages} pages"
-    Rails.logger.debug "Preloaded network ranges for #{@network_ranges_by_ip.count} unique IPs"
   end
 end

app/controllers/network_ranges_controller.rb

@@ -46,24 +46,51 @@ class NetworkRangesController < ApplicationController
     authorize @network_range
 
     if @network_range.persisted?
-      # Real network - use existing logic
-      @related_events = Event.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-                           .where("network_ranges.id = ?", @network_range.id)
-                           .recent
-                           .limit(100)
+      # Real network - use direct IP containment for consistency with stats
+      events_scope = Event.where("ip_address <<= ?", @network_range.cidr).recent
     else
       # Virtual network - find events by IP range containment
-      @related_events = Event.where("ip_address <<= ?::inet", @network_range.to_s)
-                           .recent
-                           .limit(100)
+      events_scope = Event.where("ip_address <<= ?::inet", @network_range.to_s).recent
     end
 
+    # Paginate events
+    @events_pagy, @related_events = pagy(events_scope, items: 50)
+
     @child_ranges = @network_range.child_ranges.limit(20)
     @parent_ranges = @network_range.parent_ranges.limit(10)
     @associated_rules = @network_range.persisted? ? @network_range.rules.includes(:user).order(created_at: :desc) : []
 
     # Traffic analytics (if we have events)
     @traffic_stats = calculate_traffic_stats(@network_range)
+
+    # Check if we have IPAPI data (or if parent has it)
+    @has_ipapi_data = @network_range.has_network_data_from?(:ipapi)
+    @parent_with_ipapi = nil
+
+    unless @has_ipapi_data
+      # Check if parent has IPAPI data
+      parent = @network_range.parent_with_intelligence
+      if parent&.has_network_data_from?(:ipapi)
+        @parent_with_ipapi = parent
+        @has_ipapi_data = true
+      end
+    end
+
+    # If we don't have IPAPI data anywhere and no parent has it, queue fetch job
+    if @network_range.persisted? && @network_range.should_fetch_ipapi_data?
+      @network_range.mark_as_fetching_api_data!(:ipapi)
+      FetchIpapiDataJob.perform_later(network_range_id: @network_range.id)
+      @ipapi_loading = true
+    end
+
+    # Get IPAPI data for display
+    @ipapi_data = if @parent_with_ipapi
+      @parent_with_ipapi.network_data_for(:ipapi)
+    elsif @network_range.has_network_data_from?(:ipapi)
+      @network_range.network_data_for(:ipapi)
+    else
+      nil
+    end
   end
 
   # GET /network_ranges/new
@@ -214,18 +241,27 @@ class NetworkRangesController < ApplicationController
     if network_range.persisted?
       # Real network - use cached events_count for total requests (much more performant)
       if network_range.events_count > 0
-        events = Event.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-                      .where("network_ranges.id = ?", network_range.id)
-                      .limit(1000) # Limit the sample for performance
+        # Base query for consistent IP containment logic
+        base_query = Event.where("ip_address <<= ?", network_range.cidr)
+
+        # Use separate queries: one for grouping (without ordering), one for recent activity (with ordering)
+        events_for_grouping = base_query.limit(1000)
+        events_for_activity = base_query.recent.limit(20)
+
+        # Calculate counts properly - use consistent base_query for all counts
+        total_requests = base_query.count
+        unique_ips = base_query.except(:order).distinct.count(:ip_address)
+        blocked_requests = base_query.blocked.count
+        allowed_requests = base_query.allowed.count
 
         {
-          total_requests: network_range.events_count, # Use cached count
-          unique_ips: events.distinct.count(:ip_address),
-          blocked_requests: events.blocked.count,
-          allowed_requests: events.allowed.count,
-          top_paths: events.group(:request_path).count.sort_by { |_, count| -count }.first(10),
-          top_user_agents: events.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
-          recent_activity: events.recent.limit(20)
+          total_requests: total_requests,
+          unique_ips: unique_ips,
+          blocked_requests: blocked_requests,
+          allowed_requests: allowed_requests,
+          top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
+          top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
+          recent_activity: events_for_activity
         }
       else
         # No events - return empty stats
@@ -241,20 +277,35 @@ class NetworkRangesController < ApplicationController
       end
     else
       # Virtual network - calculate stats from events within range
-      events = Event.where("ip_address <<= ?::inet", network_range.to_s)
-                   .limit(1000) # Limit the sample for performance
+      base_query = Event.where("ip_address <<= ?", network_range.cidr)
+      total_events = base_query.count
 
-      total_events = Event.where("ip_address <<= ?::inet", network_range.to_s).count
+      if total_events > 0
+        # Use separate queries: one for grouping (without ordering), one for recent activity (with ordering)
+        events_for_grouping = base_query.limit(1000)
+        events_for_activity = base_query.recent.limit(20)
 
-      {
-        total_requests: total_events,
-        unique_ips: events.distinct.count(:ip_address),
-        blocked_requests: events.blocked.count,
-        allowed_requests: events.allowed.count,
-        top_paths: events.group(:request_path).count.sort_by { |_, count| -count }.first(10),
-        top_user_agents: events.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
-        recent_activity: events.recent.limit(20)
-      }
+        {
+          total_requests: total_events,
+          unique_ips: base_query.except(:order).distinct.count(:ip_address),
+          blocked_requests: base_query.blocked.count,
+          allowed_requests: base_query.allowed.count,
+          top_paths: events_for_grouping.group(:request_path).count.sort_by { |_, count| -count }.first(10),
+          top_user_agents: events_for_grouping.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
+          recent_activity: events_for_activity
+        }
+      else
+        # No events for virtual network
+        {
+          total_requests: 0,
+          unique_ips: 0,
+          blocked_requests: 0,
+          allowed_requests: 0,
+          top_paths: {},
+          top_user_agents: {},
+          recent_activity: []
+        }
+      end
     end
   end
 end

app/controllers/rules_controller.rb

@@ -11,8 +11,8 @@ class RulesController < ApplicationController
   # GET /rules
   def index
     @pagy, @rules = pagy(policy_scope(Rule).includes(:user, :network_range).order(created_at: :desc))
-    @rule_types = Rule::RULE_TYPES
-    @actions = Rule::ACTIONS
+    @waf_rule_types = Rule.waf_rule_types
+    @waf_actions = Rule.waf_actions
   end
 
   # GET /rules/new
@@ -27,11 +27,11 @@ class RulesController < ApplicationController
     end
 
     if params[:cidr].present?
-      @rule.rule_type = 'network'
+      @rule.waf_rule_type = 'network'
     end
 
-    @rule_types = Rule::RULE_TYPES
-    @actions = Rule::ACTIONS
+    @waf_rule_types = Rule.waf_rule_types
+    @waf_actions = Rule.waf_actions
   end
 
   # POST /rules
@@ -39,8 +39,8 @@ class RulesController < ApplicationController
     authorize Rule
     @rule = Rule.new(rule_params)
     @rule.user = Current.user
-    @rule_types = Rule::RULE_TYPES
-    @actions = Rule::ACTIONS
+    @waf_rule_types = Rule.waf_rule_types
+    @waf_actions = Rule.waf_actions
 
     # Process additional form data for quick create
     process_quick_create_parameters
@@ -79,16 +79,26 @@ class RulesController < ApplicationController
   # GET /rules/:id/edit
   def edit
     authorize @rule
-    @rule_types = Rule::RULE_TYPES
-    @actions = Rule::ACTIONS
+    @waf_rule_types = Rule.waf_rule_types
+    @waf_actions = Rule.waf_actions
   end
 
   # PATCH/PUT /rules/:id
   def update
     authorize @rule
+
+    # Preserve original attributes in case validation fails
+    original_attributes = @rule.attributes.dup
+    original_network_range_id = @rule.network_range_id
+
     if @rule.update(rule_params)
       redirect_to @rule, notice: 'Rule was successfully updated.'
     else
+      # Restore original attributes to preserve form state
+      # This prevents network range dropdown from resetting
+      @rule.attributes = original_attributes
+      @rule.network_range_id = original_network_range_id
+
       render :edit, status: :unprocessable_entity
     end
   end
@@ -116,8 +126,8 @@ class RulesController < ApplicationController
 
   def rule_params
   permitted = [
-    :rule_type,
-    :action,
+    :waf_rule_type,
+    :waf_action,
     :metadata,
     :expires_at,
     :enabled,
@@ -126,7 +136,7 @@ class RulesController < ApplicationController
   ]
 
   # Only include conditions for non-network rules
-  if params[:rule][:rule_type] != 'network'
+  if params[:rule][:waf_rule_type] != 'network'
     permitted << :conditions
   end
 
@@ -136,7 +146,7 @@ end
 def calculate_rule_priority
   return unless @rule
 
-  case @rule.rule_type
+  case @rule.waf_rule_type
   when 'network'
     # For network rules, priority based on prefix specificity
     if @rule.network_range
@@ -167,20 +177,10 @@ def calculate_rule_priority
     else
       @rule.priority = 100 # Default for network rules without range
     end
-  when 'protocol_violation'
-    @rule.priority = 95
-  when 'method_enforcement'
-    @rule.priority = 90
   when 'path_pattern'
     @rule.priority = 85
-  when 'header_pattern', 'query_pattern'
-    @rule.priority = 80
-  when 'body_signature'
-    @rule.priority = 75
   when 'rate_limit'
     @rule.priority = 70
-  when 'composite'
-    @rule.priority = 65
   else
     @rule.priority = 50 # Default priority
   end
@@ -203,7 +203,7 @@ def process_quick_create_parameters
   end
 
   # Handle redirect URL
-  if @rule.action == 'redirect' && params[:redirect_url].present?
+  if @rule.redirect? && params[:redirect_url].present?
     @rule.metadata ||= {}
     if @rule.metadata.is_a?(String)
       begin
@@ -227,6 +227,24 @@ def process_quick_create_parameters
     end
   end
 
+  # Handle expires_at parsing for text input
+  if params.dig(:rule, :expires_at).present?
+    expires_at_str = params[:rule][:expires_at].strip
+    if expires_at_str.present?
+      begin
+        # Try to parse various datetime formats
+        @rule.expires_at = DateTime.parse(expires_at_str)
+      rescue ArgumentError
+        # Try specific format
+        begin
+          @rule.expires_at = DateTime.strptime(expires_at_str, '%Y-%m-%d %H:%M')
+        rescue ArgumentError
+          @rule.errors.add(:expires_at, 'must be in format YYYY-MM-DD HH:MM')
+        end
+      end
+    end
+  end
+
   # Add reason to metadata if provided
   if params.dig(:rule, :metadata).present?
     if @rule.metadata.is_a?(Hash)
@@ -245,8 +263,8 @@ end
 
   def rule_params
     permitted = [
-      :rule_type,
-      :action,
+      :waf_rule_type,
+      :waf_action,
       :metadata,
       :expires_at,
       :enabled,
@@ -255,7 +273,7 @@ end
     ]
 
     # Only include conditions for non-network rules
-    if params[:rule][:rule_type] != 'network'
+    if params[:rule][:waf_rule_type] != 'network'
       permitted << :conditions
     end
 
@@ -265,7 +283,7 @@ end
   def calculate_rule_priority
     return unless @rule
 
-    case @rule.rule_type
+    case @rule.waf_rule_type
     when 'network'
       # For network rules, priority based on prefix specificity
       if @rule.network_range
@@ -296,20 +314,10 @@ end
       else
         @rule.priority = 100 # Default for network rules without range
       end
-    when 'protocol_violation'
-      @rule.priority = 95
-    when 'method_enforcement'
-      @rule.priority = 90
     when 'path_pattern'
       @rule.priority = 85
-    when 'header_pattern', 'query_pattern'
-      @rule.priority = 80
-    when 'body_signature'
-      @rule.priority = 75
     when 'rate_limit'
       @rule.priority = 70
-    when 'composite'
-      @rule.priority = 65
     else
       @rule.priority = 50 # Default priority
     end
@@ -332,7 +340,7 @@ end
     end
 
     # Handle redirect URL
-    if @rule.action == 'redirect' && params[:redirect_url].present?
+    if @rule.redirect? && params[:redirect_url].present?
       @rule.metadata ||= {}
       if @rule.metadata.is_a?(String)
         begin

app/controllers/waf_policies_controller.rb

@@ -24,7 +24,7 @@ class WafPoliciesController < ApplicationController
 
     # Set default values from URL parameters
     @waf_policy.policy_type = params[:policy_type] if params[:policy_type].present?
-    @waf_policy.action = params[:action] if params[:action].present?
+    @waf_policy.policy_action = params[:policy_action] if params[:policy_action].present?
     @waf_policy.targets = params[:targets] if params[:targets].present?
   end
 
@@ -37,9 +37,6 @@ class WafPoliciesController < ApplicationController
     @actions = WafPolicy::ACTIONS
 
     if @waf_policy.save
-      # Trigger policy processing for existing network ranges
-      ProcessWafPoliciesJob.perform_later(waf_policy_id: @waf_policy.id)
-
       redirect_to @waf_policy, notice: 'WAF policy was successfully created.'
     else
       render :new, status: :unprocessable_entity
@@ -64,11 +61,6 @@ class WafPoliciesController < ApplicationController
     @actions = WafPolicy::ACTIONS
 
     if @waf_policy.update(waf_policy_params)
-      # Re-process policies for existing network ranges if policy was changed
-      if @waf_policy.saved_change_to_targets? || @waf_policy.saved_change_to_action?
-        ProcessWafPoliciesJob.reprocess_for_policy(@waf_policy)
-      end
-
       redirect_to @waf_policy, notice: 'WAF policy was successfully updated.'
     else
       render :edit, status: :unprocessable_entity
@@ -89,9 +81,6 @@ class WafPoliciesController < ApplicationController
   def activate
     @waf_policy.activate!
 
-    # Re-process policies for existing network ranges
-    ProcessWafPoliciesJob.reprocess_for_policy(@waf_policy)
-
     redirect_to @waf_policy, notice: 'WAF policy was activated.'
   end
 
@@ -105,7 +94,7 @@ class WafPoliciesController < ApplicationController
   # GET /waf_policies/new_country
   def new_country
     authorize WafPolicy
-    @waf_policy = WafPolicy.new(policy_type: 'country', action: 'deny')
+    @waf_policy = WafPolicy.new(policy_type: 'country', policy_action: 'deny')
     @policy_types = WafPolicy::POLICY_TYPES
     @actions = WafPolicy::ACTIONS
   end
@@ -115,24 +104,28 @@ class WafPoliciesController < ApplicationController
     authorize WafPolicy
 
     countries = params[:countries]&.reject(&:blank?) || []
-    action = params[:action] || 'deny'
+    policy_action = params[:policy_action] || 'deny'
 
     if countries.empty?
       redirect_to new_country_waf_policies_path, alert: 'Please select at least one country.'
       return
     end
 
-    @waf_policy = WafPolicy.create_country_policy(
-      countries,
-      action: action,
+    # Build the options hash with additional_data if present
+    options = {
+      policy_action: policy_action,
       user: Current.user,
       description: params[:description]
-    )
+    }
+
+    # Add additional_data if provided (for redirect/challenge actions)
+    if params[:additional_data].present?
+      options[:additional_data] = params[:additional_data].to_unsafe_hash
+    end
+
+    @waf_policy = WafPolicy.create_country_policy(countries, **options)
 
     if @waf_policy.persisted?
-      # Trigger policy processing for existing network ranges
-      ProcessWafPoliciesJob.reprocess_for_policy(@waf_policy)
-
       redirect_to @waf_policy, notice: "Country blocking policy was successfully created for #{countries.join(', ')}."
     else
       @policy_types = WafPolicy::POLICY_TYPES
@@ -144,10 +137,22 @@ class WafPoliciesController < ApplicationController
   private
 
   def set_waf_policy
-    @waf_policy = WafPolicy.find(params[:id])
-    authorize @waf_policy
-  rescue ActiveRecord::RecordNotFound
-    redirect_to waf_policies_path, alert: 'WAF policy not found.'
+    # First try to find by ID (standard Rails behavior)
+    if params[:id] =~ /^\d+$/
+      @waf_policy = WafPolicy.find_by(id: params[:id])
+    end
+
+    # If not found by ID, try to find by parameterized name
+    unless @waf_policy
+      # Try direct parameterized comparison by parameterizing existing policy names
+      @waf_policy = WafPolicy.all.find { |policy| policy.to_param == params[:id] }
+    end
+
+    if @waf_policy
+      authorize @waf_policy
+    else
+      redirect_to waf_policies_path, alert: 'WAF policy not found.'
+    end
   end
 
   def waf_policy_params
@@ -155,7 +160,7 @@ class WafPoliciesController < ApplicationController
       :name,
       :description,
       :policy_type,
-      :action,
+      :policy_action,
       :enabled,
       :expires_at,
       targets: [],

app/helpers/application_helper.rb

@@ -139,4 +139,15 @@ module ApplicationHelper
       raw: user_agent
     }
   end
+
+  # Convert country code to flag emoji
+  def country_flag(country_code)
+    return "" if country_code.blank?
+
+    # Convert ISO 3166-1 alpha-2 country code to flag emoji
+    # Each letter is converted to its regional indicator symbol
+    country_code.upcase.chars.map { |c| (c.ord + 127397).chr(Encoding::UTF_8) }.join
+  rescue
+    ""
+  end
 end

app/javascript/controllers/quick_create_rule_controller.js

@@ -38,7 +38,7 @@ export default class extends Controller {
     this.hideOptionalFields()
 
     // Show relevant fields based on rule type
-    if (["path_pattern", "header_pattern", "query_pattern", "body_signature"].includes(ruleType)) {
+    if (["path_pattern"].includes(ruleType)) {
       if (this.hasPatternFieldsTarget) {
         this.patternFieldsTarget.classList.remove("hidden")
         this.updatePatternHelpText(ruleType)
@@ -64,18 +64,6 @@ export default class extends Controller {
       path_pattern: {
         text: "Regex pattern to match URL paths (e.g.,\\.env$|wp-admin|phpmyadmin)",
         placeholder: "Example: \\.env$|\\.git|config\\.php|wp-admin"
-      },
-      header_pattern: {
-        text: 'JSON with header_name and pattern (e.g., {"header_name": "User-Agent", "pattern": "bot.*"})',
-        placeholder: 'Example: {"header_name": "User-Agent", "pattern": ".*[Bb]ot.*"}'
-      },
-      query_pattern: {
-        text: "Regex pattern to match query parameters (e.g., union.*select|<script>)",
-        placeholder: "Example: (?:union|select|insert|update|delete).*\\s+(?:union|select)"
-      },
-      body_signature: {
-        text: "Regex pattern to match request body content (e.g., OR 1=1|<script>)",
-        placeholder: "Example: (?:OR\\s+1\\s*=\\s*1|AND\\s+1\\s*=\\s*1|UNION\\s+SELECT)"
       }
     }
 

app/jobs/fetch_ipapi_data_job.rb

@@ -2,37 +2,74 @@ class FetchIpapiDataJob < ApplicationJob
   queue_as :default
 
   # Fetches IPAPI enrichment data for a NetworkRange
-  # @param network_range_id [Integer] ID of the NetworkRange to enrich
+  # @param network_range_id [Integer] ID of the tracking NetworkRange (usually /24)
   def perform(network_range_id:)
-    network_range = NetworkRange.find_by(id: network_range_id)
-    return unless network_range
-
-    # Skip if we already have IPAPI data and it's recent (< 30 days old)
-    if network_range.has_network_data_from?(:ipapi) &&
-       network_range.last_api_fetch.present? &&
-       network_range.last_api_fetch > 30.days.ago
-      Rails.logger.info "Skipping IPAPI fetch for #{network_range.cidr} - data is recent"
-      return
-    end
+    tracking_network = NetworkRange.find_by(id: network_range_id)
+    return unless tracking_network
 
     # Use the network address (first IP in range) as the representative IP
-    sample_ip = network_range.network_address.split('/').first
+    sample_ip = tracking_network.network_address.split('/').first
 
-    Rails.logger.info "Fetching IPAPI data for #{network_range.cidr} using IP #{sample_ip}"
+    Rails.logger.info "Fetching IPAPI data for #{tracking_network.cidr} using IP #{sample_ip}"
 
     ipapi_data = Ipapi.lookup(sample_ip)
 
     if ipapi_data.present? && !ipapi_data.key?('error')
-      network_range.set_network_data(:ipapi, ipapi_data)
-      network_range.last_api_fetch = Time.current
-      network_range.save!
+      # Check if IPAPI returned a different route than our tracking network
+      ipapi_route = ipapi_data.dig('asn', 'route')
+      target_network = tracking_network
 
-      Rails.logger.info "Successfully fetched IPAPI data for #{network_range.cidr}"
+      if ipapi_route.present? && ipapi_route != tracking_network.cidr
+        # IPAPI returned a different CIDR - find or create that network range
+        Rails.logger.info "IPAPI returned different route: #{ipapi_route} (requested: #{tracking_network.cidr})"
+
+        target_network = NetworkRange.find_or_create_by(network: ipapi_route) do |nr|
+          nr.source = 'api_imported'
+          nr.creation_reason = "Created from IPAPI lookup for #{tracking_network.cidr}"
+        end
+
+        Rails.logger.info "Storing IPAPI data on correct network: #{target_network.cidr}"
+      end
+
+      # Store data on the target network (wherever IPAPI said it belongs)
+      target_network.set_network_data(:ipapi, ipapi_data)
+      target_network.last_api_fetch = Time.current
+      target_network.save!
+
+      # Mark the tracking network as having been queried, with the CIDR that was returned
+      tracking_network.mark_ipapi_queried!(target_network.cidr)
+
+      Rails.logger.info "Successfully fetched IPAPI data for #{tracking_network.cidr} (stored on #{target_network.cidr})"
+
+      # Broadcast to the tracking network
+      broadcast_ipapi_update(tracking_network, ipapi_data)
     else
-      Rails.logger.warn "IPAPI returned error for #{network_range.cidr}: #{ipapi_data}"
+      Rails.logger.warn "IPAPI returned error for #{tracking_network.cidr}: #{ipapi_data}"
+      # Still mark as queried to avoid retrying immediately
+      tracking_network.mark_ipapi_queried!(tracking_network.cidr)
     end
   rescue => e
     Rails.logger.error "Failed to fetch IPAPI data for network_range #{network_range_id}: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
+  ensure
+    # Always clear the fetching status when done
+    tracking_network&.clear_fetching_status!(:ipapi)
+  end
+
+  private
+
+  def broadcast_ipapi_update(network_range, ipapi_data)
+    # Broadcast to a stream specific to this network range
+    Turbo::StreamsChannel.broadcast_replace_to(
+      "network_range_#{network_range.id}",
+      target: "ipapi_data_section",
+      partial: "network_ranges/ipapi_data",
+      locals: {
+        ipapi_data: ipapi_data,
+        network_range: network_range,
+        parent_with_ipapi: nil,
+        ipapi_loading: false
+      }
+    )
   end
 end

app/jobs/geolite_asn_import_job.rb

@@ -89,12 +89,13 @@ class GeoliteAsnImportJob < ApplicationJob
       temp_file.write(file.read)
     end
 
-    temp_file.close
+    # Close but keep the file on disk (false prevents auto-deletion)
+    temp_file.close(false)
     temp_file.path
   rescue => e
     Rails.logger.error "Error downloading file: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
-    temp_file&.close
+    temp_file&.close(false)
     temp_file&.unlink
     nil
   end

app/jobs/geolite_country_import_job.rb

@@ -89,12 +89,13 @@ class GeoliteCountryImportJob < ApplicationJob
       temp_file.write(file.read)
     end
 
-    temp_file.close
+    # Close but keep the file on disk (false prevents auto-deletion)
+    temp_file.close(false)
     temp_file.path
   rescue => e
     Rails.logger.error "Error downloading file: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
-    temp_file&.close
+    temp_file&.close(false)
     temp_file&.unlink
     nil
   end

app/jobs/process_waf_event_job.rb

@@ -10,11 +10,11 @@ class ProcessWafEventJob < ApplicationJob
     if event_data.key?('events') && event_data['events'].is_a?(Array)
       # Multiple events in an array
       events_to_process = event_data['events']
-    elsif event_data.key?('event_id')
-      # Single event
+    elsif event_data.key?('request_id') || event_data.key?('event_id') || event_data.key?('correlation_id')
+      # Single event (support new and old field names)
       events_to_process = [event_data]
     else
-      Rails.logger.warn "Invalid event data format: missing event_id or events array"
+      Rails.logger.warn "Invalid event data format: missing request_id/event_id/correlation_id or events array"
       return
     end
 
@@ -23,50 +23,70 @@ class ProcessWafEventJob < ApplicationJob
         event_start = Time.current
 
         # Generate unique event ID if not provided
-        event_id = single_event_data['event_id'] || SecureRandom.uuid
+        # Support both new (request_id) and old (event_id, correlation_id) field names during cutover
+        request_id = single_event_data['request_id'] ||
+                    single_event_data['event_id'] ||
+                    single_event_data['correlation_id'] ||
+                    SecureRandom.uuid
+
+        # Skip if event already exists (duplicate in batch or retry)
+        if Event.exists?(request_id: request_id)
+          Rails.logger.debug "Skipping duplicate event #{request_id}"
+          next
+        end
 
         # Create the WAF event record
         create_start = Time.current
-        event = Event.create_from_waf_payload!(event_id, single_event_data)
+        event = Event.create_from_waf_payload!(request_id, single_event_data)
         Rails.logger.debug "Event creation took #{((Time.current - create_start) * 1000).round(2)}ms"
 
-        # Ensure network range exists for this IP and evaluate policies if needed
-        if event.ip_address.present?
+        # Process network intelligence and policies
+        # Note: Event.before_save already created the /24 tracking network
+        # and stored it in event.network_range_id
+        if event.network_range_id.present?
           begin
             network_start = Time.current
-            # Single lookup instead of checking has_geo_data? then querying again
-            existing_range = NetworkRange.contains_ip(event.ip_address.to_s).first
-            network_range = existing_range || NetworkRangeGenerator.find_or_create_for_ip(event.ip_address)
-            Rails.logger.debug "Network range lookup/creation took #{((Time.current - network_start) * 1000).round(2)}ms"
+            # The tracking network was already created in Event.before_save
+            tracking_network = event.network_range
+            Rails.logger.debug "Using tracking network #{tracking_network.cidr} (created in before_save)"
 
-            if network_range
-              Rails.logger.debug "Network range #{network_range.cidr} for event IP #{event.ip_address}"
-
-              # Queue IPAPI enrichment if we don't have it yet
-              unless network_range.has_network_data_from?(:ipapi)
-                Rails.logger.info "Queueing IPAPI fetch for #{network_range.cidr}"
-                FetchIpapiDataJob.perform_later(network_range_id: network_range.id)
+            # Queue IPAPI enrichment based on /24 tracking
+            # The tracking network is the /24 that stores ipapi_queried_at
+            if NetworkRange.should_fetch_ipapi_for_ip?(event.ip_address)
+              # Use tracking network for fetch status to avoid race conditions
+              if tracking_network.is_fetching_api_data?(:ipapi)
+                Rails.logger.info "Skipping IPAPI fetch for #{tracking_network.cidr} - already being fetched"
+              else
+                tracking_network.mark_as_fetching_api_data!(:ipapi)
+                Rails.logger.info "Queueing IPAPI fetch for IP #{event.ip_address} (tracking network: #{tracking_network.cidr})"
+                FetchIpapiDataJob.perform_later(network_range_id: tracking_network.id)
               end
+            else
+              Rails.logger.debug "Skipping IPAPI fetch for IP #{event.ip_address} - already queried recently"
+            end
 
-              # Evaluate WAF policies inline if needed (lazy evaluation)
-              # Only runs when: network never evaluated OR policies changed since last evaluation
-              if network_range.needs_policy_evaluation?
-                policy_start = Time.current
-                result = WafPolicyMatcher.evaluate_and_mark!(network_range)
-                Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
+            # Evaluate WAF policies inline if needed (lazy evaluation)
+            # Only runs when: network never evaluated OR policies changed since last evaluation
+            if tracking_network.needs_policy_evaluation?
+              policy_start = Time.current
+              result = WafPolicyMatcher.evaluate_and_mark!(tracking_network)
+              Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
 
-                if result[:generated_rules].any?
-                  Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{network_range.cidr}"
-                end
+              if result[:generated_rules].any?
+                Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{tracking_network.cidr}"
               end
             end
+
+            Rails.logger.debug "Network processing took #{((Time.current - network_start) * 1000).round(2)}ms"
           rescue => e
             Rails.logger.warn "Failed to process network range for event #{event.id}: #{e.message}"
           end
+        elsif event.ip_address.present?
+          Rails.logger.warn "Event #{event.id} has IP but no network_range_id (private IP?)"
         end
 
         total_time = ((Time.current - event_start) * 1000).round(2)
-        Rails.logger.info "Processed WAF event #{event_id} in #{total_time}ms"
+        Rails.logger.info "Processed WAF event #{request_id} in #{total_time}ms"
       rescue ActiveRecord::RecordInvalid => e
         Rails.logger.error "Failed to create WAF event: #{e.message}"
         Rails.logger.error e.record.errors.full_messages.join(", ")

app/jobs/process_waf_policies_job.rb

@@ -9,9 +9,8 @@ class ProcessWafPoliciesJob < ApplicationJob
 
   retry_on StandardError, wait: 5.seconds, attempts: 3
 
-  def perform(network_range_id:, event_id: nil)
-    # Find the network range
-    network_range = NetworkRange.find_by(id: network_range_id)
+  def perform(network_range:, event: nil)
+    # network_range and event are passed as Global IDs and automatically deserialized
     return if network_range.nil?
 
     Rails.logger.debug "Processing WAF policies for network range #{network_range.cidr}"
@@ -55,36 +54,32 @@ class ProcessWafPoliciesJob < ApplicationJob
     network_range.update_column(:policies_evaluated_at, Time.current)
 
     # Update event record if provided
-    if event_id.present?
-      event = Event.find_by(id: event_id)
-      if event.present?
-        # Add policy match information to event metadata
-        # Handle potential nil payload or type issues
-        current_payload = event.payload || {}
+    if event.present?
+      # Add policy match information to event metadata
+      # Handle potential nil payload or type issues
+      current_payload = event.payload || {}
 
-        # Ensure payload is a hash before merging
-        unless current_payload.is_a?(Hash)
-          Rails.logger.warn "Event #{event_id} has invalid payload type: #{current_payload.class}, resetting to hash"
-          current_payload = {}
-        end
-
-        event.update!(payload: current_payload.merge({
-          policy_matches: {
-            matching_policies_count: result[:matching_policies].length,
-            generated_rules_count: result[:generated_rules].length,
-            processed_at: Time.current.iso8601
-          }
-        }))
-      else
-        Rails.logger.warn "Event #{event_id} not found for ProcessWafPoliciesJob, skipping update"
+      # Ensure payload is a hash before merging
+      unless current_payload.is_a?(Hash)
+        Rails.logger.warn "Event #{event.id} has invalid payload type: #{current_payload.class}, resetting to hash"
+        current_payload = {}
       end
+
+      event.update!(payload: current_payload.merge({
+        policy_matches: {
+          matching_policies_count: result[:matching_policies].length,
+          generated_rules_count: result[:generated_rules].length,
+          processed_at: Time.current.iso8601
+        }
+      }))
     end
   end
 
   # Class method for batch processing multiple network ranges
   def self.process_network_ranges(network_range_ids)
     network_range_ids.each do |network_range_id|
-      perform_later(network_range_id: network_range_id)
+      network_range = NetworkRange.find_by(id: network_range_id)
+      perform_later(network_range: network_range) if network_range
     end
   end
 
@@ -109,7 +104,7 @@ class ProcessWafPoliciesJob < ApplicationJob
     Rails.logger.info "Reprocessing #{network_ranges.count} network ranges for policy #{waf_policy_id}"
 
     network_ranges.find_each do |network_range|
-      perform_later(network_range_id: network_range.id)
+      perform_later(network_range: network_range)
     end
   end
 end

app/models/event.rb

@@ -4,6 +4,10 @@ class Event < ApplicationRecord
   # Normalized association for hosts (most valuable compression)
   belongs_to :request_host, optional: true
 
+  # WAF rule associations
+  belongs_to :rule, optional: true
+  has_one :waf_policy, through: :rule
+
   # Enums for fixed value sets
   enum :waf_action, {
     allow: 0,       # allow/pass
@@ -29,7 +33,7 @@ class Event < ApplicationRecord
   # This provides direct array access and efficient indexing
   attribute :tags, :json, default: -> { [] }
 
-  validates :event_id, presence: true, uniqueness: true
+  validates :request_id, presence: true, uniqueness: true
   validates :timestamp, presence: true
 
   scope :recent, -> { order(timestamp: :desc) }
@@ -55,32 +59,42 @@ class Event < ApplicationRecord
     where("tags @> ARRAY[?]", tag_array)
   }
 
-  # Network-based filtering scopes
+  # Network-based filtering scopes - now using denormalized columns
   scope :by_company, ->(company) {
-    joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-      .where("network_ranges.company ILIKE ?", "%#{company}%")
+    where("company ILIKE ?", "%#{company}%")
+  }
+
+  scope :by_country, ->(country) {
+    where(country: country)
   }
 
   scope :by_network_type, ->(type) {
-    joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-      .case(type)
-      .when("datacenter") { where("network_ranges.is_datacenter = ?", true) }
-      .when("vpn") { where("network_ranges.is_vpn = ?", true) }
-      .when("proxy") { where("network_ranges.is_proxy = ?", true) }
-      .when("standard") { where("network_ranges.is_datacenter = ? AND network_ranges.is_vpn = ? AND network_ranges.is_proxy = ?", false, false, false) }
-      .else { none }
+    case type.to_s.downcase
+    when "datacenter"
+      where(is_datacenter: true)
+    when "vpn"
+      where(is_vpn: true)
+    when "proxy"
+      where(is_proxy: true)
+    when "standard"
+      where(is_datacenter: false, is_vpn: false, is_proxy: false)
+    else
+      none
+    end
   }
 
   scope :by_asn, ->(asn) {
-    joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-      .where("network_ranges.asn = ?", asn.to_i)
+    where(asn: asn.to_i)
   }
 
   scope :by_network_cidr, ->(cidr) {
-    joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
-      .where("network_ranges.network = ?", cidr)
+    # This still requires a join since we need to match CIDR
+    joins(:network_range).where("network_ranges.network = ?", cidr)
   }
 
+  # Add association for the optional network_range_id
+  belongs_to :network_range, optional: true
+
   # Path prefix matching using range queries (uses B-tree index efficiently)
   scope :with_path_prefix, ->(prefix_segment_ids) {
     return none if prefix_segment_ids.blank?
@@ -130,13 +144,39 @@ class Event < ApplicationRecord
   # Normalize event fields after extraction
   after_validation :normalize_event_fields, if: :should_normalize?
 
-  def self.create_from_waf_payload!(event_id, payload)
+  # Populate network intelligence from IP address
+  before_save :populate_network_intelligence, if: :should_populate_network_intelligence?
+
+  # Backfill network intelligence for all events
+  def self.backfill_network_intelligence!(batch_size: 10_000)
+    total = where(country: nil).count
+    return 0 if total.zero?
+
+    puts "Backfilling network intelligence for #{total} events..."
+    processed = 0
+
+    where(country: nil).find_in_batches(batch_size: batch_size) do |batch|
+      batch.each(&:save) # Triggers before_save callback
+      processed += batch.size
+      puts "  Processed #{processed}/#{total} (#{(processed.to_f / total * 100).round(1)}%)"
+    end
+
+    processed
+  end
+
+  # Backfill network intelligence for a specific event
+  def backfill_network_intelligence!
+    populate_network_intelligence
+    save!
+  end
+
+  def self.create_from_waf_payload!(request_id, payload)
     # Normalize headers in payload during import phase
     normalized_payload = normalize_payload_headers(payload)
 
     # Create the WAF request event
     create!(
-      event_id: event_id,
+      request_id: request_id,
       timestamp: parse_timestamp(normalized_payload["timestamp"]),
       payload: normalized_payload,
 
@@ -150,7 +190,8 @@ class Event < ApplicationRecord
       response_status: normalized_payload.dig("response", "status_code"),
       response_time_ms: normalized_payload.dig("response", "duration_ms"),
       waf_action: normalize_action(normalized_payload["waf_action"]), # Normalize incoming action values
-      rule_matched: normalized_payload["rule_matched"],
+      # Support both new (rule_id) and old (rule_matched) field names during cutover
+      rule_id: normalized_payload["rule_id"] || normalized_payload["rule_matched"],
       blocked_reason: normalized_payload["blocked_reason"],
 
       # Server/Environment info
@@ -283,7 +324,7 @@ class Event < ApplicationRecord
   end
 
   def rule_matched?
-    rule_matched.present?
+    rule_id.present?
   end
 
   # New path methods for normalization
@@ -343,40 +384,39 @@ class Event < ApplicationRecord
   end
 
   def most_specific_range
-    matching_network_ranges.first
+    # Use the cached network_range_id if available (much faster)
+    return NetworkRange.find_by(id: network_range_id) if network_range_id.present?
+
+    # Fallback to expensive lookup
+    matching_network_ranges.first&.dig(:range)
   end
 
   def broadest_range
-    matching_network_ranges.last
+    matching_network_ranges.last&.dig(:range)
   end
 
   def network_intelligence
-    most_specific_range&.dig(:intelligence) || {}
+    # Use denormalized fields instead of expensive lookup
+    {
+      country: country,
+      company: company,
+      asn: asn,
+      asn_org: asn_org,
+      is_datacenter: is_datacenter,
+      is_vpn: is_vpn,
+      is_proxy: is_proxy
+    }
   end
 
-  def company
-    network_intelligence[:company]
-  end
-
-  def asn
-    network_intelligence[:asn]
-  end
-
-  def asn_org
-    network_intelligence[:asn_org]
-  end
-
-  def is_datacenter?
-    network_intelligence[:is_datacenter] || false
-  end
-
-  def is_proxy?
-    network_intelligence[:is_proxy] || false
-  end
-
-  def is_vpn?
-    network_intelligence[:is_vpn] || false
-  end
+  # Denormalized attribute accessors - these now use the columns directly
+  # No need to override - Rails provides these automatically:
+  # - country (column)
+  # - company (column)
+  # - asn (column)
+  # - asn_org (column)
+  # - is_datacenter (column)
+  # - is_vpn (column)
+  # - is_proxy (column)
 
   # IP validation
   def valid_ipv4?
@@ -480,7 +520,8 @@ class Event < ApplicationRecord
     self.request_url = request_data["url"]
     self.response_status = response_data["status_code"]
     self.response_time_ms = response_data["duration_ms"]
-    self.rule_matched = payload["rule_matched"]
+    # Support both new (rule_id) and old (rule_matched) field names during cutover
+    self.rule_id = payload["rule_id"] || payload["rule_matched"]
     self.blocked_reason = payload["blocked_reason"]
 
     # Store original values for normalization only if they don't exist yet

app/models/network_range.rb

@@ -116,7 +116,7 @@ class NetworkRange < ApplicationRecord
 
   # Parent/child relationships
   def parent_ranges
-    NetworkRange.where("network << ?::inet AND masklen(network) < ?", network.to_s, prefix_length)
+    NetworkRange.where("?::inet << network AND masklen(network) < ?", network.to_s, prefix_length)
                  .order("masklen(network) DESC")
   end
 
@@ -142,6 +142,59 @@ class NetworkRange < ApplicationRecord
                 .first
   end
 
+  # Check if this network or any parent has IPAPI data
+  def has_ipapi_data_available?
+    return true if has_network_data_from?(:ipapi)
+
+    parent_ranges.any? { |parent| parent.has_network_data_from?(:ipapi) }
+  end
+
+  # Generic API fetching status management
+  def is_fetching_api_data?(source)
+    fetching_status = network_data&.dig('fetching_status') || {}
+    fetching_status[source.to_s] &&
+      fetching_status[source.to_s]['started_at'] &&
+      fetching_status[source.to_s]['started_at'] > 5.minutes.ago.to_f
+  end
+
+  def mark_as_fetching_api_data!(source)
+    self.network_data ||= {}
+    self.network_data['fetching_status'] ||= {}
+    self.network_data['fetching_status'][source.to_s] = {
+      'started_at' => Time.current.to_f,
+      'job_id' => SecureRandom.hex(8)
+    }
+    save!
+  end
+
+  def clear_fetching_status!(source)
+    if network_data&.dig('fetching_status')&.dig(source.to_s)
+      self.network_data['fetching_status'].delete(source.to_s)
+      # Clean up empty fetching_status hash
+      self.network_data.delete('fetching_status') if self.network_data['fetching_status'].empty?
+      save!
+    end
+  end
+
+  # Check if we should fetch API data (not available and not currently being fetched)
+  def should_fetch_api_data?(source)
+    return false if send("has_network_data_from?(#{source})") if respond_to?("has_network_data_from?(#{source})")
+    return false if is_fetching_api_data?(source)
+
+    true
+  end
+
+  # Check if this network or any parent has IPAPI data available and no active fetch
+  def should_fetch_ipapi_data?
+    return false if has_ipapi_data_available?
+    return false if is_fetching_api_data?(:ipapi)
+
+    # Also check if any parent is currently fetching IPAPI data
+    return false if parent_ranges.any? { |parent| parent.is_fetching_api_data?(:ipapi) }
+
+    true
+  end
+
   def inherited_intelligence
     return own_intelligence if has_intelligence?
 
@@ -168,6 +221,12 @@ class NetworkRange < ApplicationRecord
     }
   end
 
+  def agent_tally
+    # Rails.cache.fetch("#{to_s}:agent_tally", expires_in: 5.minutes) do 
+      events.map(&:user_agent).tally
+    # end
+  end
+
   # Geographic lookup
   def geo_lookup_country!
     return if country.present?
@@ -189,6 +248,12 @@ class NetworkRange < ApplicationRecord
     where("network && ?", range_cidr)
   end
 
+  def self.findd(cidr)
+    cidr = cidr.gsub("_", "/")
+    cidr = "#{cidr}/24" unless cidr.include?("/")
+    find_by(network: cidr)
+  end
+
   def self.find_or_create_by_cidr(cidr, user: nil, source: nil, reason: nil)
     find_or_create_by(network: cidr) do |range|
       range.user = user
@@ -246,6 +311,63 @@ class NetworkRange < ApplicationRecord
     network_data&.key?(source.to_s) && network_data[source.to_s].present?
   end
 
+  # IPAPI tracking at /24 granularity
+  # Find or create the /24 network for a given IP address
+  def self.find_or_create_tracking_network_for_ip(ip_address)
+    ip = IPAddr.new(ip_address.to_s)
+
+    # Create /24 network for IPv4, /64 for IPv6
+    tracking_cidr = if ip.ipv4?
+      "#{ip.mask(24)}/24"
+    else
+      "#{ip.mask(64)}/64"
+    end
+
+    find_or_create_by(network: tracking_cidr) do |range|
+      range.source = 'auto_generated'
+      range.creation_reason = 'IPAPI tracking network'
+    end
+  end
+
+  # Check if we should fetch IPAPI data for a given IP address
+  # Uses /24 networks as the tracking unit
+  def self.should_fetch_ipapi_for_ip?(ip_address)
+    tracking_network = find_or_create_tracking_network_for_ip(ip_address)
+
+    # Check if /24 has been queried recently
+    queried_at = tracking_network.network_data&.dig('ipapi_queried_at')
+    return true if queried_at.nil?
+
+    # Check if IPAPI returned a CIDR that actually covers this IP
+    # (handles edge case where IPAPI returns /25 or more specific)
+    returned_cidr = tracking_network.network_data&.dig('ipapi_returned_cidr')
+    if returned_cidr.present?
+      begin
+        returned_range = IPAddr.new(returned_cidr)
+        ip = IPAddr.new(ip_address.to_s)
+        # If the IP is NOT covered by what IPAPI returned, fetch again
+        return true unless returned_range.include?(ip)
+      rescue IPAddr::InvalidAddressError => e
+        Rails.logger.warn "Invalid CIDR stored in ipapi_returned_cidr: #{returned_cidr}"
+      end
+    end
+
+    # Re-query after 1 year
+    Time.at(queried_at) < 1.year.ago
+  rescue => e
+    Rails.logger.error "Error checking IPAPI fetch status for #{ip_address}: #{e.message}"
+    true # Default to fetching on error
+  end
+
+  # Mark that we've queried IPAPI for this /24 network
+  # @param returned_cidr [String] The CIDR that IPAPI actually returned (may be more specific than /24)
+  def mark_ipapi_queried!(returned_cidr)
+    self.network_data ||= {}
+    self.network_data['ipapi_queried_at'] = Time.current.to_i
+    self.network_data['ipapi_returned_cidr'] = returned_cidr
+    save!
+  end
+
   # String representations
   def to_s
     cidr
@@ -261,10 +383,12 @@ class NetworkRange < ApplicationRecord
     self[:events_count] || 0
   end
 
+  def events
+    Event.where("ip_address <<= ?", cidr)
+  end
+
   def recent_events(limit: 100)
-    Event.where(ip_address: child_ranges.pluck(:network_address) + [network_address])
-         .recent
-         .limit(limit)
+    events.recent.limit(limit)
   end
 
   def blocking_rules

app/models/rule.rb

@@ -5,7 +5,11 @@
 # Rules define actions to take for matching traffic conditions.
 # Network rules are associated with NetworkRange objects for rich context.
 class Rule < ApplicationRecord
-  # Rule types and actions
+  # Rule enums
+  enum :waf_action, { allow: 0, deny: 1, rate_limit: 2, redirect: 3, log: 4, challenge: 5 }, scopes: false, prefix: true
+  enum :waf_rule_type, { network: 0, rate_limit: 1, path_pattern: 2 }, scopes: false, prefix: true
+
+  # Legacy string constants for backward compatibility
   RULE_TYPES = %w[network rate_limit path_pattern].freeze
   ACTIONS = %w[allow deny rate_limit redirect log challenge].freeze
   SOURCES = %w[manual auto:scanner_detected auto:rate_limit_exceeded auto:bot_detected imported default manual:surgical_block manual:surgical_exception policy].freeze
@@ -14,14 +18,42 @@ class Rule < ApplicationRecord
   belongs_to :user
   belongs_to :network_range, optional: true
   belongs_to :waf_policy, optional: true
+  has_many :events, dependent: :nullify
+
+  # Backward compatibility accessors for transition period
+  def action
+    waf_action
+  end
+
+  def action=(value)
+    self.waf_action = value
+    self[:action] = value  # Also set the legacy column
+  end
+
+  def rule_type
+    waf_rule_type
+  end
+
+  def rule_type=(value)
+    self.waf_rule_type = value
+    self[:rule_type] = value  # Also set the legacy column
+  end
 
   # Validations
-  validates :rule_type, presence: true, inclusion: { in: RULE_TYPES }
-  validates :action, presence: true, inclusion: { in: ACTIONS }
+  validates :waf_rule_type, presence: true, inclusion: { in: waf_rule_types.keys }
+  validates :waf_action, presence: true, inclusion: { in: waf_actions.keys }
   validates :conditions, presence: true, unless: :network_rule?
   validates :enabled, inclusion: { in: [true, false] }
   validates :source, inclusion: { in: SOURCES }
 
+  # Legacy enum definitions (disabled to prevent conflicts)
+  # enum :action, { allow: "allow", deny: "deny", rate_limit: "rate_limit", redirect: "redirect", log: "log", challenge: "challenge" }, scopes: false
+  # enum :rule_type, { network: "network", rate_limit: "rate_limit", path_pattern: "path_pattern" }, scopes: false
+
+  # Legacy validations for backward compatibility during transition
+  # validates :rule_type, presence: true, inclusion: { in: RULE_TYPES }, allow_nil: true
+  # validates :action, presence: true, inclusion: { in: ACTIONS }, allow_nil: true
+
   # Custom validations
   validate :validate_conditions_by_type
   validate :validate_metadata_by_action
@@ -33,16 +65,22 @@ class Rule < ApplicationRecord
   scope :disabled, -> { where(enabled: false) }
   scope :active, -> { enabled.where("expires_at IS NULL OR expires_at > ?", Time.current) }
   scope :expired, -> { where("expires_at IS NOT NULL AND expires_at <= ?", Time.current) }
-  scope :by_type, ->(type) { where(rule_type: type) }
-  scope :network_rules, -> { where(rule_type: "network") }
-  scope :rate_limit_rules, -> { where(rule_type: "rate_limit") }
-  scope :path_pattern_rules, -> { where(rule_type: "path_pattern") }
+  scope :by_type, ->(type) { where(waf_rule_type: type) }
+  scope :network_rules, -> { network }
+  scope :rate_limit_rules, -> { rate_limit }
+  scope :path_pattern_rules, -> { path_pattern }
   scope :by_source, ->(source) { where(source: source) }
   scope :surgical_blocks, -> { where(source: "manual:surgical_block") }
   scope :surgical_exceptions, -> { where(source: "manual:surgical_exception") }
   scope :policy_generated, -> { where(source: "policy") }
   scope :from_waf_policy, ->(waf_policy) { where(waf_policy: waf_policy) }
 
+  # Legacy scopes for backward compatibility
+  scope :by_type_legacy, ->(type) { where(rule_type: type) }
+  scope :network_rules_legacy, -> { where(rule_type: "network") }
+  scope :rate_limit_rules_legacy, -> { where(rule_type: "rate_limit") }
+  scope :path_pattern_rules_legacy, -> { where(rule_type: "path_pattern") }
+
   # Sync queries
   scope :since, ->(timestamp) { where("updated_at >= ?", Time.at(timestamp)).order(:updated_at, :id) }
   scope :sync_order, -> { order(:updated_at, :id) }
@@ -51,18 +89,19 @@ class Rule < ApplicationRecord
   before_validation :set_defaults
   before_validation :parse_json_fields
   before_save :calculate_priority_for_network_rules
+  before_save :sync_legacy_columns
 
   # Rule type checks
   def network_rule?
-    rule_type == "network"
+    waf_rule_type_network?
   end
 
   def rate_limit_rule?
-    rule_type == "rate_limit"
+    waf_rule_type_rate_limit?
   end
 
   def path_pattern_rule?
-    rule_type == "path_pattern"
+    waf_rule_type_path_pattern?
   end
 
   # Network-specific methods
@@ -104,16 +143,16 @@ class Rule < ApplicationRecord
 
   # Action-specific methods
   def redirect_action?
-    action == "redirect"
+    waf_action_redirect?
   end
 
   def challenge_action?
-    action == "challenge"
+    waf_action_challenge?
   end
 
   # Redirect/challenge convenience methods
   def redirect_url
-    metadata&.dig('redirect_url')
+    metadata_hash['redirect_url']
   end
 
   def redirect_status
@@ -162,12 +201,13 @@ class Rule < ApplicationRecord
   end
 
   def disable!(reason: nil)
+    new_metadata = metadata_hash.merge(
+      disabled_at: Time.current.iso8601,
+      disabled_reason: reason
+    )
     update!(
       enabled: false,
-      metadata: metadata.merge(
-        disabled_at: Time.current.iso8601,
-        disabled_reason: reason
-      )
+      metadata: new_metadata
     )
   end
 
@@ -180,8 +220,8 @@ class Rule < ApplicationRecord
   def to_agent_format
     format = {
       id: id,
-      rule_type: rule_type,
-      waf_action: action,  # Agents expect 'waf_action' field
+      waf_rule_type: waf_rule_type,
+      waf_action: waf_action,  # Use the enum field directly
       conditions: agent_conditions,
       priority: agent_priority,
       expires_at: expires_at&.to_i,  # Agents expect Unix timestamps
@@ -224,8 +264,8 @@ class Rule < ApplicationRecord
     network_range = NetworkRange.find_or_create_by_cidr(cidr, user: user, source: 'user_created')
 
     create!(
-      rule_type: 'network',
-      action: action,
+      waf_rule_type: 'network',
+      waf_action: action,
       network_range: network_range,
       user: user,
       **options
@@ -237,8 +277,8 @@ class Rule < ApplicationRecord
     network_range = NetworkRange.find_or_create_by_cidr(parent_cidr, user: user, source: 'user_created')
 
     block_rule = create!(
-      rule_type: 'network',
-      action: 'deny',
+      waf_rule_type: 'network',
+      waf_action: 'deny',
       network_range: network_range,
       source: 'manual:surgical_block',
       user: user,
@@ -255,8 +295,8 @@ class Rule < ApplicationRecord
     ip_network_range = NetworkRange.find_or_create_by_cidr("#{ip_address}/#{ip_address.include?(':') ? '128' : '32'}", user: user, source: 'user_created')
 
     exception_rule = create!(
-      rule_type: 'network',
-      action: 'allow',
+      waf_rule_type: 'network',
+      waf_action: 'allow',
       network_range: ip_network_range,
       source: 'manual:surgical_exception',
       user: user,
@@ -277,8 +317,8 @@ class Rule < ApplicationRecord
     network_range = NetworkRange.find_or_create_by_cidr(cidr, user: user, source: 'user_created')
 
     create!(
-      rule_type: 'rate_limit',
-      action: 'rate_limit',
+      waf_rule_type: 'rate_limit',
+      waf_action: 'rate_limit',
       network_range: network_range,
       conditions: { cidr: cidr, scope: 'ip' },
       metadata: {
@@ -307,7 +347,7 @@ class Rule < ApplicationRecord
 
     # This would need efficient IP range queries
     # For now, simple IP match
-    Event.where(ip_address: network_range.network_address)
+    Event.where("ip_address <<= ?", network_range.cidr)
          .recent
          .limit(limit)
   end
@@ -324,6 +364,18 @@ class Rule < ApplicationRecord
     }
   end
 
+  # Helper method to safely access metadata as hash
+  def metadata_hash
+    case metadata
+    when Hash
+      metadata
+    when String
+      metadata.present? ? (JSON.parse(metadata) rescue {}) : {}
+    else
+      {}
+    end
+  end
+
   private
 
   def set_defaults
@@ -361,7 +413,7 @@ class Rule < ApplicationRecord
   end
 
   def validate_conditions_by_type
-    case rule_type
+    case waf_rule_type
     when "network"
       # Network rules don't need conditions in DB - stored in network_range
       true
@@ -394,7 +446,7 @@ class Rule < ApplicationRecord
   end
 
   def validate_metadata_by_action
-    case action
+    case waf_action
     when "redirect"
       unless metadata&.dig("redirect_url").present?
         errors.add(:metadata, "must include 'redirect_url' for redirect action")
@@ -457,4 +509,14 @@ class Rule < ApplicationRecord
     self.metadata ||= {}
   end
 
+  def sync_legacy_columns
+    # Sync enum values to legacy string columns for backward compatibility
+    if waf_action.present?
+      self[:action] = waf_action
+    end
+    if waf_rule_type.present?
+      self[:rule_type] = waf_rule_type
+    end
+  end
+
   end