Many updates

app/jobs/fetch_ipapi_data_job.rb

@@ -2,37 +2,74 @@ class FetchIpapiDataJob < ApplicationJob
   queue_as :default
 
   # Fetches IPAPI enrichment data for a NetworkRange
-  # @param network_range_id [Integer] ID of the NetworkRange to enrich
+  # @param network_range_id [Integer] ID of the tracking NetworkRange (usually /24)
   def perform(network_range_id:)
-    network_range = NetworkRange.find_by(id: network_range_id)
-    return unless network_range
-
-    # Skip if we already have IPAPI data and it's recent (< 30 days old)
-    if network_range.has_network_data_from?(:ipapi) &&
-       network_range.last_api_fetch.present? &&
-       network_range.last_api_fetch > 30.days.ago
-      Rails.logger.info "Skipping IPAPI fetch for #{network_range.cidr} - data is recent"
-      return
-    end
+    tracking_network = NetworkRange.find_by(id: network_range_id)
+    return unless tracking_network
 
     # Use the network address (first IP in range) as the representative IP
-    sample_ip = network_range.network_address.split('/').first
+    sample_ip = tracking_network.network_address.split('/').first
 
-    Rails.logger.info "Fetching IPAPI data for #{network_range.cidr} using IP #{sample_ip}"
+    Rails.logger.info "Fetching IPAPI data for #{tracking_network.cidr} using IP #{sample_ip}"
 
     ipapi_data = Ipapi.lookup(sample_ip)
 
     if ipapi_data.present? && !ipapi_data.key?('error')
-      network_range.set_network_data(:ipapi, ipapi_data)
-      network_range.last_api_fetch = Time.current
-      network_range.save!
+      # Check if IPAPI returned a different route than our tracking network
+      ipapi_route = ipapi_data.dig('asn', 'route')
+      target_network = tracking_network
 
-      Rails.logger.info "Successfully fetched IPAPI data for #{network_range.cidr}"
+      if ipapi_route.present? && ipapi_route != tracking_network.cidr
+        # IPAPI returned a different CIDR - find or create that network range
+        Rails.logger.info "IPAPI returned different route: #{ipapi_route} (requested: #{tracking_network.cidr})"
+
+        target_network = NetworkRange.find_or_create_by(network: ipapi_route) do |nr|
+          nr.source = 'api_imported'
+          nr.creation_reason = "Created from IPAPI lookup for #{tracking_network.cidr}"
+        end
+
+        Rails.logger.info "Storing IPAPI data on correct network: #{target_network.cidr}"
+      end
+
+      # Store data on the target network (wherever IPAPI said it belongs)
+      target_network.set_network_data(:ipapi, ipapi_data)
+      target_network.last_api_fetch = Time.current
+      target_network.save!
+
+      # Mark the tracking network as having been queried, with the CIDR that was returned
+      tracking_network.mark_ipapi_queried!(target_network.cidr)
+
+      Rails.logger.info "Successfully fetched IPAPI data for #{tracking_network.cidr} (stored on #{target_network.cidr})"
+
+      # Broadcast to the tracking network
+      broadcast_ipapi_update(tracking_network, ipapi_data)
     else
-      Rails.logger.warn "IPAPI returned error for #{network_range.cidr}: #{ipapi_data}"
+      Rails.logger.warn "IPAPI returned error for #{tracking_network.cidr}: #{ipapi_data}"
+      # Still mark as queried to avoid retrying immediately
+      tracking_network.mark_ipapi_queried!(tracking_network.cidr)
     end
   rescue => e
     Rails.logger.error "Failed to fetch IPAPI data for network_range #{network_range_id}: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
+  ensure
+    # Always clear the fetching status when done
+    tracking_network&.clear_fetching_status!(:ipapi)
+  end
+
+  private
+
+  def broadcast_ipapi_update(network_range, ipapi_data)
+    # Broadcast to a stream specific to this network range
+    Turbo::StreamsChannel.broadcast_replace_to(
+      "network_range_#{network_range.id}",
+      target: "ipapi_data_section",
+      partial: "network_ranges/ipapi_data",
+      locals: {
+        ipapi_data: ipapi_data,
+        network_range: network_range,
+        parent_with_ipapi: nil,
+        ipapi_loading: false
+      }
+    )
   end
 end

app/jobs/geolite_asn_import_job.rb

@@ -89,12 +89,13 @@ class GeoliteAsnImportJob < ApplicationJob
       temp_file.write(file.read)
     end
 
-    temp_file.close
+    # Close but keep the file on disk (false prevents auto-deletion)
+    temp_file.close(false)
     temp_file.path
   rescue => e
     Rails.logger.error "Error downloading file: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
-    temp_file&.close
+    temp_file&.close(false)
     temp_file&.unlink
     nil
   end

app/jobs/geolite_country_import_job.rb

@@ -89,12 +89,13 @@ class GeoliteCountryImportJob < ApplicationJob
       temp_file.write(file.read)
     end
 
-    temp_file.close
+    # Close but keep the file on disk (false prevents auto-deletion)
+    temp_file.close(false)
     temp_file.path
   rescue => e
     Rails.logger.error "Error downloading file: #{e.message}"
     Rails.logger.error e.backtrace.join("\n")
-    temp_file&.close
+    temp_file&.close(false)
     temp_file&.unlink
     nil
   end

app/jobs/process_waf_event_job.rb

@@ -10,11 +10,11 @@ class ProcessWafEventJob < ApplicationJob
     if event_data.key?('events') && event_data['events'].is_a?(Array)
       # Multiple events in an array
       events_to_process = event_data['events']
-    elsif event_data.key?('event_id')
-      # Single event
+    elsif event_data.key?('request_id') || event_data.key?('event_id') || event_data.key?('correlation_id')
+      # Single event (support new and old field names)
       events_to_process = [event_data]
     else
-      Rails.logger.warn "Invalid event data format: missing event_id or events array"
+      Rails.logger.warn "Invalid event data format: missing request_id/event_id/correlation_id or events array"
       return
     end
 
@@ -23,50 +23,70 @@ class ProcessWafEventJob < ApplicationJob
         event_start = Time.current
 
         # Generate unique event ID if not provided
-        event_id = single_event_data['event_id'] || SecureRandom.uuid
+        # Support both new (request_id) and old (event_id, correlation_id) field names during cutover
+        request_id = single_event_data['request_id'] ||
+                    single_event_data['event_id'] ||
+                    single_event_data['correlation_id'] ||
+                    SecureRandom.uuid
+
+        # Skip if event already exists (duplicate in batch or retry)
+        if Event.exists?(request_id: request_id)
+          Rails.logger.debug "Skipping duplicate event #{request_id}"
+          next
+        end
 
         # Create the WAF event record
         create_start = Time.current
-        event = Event.create_from_waf_payload!(event_id, single_event_data)
+        event = Event.create_from_waf_payload!(request_id, single_event_data)
         Rails.logger.debug "Event creation took #{((Time.current - create_start) * 1000).round(2)}ms"
 
-        # Ensure network range exists for this IP and evaluate policies if needed
-        if event.ip_address.present?
+        # Process network intelligence and policies
+        # Note: Event.before_save already created the /24 tracking network
+        # and stored it in event.network_range_id
+        if event.network_range_id.present?
           begin
             network_start = Time.current
-            # Single lookup instead of checking has_geo_data? then querying again
-            existing_range = NetworkRange.contains_ip(event.ip_address.to_s).first
-            network_range = existing_range || NetworkRangeGenerator.find_or_create_for_ip(event.ip_address)
-            Rails.logger.debug "Network range lookup/creation took #{((Time.current - network_start) * 1000).round(2)}ms"
+            # The tracking network was already created in Event.before_save
+            tracking_network = event.network_range
+            Rails.logger.debug "Using tracking network #{tracking_network.cidr} (created in before_save)"
 
-            if network_range
-              Rails.logger.debug "Network range #{network_range.cidr} for event IP #{event.ip_address}"
-
-              # Queue IPAPI enrichment if we don't have it yet
-              unless network_range.has_network_data_from?(:ipapi)
-                Rails.logger.info "Queueing IPAPI fetch for #{network_range.cidr}"
-                FetchIpapiDataJob.perform_later(network_range_id: network_range.id)
+            # Queue IPAPI enrichment based on /24 tracking
+            # The tracking network is the /24 that stores ipapi_queried_at
+            if NetworkRange.should_fetch_ipapi_for_ip?(event.ip_address)
+              # Use tracking network for fetch status to avoid race conditions
+              if tracking_network.is_fetching_api_data?(:ipapi)
+                Rails.logger.info "Skipping IPAPI fetch for #{tracking_network.cidr} - already being fetched"
+              else
+                tracking_network.mark_as_fetching_api_data!(:ipapi)
+                Rails.logger.info "Queueing IPAPI fetch for IP #{event.ip_address} (tracking network: #{tracking_network.cidr})"
+                FetchIpapiDataJob.perform_later(network_range_id: tracking_network.id)
               end
+            else
+              Rails.logger.debug "Skipping IPAPI fetch for IP #{event.ip_address} - already queried recently"
+            end
 
-              # Evaluate WAF policies inline if needed (lazy evaluation)
-              # Only runs when: network never evaluated OR policies changed since last evaluation
-              if network_range.needs_policy_evaluation?
-                policy_start = Time.current
-                result = WafPolicyMatcher.evaluate_and_mark!(network_range)
-                Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
+            # Evaluate WAF policies inline if needed (lazy evaluation)
+            # Only runs when: network never evaluated OR policies changed since last evaluation
+            if tracking_network.needs_policy_evaluation?
+              policy_start = Time.current
+              result = WafPolicyMatcher.evaluate_and_mark!(tracking_network)
+              Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
 
-                if result[:generated_rules].any?
-                  Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{network_range.cidr}"
-                end
+              if result[:generated_rules].any?
+                Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{tracking_network.cidr}"
               end
             end
+
+            Rails.logger.debug "Network processing took #{((Time.current - network_start) * 1000).round(2)}ms"
           rescue => e
             Rails.logger.warn "Failed to process network range for event #{event.id}: #{e.message}"
           end
+        elsif event.ip_address.present?
+          Rails.logger.warn "Event #{event.id} has IP but no network_range_id (private IP?)"
         end
 
         total_time = ((Time.current - event_start) * 1000).round(2)
-        Rails.logger.info "Processed WAF event #{event_id} in #{total_time}ms"
+        Rails.logger.info "Processed WAF event #{request_id} in #{total_time}ms"
       rescue ActiveRecord::RecordInvalid => e
         Rails.logger.error "Failed to create WAF event: #{e.message}"
         Rails.logger.error e.record.errors.full_messages.join(", ")

app/jobs/process_waf_policies_job.rb

@@ -9,9 +9,8 @@ class ProcessWafPoliciesJob < ApplicationJob
 
   retry_on StandardError, wait: 5.seconds, attempts: 3
 
-  def perform(network_range_id:, event_id: nil)
-    # Find the network range
-    network_range = NetworkRange.find_by(id: network_range_id)
+  def perform(network_range:, event: nil)
+    # network_range and event are passed as Global IDs and automatically deserialized
     return if network_range.nil?
 
     Rails.logger.debug "Processing WAF policies for network range #{network_range.cidr}"
@@ -55,36 +54,32 @@ class ProcessWafPoliciesJob < ApplicationJob
     network_range.update_column(:policies_evaluated_at, Time.current)
 
     # Update event record if provided
-    if event_id.present?
-      event = Event.find_by(id: event_id)
-      if event.present?
-        # Add policy match information to event metadata
-        # Handle potential nil payload or type issues
-        current_payload = event.payload || {}
+    if event.present?
+      # Add policy match information to event metadata
+      # Handle potential nil payload or type issues
+      current_payload = event.payload || {}
 
-        # Ensure payload is a hash before merging
-        unless current_payload.is_a?(Hash)
-          Rails.logger.warn "Event #{event_id} has invalid payload type: #{current_payload.class}, resetting to hash"
-          current_payload = {}
-        end
-
-        event.update!(payload: current_payload.merge({
-          policy_matches: {
-            matching_policies_count: result[:matching_policies].length,
-            generated_rules_count: result[:generated_rules].length,
-            processed_at: Time.current.iso8601
-          }
-        }))
-      else
-        Rails.logger.warn "Event #{event_id} not found for ProcessWafPoliciesJob, skipping update"
+      # Ensure payload is a hash before merging
+      unless current_payload.is_a?(Hash)
+        Rails.logger.warn "Event #{event.id} has invalid payload type: #{current_payload.class}, resetting to hash"
+        current_payload = {}
       end
+
+      event.update!(payload: current_payload.merge({
+        policy_matches: {
+          matching_policies_count: result[:matching_policies].length,
+          generated_rules_count: result[:generated_rules].length,
+          processed_at: Time.current.iso8601
+        }
+      }))
     end
   end
 
   # Class method for batch processing multiple network ranges
   def self.process_network_ranges(network_range_ids)
     network_range_ids.each do |network_range_id|
-      perform_later(network_range_id: network_range_id)
+      network_range = NetworkRange.find_by(id: network_range_id)
+      perform_later(network_range: network_range) if network_range
     end
   end
 
@@ -109,7 +104,7 @@ class ProcessWafPoliciesJob < ApplicationJob
     Rails.logger.info "Reprocessing #{network_ranges.count} network ranges for policy #{waf_policy_id}"
 
     network_ranges.find_each do |network_range|
-      perform_later(network_range_id: network_range.id)
+      perform_later(network_range: network_range)
     end
   end
 end