Many updates

app/jobs/process_waf_event_job.rb

@@ -10,11 +10,11 @@ class ProcessWafEventJob < ApplicationJob
     if event_data.key?('events') && event_data['events'].is_a?(Array)
       # Multiple events in an array
       events_to_process = event_data['events']
-    elsif event_data.key?('event_id')
-      # Single event
+    elsif event_data.key?('request_id') || event_data.key?('event_id') || event_data.key?('correlation_id')
+      # Single event (support new and old field names)
       events_to_process = [event_data]
     else
-      Rails.logger.warn "Invalid event data format: missing event_id or events array"
+      Rails.logger.warn "Invalid event data format: missing request_id/event_id/correlation_id or events array"
       return
     end
 
@@ -23,50 +23,70 @@ class ProcessWafEventJob < ApplicationJob
         event_start = Time.current
 
         # Generate unique event ID if not provided
-        event_id = single_event_data['event_id'] || SecureRandom.uuid
+        # Support both new (request_id) and old (event_id, correlation_id) field names during cutover
+        request_id = single_event_data['request_id'] ||
+                    single_event_data['event_id'] ||
+                    single_event_data['correlation_id'] ||
+                    SecureRandom.uuid
+
+        # Skip if event already exists (duplicate in batch or retry)
+        if Event.exists?(request_id: request_id)
+          Rails.logger.debug "Skipping duplicate event #{request_id}"
+          next
+        end
 
         # Create the WAF event record
         create_start = Time.current
-        event = Event.create_from_waf_payload!(event_id, single_event_data)
+        event = Event.create_from_waf_payload!(request_id, single_event_data)
         Rails.logger.debug "Event creation took #{((Time.current - create_start) * 1000).round(2)}ms"
 
-        # Ensure network range exists for this IP and evaluate policies if needed
-        if event.ip_address.present?
+        # Process network intelligence and policies
+        # Note: Event.before_save already created the /24 tracking network
+        # and stored it in event.network_range_id
+        if event.network_range_id.present?
           begin
             network_start = Time.current
-            # Single lookup instead of checking has_geo_data? then querying again
-            existing_range = NetworkRange.contains_ip(event.ip_address.to_s).first
-            network_range = existing_range || NetworkRangeGenerator.find_or_create_for_ip(event.ip_address)
-            Rails.logger.debug "Network range lookup/creation took #{((Time.current - network_start) * 1000).round(2)}ms"
+            # The tracking network was already created in Event.before_save
+            tracking_network = event.network_range
+            Rails.logger.debug "Using tracking network #{tracking_network.cidr} (created in before_save)"
 
-            if network_range
-              Rails.logger.debug "Network range #{network_range.cidr} for event IP #{event.ip_address}"
-
-              # Queue IPAPI enrichment if we don't have it yet
-              unless network_range.has_network_data_from?(:ipapi)
-                Rails.logger.info "Queueing IPAPI fetch for #{network_range.cidr}"
-                FetchIpapiDataJob.perform_later(network_range_id: network_range.id)
+            # Queue IPAPI enrichment based on /24 tracking
+            # The tracking network is the /24 that stores ipapi_queried_at
+            if NetworkRange.should_fetch_ipapi_for_ip?(event.ip_address)
+              # Use tracking network for fetch status to avoid race conditions
+              if tracking_network.is_fetching_api_data?(:ipapi)
+                Rails.logger.info "Skipping IPAPI fetch for #{tracking_network.cidr} - already being fetched"
+              else
+                tracking_network.mark_as_fetching_api_data!(:ipapi)
+                Rails.logger.info "Queueing IPAPI fetch for IP #{event.ip_address} (tracking network: #{tracking_network.cidr})"
+                FetchIpapiDataJob.perform_later(network_range_id: tracking_network.id)
               end
+            else
+              Rails.logger.debug "Skipping IPAPI fetch for IP #{event.ip_address} - already queried recently"
+            end
 
-              # Evaluate WAF policies inline if needed (lazy evaluation)
-              # Only runs when: network never evaluated OR policies changed since last evaluation
-              if network_range.needs_policy_evaluation?
-                policy_start = Time.current
-                result = WafPolicyMatcher.evaluate_and_mark!(network_range)
-                Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
+            # Evaluate WAF policies inline if needed (lazy evaluation)
+            # Only runs when: network never evaluated OR policies changed since last evaluation
+            if tracking_network.needs_policy_evaluation?
+              policy_start = Time.current
+              result = WafPolicyMatcher.evaluate_and_mark!(tracking_network)
+              Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
 
-                if result[:generated_rules].any?
-                  Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{network_range.cidr}"
-                end
+              if result[:generated_rules].any?
+                Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{tracking_network.cidr}"
               end
             end
+
+            Rails.logger.debug "Network processing took #{((Time.current - network_start) * 1000).round(2)}ms"
           rescue => e
             Rails.logger.warn "Failed to process network range for event #{event.id}: #{e.message}"
           end
+        elsif event.ip_address.present?
+          Rails.logger.warn "Event #{event.id} has IP but no network_range_id (private IP?)"
         end
 
         total_time = ((Time.current - event_start) * 1000).round(2)
-        Rails.logger.info "Processed WAF event #{event_id} in #{total_time}ms"
+        Rails.logger.info "Processed WAF event #{request_id} in #{total_time}ms"
       rescue ActiveRecord::RecordInvalid => e
         Rails.logger.error "Failed to create WAF event: #{e.message}"
         Rails.logger.error e.record.errors.full_messages.join(", ")