Lots of updates

app/services/event_normalizer.rb

@@ -31,7 +31,8 @@ class EventNormalizer
     return unless hostname
 
     host = RequestHost.find_or_create_host(hostname)
-    host.increment_usage! unless host.new_record?
+    # NOTE: usage_count increment removed for performance (was adding ~50ms per event)
+    # Can be recalculated with: RequestHost.all.each { |h| h.update(usage_count: h.events.count) }
     @event.request_host = host
   end
 
@@ -83,7 +84,8 @@ class EventNormalizer
 
     segment_ids = segments.map do |segment|
       path_segment = PathSegment.find_or_create_segment(segment)
-      path_segment.increment_usage! unless path_segment.new_record?
+      # NOTE: usage_count increment removed for performance (was adding ~100ms per event for paths with many segments)
+      # Can be recalculated with: PathSegment.all.each { |ps| ps.update(usage_count: Event.where("request_segment_ids @> ARRAY[?]", ps.id).count) }
       path_segment.id
     end
 

app/services/geolite_asn_importer.rb

@@ -138,12 +138,21 @@ class GeoliteAsnImporter
     # Validate network format
     IPAddr.new(network) # This will raise if invalid
 
+    # Store raw GeoLite ASN data in network_data
+    geolite_data = {
+      asn: {
+        autonomous_system_number: asn,
+        autonomous_system_organization: asn_org
+      }
+    }
+
     NetworkRange.upsert(
       {
         network: network,
         asn: asn,
         asn_org: asn_org,
         source: 'geolite_asn',
+        network_data: { geolite: geolite_data },
         updated_at: Time.current
       },
       unique_by: :index_network_ranges_on_network_unique

app/services/geolite_country_importer.rb

@@ -210,16 +210,21 @@ class GeoliteCountryImporter
     # Get location data - prefer geoname_id, then registered_country_geoname_id
     location_data = @locations_cache[geoname_id] || @locations_cache[registered_country_geoname_id] || {}
 
-    additional_data = {
-      geoname_id: geoname_id,
-      registered_country_geoname_id: registered_country_geoname_id,
-      represented_country_geoname_id: row[:represented_country_geoname_id],
-      continent_code: location_data[:continent_code],
-      continent_name: location_data[:continent_name],
-      country_name: location_data[:country_name],
-      is_in_european_union: location_data[:is_in_european_union],
-      is_satellite_provider: is_satellite_provider,
-      is_anycast: is_anycast
+    # Store raw GeoLite country data in network_data[:geolite]
+    geolite_data = {
+      country: {
+        geoname_id: geoname_id,
+        registered_country_geoname_id: registered_country_geoname_id,
+        represented_country_geoname_id: row[:represented_country_geoname_id],
+        continent_code: location_data[:continent_code],
+        continent_name: location_data[:continent_name],
+        country_name: location_data[:country_name],
+        country_iso_code: location_data[:country_iso_code],
+        is_in_european_union: location_data[:is_in_european_union],
+        is_anonymous_proxy: is_anonymous_proxy,
+        is_satellite_provider: is_satellite_provider,
+        is_anycast: is_anycast
+      }
     }.compact
 
     NetworkRange.upsert(
@@ -228,7 +233,7 @@ class GeoliteCountryImporter
         country: location_data[:country_iso_code],
         is_proxy: is_anonymous_proxy,
         source: 'geolite_country',
-        additional_data: additional_data,
+        network_data: { geolite: geolite_data },
         updated_at: Time.current
       },
       unique_by: :index_network_ranges_on_network_unique

app/services/ipapi.rb

@@ -1,12 +1,14 @@
 class Ipapi
   include HTTParty
   BASE_URL = "https://api.ipapi.is/"
-  API_KEY = Rails.application.credentials.ipapi_key
+  
+  def api_key = Setting.ipapi_key
 
   def lookup(ip)
-  response = self.class.get("#{BASE_URL}", query: { q: ip, key: API_KEY })
-  response.parsed_response
-end
+    return unless api_key.present?
+    response = self.class.get("#{BASE_URL}", query: { q: ip, key: api_key })
+    response.parsed_response
+  end
 
   def self.lookup(ip) = new.lookup(ip)
 
@@ -19,7 +21,7 @@ end
     if ip.is_a?(Array)
       post_data(ip)
     else
-      response = self.class.get("#{BASE_URL}", query: { q: ip, key: API_KEY })
+      response = self.class.get("#{BASE_URL}", query: { q: ip, key: api_key })
       response.parsed_response
     end
   rescue JSON::ParserError
@@ -28,7 +30,7 @@ end
 
   def post_data(ips)
     response = self.class.post("#{BASE_URL}",
-      query: { key: API_KEY },
+      query: { key: api_key },
       body: { ips: ips }.to_json,
       headers: { 'Content-Type' => 'application/json' }
     )
@@ -39,7 +41,13 @@ end
       IPAddr.new(ip)
       cidr = data.dig("asn", "route")
 
-      NetworkRange.add_network(cidr).tap { |acl| acl&.update(ip_api_data: data) }
+      network_range = NetworkRange.add_network(cidr)
+      if network_range
+        network_range.set_network_data(:ipapi, data)
+        network_range.last_api_fetch = Time.current
+        network_range.save
+      end
+      network_range
     rescue IPAddr::InvalidAddressError
       puts "Skipping #{ip}"
       next

app/services/waf_policy_matcher.rb

@@ -73,6 +73,12 @@ class WafPolicyMatcher
   def match_and_generate_rules
     find_matching_policies
     generate_rules
+
+    # Return hash format expected by ProcessWafPoliciesJob
+    {
+      matching_policies: @matching_policies,
+      generated_rules: @generated_rules
+    }
   end
 
   # Class methods for batch processing
@@ -81,6 +87,20 @@ class WafPolicyMatcher
     matcher.match_and_generate_rules
   end
 
+  # Evaluate a network range against policies and mark it as evaluated
+  # This is the main entry point for inline policy evaluation
+  def self.evaluate_and_mark!(network_range)
+    return { matching_policies: [], generated_rules: [] } unless network_range
+
+    matcher = new(network_range: network_range)
+    result = matcher.match_and_generate_rules
+
+    # Mark this network range as evaluated
+    network_range.update_column(:policies_evaluated_at, Time.current)
+
+    result
+  end
+
   def self.batch_process_network_ranges(network_ranges)
     results = []