Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

This commit is contained in:
Dan Milne
2025-11-18 16:40:05 +11:00
parent ef56779584
commit 3f274c842c
37 changed files with 3522 additions and 151 deletions

View File

@@ -53,16 +53,15 @@ class ProcessWafEventJob < ApplicationJob
# Queue IPAPI enrichment based on /24 tracking
# The tracking network is the /24 that stores ipapi_queried_at
if NetworkRange.should_fetch_ipapi_for_ip?(event.ip_address)
# Use tracking network for fetch status to avoid race conditions
if tracking_network.is_fetching_api_data?(:ipapi)
Rails.logger.info "Skipping IPAPI fetch for #{tracking_network.cidr} - already being fetched"
else
tracking_network.mark_as_fetching_api_data!(:ipapi)
# Atomically mark as fetching - this prevents duplicate jobs via database lock
if tracking_network.mark_as_fetching_api_data!(:ipapi)
Rails.logger.info "Queueing IPAPI fetch for IP #{event.ip_address} (tracking network: #{tracking_network.cidr})"
FetchIpapiDataJob.perform_later(network_range_id: tracking_network.id)
else
Rails.logger.info "Skipping IPAPI fetch for #{tracking_network.cidr} - another job already started"
end
else
Rails.logger.debug "Skipping IPAPI fetch for IP #{event.ip_address} - already queried recently"
Rails.logger.debug "Skipping IPAPI fetch for IP #{event.ip_address} - already queried or being fetched"
end
# Evaluate WAF policies inline if needed (lazy evaluation)