Smarter backfil

app/controllers/events_controller.rb

@@ -20,37 +20,71 @@ class EventsController < ApplicationController
   end
 
   def index
-    @events = Event.includes(:network_range, :rule).order(timestamp: :desc)
-    Rails.logger.debug "Found #{@events.count} total events"
-    Rails.logger.debug "Action: #{params[:waf_action]}"
+    # Build filters hash from params
+    filters = {}
+    filters[:ip] = params[:ip] if params[:ip].present?
+    filters[:waf_action] = params[:waf_action] if params[:waf_action].present?
+    filters[:country] = params[:country] if params[:country].present?
+    filters[:rule_id] = params[:rule_id] if params[:rule_id].present?
+    filters[:company] = params[:company] if params[:company].present?
+    filters[:network_type] = params[:network_type] if params[:network_type].present?
+    filters[:asn] = params[:asn] if params[:asn].present?
+    filters[:exclude_bots] = params[:exclude_bots] if params[:exclude_bots] == "true"
 
-    # Apply filters
-    @events = @events.by_ip(params[:ip]) if params[:ip].present?
-    @events = @events.by_waf_action(params[:waf_action]) if params[:waf_action].present?
-    @events = @events.by_country(params[:country]) if params[:country].present?
-    @events = @events.where(rule_id: params[:rule_id]) if params[:rule_id].present?
+    # Handle network_cidr filter (requires NetworkRange lookup)
+    if params[:network_cidr].present?
+      range = NetworkRange.find_by(network: params[:network_cidr])
+      filters[:network_range_id] = range.id if range
+    end
 
-    # Network-based filters (now using denormalized columns)
-    @events = @events.by_company(params[:company]) if params[:company].present?
-    @events = @events.by_network_type(params[:network_type]) if params[:network_type].present?
-    @events = @events.by_asn(params[:asn]) if params[:asn].present?
-    @events = @events.by_network_cidr(params[:network_cidr]) if params[:network_cidr].present?
+    # Try DuckDB first, fallback to PostgreSQL if unavailable
+    result = EventDdb.search(filters, page: params[:page]&.to_i || 1, per_page: 50)
 
-    # Bot filtering
-    @events = @events.exclude_bots if params[:exclude_bots] == "true"
+    if result
+      # DuckDB query succeeded
+      @pagy = Pagy.new(count: result[:total_count], page: result[:page], items: result[:per_page])
+      @events = result[:events]
 
-    Rails.logger.debug "Events count after filtering: #{@events.count}"
+      # Load network_range associations for events that have network_range_id
+      network_range_ids = @events.map(&:network_range_id).compact.uniq
+      if network_range_ids.any?
+        network_ranges = NetworkRange.where(id: network_range_ids).index_by(&:id)
+        @events.each do |event|
+          event.network_range = network_ranges[event.network_range_id] if event.network_range_id
+        end
+      end
 
-    # Debug info
-    Rails.logger.debug "Events count before pagination: #{@events.count}"
+      # Load rule associations if needed
+      rule_ids = @events.map(&:rule_id).compact.uniq
+      if rule_ids.any?
+        rules = Rule.where(id: rule_ids).index_by(&:id)
+        @events.each do |event|
+          event.rule = rules[event.rule_id] if event.rule_id
+        end
+      end
 
-    # Paginate
-    @pagy, @events = pagy(@events, items: 50)
+      Rails.logger.debug "[DuckDB] Found #{result[:total_count]} total events, showing page #{result[:page]}"
+    else
+      # Fallback to PostgreSQL
+      Rails.logger.warn "[EventsController] DuckDB unavailable, falling back to PostgreSQL"
 
-    # Network ranges are now preloaded via includes(:network_range)
-    # The denormalized network_range_id makes this much faster than IP containment lookups
+      @events = Event.includes(:network_range, :rule).order(timestamp: :desc)
 
-    Rails.logger.debug "Events count after pagination: #{@events.count}"
-    Rails.logger.debug "Pagy info: #{@pagy.count} total, #{@pagy.pages} pages"
+      # Apply filters using ActiveRecord scopes
+      @events = @events.by_ip(params[:ip]) if params[:ip].present?
+      @events = @events.by_waf_action(params[:waf_action]) if params[:waf_action].present?
+      @events = @events.by_country(params[:country]) if params[:country].present?
+      @events = @events.where(rule_id: params[:rule_id]) if params[:rule_id].present?
+      @events = @events.by_company(params[:company]) if params[:company].present?
+      @events = @events.by_network_type(params[:network_type]) if params[:network_type].present?
+      @events = @events.by_asn(params[:asn]) if params[:asn].present?
+      @events = @events.by_network_cidr(params[:network_cidr]) if params[:network_cidr].present?
+      @events = @events.exclude_bots if params[:exclude_bots] == "true"
+
+      # Paginate
+      @pagy, @events = pagy(@events, items: 50)
+
+      Rails.logger.debug "[PostgreSQL] Events count: #{@pagy.count} total, #{@pagy.pages} pages"
+    end
   end
 end