Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

app/jobs/cleanup_old_events_job.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# CleanupOldEventsJob - Removes events older than the configured retention period
+#
+# This job runs periodically (hourly) to clean up old events based on the
+# event_retention_days setting. This helps keep the database size manageable
+# and improves query performance.
+#
+# The retention period is configurable via the 'event_retention_days' setting
+# (default: 90 days). This allows administrators to balance between historical
+# data retention and database performance.
+#
+# Schedule: Every hour (configured in config/recurring.yml)
+class CleanupOldEventsJob < ApplicationJob
+  queue_as :background
+
+  def perform
+    retention_days = Setting.event_retention_days
+
+    # Don't delete if retention is set to 0 or negative (disabled)
+    if retention_days <= 0
+      Rails.logger.info "CleanupOldEventsJob: Event retention disabled (retention_days: #{retention_days})"
+      return 0
+    end
+
+    cutoff_date = retention_days.days.ago
+
+    # Count events to be deleted
+    old_events = Event.where('timestamp < ?', cutoff_date)
+    count = old_events.count
+
+    if count.zero?
+      Rails.logger.info "CleanupOldEventsJob: No events older than #{retention_days} days found"
+      return 0
+    end
+
+    Rails.logger.info "CleanupOldEventsJob: Deleting #{count} events older than #{retention_days} days (before #{cutoff_date})"
+
+    # Delete in batches to avoid long-running transactions
+    deleted_count = 0
+    batch_size = 10_000
+
+    old_events.in_batches(of: batch_size) do |batch|
+      batch_count = batch.delete_all
+      deleted_count += batch_count
+      Rails.logger.info "CleanupOldEventsJob: Deleted batch of #{batch_count} events (total: #{deleted_count}/#{count})"
+    end
+
+    Rails.logger.info "CleanupOldEventsJob: Successfully deleted #{deleted_count} events"
+    deleted_count
+  end
+end