Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

test/jobs/cleanup_old_events_job_test.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CleanupOldEventsJobTest < ActiveJob::TestCase
+  setup do
+    # Clear any existing events
+    Event.delete_all
+    # Set default retention to 90 days
+    Setting.set('event_retention_days', '90')
+  end
+
+  test "deletes events older than retention period" do
+    # Create old event (100 days ago - should be deleted)
+    old_event = Event.create!(
+      request_id: "old-request-#{SecureRandom.uuid}",
+      timestamp: 100.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    # Create recent event (30 days ago - should be kept)
+    recent_event = Event.create!(
+      request_id: "recent-request-#{SecureRandom.uuid}",
+      timestamp: 30.days.ago,
+      ip_address: "5.6.7.8",
+      payload: { request: { ip: "5.6.7.8" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 1, count
+    assert_raises(ActiveRecord::RecordNotFound) { old_event.reload }
+    assert_nothing_raised { recent_event.reload }
+  end
+
+  test "respects custom retention period" do
+    # Set retention to 30 days
+    Setting.set('event_retention_days', '30')
+
+    # Create event that's 40 days old (should be deleted with 30-day retention)
+    old_event = Event.create!(
+      request_id: "old-request-#{SecureRandom.uuid}",
+      timestamp: 40.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    # Create event that's 20 days old (should be kept)
+    recent_event = Event.create!(
+      request_id: "recent-request-#{SecureRandom.uuid}",
+      timestamp: 20.days.ago,
+      ip_address: "5.6.7.8",
+      payload: { request: { ip: "5.6.7.8" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 1, count
+    assert_raises(ActiveRecord::RecordNotFound) { old_event.reload }
+    assert_nothing_raised { recent_event.reload }
+  end
+
+  test "does not delete when retention is zero" do
+    Setting.set('event_retention_days', '0')
+
+    old_event = Event.create!(
+      request_id: "old-request-#{SecureRandom.uuid}",
+      timestamp: 100.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 0, count
+    assert_nothing_raised { old_event.reload }
+  end
+
+  test "does not delete when retention is negative" do
+    Setting.set('event_retention_days', '-1')
+
+    old_event = Event.create!(
+      request_id: "old-request-#{SecureRandom.uuid}",
+      timestamp: 100.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 0, count
+    assert_nothing_raised { old_event.reload }
+  end
+
+  test "returns zero when no old events exist" do
+    # Create only recent events
+    Event.create!(
+      request_id: "recent-request-#{SecureRandom.uuid}",
+      timestamp: 30.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 0, count
+  end
+
+  test "returns zero when no events exist" do
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 0, count
+  end
+
+  test "deletes multiple old events" do
+    # Create 5 old events
+    5.times do |i|
+      Event.create!(
+        request_id: "old-request-#{i}-#{SecureRandom.uuid}",
+        timestamp: 100.days.ago,
+        ip_address: "1.2.3.#{i}",
+        payload: { request: { ip: "1.2.3.#{i}" } }
+      )
+    end
+
+    # Create 3 recent events
+    3.times do |i|
+      Event.create!(
+        request_id: "recent-request-#{i}-#{SecureRandom.uuid}",
+        timestamp: 30.days.ago,
+        ip_address: "5.6.7.#{i}",
+        payload: { request: { ip: "5.6.7.#{i}" } }
+      )
+    end
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 5, count
+    assert_equal 3, Event.count
+  end
+
+  test "uses default retention when setting not configured" do
+    # Remove the setting
+    Setting.find_by(key: 'event_retention_days')&.destroy
+
+    # Create event that's 100 days old (should be deleted with default 90-day retention)
+    old_event = Event.create!(
+      request_id: "old-request-#{SecureRandom.uuid}",
+      timestamp: 100.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    # Create event that's 80 days old (should be kept with default 90-day retention)
+    recent_event = Event.create!(
+      request_id: "recent-request-#{SecureRandom.uuid}",
+      timestamp: 80.days.ago,
+      ip_address: "5.6.7.8",
+      payload: { request: { ip: "5.6.7.8" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 1, count
+    assert_raises(ActiveRecord::RecordNotFound) { old_event.reload }
+    assert_nothing_raised { recent_event.reload }
+  end
+
+  test "handles events at exact cutoff boundary correctly" do
+    Setting.set('event_retention_days', '90')
+
+    # Create event exactly at cutoff (should be deleted - uses < comparison)
+    cutoff_event = Event.create!(
+      request_id: "cutoff-request-#{SecureRandom.uuid}",
+      timestamp: 90.days.ago,
+      ip_address: "1.2.3.4",
+      payload: { request: { ip: "1.2.3.4" } }
+    )
+
+    # Create event just inside cutoff (should be kept)
+    inside_event = Event.create!(
+      request_id: "inside-request-#{SecureRandom.uuid}",
+      timestamp: 89.days.ago,
+      ip_address: "5.6.7.8",
+      payload: { request: { ip: "5.6.7.8" } }
+    )
+
+    count = CleanupOldEventsJob.perform_now
+
+    assert_equal 1, count
+    assert_raises(ActiveRecord::RecordNotFound) { cutoff_event.reload }
+    assert_nothing_raised { inside_event.reload }
+  end
+end