Use only parquet files for events

This commit is contained in:
Dan Milne
2025-12-03 17:16:38 +11:00
parent 032243ba6a
commit 693851f664
12 changed files with 673 additions and 165 deletions

View File

@@ -0,0 +1,67 @@
# frozen_string_literal: true
class RemovePostgresEventsCountFromNetworkRanges < ActiveRecord::Migration[8.1]
def up
# Drop triggers first
execute <<-SQL
DROP TRIGGER IF EXISTS update_network_ranges_events_count_after_insert ON events;
DROP TRIGGER IF EXISTS update_network_ranges_events_count_after_delete ON events;
DROP FUNCTION IF EXISTS update_network_range_events_count();
SQL
# Remove index and column
remove_index :network_ranges, :events_count
remove_column :network_ranges, :events_count
end
def down
# Add column back (for rollback)
add_column :network_ranges, :events_count, :integer, null: false, default: 0
add_index :network_ranges, :events_count
# Recreate trigger function
execute <<-SQL
CREATE OR REPLACE FUNCTION update_network_range_events_count()
RETURNS TRIGGER AS $$
BEGIN
-- Update all network ranges that contain IP address
UPDATE network_ranges
SET events_count = events_count +
CASE
WHEN TG_OP = 'INSERT' THEN 1
WHEN TG_OP = 'DELETE' THEN -1
ELSE 0
END
WHERE network >>= NEW.ip_address::inet;
RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;
SQL
# Recreate triggers
execute <<-SQL
CREATE TRIGGER update_network_ranges_events_count_after_insert
AFTER INSERT ON events
FOR EACH ROW
EXECUTE FUNCTION update_network_range_events_count();
SQL
execute <<-SQL
CREATE TRIGGER update_network_ranges_events_count_after_delete
AFTER DELETE ON events
FOR EACH ROW
EXECUTE FUNCTION update_network_range_events_count();
SQL
# Backfill existing counts
execute <<-SQL
UPDATE network_ranges
SET events_count = (
SELECT COUNT(*)
FROM events
WHERE events.ip_address <<= network_ranges.network
);
SQL
end
end

View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.1].define(version: 2025_11_20_003554) do
ActiveRecord::Schema[8.1].define(version: 2025_12_02_070000) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
@@ -128,7 +128,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_20_003554) do
t.string "country"
t.datetime "created_at", null: false
t.text "creation_reason"
t.integer "events_count", default: 0, null: false
t.boolean "is_datacenter", default: false
t.boolean "is_proxy", default: false
t.boolean "is_vpn", default: false
@@ -143,7 +142,6 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_20_003554) do
t.index ["asn_org"], name: "index_network_ranges_on_asn_org"
t.index ["company"], name: "index_network_ranges_on_company"
t.index ["country"], name: "index_network_ranges_on_country"
t.index ["events_count"], name: "index_network_ranges_on_events_count"
t.index ["is_datacenter", "is_proxy", "is_vpn"], name: "idx_network_flags"
t.index ["is_datacenter"], name: "index_network_ranges_on_is_datacenter"
t.index ["network"], name: "index_network_ranges_on_network", opclass: :inet_ops, using: :gist