Baffle-WAF/baffle-hub
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Accepts incoming events and correctly parses them into events. GeoLite2 integration complete"

Browse Source
This commit is contained in:
Dan Milne
2025-11-04 00:11:10 +11:00
parent 0cbd462e7c
commit 5ff166613e
49 changed files with 4489 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
db/migrate/20251103080823_enhance_rules_table_for_sync.rb Normal file
View File

@@ -0,0 +1,56 @@
class EnhanceRulesTableForSync < ActiveRecord::Migration[8.1]
def change
# Remove rule_sets relationship (we're skipping rule sets for Phase 1)
if foreign_key_exists?(:rules, :rule_sets)
remove_foreign_key :rules, :rule_sets
end
if column_exists?(:rules, :rule_set_id)
remove_column :rules, :rule_set_id
end
change_table :rules do |t|
# Add source field to track rule origin
unless column_exists?(:rules, :source)
t.string :source, limit: 100
end
# Ensure core fields exist with proper types
unless column_exists?(:rules, :rule_type)
t.string :rule_type, null: false
end
unless column_exists?(:rules, :action)
t.string :action, null: false
end
unless column_exists?(:rules, :conditions)
t.json :conditions, null: false, default: {}
end
unless column_exists?(:rules, :metadata)
t.json :metadata, default: {}
end
unless column_exists?(:rules, :priority)
t.integer :priority
end
unless column_exists?(:rules, :expires_at)
t.datetime :expires_at
end
unless column_exists?(:rules, :enabled)
t.boolean :enabled, default: true, null: false
end
end
# Add indexes for efficient sync queries
add_index :rules, [:updated_at, :id], if_not_exists: true, name: "idx_rules_sync"
add_index :rules, :enabled, if_not_exists: true
add_index :rules, :expires_at, if_not_exists: true
add_index :rules, :source, if_not_exists: true
add_index :rules, :rule_type, if_not_exists: true
add_index :rules, [:rule_type, :enabled], if_not_exists: true, name: "idx_rules_type_enabled"
end
end

70
db/migrate/20251103093205_split_network_ranges_into_ipv4_and_ipv6.rb Normal file
View File

@@ -0,0 +1,70 @@
class SplitNetworkRangesIntoIpv4AndIpv6 < ActiveRecord::Migration[8.1]
def change
# Drop the old network_ranges table (no data to preserve)
drop_table :network_ranges, if_exists: true
# Create optimized IPv4 ranges table
create_table :ipv4_ranges do |t|
# Range fields for fast lookups
t.integer :network_start, limit: 8, null: false
t.integer :network_end, limit: 8, null: false
t.integer :network_prefix, null: false
# IP intelligence metadata
t.string :company
t.integer :asn
t.string :asn_org
t.boolean :is_datacenter, default: false
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.string :ip_api_country
t.string :geo2_country
t.text :abuser_scores
t.text :additional_data
t.timestamp :last_api_fetch
t.timestamps
end
# Optimized indexes for IPv4
add_index :ipv4_ranges, [:network_start, :network_end, :network_prefix],
name: "idx_ipv4_range_lookup"
add_index :ipv4_ranges, :asn, name: "idx_ipv4_asn"
add_index :ipv4_ranges, :company, name: "idx_ipv4_company"
add_index :ipv4_ranges, :ip_api_country, name: "idx_ipv4_country"
add_index :ipv4_ranges, [:is_datacenter, :is_proxy, :is_vpn],
name: "idx_ipv4_flags"
# Create optimized IPv6 ranges table
create_table :ipv6_ranges do |t|
# Range fields for fast lookups (binary for 128-bit addresses)
t.binary :network_start, limit: 16, null: false
t.binary :network_end, limit: 16, null: false
t.integer :network_prefix, null: false
# IP intelligence metadata (same as IPv4)
t.string :company
t.integer :asn
t.string :asn_org
t.boolean :is_datacenter, default: false
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.string :ip_api_country
t.string :geo2_country
t.text :abuser_scores
t.text :additional_data
t.timestamp :last_api_fetch
t.timestamps
end
# Optimized indexes for IPv6
add_index :ipv6_ranges, [:network_start, :network_end, :network_prefix],
name: "idx_ipv6_range_lookup"
add_index :ipv6_ranges, :asn, name: "idx_ipv6_asn"
add_index :ipv6_ranges, :company, name: "idx_ipv6_company"
add_index :ipv6_ranges, :ip_api_country, name: "idx_ipv6_country"
add_index :ipv6_ranges, [:is_datacenter, :is_proxy, :is_vpn],
name: "idx_ipv6_flags"
end
end

16
db/migrate/20251103103521_create_geo_ip_databases.rb Normal file
View File

@@ -0,0 +1,16 @@
class CreateGeoIpDatabases < ActiveRecord::Migration[8.1]
def change
create_table :geo_ip_databases do |t|
t.string :database_type
t.string :version
t.string :file_path
t.integer :file_size
t.string :checksum_md5
t.datetime :downloaded_at
t.datetime :last_checked_at
t.boolean :is_active
t.timestamps
end
end
end

25
db/migrate/20251103105609_drop_geo_ip_databases_table.rb Normal file
View File

@@ -0,0 +1,25 @@
# frozen_string_literal: true
class DropGeoIpDatabasesTable < ActiveRecord::Migration[8.1]
def up
drop_table :geo_ip_databases
end
def down
create_table :geo_ip_databases do |t|
t.string :database_type, null: false
t.string :version, null: false
t.string :file_path, null: false
t.integer :file_size, null: false
t.string :checksum_md5, null: false
t.datetime :downloaded_at, null: false
t.datetime :last_checked_at
t.boolean :is_active, default: true
t.timestamps
end
add_index :geo_ip_databases, :is_active
add_index :geo_ip_databases, :database_type
add_index :geo_ip_databases, :file_path, unique: true
end
end

74
db/migrate/20251103130430_change_request_method_to_integer_in_events.rb Normal file
View File

@@ -0,0 +1,74 @@
class ChangeRequestMethodToIntegerInEvents < ActiveRecord::Migration[8.1]
def change
# Convert enum columns from string to integer for proper enum support
reversible do |dir|
dir.up do
# Map request_method string values to enum integers
execute <<-SQL
UPDATE events
SET request_method = CASE
WHEN LOWER(request_method) = 'get' THEN '0'
WHEN LOWER(request_method) = 'post' THEN '1'
WHEN LOWER(request_method) = 'put' THEN '2'
WHEN LOWER(request_method) = 'patch' THEN '3'
WHEN LOWER(request_method) = 'delete' THEN '4'
WHEN LOWER(request_method) = 'head' THEN '5'
WHEN LOWER(request_method) = 'options' THEN '6'
ELSE '0' -- Default to GET for unknown values
END
WHERE request_method IS NOT NULL;
SQL
# Map waf_action string values to enum integers
execute <<-SQL
UPDATE events
SET waf_action = CASE
WHEN LOWER(waf_action) = 'allow' THEN '0'
WHEN LOWER(waf_action) IN ('deny', 'block') THEN '1'
WHEN LOWER(waf_action) = 'redirect' THEN '2'
WHEN LOWER(waf_action) = 'challenge' THEN '3'
ELSE '0' -- Default to allow for unknown values
END
WHERE waf_action IS NOT NULL;
SQL
# Change column types to integer
change_column :events, :request_method, :integer
change_column :events, :waf_action, :integer
end
dir.down do
# Convert back to string values
change_column :events, :request_method, :string
change_column :events, :waf_action, :string
execute <<-SQL
UPDATE events
SET request_method = CASE request_method
WHEN 0 THEN 'get'
WHEN 1 THEN 'post'
WHEN 2 THEN 'put'
WHEN 3 THEN 'patch'
WHEN 4 THEN 'delete'
WHEN 5 THEN 'head'
WHEN 6 THEN 'options'
ELSE 'get' -- Default to GET for unknown values
END
WHERE request_method IS NOT NULL;
SQL
execute <<-SQL
UPDATE events
SET waf_action = CASE waf_action
WHEN 0 THEN 'allow'
WHEN 1 THEN 'deny'
WHEN 2 THEN 'redirect'
WHEN 3 THEN 'challenge'
ELSE 'allow' -- Default to allow for unknown values
END
WHERE waf_action IS NOT NULL;
SQL
end
end
end
end

64
db/schema.rb
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
ActiveRecord::Schema[8.1].define(version: 2025_11_03_130430) do
create_table "events", force: :cascade do |t|
t.string "agent_name"
t.string "agent_version"
@@ -24,7 +24,7 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.json "payload"
t.integer "project_id", null: false
t.integer "request_host_id"
t.string "request_method"
t.integer "request_method"
t.string "request_path"
t.string "request_protocol"
t.string "request_segment_ids"
@@ -36,7 +36,7 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.datetime "timestamp", null: false
t.datetime "updated_at", null: false
t.text "user_agent"
t.string "waf_action"
t.integer "waf_action"
t.index ["event_id"], name: "index_events_on_event_id", unique: true
t.index ["ip_address"], name: "index_events_on_ip_address"
t.index ["project_id", "ip_address"], name: "index_events_on_project_id_and_ip_address"
@@ -50,7 +50,7 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.index ["waf_action"], name: "index_events_on_waf_action"
end
create_table "network_ranges", force: :cascade do |t|
create_table "ipv4_ranges", force: :cascade do |t|
t.text "abuser_scores"
t.text "additional_data"
t.integer "asn"
@@ -58,21 +58,44 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.string "company"
t.datetime "created_at", null: false
t.string "geo2_country"
t.binary "ip_address", null: false
t.string "ip_api_country"
t.integer "ip_version", null: false
t.boolean "is_datacenter", default: false
t.boolean "is_proxy", default: false
t.boolean "is_vpn", default: false
t.datetime "last_api_fetch"
t.integer "network_end", limit: 8, null: false
t.integer "network_prefix", null: false
t.integer "network_start", limit: 8, null: false
t.datetime "updated_at", null: false
t.index ["asn"], name: "idx_network_ranges_asn"
t.index ["company"], name: "idx_network_ranges_company"
t.index ["ip_address", "network_prefix"], name: "idx_network_ranges_ip_range"
t.index ["ip_api_country"], name: "idx_network_ranges_country"
t.index ["ip_version"], name: "idx_network_ranges_version"
t.index ["is_datacenter", "is_proxy", "is_vpn"], name: "idx_network_ranges_flags"
t.index ["asn"], name: "idx_ipv4_asn"
t.index ["company"], name: "idx_ipv4_company"
t.index ["ip_api_country"], name: "idx_ipv4_country"
t.index ["is_datacenter", "is_proxy", "is_vpn"], name: "idx_ipv4_flags"
t.index ["network_start", "network_end", "network_prefix"], name: "idx_ipv4_range_lookup"
end
create_table "ipv6_ranges", force: :cascade do |t|
t.text "abuser_scores"
t.text "additional_data"
t.integer "asn"
t.string "asn_org"
t.string "company"
t.datetime "created_at", null: false
t.string "geo2_country"
t.string "ip_api_country"
t.boolean "is_datacenter", default: false
t.boolean "is_proxy", default: false
t.boolean "is_vpn", default: false
t.datetime "last_api_fetch"
t.binary "network_end", limit: 16, null: false
t.integer "network_prefix", null: false
t.binary "network_start", limit: 16, null: false
t.datetime "updated_at", null: false
t.index ["asn"], name: "idx_ipv6_asn"
t.index ["company"], name: "idx_ipv6_company"
t.index ["ip_api_country"], name: "idx_ipv6_country"
t.index ["is_datacenter", "is_proxy", "is_vpn"], name: "idx_ipv6_flags"
t.index ["network_start", "network_end", "network_prefix"], name: "idx_ipv6_range_lookup"
end
create_table "path_segments", force: :cascade do |t|
@@ -101,6 +124,13 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.index ["slug"], name: "index_projects_on_slug", unique: true
end
create_table "request_actions", force: :cascade do |t|
t.string "action", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["action"], name: "index_request_actions_on_action", unique: true
end
create_table "request_hosts", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "first_seen_at", null: false
@@ -148,14 +178,18 @@ ActiveRecord::Schema[8.1].define(version: 2025_11_03_035249) do
t.datetime "expires_at"
t.json "metadata"
t.integer "priority"
t.integer "rule_set_id", null: false
t.string "rule_type"
t.string "source", limit: 100
t.string "target"
t.datetime "updated_at", null: false
t.index ["rule_set_id"], name: "index_rules_on_rule_set_id"
t.index ["enabled"], name: "index_rules_on_enabled"
t.index ["expires_at"], name: "index_rules_on_expires_at"
t.index ["rule_type", "enabled"], name: "idx_rules_type_enabled"
t.index ["rule_type"], name: "index_rules_on_rule_type"
t.index ["source"], name: "index_rules_on_source"
t.index ["updated_at", "id"], name: "idx_rules_sync"
end
add_foreign_key "events", "projects"
add_foreign_key "events", "request_hosts"
add_foreign_key "rules", "rule_sets"
end