Lots of updates
This commit is contained in:
Dan Milne
38
app/jobs/fetch_ipapi_data_job.rb
Normal file
38
app/jobs/fetch_ipapi_data_job.rb
Normal file
@@ -0,0 +1,38 @@
|
||||
class FetchIpapiDataJob < ApplicationJob
|
||||
queue_as :default
|
||||
|
||||
# Fetches IPAPI enrichment data for a NetworkRange
|
||||
# @param network_range_id [Integer] ID of the NetworkRange to enrich
|
||||
def perform(network_range_id:)
|
||||
network_range = NetworkRange.find_by(id: network_range_id)
|
||||
return unless network_range
|
||||
|
||||
# Skip if we already have IPAPI data and it's recent (< 30 days old)
|
||||
if network_range.has_network_data_from?(:ipapi) &&
|
||||
network_range.last_api_fetch.present? &&
|
||||
network_range.last_api_fetch > 30.days.ago
|
||||
Rails.logger.info "Skipping IPAPI fetch for #{network_range.cidr} - data is recent"
|
||||
return
|
||||
end
|
||||
|
||||
# Use the network address (first IP in range) as the representative IP
|
||||
sample_ip = network_range.network_address.split('/').first
|
||||
|
||||
Rails.logger.info "Fetching IPAPI data for #{network_range.cidr} using IP #{sample_ip}"
|
||||
|
||||
ipapi_data = Ipapi.lookup(sample_ip)
|
||||
|
||||
if ipapi_data.present? && !ipapi_data.key?('error')
|
||||
network_range.set_network_data(:ipapi, ipapi_data)
|
||||
network_range.last_api_fetch = Time.current
|
||||
network_range.save!
|
||||
|
||||
Rails.logger.info "Successfully fetched IPAPI data for #{network_range.cidr}"
|
||||
else
|
||||
Rails.logger.warn "IPAPI returned error for #{network_range.cidr}: #{ipapi_data}"
|
||||
end
|
||||
rescue => e
|
||||
Rails.logger.error "Failed to fetch IPAPI data for network_range #{network_range_id}: #{e.message}"
|
||||
Rails.logger.error e.backtrace.join("\n")
|
||||
end
|
||||
end
|
||||
@@ -1,163 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class GenerateWafRulesJob < ApplicationJob
|
||||
queue_as :waf_rules
|
||||
|
||||
def perform(event_id:)
|
||||
event = Event.find(event_id)
|
||||
|
||||
# Only analyze blocked events for rule generation
|
||||
return unless event.blocked?
|
||||
|
||||
# Generate different types of rules based on patterns
|
||||
generate_ip_rules(event)
|
||||
generate_path_rules(event)
|
||||
generate_user_agent_rules(event)
|
||||
generate_parameter_rules(event)
|
||||
|
||||
# Broadcast rule updates globally
|
||||
ActionCable.server.broadcast("rules", { type: "refresh" })
|
||||
|
||||
rescue => e
|
||||
Rails.logger.error "Error generating WAF rules: #{e.message}"
|
||||
Rails.logger.error e.backtrace.join("\n")
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_ip_rules(event)
|
||||
return unless event.ip_address.present?
|
||||
|
||||
# Check if this IP has multiple violations
|
||||
violation_count = Event
|
||||
.by_ip(event.ip_address)
|
||||
.blocked
|
||||
.where(timestamp: 24.hours.ago..Time.current)
|
||||
.count
|
||||
|
||||
# Log high-violation IPs - no automatic blocking without projects
|
||||
if violation_count >= 10
|
||||
Rails.logger.info "IP with high violation count: #{event.ip_address} (#{violation_count} violations in 24 hours)"
|
||||
end
|
||||
end
|
||||
|
||||
def generate_path_rules(event)
|
||||
return unless event.request_path.present?
|
||||
|
||||
# Look for repeated attack patterns on specific paths
|
||||
path_violations = project.events
|
||||
.where(request_path: event.request_path)
|
||||
.blocked
|
||||
.where(timestamp: 1.hour.ago..Time.current)
|
||||
.count
|
||||
|
||||
# Suggest path rules if 20+ violations on same path
|
||||
if path_violations >= 20
|
||||
suggest_path_rule(project, event.request_path, path_violations)
|
||||
end
|
||||
end
|
||||
|
||||
def generate_user_agent_rules(event)
|
||||
return unless event.user_agent.present?
|
||||
|
||||
# Look for malicious user agents
|
||||
ua_violations = project.events
|
||||
.by_user_agent(event.user_agent)
|
||||
.blocked
|
||||
.where(timestamp: 1.hour.ago..Time.current)
|
||||
.count
|
||||
|
||||
# Suggest user agent rules if 15+ violations from same UA
|
||||
if ua_violations >= 15
|
||||
suggest_user_agent_rule(project, event.user_agent, ua_violations)
|
||||
end
|
||||
end
|
||||
|
||||
def generate_parameter_rules(event)
|
||||
params = event.query_params
|
||||
return unless params.present?
|
||||
|
||||
# Look for suspicious parameter patterns
|
||||
params.each do |key, value|
|
||||
next unless value.is_a?(String)
|
||||
|
||||
# Check for common attack patterns in parameter values
|
||||
if contains_attack_pattern?(value)
|
||||
param_violations = project.events
|
||||
.where("payload LIKE ?", "%#{key}%#{value}%")
|
||||
.blocked
|
||||
.where(timestamp: 6.hours.ago..Time.current)
|
||||
.count
|
||||
|
||||
if param_violations >= 5
|
||||
suggest_parameter_rule(project, key, value, param_violations)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def suggest_path_rule(project, path, violation_count)
|
||||
# Create an issue for manual review
|
||||
Issue.create!(
|
||||
project: project,
|
||||
title: "Suggested Path Rule",
|
||||
description: "Path '#{path}' has #{violation_count} violations in 1 hour",
|
||||
severity: "low",
|
||||
metadata: {
|
||||
type: "path_rule",
|
||||
path: path,
|
||||
violation_count: violation_count,
|
||||
suggested_action: "block"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def suggest_user_agent_rule(project, user_agent, violation_count)
|
||||
# Create an issue for manual review
|
||||
Issue.create!(
|
||||
project: project,
|
||||
title: "Suggested User Agent Rule",
|
||||
description: "User Agent '#{user_agent}' has #{violation_count} violations in 1 hour",
|
||||
severity: "low",
|
||||
metadata: {
|
||||
type: "user_agent_rule",
|
||||
user_agent: user_agent,
|
||||
violation_count: violation_count,
|
||||
suggested_action: "block"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def suggest_parameter_rule(project, param_name, param_value, violation_count)
|
||||
# Create an issue for manual review
|
||||
Issue.create!(
|
||||
project: project,
|
||||
title: "Suggested Parameter Rule",
|
||||
description: "Parameter '#{param_name}' with suspicious values has #{violation_count} violations",
|
||||
severity: "medium",
|
||||
metadata: {
|
||||
type: "parameter_rule",
|
||||
param_name: param_name,
|
||||
param_value: param_value,
|
||||
violation_count: violation_count,
|
||||
suggested_action: "block"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def contains_attack_pattern?(value)
|
||||
# Common attack patterns
|
||||
attack_patterns = [
|
||||
/<script/i, # XSS
|
||||
/union.*select/i, # SQL injection
|
||||
/\.\./, # Directory traversal
|
||||
/\/etc\/passwd/i, # File inclusion
|
||||
/cmd\.exe/i, # Command injection
|
||||
/eval\(/i, # Code injection
|
||||
/javascript:/i, # JavaScript protocol
|
||||
/onload=/i, # Event handler injection
|
||||
]
|
||||
|
||||
attack_patterns.any? { |pattern| value.match?(pattern) }
|
||||
end
|
||||
end
|
||||
@@ -1,125 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class ProcessWafAnalyticsJob < ApplicationJob
|
||||
queue_as :waf_analytics
|
||||
|
||||
def perform(event_id:)
|
||||
event = Event.find(event_id)
|
||||
|
||||
# Analyze event patterns
|
||||
analyze_traffic_patterns(event)
|
||||
analyze_geographic_distribution(event)
|
||||
analyze_attack_vectors(event)
|
||||
|
||||
# Update global analytics cache
|
||||
update_analytics_cache
|
||||
|
||||
rescue => e
|
||||
Rails.logger.error "Error processing WAF analytics: #{e.message}"
|
||||
Rails.logger.error e.backtrace.join("\n")
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def analyze_traffic_patterns(event)
|
||||
# Look for unusual traffic spikes
|
||||
recent_events = Event.where(timestamp: 5.minutes.ago..Time.current)
|
||||
|
||||
# Use a default threshold since we no longer have project-specific thresholds
|
||||
threshold = 1000 # Default threshold
|
||||
if recent_events.count > threshold * 5
|
||||
# High traffic detected - create an issue
|
||||
Issue.create!(
|
||||
title: "High Traffic Spike Detected",
|
||||
description: "Detected #{recent_events.count} requests in the last 5 minutes",
|
||||
severity: "medium",
|
||||
event_id: event.id,
|
||||
metadata: {
|
||||
event_count: recent_events.count,
|
||||
time_window: "5 minutes",
|
||||
threshold: threshold * 5
|
||||
}
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def analyze_geographic_distribution(event)
|
||||
return unless event.has_geo_data?
|
||||
|
||||
country_code = event.lookup_country
|
||||
return unless country_code.present?
|
||||
|
||||
# Check if this country is unusual globally by joining through network ranges
|
||||
country_events = Event
|
||||
.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network")
|
||||
.where("network_ranges.country = ?", country_code)
|
||||
.where(timestamp: 1.hour.ago..Time.current)
|
||||
|
||||
# If this is the first event from this country or unusual spike
|
||||
if country_events.count == 1 || country_events.count > 100
|
||||
Rails.logger.info "Unusual geographic activity from #{country_code}"
|
||||
end
|
||||
end
|
||||
|
||||
def analyze_attack_vectors(event)
|
||||
return unless event.blocked?
|
||||
|
||||
# Analyze common attack patterns
|
||||
analyze_ip_reputation(event)
|
||||
analyze_user_agent_patterns(event)
|
||||
analyze_path_attacks(event)
|
||||
end
|
||||
|
||||
def analyze_ip_reputation(event)
|
||||
return unless event.ip_address.present?
|
||||
|
||||
# Count recent blocks from this IP
|
||||
recent_blocks = Event
|
||||
.by_ip(event.ip_address)
|
||||
.blocked
|
||||
.where(timestamp: 1.hour.ago..Time.current)
|
||||
|
||||
if recent_blocks.count >= 5
|
||||
# Log IP reputation issue - no automatic IP blocking without projects
|
||||
Rails.logger.warn "IP with poor reputation detected: #{event.ip_address} (#{recent_blocks.count} blocks in 1 hour)"
|
||||
end
|
||||
end
|
||||
|
||||
def analyze_user_agent_patterns(event)
|
||||
return unless event.user_agent.present?
|
||||
|
||||
# Look for common bot/user agent patterns
|
||||
suspicious_patterns = [
|
||||
/bot/i, /crawler/i, /spider/i, /scanner/i,
|
||||
/python/i, /curl/i, /wget/i, /nmap/i
|
||||
]
|
||||
|
||||
if suspicious_patterns.any? { |pattern| event.user_agent.match?(pattern) }
|
||||
# Log suspicious user agent for potential rule generation
|
||||
Rails.logger.info "Suspicious user agent detected: #{event.user_agent}"
|
||||
end
|
||||
end
|
||||
|
||||
def analyze_path_attacks(event)
|
||||
return unless event.request_path.present?
|
||||
|
||||
# Look for common attack paths
|
||||
attack_patterns = [
|
||||
/\.\./, # Directory traversal
|
||||
/admin/i, # Admin panel access
|
||||
/wp-admin/i, # WordPress admin
|
||||
/\.php/i, # PHP files
|
||||
/union.*select/i, # SQL injection
|
||||
/script.*>/i # XSS attempts
|
||||
]
|
||||
|
||||
if attack_patterns.any? { |pattern| event.request_path.match?(pattern) }
|
||||
Rails.logger.info "Potential attack path detected: #{event.request_path}"
|
||||
end
|
||||
end
|
||||
|
||||
def update_analytics_cache
|
||||
# Update cached analytics for faster dashboard loading
|
||||
Rails.cache.delete("global_analytics")
|
||||
end
|
||||
end
|
||||
@@ -20,47 +20,53 @@ class ProcessWafEventJob < ApplicationJob
|
||||
|
||||
events_to_process.each do |single_event_data|
|
||||
begin
|
||||
event_start = Time.current
|
||||
|
||||
# Generate unique event ID if not provided
|
||||
event_id = single_event_data['event_id'] || SecureRandom.uuid
|
||||
|
||||
# Create the WAF event record
|
||||
create_start = Time.current
|
||||
event = Event.create_from_waf_payload!(event_id, single_event_data)
|
||||
Rails.logger.debug "Event creation took #{((Time.current - create_start) * 1000).round(2)}ms"
|
||||
|
||||
# Log geo-location data status (uses NetworkRange delegation)
|
||||
if event.ip_address.present?
|
||||
begin
|
||||
unless event.has_geo_data?
|
||||
Rails.logger.debug "No geo data available for event #{event.id} with IP #{event.ip_address}"
|
||||
end
|
||||
rescue => e
|
||||
Rails.logger.warn "Failed to check geo data for event #{event.id}: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
# Ensure network range exists for this IP and process policies
|
||||
# Ensure network range exists for this IP and evaluate policies if needed
|
||||
if event.ip_address.present?
|
||||
begin
|
||||
network_start = Time.current
|
||||
# Single lookup instead of checking has_geo_data? then querying again
|
||||
existing_range = NetworkRange.contains_ip(event.ip_address.to_s).first
|
||||
network_range = existing_range || NetworkRangeGenerator.find_or_create_for_ip(event.ip_address)
|
||||
Rails.logger.debug "Network range lookup/creation took #{((Time.current - network_start) * 1000).round(2)}ms"
|
||||
|
||||
if network_range
|
||||
Rails.logger.debug "Network range #{network_range.cidr} for event IP #{event.ip_address}"
|
||||
|
||||
# Process WAF policies for this network range
|
||||
ProcessWafPoliciesJob.perform_later(network_range_id: network_range.id, event_id: event.id)
|
||||
# Queue IPAPI enrichment if we don't have it yet
|
||||
unless network_range.has_network_data_from?(:ipapi)
|
||||
Rails.logger.info "Queueing IPAPI fetch for #{network_range.cidr}"
|
||||
FetchIpapiDataJob.perform_later(network_range_id: network_range.id)
|
||||
end
|
||||
|
||||
# Evaluate WAF policies inline if needed (lazy evaluation)
|
||||
# Only runs when: network never evaluated OR policies changed since last evaluation
|
||||
if network_range.needs_policy_evaluation?
|
||||
policy_start = Time.current
|
||||
result = WafPolicyMatcher.evaluate_and_mark!(network_range)
|
||||
Rails.logger.debug "Policy evaluation took #{((Time.current - policy_start) * 1000).round(2)}ms"
|
||||
|
||||
if result[:generated_rules].any?
|
||||
Rails.logger.info "Generated #{result[:generated_rules].length} rules for #{network_range.cidr}"
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue => e
|
||||
Rails.logger.warn "Failed to create network range for event #{event.id}: #{e.message}"
|
||||
Rails.logger.warn "Failed to process network range for event #{event.id}: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
# Trigger analytics processing
|
||||
ProcessWafAnalyticsJob.perform_later(event_id: event.id)
|
||||
|
||||
# Check for automatic rule generation opportunities
|
||||
GenerateWafRulesJob.perform_later(event_id: event.id)
|
||||
|
||||
Rails.logger.info "Processed WAF event #{event_id}"
|
||||
total_time = ((Time.current - event_start) * 1000).round(2)
|
||||
Rails.logger.info "Processed WAF event #{event_id} in #{total_time}ms"
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
Rails.logger.error "Failed to create WAF event: #{e.message}"
|
||||
Rails.logger.error e.record.errors.full_messages.join(", ")
|
||||
@@ -70,9 +76,6 @@ class ProcessWafEventJob < ApplicationJob
|
||||
end
|
||||
end
|
||||
|
||||
# Broadcast real-time updates once per batch
|
||||
ActionCable.server.broadcast("events", { type: "refresh" })
|
||||
|
||||
Rails.logger.info "Processed #{events_to_process.count} WAF events"
|
||||
end
|
||||
end
|
||||
@@ -17,8 +17,13 @@ class ProcessWafPoliciesJob < ApplicationJob
|
||||
Rails.logger.debug "Processing WAF policies for network range #{network_range.cidr}"
|
||||
|
||||
# Use WafPolicyMatcher to find and generate rules
|
||||
matcher = WafPolicyMatcher.new(network_range: network_range)
|
||||
result = matcher.match_and_generate_rules
|
||||
begin
|
||||
matcher = WafPolicyMatcher.new(network_range: network_range)
|
||||
result = matcher.match_and_generate_rules
|
||||
rescue => e
|
||||
Rails.logger.error "WafPolicyMatcher failed for network range #{network_range.cidr}: #{e.message}"
|
||||
result = { matching_policies: [], generated_rules: [] }
|
||||
end
|
||||
|
||||
# Log results
|
||||
if result[:matching_policies].any?
|
||||
@@ -42,27 +47,36 @@ class ProcessWafPoliciesJob < ApplicationJob
|
||||
Rails.logger.info " Challenge type: #{rule.challenge_type}"
|
||||
end
|
||||
end
|
||||
|
||||
# Trigger agent sync for new rules if there are any
|
||||
if result[:generated_rules].any?
|
||||
RulesSyncJob.perform_later
|
||||
end
|
||||
else
|
||||
Rails.logger.debug "No matching policies found for network range #{network_range.cidr}"
|
||||
end
|
||||
|
||||
# Mark network range as evaluated
|
||||
network_range.update_column(:policies_evaluated_at, Time.current)
|
||||
|
||||
# Update event record if provided
|
||||
if event_id.present?
|
||||
event = Event.find_by(id: event_id)
|
||||
if event.present?
|
||||
# Add policy match information to event metadata
|
||||
event.update!(payload: event.payload.merge({
|
||||
# Handle potential nil payload or type issues
|
||||
current_payload = event.payload || {}
|
||||
|
||||
# Ensure payload is a hash before merging
|
||||
unless current_payload.is_a?(Hash)
|
||||
Rails.logger.warn "Event #{event_id} has invalid payload type: #{current_payload.class}, resetting to hash"
|
||||
current_payload = {}
|
||||
end
|
||||
|
||||
event.update!(payload: current_payload.merge({
|
||||
policy_matches: {
|
||||
matching_policies_count: result[:matching_policies].length,
|
||||
generated_rules_count: result[:generated_rules].length,
|
||||
processed_at: Time.current.iso8601
|
||||
}
|
||||
}))
|
||||
else
|
||||
Rails.logger.warn "Event #{event_id} not found for ProcessWafPoliciesJob, skipping update"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user