First commit!

app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

app/jobs/event_normalization_job.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+class EventNormalizationJob < ApplicationJob
+  queue_as :default
+
+  # Normalize all existing events
+  def perform_all_events(batch_size: 1000)
+    total_events = Event.where(request_host_id: nil).count
+    Rails.logger.info "Starting normalization of #{total_events} events"
+
+    offset = 0
+    processed = 0
+
+    loop do
+      events = Event.where(request_host_id: nil)
+                   .limit(batch_size)
+                   .offset(offset)
+                   .includes(:project)
+
+      break if events.empty?
+
+      events.each do |event|
+        begin
+          EventNormalizer.normalize_event!(event)
+          event.save!
+          processed += 1
+        rescue => e
+          Rails.logger.error "Failed to normalize event #{event.id}: #{e.message}"
+        end
+      end
+
+      Rails.logger.info "Processed #{processed}/#{total_events} events"
+      offset += batch_size
+    end
+
+    Rails.logger.info "Completed normalization of #{processed} events"
+  end
+
+  # Normalize a specific event
+  def perform(event_id)
+    event = Event.find(event_id)
+
+    EventNormalizer.normalize_event!(event)
+    event.save!
+
+    Rails.logger.info "Successfully normalized event #{event_id}"
+  rescue ActiveRecord::RecordNotFound
+    Rails.logger.error "Event #{event_id} not found for normalization"
+  rescue => e
+    Rails.logger.error "Failed to normalize event #{event_id}: #{e.message}"
+    raise
+  end
+
+  # Normalize events for a specific project
+  def perform_for_project(project_id, batch_size: 1000)
+    project = Project.find(project_id)
+    total_events = project.events.where(request_host_id: nil).count
+    Rails.logger.info "Starting normalization of #{total_events} events for project #{project.name}"
+
+    offset = 0
+    processed = 0
+
+    loop do
+      events = project.events
+                      .where(request_host_id: nil)
+                      .limit(batch_size)
+                      .offset(offset)
+
+      break if events.empty?
+
+      events.each do |event|
+        begin
+          EventNormalizer.normalize_event!(event)
+          event.save!
+          processed += 1
+        rescue => e
+          Rails.logger.error "Failed to normalize event #{event.id}: #{e.message}"
+        end
+      end
+
+      Rails.logger.info "Processed #{processed}/#{total_events} events for project #{project.name}"
+      offset += batch_size
+    end
+
+    Rails.logger.info "Completed normalization of #{processed} events for project #{project.name}"
+  end
+end

app/jobs/generate_waf_rules_job.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+class GenerateWafRulesJob < ApplicationJob
+  queue_as :waf_rules
+
+  def perform(project_id:, event_id:)
+    project = Project.find(project_id)
+    event = Event.find(event_id)
+
+    # Only analyze blocked events for rule generation
+    return unless event.blocked?
+
+    # Generate different types of rules based on patterns
+    generate_ip_rules(project, event)
+    generate_path_rules(project, event)
+    generate_user_agent_rules(project, event)
+    generate_parameter_rules(project, event)
+
+    # Notify project of new rules
+    project.broadcast_rules_refresh
+
+  rescue => e
+    Rails.logger.error "Error generating WAF rules: #{e.message}"
+    Rails.logger.error e.backtrace.join("\n")
+  end
+
+  private
+
+  def generate_ip_rules(project, event)
+    return unless event.ip_address.present?
+
+    # Check if this IP has multiple violations
+    violation_count = project.events
+      .by_ip(event.ip_address)
+      .blocked
+      .where(timestamp: 24.hours.ago..Time.current)
+      .count
+
+    # Auto-block IPs with 10+ violations in 24 hours
+    if violation_count >= 10 && !project.blocked_ips.include?(event.ip_address)
+      project.add_ip_rule(
+        event.ip_address,
+        'block',
+        expires_at: 7.days.from_now,
+        reason: "Auto-generated: #{violation_count} violations in 24 hours"
+      )
+
+      Rails.logger.info "Auto-blocked IP #{event.ip_address} for project #{project.slug}"
+    end
+  end
+
+  def generate_path_rules(project, event)
+    return unless event.request_path.present?
+
+    # Look for repeated attack patterns on specific paths
+    path_violations = project.events
+      .where(request_path: event.request_path)
+      .blocked
+      .where(timestamp: 1.hour.ago..Time.current)
+      .count
+
+    # Suggest path rules if 20+ violations on same path
+    if path_violations >= 20
+      suggest_path_rule(project, event.request_path, path_violations)
+    end
+  end
+
+  def generate_user_agent_rules(project, event)
+    return unless event.user_agent.present?
+
+    # Look for malicious user agents
+    ua_violations = project.events
+      .by_user_agent(event.user_agent)
+      .blocked
+      .where(timestamp: 1.hour.ago..Time.current)
+      .count
+
+    # Suggest user agent rules if 15+ violations from same UA
+    if ua_violations >= 15
+      suggest_user_agent_rule(project, event.user_agent, ua_violations)
+    end
+  end
+
+  def generate_parameter_rules(project, event)
+    params = event.query_params
+    return unless params.present?
+
+    # Look for suspicious parameter patterns
+    params.each do |key, value|
+      next unless value.is_a?(String)
+
+      # Check for common attack patterns in parameter values
+      if contains_attack_pattern?(value)
+        param_violations = project.events
+          .where("payload LIKE ?", "%#{key}%#{value}%")
+          .blocked
+          .where(timestamp: 6.hours.ago..Time.current)
+          .count
+
+        if param_violations >= 5
+          suggest_parameter_rule(project, key, value, param_violations)
+        end
+      end
+    end
+  end
+
+  def suggest_path_rule(project, path, violation_count)
+    # Create an issue for manual review
+    Issue.create!(
+      project: project,
+      title: "Suggested Path Rule",
+      description: "Path '#{path}' has #{violation_count} violations in 1 hour",
+      severity: "low",
+      metadata: {
+        type: "path_rule",
+        path: path,
+        violation_count: violation_count,
+        suggested_action: "block"
+      }
+    )
+  end
+
+  def suggest_user_agent_rule(project, user_agent, violation_count)
+    # Create an issue for manual review
+    Issue.create!(
+      project: project,
+      title: "Suggested User Agent Rule",
+      description: "User Agent '#{user_agent}' has #{violation_count} violations in 1 hour",
+      severity: "low",
+      metadata: {
+        type: "user_agent_rule",
+        user_agent: user_agent,
+        violation_count: violation_count,
+        suggested_action: "block"
+      }
+    )
+  end
+
+  def suggest_parameter_rule(project, param_name, param_value, violation_count)
+    # Create an issue for manual review
+    Issue.create!(
+      project: project,
+      title: "Suggested Parameter Rule",
+      description: "Parameter '#{param_name}' with suspicious values has #{violation_count} violations",
+      severity: "medium",
+      metadata: {
+        type: "parameter_rule",
+        param_name: param_name,
+        param_value: param_value,
+        violation_count: violation_count,
+        suggested_action: "block"
+      }
+    )
+  end
+
+  def contains_attack_pattern?(value)
+    # Common attack patterns
+    attack_patterns = [
+      /<script/i,                    # XSS
+      /union.*select/i,             # SQL injection
+      /\.\./,                       # Directory traversal
+      /\/etc\/passwd/i,             # File inclusion
+      /cmd\.exe/i,                  # Command injection
+      /eval\(/i,                    # Code injection
+      /javascript:/i,               # JavaScript protocol
+      /onload=/i,                   # Event handler injection
+    ]
+
+    attack_patterns.any? { |pattern| value.match?(pattern) }
+  end
+end

app/jobs/process_waf_analytics_job.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+class ProcessWafAnalyticsJob < ApplicationJob
+  queue_as :waf_analytics
+
+  def perform(project_id:, event_id:)
+    project = Project.find(project_id)
+    event = Event.find(event_id)
+
+    # Analyze event patterns
+    analyze_traffic_patterns(project, event)
+    analyze_geographic_distribution(project, event)
+    analyze_attack_vectors(project, event)
+
+    # Update project analytics cache
+    update_project_analytics(project)
+
+  rescue => e
+    Rails.logger.error "Error processing WAF analytics: #{e.message}"
+    Rails.logger.error e.backtrace.join("\n")
+  end
+
+  private
+
+  def analyze_traffic_patterns(project, event)
+    # Look for unusual traffic spikes
+    recent_events = project.events.where(timestamp: 5.minutes.ago..Time.current)
+
+    if recent_events.count > project.rate_limit_threshold * 5
+      # High traffic detected - create an issue
+      Issue.create!(
+        project: project,
+        title: "High Traffic Spike Detected",
+        description: "Detected #{recent_events.count} requests in the last 5 minutes",
+        severity: "medium",
+        event_id: event.id,
+        metadata: {
+          event_count: recent_events.count,
+          time_window: "5 minutes",
+          threshold: project.rate_limit_threshold * 5
+        }
+      )
+    end
+  end
+
+  def analyze_geographic_distribution(project, event)
+    return unless event.country_code.present?
+
+    # Check if this country is unusual for this project
+    country_events = project.events
+      .where(country_code: event.country_code)
+      .where(timestamp: 1.hour.ago..Time.current)
+
+    # If this is the first event from this country or unusual spike
+    if country_events.count == 1 || country_events.count > 100
+      Rails.logger.info "Unusual geographic activity from #{event.country_code} for project #{project.slug}"
+    end
+  end
+
+  def analyze_attack_vectors(project, event)
+    return unless event.blocked?
+
+    # Analyze common attack patterns
+    analyze_ip_reputation(project, event)
+    analyze_user_agent_patterns(project, event)
+    analyze_path_attacks(project, event)
+  end
+
+  def analyze_ip_reputation(project, event)
+    return unless event.ip_address.present?
+
+    # Count recent blocks from this IP
+    recent_blocks = project.events
+      .by_ip(event.ip_address)
+      .blocked
+      .where(timestamp: 1.hour.ago..Time.current)
+
+    if recent_blocks.count >= 5
+      # Suggest automatic IP block
+      project.add_ip_rule(
+        event.ip_address,
+        'block',
+        expires_at: 24.hours.from_now,
+        reason: "Automated block: #{recent_blocks.count} violations in 1 hour"
+      )
+    end
+  end
+
+  def analyze_user_agent_patterns(project, event)
+    return unless event.user_agent.present?
+
+    # Look for common bot/user agent patterns
+    suspicious_patterns = [
+      /bot/i, /crawler/i, /spider/i, /scanner/i,
+      /python/i, /curl/i, /wget/i, /nmap/i
+    ]
+
+    if suspicious_patterns.any? { |pattern| event.user_agent.match?(pattern) }
+      # Log suspicious user agent for potential rule generation
+      Rails.logger.info "Suspicious user agent detected: #{event.user_agent}"
+    end
+  end
+
+  def analyze_path_attacks(project, event)
+    return unless event.request_path.present?
+
+    # Look for common attack paths
+    attack_patterns = [
+      /\.\./,         # Directory traversal
+      /admin/i,      # Admin panel access
+      /wp-admin/i,   # WordPress admin
+      /\.php/i,      # PHP files
+      /union.*select/i,  # SQL injection
+      /script.*>/i   # XSS attempts
+    ]
+
+    if attack_patterns.any? { |pattern| event.request_path.match?(pattern) }
+      Rails.logger.info "Potential attack path detected: #{event.request_path}"
+    end
+  end
+
+  def update_project_analytics(project)
+    # Update cached analytics for faster dashboard loading
+    Rails.cache.delete("project_#{project.id}_analytics")
+  end
+end

app/jobs/process_waf_event_job.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class ProcessWafEventJob < ApplicationJob
+  queue_as :waf_events
+
+  def perform(project_id:, event_data:, headers:)
+    project = Project.find(project_id)
+
+    # Handle both single event and events array
+    events_to_process = []
+
+    if event_data.key?('events') && event_data['events'].is_a?(Array)
+      # Multiple events in an array
+      events_to_process = event_data['events']
+    elsif event_data.key?('event_id')
+      # Single event
+      events_to_process = [event_data]
+    else
+      Rails.logger.warn "Invalid event data format: missing event_id or events array"
+      return
+    end
+
+    events_to_process.each do |single_event_data|
+      begin
+        # Generate unique event ID if not provided
+        event_id = single_event_data['event_id'] || SecureRandom.uuid
+
+        # Create the WAF event record
+        event = Event.create_from_waf_payload!(event_id, single_event_data, project)
+
+        # Trigger analytics processing
+        ProcessWafAnalyticsJob.perform_later(project_id: project_id, event_id: event.id)
+
+        # Check for automatic rule generation opportunities
+        GenerateWafRulesJob.perform_later(project_id: project_id, event_id: event.id)
+
+        Rails.logger.info "Processed WAF event #{event_id} for project #{project.slug}"
+      rescue ActiveRecord::RecordInvalid => e
+        Rails.logger.error "Failed to create WAF event: #{e.message}"
+        Rails.logger.error e.record.errors.full_messages.join(", ")
+      rescue => e
+        Rails.logger.error "Error processing WAF event: #{e.message}"
+        Rails.logger.error e.backtrace.join("\n")
+      end
+    end
+
+    # Broadcast real-time updates once per batch
+    project.broadcast_events_refresh
+
+    Rails.logger.info "Processed #{events_to_process.count} WAF events for project #{project.slug}"
+  end
+end