From cc8213f87a7435f2e12091d6fd82cd63d7f10f40 Mon Sep 17 00:00:00 2001 From: Dan Milne Date: Tue, 11 Nov 2025 16:54:52 +1100 Subject: [PATCH] Lots of updates --- app/controllers/analytics_controller.rb | 260 +++++++++++--- app/controllers/dsns_controller.rb | 37 +- app/controllers/events_controller.rb | 11 + app/controllers/settings_controller.rb | 31 ++ app/helpers/application_helper.rb | 50 +++ .../controllers/dashboard_controller.js | 4 +- .../quick_create_rule_controller.js | 119 +++++++ app/jobs/fetch_ipapi_data_job.rb | 38 ++ app/jobs/generate_waf_rules_job.rb | 163 --------- app/jobs/process_waf_analytics_job.rb | 125 ------- app/jobs/process_waf_event_job.rb | 53 +-- app/jobs/process_waf_policies_job.rb | 30 +- app/models/dsn.rb | 6 +- app/models/network_range.rb | 59 ++- app/models/setting.rb | 18 + app/services/event_normalizer.rb | 6 +- app/services/geolite_asn_importer.rb | 9 + app/services/geolite_country_importer.rb | 27 +- app/services/ipapi.rb | 22 +- app/services/waf_policy_matcher.rb | 20 ++ app/views/analytics/index.html.erb | 63 +++- app/views/data_imports/index.html.erb | 5 +- app/views/data_imports/show.html.erb | 5 +- app/views/dsns/index.html.erb | 61 +--- app/views/dsns/show.html.erb | 9 +- app/views/events/index.html.erb | 44 ++- app/views/events/show.html.erb | 337 ++++++++++++++++++ app/views/layouts/application.html.erb | 4 + app/views/network_ranges/show.html.erb | 171 ++++----- app/views/settings/index.html.erb | 60 ++++ config/environments/development.rb | 2 + config/environments/production.rb | 2 + config/importmap.rb | 4 + config/initializers/sentry.rb | 156 ++++++++ config/recurring.yml | 12 +- config/routes.rb | 16 +- config/storage.yml | 2 +- config/tailwind.config.js | 13 + ...policies_evaluated_at_to_network_ranges.rb | 6 + db/migrate/20251111053159_create_settings.rb | 11 + ...convert_additional_data_to_network_data.rb | 6 + 41 files changed, 1463 insertions(+), 614 deletions(-) create mode 100644 app/controllers/settings_controller.rb create mode 100644 app/javascript/controllers/quick_create_rule_controller.js create mode 100644 app/jobs/fetch_ipapi_data_job.rb delete mode 100644 app/jobs/generate_waf_rules_job.rb delete mode 100644 app/jobs/process_waf_analytics_job.rb create mode 100644 app/models/setting.rb create mode 100644 app/views/events/show.html.erb create mode 100644 app/views/settings/index.html.erb create mode 100644 config/initializers/sentry.rb create mode 100644 config/tailwind.config.js create mode 100644 db/migrate/20251111024931_add_policies_evaluated_at_to_network_ranges.rb create mode 100644 db/migrate/20251111053159_create_settings.rb create mode 100644 db/migrate/20251111054933_convert_additional_data_to_network_data.rb diff --git a/app/controllers/analytics_controller.rb b/app/controllers/analytics_controller.rb index ec73907..4c3c7e6 100644 --- a/app/controllers/analytics_controller.rb +++ b/app/controllers/analytics_controller.rb @@ -11,17 +11,41 @@ class AnalyticsController < ApplicationController @time_period = params[:period]&.to_sym || :day @start_time = calculate_start_time(@time_period) - # Core statistics - @total_events = Event.where("timestamp >= ?", @start_time).count - @total_rules = Rule.enabled.count - @network_ranges_with_events = NetworkRange.with_events.count - @total_network_ranges = NetworkRange.count + # Cache TTL based on time period + cache_ttl = case @time_period + when :hour then 5.minutes + when :day then 1.hour + when :week then 6.hours + when :month then 12.hours + else 1.hour + end - # Event breakdown by action - @event_breakdown = Event.where("timestamp >= ?", @start_time) - .group(:waf_action) - .count - .transform_keys do |action_id| + # Cache key includes period and start_time (hour-aligned for consistency) + cache_key_base = "analytics/#{@time_period}/#{@start_time.to_i}" + + # Core statistics - cached + @total_events = Rails.cache.fetch("#{cache_key_base}/total_events", expires_in: cache_ttl) do + Event.where("timestamp >= ?", @start_time).count + end + + @total_rules = Rails.cache.fetch("analytics/total_rules", expires_in: 5.minutes) do + Rule.enabled.count + end + + @network_ranges_with_events = Rails.cache.fetch("analytics/network_ranges_with_events", expires_in: 5.minutes) do + NetworkRange.with_events.count + end + + @total_network_ranges = Rails.cache.fetch("analytics/total_network_ranges", expires_in: 5.minutes) do + NetworkRange.count + end + + # Event breakdown by action - cached + @event_breakdown = Rails.cache.fetch("#{cache_key_base}/event_breakdown", expires_in: cache_ttl) do + Event.where("timestamp >= ?", @start_time) + .group(:waf_action) + .count + .transform_keys do |action_id| case action_id when 0 then 'allow' when 1 then 'deny' @@ -30,45 +54,64 @@ class AnalyticsController < ApplicationController else 'unknown' end end + end - # Top countries by event count - @top_countries = Event.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network") - .where("timestamp >= ? AND network_ranges.country IS NOT NULL", @start_time) - .group("network_ranges.country") - .count - .sort_by { |_, count| -count } - .first(10) + # Top countries by event count - cached (this is the expensive JOIN query) + @top_countries = Rails.cache.fetch("#{cache_key_base}/top_countries", expires_in: cache_ttl) do + Event.joins("JOIN network_ranges ON events.ip_address <<= network_ranges.network") + .where("timestamp >= ? AND network_ranges.country IS NOT NULL", @start_time) + .group("network_ranges.country") + .count + .sort_by { |_, count| -count } + .first(10) + end - # Top blocked IPs - @top_blocked_ips = Event.where("timestamp >= ?", @start_time) - .where(waf_action: 1) # deny action in enum - .group(:ip_address) - .count - .sort_by { |_, count| -count } - .first(10) + # Top blocked IPs - cached + @top_blocked_ips = Rails.cache.fetch("#{cache_key_base}/top_blocked_ips", expires_in: cache_ttl) do + Event.where("timestamp >= ?", @start_time) + .where(waf_action: 1) # deny action in enum + .group(:ip_address) + .count + .sort_by { |_, count| -count } + .first(10) + end - # Network range intelligence breakdown - @network_intelligence = { - datacenter_ranges: NetworkRange.datacenter.count, - vpn_ranges: NetworkRange.vpn.count, - proxy_ranges: NetworkRange.proxy.count, - total_ranges: NetworkRange.count - } + # Network range intelligence breakdown - cached + @network_intelligence = Rails.cache.fetch("analytics/network_intelligence", expires_in: 10.minutes) do + { + datacenter_ranges: NetworkRange.datacenter.count, + vpn_ranges: NetworkRange.vpn.count, + proxy_ranges: NetworkRange.proxy.count, + total_ranges: NetworkRange.count + } + end - # Recent activity - @recent_events = Event.recent.limit(10) - @recent_rules = Rule.order(created_at: :desc).limit(5) + # Recent activity - minimal cache for freshness + @recent_events = Rails.cache.fetch("analytics/recent_events", expires_in: 1.minute) do + Event.recent.limit(10).to_a + end - # System health indicators - @system_health = { - total_users: User.count, - active_rules: Rule.enabled.count, - disabled_rules: Rule.where(enabled: false).count, - recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 1).count # 1 = deny - } + @recent_rules = Rails.cache.fetch("analytics/recent_rules", expires_in: 5.minutes) do + Rule.order(created_at: :desc).limit(5).to_a + end - # Prepare data for charts - @chart_data = prepare_chart_data + # System health indicators - cached + @system_health = Rails.cache.fetch("#{cache_key_base}/system_health", expires_in: cache_ttl) do + { + total_users: User.count, + active_rules: Rule.enabled.count, + disabled_rules: Rule.where(enabled: false).count, + recent_errors: Event.where("timestamp >= ? AND waf_action = ?", @start_time, 1).count # 1 = deny + } + end + + # Job queue statistics - short cache for near real-time + @job_statistics = Rails.cache.fetch("analytics/job_statistics", expires_in: 30.seconds) do + calculate_job_statistics + end + + # Prepare data for charts - split caching for current vs historical data + @chart_data = prepare_chart_data_with_split_cache(cache_key_base, cache_ttl) respond_to do |format| format.html @@ -130,30 +173,99 @@ class AnalyticsController < ApplicationController private def calculate_start_time(period) + # Snap to hour/day boundaries for cacheability + # Instead of rolling windows that change every second, use fixed boundaries case period when :hour - 1.hour.ago + # Last complete hour: if it's 13:45, show 12:00-13:00 + 1.hour.ago.beginning_of_hour when :day - 24.hours.ago + # Last 24 complete hours from current hour boundary + 24.hours.ago.beginning_of_hour when :week - 1.week.ago + # Last 7 complete days from today's start + 7.days.ago.beginning_of_day when :month - 1.month.ago + # Last 30 complete days from today's start + 30.days.ago.beginning_of_day else - 24.hours.ago + 24.hours.ago.beginning_of_hour end end + def prepare_chart_data_with_split_cache(cache_key_base, cache_ttl) + # Split timeline into historical (completed hours) and current (incomplete hour) + # Historical hours are cached for full TTL, current hour cached briefly for freshness + + # Cache historical hours (1-23 hours ago) - these are complete and won't change + # No expiration - will stick around until evicted by cache store + historical_timeline = Rails.cache.fetch("#{cache_key_base}/chart_historical") do + historical_start = 23.hours.ago.beginning_of_hour + events_by_hour = Event.where("timestamp >= ? AND timestamp < ?", historical_start, Time.current.beginning_of_hour) + .group("DATE_TRUNC('hour', timestamp)") + .count + + (1..23).map do |hour_ago| + hour_time = hour_ago.hours.ago.beginning_of_hour + hour_key = hour_time.utc + { + time_iso: hour_time.iso8601, + total: events_by_hour[hour_key] || 0 + } + end + end + + # Current hour (0 hours ago) - cache very briefly since it's actively accumulating + current_hour_data = Rails.cache.fetch("#{cache_key_base}/chart_current_hour", expires_in: 1.minute) do + hour_time = Time.current.beginning_of_hour + count = Event.where("timestamp >= ?", hour_time).count + { + time_iso: hour_time.iso8601, + total: count + } + end + + # Combine current + historical for full 24-hour timeline + timeline_data = [current_hour_data] + historical_timeline + + # Action distribution and other chart data (cached with main cache) + other_chart_data = Rails.cache.fetch("#{cache_key_base}/chart_metadata", expires_in: cache_ttl) do + action_distribution = @event_breakdown.map do |action, count| + { + action: action.humanize, + count: count, + percentage: ((count.to_f / [@total_events, 1].max) * 100).round(1) + } + end + + { + actions: action_distribution, + countries: @top_countries.map { |country, count| { country: country, count: count } }, + network_types: [ + { type: "Datacenter", count: @network_intelligence[:datacenter_ranges] }, + { type: "VPN", count: @network_intelligence[:vpn_ranges] }, + { type: "Proxy", count: @network_intelligence[:proxy_ranges] }, + { type: "Standard", count: @network_intelligence[:total_ranges] - @network_intelligence[:datacenter_ranges] - @network_intelligence[:vpn_ranges] - @network_intelligence[:proxy_ranges] } + ] + } + end + + # Merge timeline with other chart data + other_chart_data.merge(timeline: timeline_data) + end + def prepare_chart_data - # Events over time (hourly buckets for last 24 hours) - events_by_hour = Event.where("timestamp >= ?", 24.hours.ago) + # Legacy method - kept for reference but no longer used + # Events over time (hourly buckets) - use @start_time for consistency + events_by_hour = Event.where("timestamp >= ?", @start_time) .group("DATE_TRUNC('hour', timestamp)") .count - # Convert to chart format - keep everything in UTC for consistency + # Convert to chart format - snap to hour boundaries for cacheability timeline_data = (0..23).map do |hour_ago| - hour_time = hour_ago.hours.ago - hour_key = hour_time.utc.beginning_of_hour + # Use hour boundaries instead of rolling times + hour_time = hour_ago.hours.ago.beginning_of_hour + hour_key = hour_time.utc { # Store as ISO string for JavaScript to handle timezone conversion @@ -311,4 +423,46 @@ class AnalyticsController < ApplicationController suspicious_patterns: @suspicious_patterns } end + + def calculate_job_statistics + # Get job queue information from SolidQueue + begin + total_jobs = SolidQueue::Job.count + pending_jobs = SolidQueue::Job.where(finished_at: nil).count + recent_jobs = SolidQueue::Job.where('created_at > ?', 1.hour.ago).count + + # Get jobs by queue name + queue_breakdown = SolidQueue::Job.group(:queue_name).count + + # Get recent job activity + recent_enqueued = SolidQueue::Job.where('created_at > ?', 1.hour.ago).count + + # Calculate health status + health_status = if pending_jobs > 100 + 'warning' + elsif pending_jobs > 500 + 'critical' + else + 'healthy' + end + + { + total_jobs: total_jobs, + pending_jobs: pending_jobs, + recent_enqueued: recent_enqueued, + queue_breakdown: queue_breakdown, + health_status: health_status + } + rescue => e + Rails.logger.error "Failed to calculate job statistics: #{e.message}" + { + total_jobs: 0, + pending_jobs: 0, + recent_enqueued: 0, + queue_breakdown: {}, + health_status: 'error', + error: e.message + } + end + end end \ No newline at end of file diff --git a/app/controllers/dsns_controller.rb b/app/controllers/dsns_controller.rb index 6704a60..4b0044c 100644 --- a/app/controllers/dsns_controller.rb +++ b/app/controllers/dsns_controller.rb @@ -2,19 +2,12 @@ class DsnsController < ApplicationController before_action :require_authentication - before_action :set_dsn, only: [:show, :edit, :update, :disable, :enable] + before_action :set_dsn, only: [:show, :edit, :update, :disable, :enable, :destroy] before_action :authorize_dsn_management, except: [:index, :show] # GET /dsns def index @dsns = policy_scope(Dsn).order(created_at: :desc) - - # Generate environment DSNs using default DSN key or first enabled DSN - default_dsn = Dsn.enabled.first - if default_dsn - @external_dsn = generate_external_dsn(default_dsn.key) - @internal_dsn = generate_internal_dsn(default_dsn.key) - end end # GET /dsns/new @@ -64,6 +57,20 @@ class DsnsController < ApplicationController redirect_to @dsn, notice: 'DSN was enabled.' end + # DELETE /dsns/:id + def destroy + # Only allow deletion of disabled DSNs for safety + if @dsn.enabled? + redirect_to @dsn, alert: 'Cannot delete an enabled DSN. Please disable it first.' + return + end + + dsn_name = @dsn.name + @dsn.destroy + + redirect_to dsns_path, notice: "DSN '#{dsn_name}' was successfully deleted." + end + private def set_dsn @@ -78,18 +85,4 @@ class DsnsController < ApplicationController # Only allow admins to manage DSNs redirect_to root_path, alert: 'Access denied' unless Current.user&.admin? end - - def generate_external_dsn(key) - host = ENV.fetch("BAFFLE_HOST", "localhost:3000") - protocol = host.include?("localhost") ? "http" : "https" - "#{protocol}://#{key}@#{host}" - end - - def generate_internal_dsn(key) - internal_host = ENV.fetch("BAFFLE_INTERNAL_HOST", nil) - return nil unless internal_host.present? - - protocol = "http" # Internal connections use HTTP - "#{protocol}://#{key}@#{internal_host}" - end end \ No newline at end of file diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index ead1215..305fd16 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,6 +1,17 @@ # frozen_string_literal: true class EventsController < ApplicationController + def show + @event = Event.find(params[:id]) + @network_range = NetworkRange.contains_ip(@event.ip_address.to_s).first + + # Auto-generate network range if no match found + unless @network_range + @network_range = NetworkRangeGenerator.find_or_create_for_ip(@event.ip_address) + Rails.logger.debug "Auto-generated network range #{@network_range&.cidr} for IP #{@event.ip_address}" if @network_range + end + end + def index @events = Event.order(timestamp: :desc) Rails.logger.debug "Found #{@events.count} total events" diff --git a/app/controllers/settings_controller.rb b/app/controllers/settings_controller.rb new file mode 100644 index 0000000..6486572 --- /dev/null +++ b/app/controllers/settings_controller.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class SettingsController < ApplicationController + before_action :require_authentication + before_action :authorize_settings_management + + # GET /settings + def index + @settings = Setting.all.index_by(&:key) + end + + # PATCH /settings + def update + setting_key = params[:key] + setting_value = params[:value] + + if setting_key.present? + Setting.set(setting_key, setting_value) + redirect_to settings_path, notice: 'Setting was successfully updated.' + else + redirect_to settings_path, alert: 'Invalid setting key.' + end + end + + private + + def authorize_settings_management + # Only allow admins to manage settings + redirect_to root_path, alert: 'Access denied' unless Current.user&.admin? + end +end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 671a85e..40b6b3b 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -89,4 +89,54 @@ module ApplicationHelper raw html end + + # Helper methods for job queue status colors + def job_queue_status_color(status) + case status.to_s + when 'healthy' + 'bg-green-500' + when 'warning' + 'bg-yellow-500' + when 'critical' + 'bg-red-500' + when 'error' + 'bg-gray-500' + else + 'bg-blue-500' + end + end + + def job_queue_status_text_color(status) + case status.to_s + when 'healthy' + 'text-green-600' + when 'warning' + 'text-yellow-600' + when 'critical' + 'text-red-600' + when 'error' + 'text-gray-600' + else + 'text-blue-600' + end + end + + # Parse user agent string into readable components + def parse_user_agent(user_agent) + return nil if user_agent.blank? + + client = DeviceDetector.new(user_agent) + + { + name: client.name, + version: client.full_version, + os_name: client.os_name, + os_version: client.os_full_version, + device_type: client.device_type || "desktop", + device_name: client.device_name, + bot: client.bot?, + bot_name: client.bot_name, + raw: user_agent + } + end end diff --git a/app/javascript/controllers/dashboard_controller.js b/app/javascript/controllers/dashboard_controller.js index 78c940a..edb3ab8 100644 --- a/app/javascript/controllers/dashboard_controller.js +++ b/app/javascript/controllers/dashboard_controller.js @@ -8,7 +8,9 @@ export default class extends Controller { } connect() { - this.startRefreshing() + // TEMPORARILY DISABLED: Auto-refresh causes performance issues with slow queries (30s+ load times) + // TODO: Re-enable after optimizing analytics queries + // this.startRefreshing() } disconnect() { diff --git a/app/javascript/controllers/quick_create_rule_controller.js b/app/javascript/controllers/quick_create_rule_controller.js new file mode 100644 index 0000000..0d1bd29 --- /dev/null +++ b/app/javascript/controllers/quick_create_rule_controller.js @@ -0,0 +1,119 @@ +// QuickCreateRuleController - Handles the quick create rule form functionality +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["form", "toggle", "ruleTypeSelect", "actionSelect", "patternFields", "rateLimitFields", "redirectFields", "helpText", "conditionsField"] + + connect() { + this.setupEventListeners() + this.initializeFieldVisibility() + } + + toggle() { + this.formTarget.classList.toggle("hidden") + + if (this.formTarget.classList.contains("hidden")) { + this.resetForm() + } + } + + updateRuleTypeFields() { + if (!this.hasRuleTypeSelectTarget || !this.hasActionSelectTarget) return + + const ruleType = this.ruleTypeSelectTarget.value + const action = this.actionSelectTarget.value + + // Hide all optional fields + this.hideOptionalFields() + + // Show relevant fields based on rule type + if (["path_pattern", "header_pattern", "query_pattern", "body_signature"].includes(ruleType)) { + if (this.hasPatternFieldsTarget) { + this.patternFieldsTarget.classList.remove("hidden") + this.updatePatternHelpText(ruleType) + } + } else if (ruleType === "rate_limit") { + if (this.hasRateLimitFieldsTarget) { + this.rateLimitFieldsTarget.classList.remove("hidden") + } + } + + // Show redirect fields if action is redirect + if (action === "redirect") { + if (this.hasRedirectFieldsTarget) { + this.redirectFieldsTarget.classList.remove("hidden") + } + } + } + + updatePatternHelpText(ruleType) { + if (!this.hasHelpTextTarget || !this.hasConditionsFieldTarget) return + + const helpTexts = { + path_pattern: { + text: "Regex pattern to match URL paths (e.g.,\\.env$|wp-admin|phpmyadmin)", + placeholder: "Example: \\.env$|\\.git|config\\.php|wp-admin" + }, + header_pattern: { + text: 'JSON with header_name and pattern (e.g., {"header_name": "User-Agent", "pattern": "bot.*"})', + placeholder: 'Example: {"header_name": "User-Agent", "pattern": ".*[Bb]ot.*"}' + }, + query_pattern: { + text: "Regex pattern to match query parameters (e.g., union.*select| \ No newline at end of file diff --git a/app/views/settings/index.html.erb b/app/views/settings/index.html.erb new file mode 100644 index 0000000..9601fec --- /dev/null +++ b/app/views/settings/index.html.erb @@ -0,0 +1,60 @@ +<% content_for :title, "Settings" %> + +
+ +
+
+
+

Settings

+

Manage system configuration and API keys

+
+
+
+ + +
+
+

API Configuration

+ + +
+ <%= form_with url: settings_path, method: :patch, class: "space-y-4" do |f| %> + <%= hidden_field_tag :key, 'ipapi_key' %> + +
+ +
+ <%= text_field_tag :value, + @settings['ipapi_key']&.value || ENV['IPAPI_KEY'], + class: "flex-1 min-w-0 block w-full px-3 py-2 rounded-md border-gray-300 focus:ring-blue-500 focus:border-blue-500 sm:text-sm", + placeholder: "Enter your ipapi.is API key" %> + <%= f.submit "Update", class: "ml-3 inline-flex items-center px-4 py-2 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500" %> +
+

+ <% if @settings['ipapi_key']&.value.present? %> + โœ“ Configured in database + <% elsif ENV['IPAPI_KEY'].present? %> + Using environment variable (IPAPI_KEY) + <% else %> + ipapi.is not active + <% end %> +

+

+ Get your API key from ipapi.is +

+
+ <% end %> +
+
+
+ + +
+
+

Additional Settings

+

More configuration options will be added here as needed.

+
+
+
diff --git a/config/environments/development.rb b/config/environments/development.rb index 6bafe50..b846721 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -77,4 +77,6 @@ Rails.application.configure do # config.generators.apply_rubocop_autocorrect_after_generate! config.active_job.queue_adapter = :solid_queue config.solid_queue.connects_to = { database: { writing: :queue } } + # Don't store finished jobs - we don't need job history, saves DB space + config.solid_queue.preserve_finished_jobs = false end diff --git a/config/environments/production.rb b/config/environments/production.rb index ad16af7..48f23ac 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -52,6 +52,8 @@ Rails.application.configure do # Replace the default in-process and non-durable queuing backend for Active Job. config.active_job.queue_adapter = :solid_queue config.solid_queue.connects_to = { database: { writing: :queue } } + # Don't store finished jobs - we don't need job history, saves DB space + config.solid_queue.preserve_finished_jobs = false # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. diff --git a/config/importmap.rb b/config/importmap.rb index 909dfc5..b52bae5 100644 --- a/config/importmap.rb +++ b/config/importmap.rb @@ -5,3 +5,7 @@ pin "@hotwired/turbo-rails", to: "turbo.min.js" pin "@hotwired/stimulus", to: "stimulus.min.js" pin "@hotwired/stimulus-loading", to: "stimulus-loading.js" pin_all_from "app/javascript/controllers", under: "controllers" + +# Tom Select for enhanced multi-select +pin "tom-select", to: "https://cdn.jsdelivr.net/npm/tom-select@2.3.1/dist/js/tom-select.complete.min.js" +pin "tom-select-css", to: "https://cdn.jsdelivr.net/npm/tom-select@2.3.1/dist/css/tom-select.css" diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb new file mode 100644 index 0000000..d451415 --- /dev/null +++ b/config/initializers/sentry.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +# Sentry configuration for error tracking and performance monitoring +# Only initializes if SENTRY_DSN is configured + +return unless ENV['SENTRY_DSN'].present? + +require 'sentry-rails' + +Sentry.init do |config| + config.dsn = ENV['SENTRY_DSN'] + + # Configure sampling for production (lower in production, higher in staging) + config.traces_sample_rate = case Rails.env + when 'production' then ENV.fetch('SENTRY_TRACES_SAMPLE_RATE', '0.05').to_f + when 'staging' then ENV.fetch('SENTRY_TRACES_SAMPLE_RATE', '0.2').to_f + else ENV.fetch('SENTRY_TRACES_SAMPLE_RATE', '0.1').to_f + end + + # Enable breadcrumbs for better debugging + config.breadcrumbs_logger = [:active_support_logger, :http_logger] + + # Send PII data (like IPs) to Sentry for debugging (disable in production if needed) + config.send_default_pii = ENV.fetch('SENTRY_SEND_PII', 'false') == 'true' + + # Set environment + config.environment = Rails.env + + # Configure release info from Docker tag or Git + if ENV['GIT_COMMIT_SHA'] + config.release = ENV['GIT_COMMIT_SHA'][0..7] + elsif ENV['APP_VERSION'] + config.release = ENV['APP_VERSION'] + end + + # Set server name for multi-instance environments + config.server_name = ENV.fetch('SERVER_NAME', 'baffle-agent') + + # Filter out certain errors to reduce noise and add tags + config.before_send = lambda do |event, hint| + # Filter out 404 errors and other expected HTTP errors + if event.contexts.dig(:response, :status_code) == 404 + nil + # Filter out validation errors in development + elsif Rails.env.development? && event.exception&.message&.include?("Validation failed") + nil + # Filter out specific noisy exceptions + elsif %w[ActionController::RoutingError + ActionController::InvalidAuthenticityToken + ActionController::UnknownFormat + ActionDispatch::Http::Parameters::ParseError].include?(event.exception&.class&.name) + nil + else + # Add tags for better filtering in Sentry + event.tags.merge!({ + ruby_version: RUBY_VERSION, + rails_version: Rails.version, + environment: Rails.env + }) + event + end + end + + # Configure exception class exclusion + config.excluded_exceptions += [ + 'ActionController::RoutingError', + 'ActionController::InvalidAuthenticityToken', + 'CGI::Session::CookieStore::TamperedWithCookie', + 'ActionController::UnknownFormat', + 'ActionDispatch::Http::Parameters::ParseError', + 'Mongoid::Errors::DocumentNotFound' + ] +end + +# SolidQueue monitoring will be automatically handled by sentry-solid_queue gem + +# Add correlation ID to Sentry context +ActiveSupport::Notifications.subscribe('action_controller.process_action') do |name, start, finish, id, payload| + controller = payload[:controller] + action = payload[:action] + request_id = payload[:request]&.request_id + + if controller && action && request_id + Sentry.set_context(:request, { + correlation_id: request_id, + controller: controller.controller_name, + action: action.action_name, + ip: request&.remote_ip + }) + end +end + +# Add ActiveJob context to all job transactions +ActiveSupport::Notifications.subscribe('perform.active_job') do |name, start, finish, id, payload| + job = payload[:job] + + Sentry.configure_scope do |scope| + scope.set_tag(:job_class, job.class.name) + scope.set_tag(:job_queue, job.queue_name) + scope.set_tag(:job_id, job.job_id) + + scope.set_context(:job, { + job_id: job.job_id, + job_class: job.class.name, + queue_name: job.queue_name, + arguments: job.arguments.to_s, + enqueued_at: job.enqueued_at, + executions: job.executions + }) + end +end + +# Monitor SolidQueue job failures +ActiveSupport::Notifications.subscribe('solid_queue.error') do |name, start, finish, id, payload| + job = payload[:job] + error = payload[:error] + + Sentry.with_scope do |scope| + scope.set_tag(:job_class, job.class_name) + scope.set_tag(:job_queue, job.queue_name) + scope.set_context(:job, { + job_id: job.active_job_id, + arguments: job.arguments.to_s, + queue_name: job.queue_name, + created_at: job.created_at + }) + scope.set_context(:error, { + error_class: error.class.name, + error_message: error.message + }) + + Sentry.capture_exception(error) + end +end + +# Set user context when available +if defined?(Current) && Current.user + Sentry.set_user(id: Current.user.id, email: Current.user.email) +end + +# Add application-specific context +app_version = begin + File.read(Rails.root.join('VERSION')).strip +rescue + ENV['APP_VERSION'] || ENV['GIT_COMMIT_SHA']&.[](0..7) || 'unknown' +end + +Sentry.set_context('application', { + name: 'BaffleHub', + version: app_version, + environment: Rails.env, + database: ActiveRecord::Base.connection.adapter_name, + queue_adapter: Rails.application.config.active_job.queue_adapter +}) + +Rails.logger.info "Sentry configured for environment: #{Rails.env}" \ No newline at end of file diff --git a/config/recurring.yml b/config/recurring.yml index b4207f9..b7bbe5c 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -9,7 +9,11 @@ # priority: 2 # schedule: at 5am every day -production: - clear_solid_queue_finished_jobs: - command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" - schedule: every hour at minute 12 +# No recurring tasks configured yet +# (previously had clear_solid_queue_finished_jobs, but now preserve_finished_jobs: false in queue.yml) + +# Clean up failed jobs older than 1 day +cleanup_failed_jobs: + command: "SolidQueue::FailedExecution.where('created_at < ?', 1.day.ago).delete_all" + queue: background + schedule: every 6 hours diff --git a/config/routes.rb b/config/routes.rb index 9a1673b..81e1441 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -11,6 +11,13 @@ Rails.application.routes.draw do # Admin user management (admin only) resources :users, only: [:index, :show, :edit, :update] + # Settings management (admin only) + resources :settings, only: [:index] do + collection do + patch :update + end + end + # DSN management (admin only) resources :dsns do member do @@ -44,7 +51,7 @@ Rails.application.routes.draw do root "analytics#index" # Event management - resources :events, only: [:index] + resources :events, only: [:index, :show] # Network range management resources :network_ranges, only: [:index, :show, :new, :create, :edit, :update, :destroy] do @@ -79,4 +86,11 @@ Rails.application.routes.draw do post :create_country end end + + # GeoLite2 data import management (admin only) + resources :data_imports, only: [:index, :new, :create, :show, :destroy] do + member do + get :progress + end + end end diff --git a/config/storage.yml b/config/storage.yml index 927dc53..82e3cac 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -4,7 +4,7 @@ test: local: service: Disk - root: <%= Rails.root.join("storage") %> + root: <%= Rails.root.join("storage", "uploads") %> # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) # amazon: diff --git a/config/tailwind.config.js b/config/tailwind.config.js new file mode 100644 index 0000000..e329df4 --- /dev/null +++ b/config/tailwind.config.js @@ -0,0 +1,13 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + content: [ + './app/helpers/**/*.rb', + './app/javascript/**/*.js', + './app/views/**/*.erb', + './app/views/**/*.html.erb' + ], + theme: { + extend: {}, + }, + plugins: [], +} \ No newline at end of file diff --git a/db/migrate/20251111024931_add_policies_evaluated_at_to_network_ranges.rb b/db/migrate/20251111024931_add_policies_evaluated_at_to_network_ranges.rb new file mode 100644 index 0000000..e9f5207 --- /dev/null +++ b/db/migrate/20251111024931_add_policies_evaluated_at_to_network_ranges.rb @@ -0,0 +1,6 @@ +class AddPoliciesEvaluatedAtToNetworkRanges < ActiveRecord::Migration[8.1] + def change + add_column :network_ranges, :policies_evaluated_at, :datetime + add_index :network_ranges, :policies_evaluated_at + end +end diff --git a/db/migrate/20251111053159_create_settings.rb b/db/migrate/20251111053159_create_settings.rb new file mode 100644 index 0000000..8f2cdd1 --- /dev/null +++ b/db/migrate/20251111053159_create_settings.rb @@ -0,0 +1,11 @@ +class CreateSettings < ActiveRecord::Migration[8.1] + def change + create_table :settings do |t| + t.string :key + t.string :value + + t.timestamps + end + add_index :settings, :key, unique: true + end +end diff --git a/db/migrate/20251111054933_convert_additional_data_to_network_data.rb b/db/migrate/20251111054933_convert_additional_data_to_network_data.rb new file mode 100644 index 0000000..f9cfa5b --- /dev/null +++ b/db/migrate/20251111054933_convert_additional_data_to_network_data.rb @@ -0,0 +1,6 @@ +class ConvertAdditionalDataToNetworkData < ActiveRecord::Migration[8.1] + def change + add_column :network_ranges, :network_data, :jsonb, default: {} + add_index :network_ranges, :network_data, using: :gin + end +end