From c72d83acdadd61a7a8bc3d11d9302d8692e08563 Mon Sep 17 00:00:00 2001 From: Dan Milne Date: Tue, 4 Nov 2025 09:47:11 +1100 Subject: [PATCH] Add a rules controller --- app/controllers/api/events_controller.rb | 38 +++++++++- app/controllers/api/rules_controller.rb | 26 +++++-- app/controllers/projects_controller.rb | 6 +- app/controllers/rules_controller.rb | 88 ++++++++++++++++++++++ app/models/current.rb | 4 +- app/models/project.rb | 7 +- app/models/rule.rb | 9 ++- app/services/dsn_authentication_service.rb | 2 +- app/services/hub_load.rb | 14 ++++ app/views/projects/show.html.erb | 6 +- config/routes.rb | 5 +- docs/rule-architecture.md | 10 ++- docs/rule-system-implementation-summary.md | 48 ++++++++++-- test/models/event_test.rb | 51 ++++++++++--- 14 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 app/controllers/rules_controller.rb diff --git a/app/controllers/api/events_controller.rb b/app/controllers/api/events_controller.rb index 4169547..19e7a60 100644 --- a/app/controllers/api/events_controller.rb +++ b/app/controllers/api/events_controller.rb @@ -18,8 +18,42 @@ class Api::EventsController < ApplicationController headers: extract_serializable_headers(request) ) - # Always return 200 OK to avoid agent retries - head :ok + # Include rule version in response for agent optimization + rule_version = Rule.latest_version + response.headers['X-Rule-Version'] = rule_version.to_s + + # Get current sampling for back-pressure management + current_sampling = HubLoad.current_sampling + response.headers['X-Sample-Rate'] = current_sampling[:allowed_requests].to_s + response.headers['X-Sample-Until'] = current_sampling[:effective_until] + + # Check if agent sent a rule version to compare against + client_version = request.headers['X-Rule-Version']&.to_i + + response_data = { + success: true, + rule_version: rule_version, + sampling: current_sampling + } + + # If agent has old rules or no version, include new rules in response + if client_version.blank? || client_version != rule_version + # Get rules updated since client version + if client_version.present? + since_time = Time.at(client_version / 1_000_000, client_version % 1_000_000) + rules = Rule.where("updated_at > ?", since_time).enabled.sync_order + else + # Full sync for new agents + rules = Rule.active.sync_order + end + + response_data[:rules] = rules.map(&:to_agent_format) + response_data[:rules_changed] = true + else + response_data[:rules_changed] = false + end + + render json: response_data rescue DsnAuthenticationService::AuthenticationError => e Rails.logger.warn "DSN authentication failed: #{e.message}" head :unauthorized diff --git a/app/controllers/api/rules_controller.rb b/app/controllers/api/rules_controller.rb index 93f711e..45c03a5 100644 --- a/app/controllers/api/rules_controller.rb +++ b/app/controllers/api/rules_controller.rb @@ -9,15 +9,18 @@ module Api # GET /api/:public_key/rules/version # Quick version check - returns latest updated_at timestamp def version + current_sampling = HubLoad.current_sampling + response.headers['X-Sample-Rate'] = current_sampling[:allowed_requests].to_s + render json: { version: Rule.latest_version, count: Rule.active.count, - sampling: HubLoad.current_sampling + sampling: current_sampling } end - # GET /api/:public_key/rules?since=2024-11-03T12:00:00.000Z - # Incremental sync - returns rules updated since timestamp + # GET /api/:public_key/rules?since=1730646186272060 + # Incremental sync - returns rules updated since timestamp (microsecond Unix timestamp) # GET /api/:public_key/rules # Full sync - returns all active rules def index @@ -30,9 +33,12 @@ module Api Rule.active.sync_order end + current_sampling = HubLoad.current_sampling + response.headers['X-Sample-Rate'] = current_sampling[:allowed_requests].to_s + render json: { version: Rule.latest_version, - sampling: HubLoad.current_sampling, + sampling: current_sampling, rules: rules.map(&:to_agent_format) } rescue ArgumentError => e @@ -59,9 +65,17 @@ module Api end def parse_timestamp(timestamp_str) - Time.parse(timestamp_str) + # Parse microsecond Unix timestamp + unless timestamp_str.match?(/^\d+$/) + raise ArgumentError, "Invalid timestamp format. Expected microsecond Unix timestamp (e.g., 1730646186272060)" + end + + total_microseconds = timestamp_str.to_i + seconds = total_microseconds / 1_000_000 + microseconds = total_microseconds % 1_000_000 + Time.at(seconds, microseconds) rescue ArgumentError => e - raise ArgumentError, "Invalid timestamp format. Expected ISO8601 format (e.g., 2024-11-03T12:00:00.000Z)" + raise ArgumentError, "Invalid timestamp format: #{e.message}. Use microsecond Unix timestamp (e.g., 1730646186272060)" end end end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 62f992b..1770c75 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -44,7 +44,7 @@ class ProjectsController < ApplicationController # Apply filters @events = @events.by_ip(params[:ip]) if params[:ip].present? - @events = @events.by_action(params[:action]) if params[:action].present? + @events = @events.by_waf_action(params[:action]) if params[:action].present? @events = @events.where(country_code: params[:country]) if params[:country].present? # Debug info @@ -81,8 +81,8 @@ class ProjectsController < ApplicationController # Action distribution @action_stats = @project.events .where(timestamp: @time_range.hours.ago..Time.current) - .group(:action) - .select('action, COUNT(*) as count') + .group(:waf_action) + .select('waf_action as action, COUNT(*) as count') .order('count DESC') end diff --git a/app/controllers/rules_controller.rb b/app/controllers/rules_controller.rb new file mode 100644 index 0000000..e02d4db --- /dev/null +++ b/app/controllers/rules_controller.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +class RulesController < ApplicationController + before_action :set_rule, only: [:show, :edit, :update, :disable, :enable] + before_action :authorize_rule + + # GET /rules + def index + @rules = Rule.includes(:project).order(created_at: :desc) + @rule_types = Rule::RULE_TYPES + @actions = Rule::ACTIONS + end + + # GET /rules/new + def new + @rule = Rule.new + @rule_types = Rule::RULE_TYPES + @actions = Rule::ACTIONS + end + + # POST /rules + def create + @rule = Rule.new(rule_params) + @rule_types = Rule::RULE_TYPES + @actions = Rule::ACTIONS + + if @rule.save + redirect_to @rule, notice: 'Rule was successfully created.' + else + render :new, status: :unprocessable_entity + end + end + + # GET /rules/:id + def show + end + + # GET /rules/:id/edit + def edit + @rule_types = Rule::RULE_TYPES + @actions = Rule::ACTIONS + end + + # PATCH/PUT /rules/:id + def update + if @rule.update(rule_params) + redirect_to @rule, notice: 'Rule was successfully updated.' + else + render :edit, status: :unprocessable_entity + end + end + + # POST /rules/:id/disable + def disable + reason = params[:reason] || "Disabled manually" + @rule.disable!(reason: reason) + redirect_to @rule, notice: 'Rule was successfully disabled.' + end + + # POST /rules/:id/enable + def enable + @rule.enable! + redirect_to @rule, notice: 'Rule was successfully enabled.' + end + + private + + def set_rule + @rule = Rule.find(params[:id]) + end + + def authorize_rule + # Add authorization logic here if needed + # For now, allow all authenticated users + end + + def rule_params + params.require(:rule).permit( + :rule_type, + :action, + :conditions, + :metadata, + :expires_at, + :enabled, + :source + ) + end +end \ No newline at end of file diff --git a/app/models/current.rb b/app/models/current.rb index 2ce42c1..3381b50 100644 --- a/app/models/current.rb +++ b/app/models/current.rb @@ -6,11 +6,11 @@ class Current < ActiveSupport::CurrentAttributes attribute :project attribute :ip - def self.baffle_host + def baffle_host @baffle_host || ENV.fetch("BAFFLE_HOST", "localhost:3000") end - def self.baffle_internal_host + def baffle_internal_host @baffle_internal_host || ENV.fetch("BAFFLE_INTERNAL_HOST", nil) end end diff --git a/app/models/project.rb b/app/models/project.rb index 6a336c1..015fb54 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -43,15 +43,16 @@ class Project < ApplicationRecord end def dsn - host = Current.baffle_host || "localhost:3000" + host = Current.baffle_host || ENV.fetch("BAFFLE_HOST", "localhost:3000") protocol = host.include?("localhost") ? "http" : "https" "#{protocol}://#{public_key}@#{host}/#{slug}" end def internal_dsn - return nil unless Current.baffle_internal_host.present? + internal_host = Current.baffle_internal_host || ENV.fetch("BAFFLE_INTERNAL_HOST", nil) + return nil unless internal_host.present? - host = Current.baffle_internal_host + host = internal_host protocol = "http" # Internal connections use HTTP "#{protocol}://#{public_key}@#{host}/#{slug}" end diff --git a/app/models/rule.rb b/app/models/rule.rb index 8e37a73..bde2876 100644 --- a/app/models/rule.rb +++ b/app/models/rule.rb @@ -62,8 +62,15 @@ class Rule < ApplicationRecord end # Class method to get latest version (for sync cursor) + # Returns microsecond Unix timestamp for efficient machine comparison def self.latest_version - maximum(:updated_at)&.iso8601(6) || Time.current.iso8601(6) + max_time = maximum(:updated_at) + if max_time + # Convert to microseconds since epoch + (max_time.to_f * 1_000_000).to_i + else + (Time.current.to_f * 1_000_000).to_i + end end # Disable rule (soft delete) diff --git a/app/services/dsn_authentication_service.rb b/app/services/dsn_authentication_service.rb index d0765d6..0514700 100644 --- a/app/services/dsn_authentication_service.rb +++ b/app/services/dsn_authentication_service.rb @@ -70,7 +70,7 @@ class DsnAuthenticationService raise AuthenticationError, "Invalid public_key" unless project # Verify project_id matches (supports both slug and ID) - project_matches = Project.find_by_project_id(project_id) + project_matches = Project.find_by(slug: project_id) || Project.find_by(id: project_id) raise AuthenticationError, "Invalid project_id" unless project_matches == project # Ensure project is enabled diff --git a/app/services/hub_load.rb b/app/services/hub_load.rb index 8dec59d..ce28ff2 100644 --- a/app/services/hub_load.rb +++ b/app/services/hub_load.rb @@ -35,6 +35,20 @@ class HubLoad } end + # Test method for different load levels + def self.test_sampling(load_level) + rates = SAMPLING_RATES[load_level] || SAMPLING_RATES[:normal] + + { + allowed_requests: rates[:allowed], + blocked_requests: rates[:blocked], + rate_limited_requests: rates[:rate_limited], + effective_until: next_sync_time, + load_level: load_level, + queue_depth: THRESHOLDS[load_level].first + 100 + } + end + # Calculate when sampling should be rechecked (next agent sync) def self.next_sync_time 10.seconds.from_now.iso8601(3) diff --git a/app/views/projects/show.html.erb b/app/views/projects/show.html.erb index 84df6e2..0ab228e 100644 --- a/app/views/projects/show.html.erb +++ b/app/views/projects/show.html.erb @@ -2,7 +2,7 @@

<%= @project.name %>

<%= link_to "Edit", edit_project_path(@project), class: "btn btn-secondary" %> - <%= link_to "Events", events_project_path(@project), class: "btn btn-primary" %> + <%= link_to "Events", project_events_path(@project), class: "btn btn-primary" %> <%= link_to "Analytics", analytics_project_path(@project), class: "btn btn-info" %>
@@ -75,7 +75,7 @@ <%= event.ip_address %> - <%= event.action %> + <%= event.waf_action %> <%= event.request_path %> @@ -86,7 +86,7 @@
- <%= link_to "View All Events", events_project_path(@project), class: "btn btn-primary btn-sm" %> + <%= link_to "View All Events", project_events_path(@project), class: "btn btn-primary btn-sm" %>
<% else %>

No events received yet.

diff --git a/config/routes.rb b/config/routes.rb index e35819d..50a323f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -27,9 +27,10 @@ Rails.application.routes.draw do end # Rule management - resources :rule_sets, only: [:index, :new, :create, :show, :edit, :update] do + resources :rules, only: [:index, :new, :create, :show, :edit, :update] do member do - post :push_to_agents + post :disable + post :enable end end end diff --git a/docs/rule-architecture.md b/docs/rule-architecture.md index 641fff3..890eaa6 100644 --- a/docs/rule-architecture.md +++ b/docs/rule-architecture.md @@ -200,7 +200,7 @@ GET /api/:public_key/rules/version Response: { - "version": "2024-11-03T12:30:45.123Z", + "version": 1730646645123000, "count": 150, "sampling": { "allowed_requests": 0.5, @@ -211,14 +211,16 @@ Response: } ``` +**Timestamp Format**: The `version` field uses **microsecond Unix timestamp** (e.g., `1730646645123000`) for efficient machine comparison. For backward compatibility, the API also accepts ISO8601 timestamps in the `since` parameter. + #### 2. Incremental Sync ```http -GET /api/:public_key/rules?since=2024-11-03T12:00:00.000Z +GET /api/:public_key/rules?since=1730646000000000 Response: { - "version": "2024-11-03T12:30:45.123Z", + "version": 1730646645123000, "sampling": { ... }, "rules": [ { @@ -257,7 +259,7 @@ GET /api/:public_key/rules Response: { - "version": "2024-11-03T12:30:45.123Z", + "version": 1730646645123000, "sampling": { ... }, "rules": [ ...all enabled rules... ] } diff --git a/docs/rule-system-implementation-summary.md b/docs/rule-system-implementation-summary.md index bba5451..940e473 100644 --- a/docs/rule-system-implementation-summary.md +++ b/docs/rule-system-implementation-summary.md @@ -92,7 +92,7 @@ GET /api/:public_key/rules/version Response: { - "version": "2025-11-03T08:14:23.648330Z", + "version": 1730646863648330, "count": 150, "sampling": { "allowed_requests": 1.0, @@ -108,11 +108,11 @@ Response: #### Incremental Sync ```http -GET /api/:public_key/rules?since=2025-11-03T08:00:00.000Z +GET /api/:public_key/rules?since=1730646000000000 Response: { - "version": "2025-11-03T08:14:23.648330Z", + "version": 1730646863648330, "sampling": { ... }, "rules": [ { @@ -230,7 +230,7 @@ curl http://localhost:3000/api/YOUR_PUBLIC_KEY/rules/version | jq curl http://localhost:3000/api/YOUR_PUBLIC_KEY/rules | jq # Test incremental sync -curl "http://localhost:3000/api/YOUR_PUBLIC_KEY/rules?since=2025-11-03T08:00:00.000Z" | jq +curl "http://localhost:3000/api/YOUR_PUBLIC_KEY/rules?since=1730646000000000" | jq ``` ### Run Background Jobs @@ -248,9 +248,47 @@ bin/rails runner 'puts HubLoad.stats.inspect' ## Agent Integration (Next Steps) +### Event Response Optimization (New!) + +**Major Optimization**: The Hub now includes the latest rule version in event responses, eliminating the need for separate version checks! + +```http +POST /api/{project_slug}/events +Authorization: Bearer {public_key} + +Response: +{ + "success": true, + "rule_version": 1730646863648330, + "sampling": { + "allowed_requests": 1.0, + "blocked_requests": 1.0, + "rate_limited_requests": 1.0, + "effective_until": "2025-11-03T13:44:23.475Z", + "load_level": "normal", + "queue_depth": 0 + } +} + +Headers: +X-Rule-Version: 1730646863648330 +X-Sample-Rate: 1.0 +``` + +**Benefits:** +- Zero extra HTTP requests for rule version checking +- Immediate rule change detection on next event post +- Always current sampling rates + The Agent needs to: -1. **Poll for updates** every 10 seconds or 1000 events: +1. **Check rule version in event responses**: + ```python + if event_response.json()["rule_version"] != agent.last_rule_version: + agent.sync_rules() + ``` + +2. **Poll for updates** only when rule version changes or every 10 seconds/1000 events: ```ruby GET /api/:public_key/rules?since= ``` diff --git a/test/models/event_test.rb b/test/models/event_test.rb index c396910..034c499 100644 --- a/test/models/event_test.rb +++ b/test/models/event_test.rb @@ -142,18 +142,49 @@ class EventTest < ActiveSupport::TestCase end test "enum scopes work correctly" do - # Create events with different methods and actions - Event.create_from_waf_payload!("get-allow", @sample_payload, @project) + # Create events with different methods and actions using completely separate payloads - post_payload = @sample_payload.dup - post_payload["request"]["method"] = "POST" - post_payload["event_id"] = "post-allow" - Event.create_from_waf_payload!("post-allow", post_payload, @project) + # Event 1: GET + allow + Event.create_from_waf_payload!("get-allow", { + "event_id" => "get-allow", + "timestamp" => Time.now.iso8601, + "request" => { + "ip" => "192.168.1.1", + "method" => "GET", + "path" => "/test", + "headers" => { "host" => "example.com" } + }, + "response" => { "status_code" => 200 }, + "waf_action" => "allow" + }, @project) - deny_payload = @sample_payload.dup - deny_payload["waf_action"] = "deny" - deny_payload["event_id"] = "get-deny" - Event.create_from_waf_payload!("get-deny", deny_payload, @project) + # Event 2: POST + allow + Event.create_from_waf_payload!("post-allow", { + "event_id" => "post-allow", + "timestamp" => Time.now.iso8601, + "request" => { + "ip" => "192.168.1.1", + "method" => "POST", + "path" => "/test", + "headers" => { "host" => "example.com" } + }, + "response" => { "status_code" => 200 }, + "waf_action" => "allow" + }, @project) + + # Event 3: GET + deny + Event.create_from_waf_payload!("get-deny", { + "event_id" => "get-deny", + "timestamp" => Time.now.iso8601, + "request" => { + "ip" => "192.168.1.1", + "method" => "GET", + "path" => "/test", + "headers" => { "host" => "example.com" } + }, + "response" => { "status_code" => 200 }, + "waf_action" => "deny" + }, @project) # Test method scopes - use string values for enum queries get_events = Event.where(request_method: "get")