Bot Network Ranges

Import and manage official network ranges for search crawlers and API bots

Available Sources

<% @bot_sources.each do |key, source| %>

<%= source[:name] %>

<%= source[:url] ? 'Available' : 'Manual' %>

<%= source[:description] %>

<%= form_with url: import_bot_network_ranges_path, method: :post, class: "inline" do |f| %> <%= hidden_field_tag :source, key %> <%= f.submit "Import Now", class: "px-3 py-1 text-xs font-medium text-white bg-blue-600 rounded hover:bg-blue-700 transition-colors disabled:opacity-50", disabled: !source[:url] %> <% end %> <%= form_with url: import_async_bot_network_ranges_path, method: :post, class: "inline" do |f| %> <%= hidden_field_tag :source, key %> <%= f.submit "Import Async", class: "px-3 py-1 text-xs font-medium text-white bg-purple-600 rounded hover:bg-purple-700 transition-colors disabled:opacity-50", disabled: !source[:url] %> <% end %> <%= link_to "View", bot_network_range_path(key), class: "px-3 py-1 text-xs font-medium text-gray-700 bg-gray-200 rounded hover:bg-gray-300 transition-colors" %>

<% end %>

Batch Import

Import from all available sources (this may take several minutes).

<%= form_with url: import_all_bot_network_ranges_path, method: :post do |f| %>

<%= f.submit "Import All Sources", class: "px-6 py-2 font-medium text-white bg-green-600 rounded hover:bg-green-700 transition-colors", confirm: "This will import from all available sources and may take several minutes. Continue?" %>

<% end %>

<% if @recent_imports.any? %>

Recent Imports

<% @recent_imports.each do |import| %> <% end %>

Source	Status	Records	Date	Notes
<%= import.source.titleize %>	<%= import.status.titleize %>	<%= import.records_processed&.to_s \|\| '0' %>	<%= import.created_at.strftime('%Y-%m-%d %H:%M') %>	<%= import.notes %>

<% end %> <% if @bot_network_ranges.any? %>

Recently Imported Bot Ranges

<% @bot_network_ranges.each do |range| %> <% end %>

Network	Source	Company	Created	Details
<%= range.network %>	<%= range.source.gsub('bot_import_', '').titleize %>	<%= range.company \|\| 'Unknown' %>	<%= range.created_at.strftime('%Y-%m-%d %H:%M') %>	<% if range.additional_data.present? %> <% data = JSON.parse(range.additional_data) rescue {} %> <% if data['crawler_type'] %> <%= data['crawler_type'].titleize %> <% end %> <% if data['aws_service'] %> <%= data['aws_service'] %> <% end %> <% end %>

<% end %>