Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

app/services/ipapi.rb

@@ -53,4 +53,107 @@ class Ipapi
       next
     end
   end
+
+  # Parse company/datacenter network range from IPAPI data
+  # Handles "X.X.X.X - Y.Y.Y.Y" format and converts to CIDR
+  def self.parse_company_network_range(ipapi_data)
+    # Try company.network first, then datacenter.network
+    network_range = ipapi_data.dig('company', 'network') || ipapi_data.dig('datacenter', 'network')
+    return nil if network_range.blank?
+
+    # Parse "X.X.X.X - Y.Y.Y.Y" format
+    if network_range.include?(' - ')
+      start_ip_str, end_ip_str = network_range.split(' - ').map(&:strip)
+
+      begin
+        start_ip = IPAddr.new(start_ip_str)
+        end_ip = IPAddr.new(end_ip_str)
+
+        # Calculate the number of IPs in the range
+        num_ips = end_ip.to_i - start_ip.to_i + 1
+
+        # Calculate prefix length from number of IPs
+        # num_ips = 2^(32 - prefix_length) for IPv4
+        prefix_length = 32 - Math.log2(num_ips).to_i
+
+        # Verify it's a valid CIDR block (power of 2)
+        if 2**(32 - prefix_length) == num_ips
+          cidr = "#{start_ip_str}/#{prefix_length}"
+          Rails.logger.debug "Parsed company network range: #{network_range} -> #{cidr}"
+          return cidr
+        else
+          Rails.logger.warn "Network range #{network_range} is not a valid CIDR block (#{num_ips} IPs)"
+          return nil
+        end
+      rescue IPAddr::InvalidAddressError => e
+        Rails.logger.error "Invalid IP in company network range: #{network_range} (#{e.message})"
+        return nil
+      end
+    elsif network_range.include?('/')
+      # Already in CIDR format
+      return network_range
+    else
+      Rails.logger.warn "Unknown network range format: #{network_range}"
+      return nil
+    end
+  end
+
+  # Populate NetworkRange attributes from IPAPI data
+  def self.populate_network_attributes(network_range, ipapi_data)
+    network_range.asn = ipapi_data.dig('asn', 'asn')
+    network_range.asn_org = ipapi_data.dig('asn', 'org') || ipapi_data.dig('company', 'name')
+    network_range.company = ipapi_data.dig('company', 'name')
+    network_range.country = ipapi_data.dig('location', 'country_code')
+    network_range.is_datacenter = ipapi_data['is_datacenter'] || false
+    network_range.is_vpn = ipapi_data['is_vpn'] || false
+    network_range.is_proxy = ipapi_data['is_proxy'] || false
+  end
+
+  # Process IPAPI data and create network ranges
+  # Returns array of created/updated NetworkRange objects
+  def self.process_ipapi_data(ipapi_data, tracking_network)
+    created_networks = []
+
+    # Extract and create company/datacenter network range if present
+    company_network_cidr = parse_company_network_range(ipapi_data)
+    if company_network_cidr.present?
+      company_range = NetworkRange.find_or_create_by(network: company_network_cidr) do |nr|
+        nr.source = 'api_imported'
+        nr.creation_reason = "Company allocation from IPAPI for #{tracking_network.cidr}"
+      end
+
+      # Always update attributes (whether new or existing)
+      populate_network_attributes(company_range, ipapi_data)
+      company_range.set_network_data(:ipapi, ipapi_data)
+      company_range.last_api_fetch = Time.current
+      company_range.save!
+
+      created_networks << company_range
+      Rails.logger.info "Created/updated company network: #{company_range.cidr}"
+    end
+
+    # Extract and create ASN route network if present
+    ipapi_route = ipapi_data.dig('asn', 'route')
+    if ipapi_route.present? && ipapi_route != tracking_network.cidr
+      route_network = NetworkRange.find_or_create_by(network: ipapi_route) do |nr|
+        nr.source = 'api_imported'
+        nr.creation_reason = "BGP route from IPAPI lookup for #{tracking_network.cidr}"
+      end
+
+      # Always update attributes (whether new or existing)
+      populate_network_attributes(route_network, ipapi_data)
+      route_network.set_network_data(:ipapi, ipapi_data)
+      route_network.last_api_fetch = Time.current
+      route_network.save!
+
+      created_networks << route_network
+      Rails.logger.info "Created/updated BGP route network: #{route_network.cidr}"
+    end
+
+    # Return both the created networks and the broadest CIDR for deduplication
+    {
+      networks: created_networks,
+      broadest_cidr: company_network_cidr.presence || ipapi_route || tracking_network.cidr
+    }
+  end
 end