Add 'tags' to event model. Add a dataimport system - currently for MaxMind zip files
This commit is contained in:
101
app/jobs/geolite_country_import_job.rb
Normal file
101
app/jobs/geolite_country_import_job.rb
Normal file
@@ -0,0 +1,101 @@
|
||||
class GeoliteCountryImportJob < ApplicationJob
|
||||
queue_as :default
|
||||
|
||||
# No retry needed for CSV processing - either works or fails immediately
|
||||
|
||||
def perform(data_import)
|
||||
Rails.logger.info "Starting GeoLite Country import job for DataImport #{data_import.id}"
|
||||
|
||||
# Check if file is attached
|
||||
unless data_import.file.attached?
|
||||
Rails.logger.error "No file attached to DataImport #{data_import.id}"
|
||||
data_import.fail!("No file attached")
|
||||
return
|
||||
end
|
||||
|
||||
# Download the file to a temporary location
|
||||
temp_file = download_to_temp_file(data_import.file.blob)
|
||||
if temp_file.nil?
|
||||
Rails.logger.error "Failed to download file from storage"
|
||||
data_import.fail!("Failed to download file from storage")
|
||||
return
|
||||
end
|
||||
|
||||
Rails.logger.info "File downloaded to: #{temp_file}"
|
||||
Rails.logger.info "File exists: #{File.exist?(temp_file)}"
|
||||
Rails.logger.info "File size: #{File.size(temp_file)} bytes" if File.exist?(temp_file)
|
||||
|
||||
# Mark as processing
|
||||
data_import.start_processing!
|
||||
|
||||
importer = nil
|
||||
begin
|
||||
Rails.logger.info "Creating GeoliteCountryImporter"
|
||||
importer = GeoliteCountryImporter.new(temp_file, data_import: data_import)
|
||||
Rails.logger.info "Calling importer.import"
|
||||
result = importer.import
|
||||
|
||||
# Update final stats
|
||||
data_import.update_progress(
|
||||
processed: result[:processed_records],
|
||||
failed: result[:failed_records],
|
||||
stats: {
|
||||
total_records: result[:total_records],
|
||||
errors: result[:errors].last(10), # Keep last 10 errors
|
||||
completed_at: Time.current
|
||||
}
|
||||
)
|
||||
|
||||
data_import.complete!
|
||||
|
||||
# Log completion
|
||||
Rails.logger.info "GeoLite Country import completed: #{result[:processed_records]} processed, #{result[:failed_records]} failed"
|
||||
|
||||
rescue => e
|
||||
Rails.logger.error "GeoLite Country import failed: #{e.message}"
|
||||
Rails.logger.error e.backtrace.join("\n")
|
||||
|
||||
# Update final stats even on failure
|
||||
if importer
|
||||
data_import.update_progress(
|
||||
processed: importer.instance_variable_get(:@processed_records),
|
||||
failed: importer.instance_variable_get(:@failed_records),
|
||||
stats: {
|
||||
total_records: importer.instance_variable_get(:@total_records),
|
||||
current_file: File.basename(temp_file),
|
||||
errors: importer.instance_variable_get(:@errors).last(10),
|
||||
failed_at: Time.current
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
data_import.fail!(e.message)
|
||||
raise
|
||||
ensure
|
||||
# Cleanup temporary files
|
||||
File.delete(temp_file) if temp_file && File.exist?(temp_file)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def download_to_temp_file(blob)
|
||||
# Create a temporary file with the original filename
|
||||
temp_file = Tempfile.new([blob.filename.to_s])
|
||||
temp_file.binmode
|
||||
|
||||
# Download the blob content
|
||||
blob.open do |file|
|
||||
temp_file.write(file.read)
|
||||
end
|
||||
|
||||
temp_file.close
|
||||
temp_file.path
|
||||
rescue => e
|
||||
Rails.logger.error "Error downloading file: #{e.message}"
|
||||
Rails.logger.error e.backtrace.join("\n")
|
||||
temp_file&.close
|
||||
temp_file&.unlink
|
||||
nil
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user