require 'csv' def get_column_lookup columns = [ :name, :name_on_ribica, :picture, :brand, :code, :current_input_price, :list_price, :units_in_pack, :description, :stock, :on_display, :tags, :traits, :weight, :delivery_time_estimation, :name_unit, :short_name_unit, :description_suffix_unit, :number_of_pieces_suffix_unit, :name_sub_category, :order_sub_category, :name_supplier, :address_supplier, :postal_code_supplier, :town_supplier, :phone_supplier, :contact_person_supplier, :email_supplier, :note_supplier ] columns.map.with_index { |x, i| [x, i] }.to_h end def lookup_delivery_time_estimation(row, lookup) dte = DeliveryTimeEstimation.where( duration_in_days: row[lookup[:delivery_time_estimation]] ).take if dte.nil? dte = DeliveryTimeEstimation.new dte.duration_in_days = row[lookup[:delivery_time_estimation]] dte.save! end if dte.duration_in_days == 0 raise "Delivery time estimation input is missing or incorrect: #{dte.delivery_time_estimation}" end return dte end def lookup_unit(row, lookup) unit = Unit.where( name: row[lookup[:name_unit]], short_name: row[lookup[:short_name_unit]], description_suffix: row[lookup[:description_suffix_unit]], number_of_pieces_suffix: row[lookup[:number_of_pieces_suffix_unit]]).take if unit.nil? new_unit = Unit.new new_unit.name = row[lookup[:name_unit]] new_unit.short_name = "kom" #row[lookup[:short_name_unit]] new_unit.description_suffix = row[lookup[:description_suffix_unit]] new_unit.number_of_pieces_suffix = row[lookup[:number_of_pieces_suffix_unit]] new_unit.save! new_unit else unit end end def lookup_brand(row, lookup) brand = Brand.find_by name: row[lookup[:brand]] if brand.nil? new_brand = Brand.new new_brand.name = row[lookup[:brand]] new_brand.save! new_brand else brand end end def lookup_supplier(row, lookup) supplier = Supplier.find_by name: row[lookup[:name_supplier]] if supplier.nil? new_supplier = Supplier.new new_supplier.name = row[lookup[:name_supplier]] new_supplier.address = row[lookup[:address_supplier]] new_supplier.postal_code = row[lookup[:postal_code_supplier]] new_supplier.town = row[lookup[:town_supplier]] new_supplier.phone = row[lookup[:phone_supplier]] new_supplier.contact_person = row[lookup[:contact_person_supplier]] new_supplier.email = row[lookup[:email_supplier]] new_supplier.note = row[lookup[:note_supplier]] new_supplier.save! new_supplier else supplier end end def resolve_subcategory(str) raise "Subcategory is missing!" if str.nil? parts = str.split(">").map{ |s| s.strip } subcategory = parts[2] category = parts[1] section = parts[0] if subcategory.to_s == '' || category.to_s == '' || section.to_s == '' raise "Invalid subcategory : #{str}" end subcategory = subcategory.downcase category = category.downcase section = section.downcase sc = SubCategory.eager_load(category: :section) .where("lower(sub_categories.name) = '#{subcategory}' and lower(categories.name) = '#{category}' and lower(sections.name) = '#{section}'") .take return sc unless sc.nil? section_in_db = Section.where("lower(name) = '#{section}'").take if section_in_db.nil? section_in_db = Section.new section_in_db.name = section.capitalize section_in_db.save! end cat_in_db = Category .eager_load(:section) .where("lower(categories.name) = '#{category}' and lower(sections.name) = '#{section}'") .take if cat_in_db.nil? cat_in_db = Category.new cat_in_db.name = category.capitalize cat_in_db.section_id = section_in_db.id cat_in_db.save! end sub_cat_in_db = SubCategory .eager_load(:category) .where("lower(sub_categories.name) = '#{subcategory}' and lower(categories.name) = '#{category}'") .take if sub_cat_in_db.nil? sub_cat_in_db = SubCategory.new sub_cat_in_db.name = subcategory.capitalize sub_cat_in_db.category_id = cat_in_db.id sub_cat_in_db.save! end sub_cat_in_db #item.sub_category.order = row[lookup[:order_sub_category]] end def handle_multimedia(item, row, index, logger, lookup) multimedia = row[lookup[:picture]] return if multimedia.to_s == "" parts = multimedia.split(";").map{ |s| s.strip } parts.each do |part| mmd = MultiMediaDescription.new if part != "" and not part.start_with? "https://" raise "Invalid url for the image: #{part}" end mmd.url = part item.multi_media_descriptions << mmd end end def import_single_item(row, index, logger) succes = true begin lookup = get_column_lookup code = row[lookup[:code]] item = (Item.find_by code: code) || Item.new item.name = row[lookup[:name_on_ribica]] item.code = code item.current_input_price = row[lookup[:current_input_price]] || 11.00 item.list_price = row[lookup[:list_price]] || 12.00 item.units_in_pack = row[lookup[:units_in_pack]] item.description = row[lookup[:description]] || "default description" item.stock = row[lookup[:stock]] item.on_display = row[lookup[:on_display]] item.tags = row[lookup[:tags]] item.traits = row[lookup[:traits]] item.weight = row[lookup[:weight]] || 1.0 item.delivery_time_estimation = lookup_delivery_time_estimation(row, lookup) item.sub_category = resolve_subcategory(row[lookup[:name_sub_category]]) # todo multimedia, item groups handle_multimedia(item, row, index, logger, lookup) item.unit = lookup_unit(row, lookup) item.supplier = lookup_supplier(row, lookup) item.brand = lookup_brand(row, lookup) item.save(validate: false) # logger.info "Successfully imported item with on row position #{index}: #{row[lookup[:name_on_ribica]]}" success = true rescue Exception => e logger.error "Could not import item on row number #{index} (#{row[lookup[:name_on_ribica]]}), reason: #{e}" success = false end success end def trim_whitespace(row) row.map! do |value| trimmed = value.strip if not value.nil? # value.to_s.strip # puts "trimming '#{value}' to '#{trimmed}'" trimmed end end def do_import(validate_only) begin input_file = ENV['INPUT'] if input_file.to_s == "" puts "Input file is missing! Please provide input file in form INPUT=somefile.csv" return end lookup = get_column_lookup path = Rails.root.join(input_file) log_filename = "import.log" log_filename = "import_validate.log" if validate_only logger = Logger.new(log_filename) logger.info "Item import starting at #{Time.now}" logger.info "Will be importing items from #{path}" i = 1 should_rollback = false Item.transaction do CSV.foreach(path) do |row| if i != 1 trim_whitespace(row) if import_single_item(row, i, logger) == false should_rollback = true end end i += 1 end if validate_only || should_rollback if should_rollback puts "Import failed, please check the import log file for error details." logger.info "Rolling back because of errors" end raise ActiveRecord::Rollback end end rescue Exception => e puts "Import failed, please check the import log file for error details." logger.error "Error while importing: #{e}" end logger.info "Import done" end namespace :ribica do task copy_sections_to_menu: :environment do Section.transaction do Section.all.each do |section| mi = MenuItem.new mi.title = section.name mi.url = "/sekcija/#{section.id}/#{section.name}" mi.ordinal = section.order section.categories.each do |category| msi = MenuSubItem.new msi.title = category.name msi.url = "/sekcija/#{section.name}/kategorija/#{category.id}/#{category.name}" msi.ordinal = section.order mi.menu_sub_items << msi category.sub_categories.each do |sub_category| mssi = MenuSubSubItem.new mssi.title = sub_category.name mssi.url = "/podkategorija/#{sub_category.id}/#{sub_category.name}" mssi.ordinal = sub_category.order msi.menu_sub_sub_items << mssi end end mi.save! end end end end namespace :ribica do task clear_database: :environment do conn = ActiveRecord::Base.connection tables = conn.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';").map { |r| r["table_name"] } tables.delete "schema_migrations" tables.each { |t| conn.execute("TRUNCATE TABLE #{t}") } end end namespace :ribica do task validate_items: :environment do do_import true end end namespace :ribica do task import_items: :environment do do_import false end end namespace :ribica do task reindex: :environment do es_client = Elasticsearch::Client.new log: true # first delete the index begin es_client.indices.delete index: 'ribica' rescue logger.warn "Ribica index could not be deleted. Continuing with indexing operation..." end # now index items all_items = Item.includes(sub_category: { category: :section }).all.to_a all_items.each do |item| es_client.index index: 'ribica', type: 'items', id: item.id, body: { title: 'Test', name: item.name, code: item.code, description: item.description, sub_category: item.sub_category.name, category: item.sub_category.category.name, section: item.sub_category.category.section.name, brand: item.brand.name } end puts "ok!" end end