From c6200cf181d47ede15ef8457caebc21532d466cc Mon Sep 17 00:00:00 2001 From: ahmedsosic Date: Mon, 16 May 2022 07:01:22 +0200 Subject: [PATCH] add cars model --- .../allcrawlers/autocrawler/crawleritem.py | 14 +-- .../allcrawlers/autocrawler/olxautocrawler.py | 11 ++- .../allcrawlers/autocrawler/test_auto.py | 2 + kivi_cars/cars/__init__.py | 0 kivi_cars/cars/admin.py | 3 + kivi_cars/cars/apps.py | 6 ++ kivi_cars/cars/migrations/0001_initial.py | 99 +++++++++++++++++++ kivi_cars/cars/migrations/__init__.py | 0 kivi_cars/cars/models.py | 84 ++++++++++++++++ kivi_cars/cars/tests.py | 3 + kivi_cars/cars/views.py | 3 + kivi_cars/kivi_cars/settings.py | 5 +- 12 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 kivi_cars/cars/__init__.py create mode 100644 kivi_cars/cars/admin.py create mode 100644 kivi_cars/cars/apps.py create mode 100644 kivi_cars/cars/migrations/0001_initial.py create mode 100644 kivi_cars/cars/migrations/__init__.py create mode 100644 kivi_cars/cars/models.py create mode 100644 kivi_cars/cars/tests.py create mode 100644 kivi_cars/cars/views.py diff --git a/kivi_cars/allcrawlers/autocrawler/crawleritem.py b/kivi_cars/allcrawlers/autocrawler/crawleritem.py index a5e2694..80e58c2 100644 --- a/kivi_cars/allcrawlers/autocrawler/crawleritem.py +++ b/kivi_cars/allcrawlers/autocrawler/crawleritem.py @@ -1,4 +1,5 @@ # ----------Imports------------ +from ast import Num from datetime import date from traceback import print_tb from unittest import result @@ -156,7 +157,7 @@ for i in range(1,pages_number_to_crawl): for i in range(0, 1): #len(olx_id) # Artikal olx_link # artikal_link = 'https://www.olx.ba/artikal/' + '35731053' - artikal_link = 'https://www.olx.ba/artikal/' + '36976713' + artikal_link = 'https://www.olx.ba/artikal/' + '46962030' podaci["Olx_id"] = artikal_link @@ -265,14 +266,15 @@ for i in range(1,pages_number_to_crawl): # Vrijeme i datum - if (result_item.find('time', {'class' : 'entry-date'})): - date_time_div = result_item.find('time', {'class' : 'entry-date'}).get_text().split() - datum = date_time_div[0] - vrijeme = date_time_div[2] + if (result_item.find('div', {'class' : 'ob'})): + date_time_div = result_item.find('div', {'class' : 'ob'}).attrs + datum_vrijeme = date_time_div["data-content"].split() + datum = datum_vrijeme[0] + vrijeme = datum_vrijeme[2] podaci["Datum"] = datum podaci["Vrijeme"] = vrijeme - # print(podaci["Datum"], podaci["Vrijeme"]) + #print(datum,vrijeme) # Vrsta oglasa diff --git a/kivi_cars/allcrawlers/autocrawler/olxautocrawler.py b/kivi_cars/allcrawlers/autocrawler/olxautocrawler.py index 1583929..968bb98 100644 --- a/kivi_cars/allcrawlers/autocrawler/olxautocrawler.py +++ b/kivi_cars/allcrawlers/autocrawler/olxautocrawler.py @@ -258,10 +258,11 @@ for i in range(1,pages_number_to_crawl): # Vrijeme i datum - if (result_item.find('time', {'class' : 'entry-date'})): - date_time_div = result_item.find('time', {'class' : 'entry-date'}).get_text().split() - datum = date_time_div[0] - vrijeme = date_time_div[2] + if (result_item.find('div', {'class' : 'ob'})): + date_time_div = result_item.find('div', {'class' : 'ob'}).attrs + datum_vrijeme = date_time_div["data-content"].split() + datum = datum_vrijeme[0] + vrijeme = datum_vrijeme[2] podaci["Datum"] = datum podaci["Vrijeme"] = vrijeme # print(podaci["Datum"], podaci["Vrijeme"]) @@ -281,7 +282,7 @@ for i in range(1,pages_number_to_crawl): olx_db = pd.DataFrame(podaci_db) # treba biti niz # print(olx_db) -olx_db.to_excel('proba2.xlsx',index=False) +olx_db.to_excel('proba231112.xlsx',index=False) print("Zavrseno!!") diff --git a/kivi_cars/allcrawlers/autocrawler/test_auto.py b/kivi_cars/allcrawlers/autocrawler/test_auto.py index bec1b61..023fd54 100644 --- a/kivi_cars/allcrawlers/autocrawler/test_auto.py +++ b/kivi_cars/allcrawlers/autocrawler/test_auto.py @@ -276,6 +276,8 @@ for i in range(1,pages_number_to_crawl): podaci["Datum"] = datum podaci["Vrijeme"] = vrijeme # print(podaci["Datum"], podaci["Vrijeme"]) + + # Insert datas to database dictionary_copy = podaci.copy() diff --git a/kivi_cars/cars/__init__.py b/kivi_cars/cars/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kivi_cars/cars/admin.py b/kivi_cars/cars/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/kivi_cars/cars/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/kivi_cars/cars/apps.py b/kivi_cars/cars/apps.py new file mode 100644 index 0000000..51be0d4 --- /dev/null +++ b/kivi_cars/cars/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CarsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'cars' diff --git a/kivi_cars/cars/migrations/0001_initial.py b/kivi_cars/cars/migrations/0001_initial.py new file mode 100644 index 0000000..6e75642 --- /dev/null +++ b/kivi_cars/cars/migrations/0001_initial.py @@ -0,0 +1,99 @@ +# Generated by Django 4.0.4 on 2022-05-16 04:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Car', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('Olx_id', models.CharField(max_length=10)), + ('Kategorija', models.CharField(max_length=30)), + ('Cijena', models.CharField(max_length=30)), + ('Stanje', models.CharField(max_length=30)), + ('Lokacija_kanton', models.CharField(max_length=50)), + ('Lokacija_grad', models.CharField(max_length=50)), + ('Proizvođač', models.CharField(max_length=30)), + ('Model', models.CharField(max_length=30)), + ('Godište', models.CharField(max_length=30)), + ('Kilometraža', models.CharField(max_length=30)), + ('Kilovata_KW', models.CharField(max_length=30)), + ('Kubikaža', models.CharField(max_length=30)), + ('Gorivo', models.CharField(max_length=30)), + ('Broj_vrata', models.CharField(max_length=30)), + ('Konjskih_snaga', models.CharField(max_length=30)), + ('Metalik', models.CharField(max_length=30)), + ('Masa_Težina_kg', models.CharField(max_length=30)), + ('Tip', models.CharField(max_length=30)), + ('Pogon', models.CharField(max_length=30)), + ('Emisioni_standard', models.CharField(max_length=30)), + ('Veličina_felgi', models.CharField(max_length=30)), + ('Transmisija', models.CharField(max_length=30)), + ('Broj_stepeni_prijenosa', models.CharField(max_length=30)), + ('Boja', models.CharField(max_length=30)), + ('Muzika_ozvučenje', models.CharField(max_length=30)), + ('Parking_senzori', models.CharField(max_length=30)), + ('Parking_kamera', models.CharField(max_length=30)), + ('Registrovan_do', models.CharField(max_length=30)), + ('Godina_prve_registracije', models.CharField(max_length=30)), + ('Broj_prethodnih_vlasnik', models.CharField(max_length=30)), + ('Posjeduje_gume', models.CharField(max_length=30)), + ('Višezonska_klima', models.CharField(max_length=30)), + ('Rolo_zavjese', models.CharField(max_length=30)), + ('Svjetla', models.CharField(max_length=30)), + ('Zaštita_Blokada', models.CharField(max_length=30)), + ('Sjedećih_mjesta', models.CharField(max_length=30)), + ('Turbo', models.CharField(max_length=30)), + ('DPF_FAP_filter', models.CharField(max_length=30)), + ('Strane_tablice', models.CharField(max_length=30)), + ('Ocarinjen', models.CharField(max_length=30)), + ('Prilagođen_invalidima', models.CharField(max_length=30)), + ('Servo_volan', models.CharField(max_length=30)), + ('Tempomat', models.CharField(max_length=30)), + ('ESP', models.CharField(max_length=30)), + ('El_podizači_stakala', models.CharField(max_length=30)), + ('Senzor_mrtvog_ugla', models.CharField(max_length=30)), + ('Digitalna_klima', models.CharField(max_length=30)), + ('Touch_screen_ekran', models.CharField(max_length=30)), + ('Panorama_krov', models.CharField(max_length=30)), + ('Koža', models.CharField(max_length=30)), + ('Masaža_sjedišta', models.CharField(max_length=30)), + ('El_pomjeranje_sjedišta', models.CharField(max_length=30)), + ('Senzor_auto_svjetla', models.CharField(max_length=30)), + ('Alarm', models.CharField(max_length=30)), + ('Daljinsko_otključavanje', models.CharField(max_length=30)), + ('Auto_kuka', models.CharField(max_length=30)), + ('Udaren', models.CharField(max_length=30)), + ('Start_Stop_sistem', models.CharField(max_length=30)), + ('Park_assist', models.CharField(max_length=30)), + ('Registrovan', models.CharField(max_length=30)), + ('Na_lizingu', models.CharField(max_length=30)), + ('Servisna_knjiga', models.CharField(max_length=30)), + ('Komande_na_volanu', models.CharField(max_length=30)), + ('ABS', models.CharField(max_length=30)), + ('Airbag', models.CharField(max_length=30)), + ('Električni_retrovizori', models.CharField(max_length=30)), + ('Klima', models.CharField(max_length=30)), + ('Navigacija', models.CharField(max_length=30)), + ('Šiber', models.CharField(max_length=30)), + ('Naslon_za_ruku', models.CharField(max_length=30)), + ('Hlađenje_sjedišta', models.CharField(max_length=30)), + ('Grijanje_sjedišta', models.CharField(max_length=30)), + ('Memorija_sjedišta', models.CharField(max_length=30)), + ('Alu_felge', models.CharField(max_length=30)), + ('Centralna_brava', models.CharField(max_length=30)), + ('Oldtimer', models.CharField(max_length=30)), + ('ISOFIX', models.CharField(max_length=30)), + ('Datum', models.DateField()), + ('Vrijeme', models.TimeField()), + ], + ), + ] diff --git a/kivi_cars/cars/migrations/__init__.py b/kivi_cars/cars/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kivi_cars/cars/models.py b/kivi_cars/cars/models.py new file mode 100644 index 0000000..d1b3735 --- /dev/null +++ b/kivi_cars/cars/models.py @@ -0,0 +1,84 @@ +from django.db import models + + +# Create your models here. +class Car (models.Model): + Olx_id = models.CharField(max_length=10) + Kategorija = models.CharField(max_length=30) + Cijena = models.CharField(max_length=30) + Stanje = models.CharField(max_length=30) + Lokacija_kanton = models.CharField(max_length=50) + Lokacija_grad = models.CharField(max_length=50) + Proizvođač = models.CharField(max_length=30) + Model = models.CharField(max_length=30) + Godište = models.CharField(max_length=30) + Kilometraža = models.CharField(max_length=30) + Kilovata_KW = models.CharField(max_length=30) + Kubikaža = models.CharField(max_length=30) + Gorivo = models.CharField(max_length=30) + Broj_vrata = models.CharField(max_length=30) + Konjskih_snaga = models.CharField(max_length=30) + Metalik = models.CharField(max_length=30) + Masa_Težina_kg = models.CharField(max_length=30) + Tip = models.CharField(max_length=30) + Pogon = models.CharField(max_length=30) + Emisioni_standard = models.CharField(max_length=30) + Veličina_felgi = models.CharField(max_length=30) + Transmisija = models.CharField(max_length=30) + Broj_stepeni_prijenosa = models.CharField(max_length=30) + Boja = models.CharField(max_length=30) + Muzika_ozvučenje = models.CharField(max_length=30) + Parking_senzori= models.CharField(max_length=30) + Parking_kamera = models.CharField(max_length=30) + Registrovan_do = models.CharField(max_length=30) + Godina_prve_registracije = models.CharField(max_length=30) + Broj_prethodnih_vlasnik = models.CharField(max_length=30) + Posjeduje_gume = models.CharField(max_length=30) + Višezonska_klima = models.CharField(max_length=30) + Rolo_zavjese = models.CharField(max_length=30) + Svjetla = models.CharField(max_length=30) + Zaštita_Blokada = models.CharField(max_length=30) + Sjedećih_mjesta = models.CharField(max_length=30) + Turbo = models.CharField(max_length=30) + DPF_FAP_filter = models.CharField(max_length=30) + Strane_tablice = models.CharField(max_length=30) + Ocarinjen = models.CharField(max_length=30) + Prilagođen_invalidima = models.CharField(max_length=30) + Servo_volan = models.CharField(max_length=30) + Tempomat = models.CharField(max_length=30) + ESP = models.CharField(max_length=30) + El_podizači_stakala = models.CharField(max_length=30) + Senzor_mrtvog_ugla = models.CharField(max_length=30) + Digitalna_klima = models.CharField(max_length=30) + Touch_screen_ekran = models.CharField(max_length=30) + Panorama_krov = models.CharField(max_length=30) + Koža = models.CharField(max_length=30) + Masaža_sjedišta = models.CharField(max_length=30) + El_pomjeranje_sjedišta= models.CharField(max_length=30) + Senzor_auto_svjetla = models.CharField(max_length=30) + Alarm = models.CharField(max_length=30) + Daljinsko_otključavanje = models.CharField(max_length=30) + Auto_kuka = models.CharField(max_length=30) + Udaren = models.CharField(max_length=30) + Start_Stop_sistem = models.CharField(max_length=30) + Park_assist = models.CharField(max_length=30) + Registrovan = models.CharField(max_length=30) + Na_lizingu = models.CharField(max_length=30) + Servisna_knjiga = models.CharField(max_length=30) + Komande_na_volanu = models.CharField(max_length=30) + ABS = models.CharField(max_length=30) + Airbag = models.CharField(max_length=30) + Električni_retrovizori = models.CharField(max_length=30) + Klima = models.CharField(max_length=30) + Navigacija = models.CharField(max_length=30) + Šiber= models.CharField(max_length=30) + Naslon_za_ruku= models.CharField(max_length=30) + Hlađenje_sjedišta = models.CharField(max_length=30) + Grijanje_sjedišta = models.CharField(max_length=30) + Memorija_sjedišta = models.CharField(max_length=30) + Alu_felge = models.CharField(max_length=30) + Centralna_brava = models.CharField(max_length=30) + Oldtimer = models.CharField(max_length=30) + ISOFIX = models.CharField(max_length=30) + Datum = models.DateField() + Vrijeme = models.TimeField() \ No newline at end of file diff --git a/kivi_cars/cars/tests.py b/kivi_cars/cars/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/kivi_cars/cars/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/kivi_cars/cars/views.py b/kivi_cars/cars/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/kivi_cars/cars/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/kivi_cars/kivi_cars/settings.py b/kivi_cars/kivi_cars/settings.py index dacdffe..d46c43f 100644 --- a/kivi_cars/kivi_cars/settings.py +++ b/kivi_cars/kivi_cars/settings.py @@ -40,7 +40,8 @@ INSTALLED_APPS = [ 'rest_framework', - 'Generator' + 'Generator', + 'cars' ] MIDDLEWARE = [ @@ -83,7 +84,7 @@ DATABASES = { 'NAME': 'kivi_cars', 'USER': 'postgres', 'HOST': 'localhost', - 'PASSWORD': 'b18' + 'PASSWORD': 'postgres' } } -- 2.47.3