create new crawler and Postgres saver

This commit is contained in:
Bilal Catic
2019-09-18 15:32:48 +02:00
parent 2e13763939
commit 3d46c82d3d
4 changed files with 363 additions and 164 deletions

View File

@@ -1,8 +1,6 @@
class PostgresSaver {
constructor(url) {
this.url = url;
}
const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
class PostgresSaver {
connect() {
//TODO: It seems we never worry about open/close connection with Sequelize ?
//TODO: Check if postgres is ready
@@ -10,13 +8,8 @@ class PostgresSaver {
}
async save(results) {
let resultsForMongo = Object.keys(results).map(key => {
return results[key];
});
for (const doc of resultsForMongo) {
this.collection.update({ url: doc.url }, doc, { upsert: true });
}
console.log("[POSTGRES] Saving...");
await bulkUpsertRealEstates(results);
}
close() {