replace old crawler, without specific crawler and saver implementation

This commit is contained in:
Bilal Catic
2019-09-16 15:59:53 +02:00
parent b28b160852
commit 76a989fa37
7 changed files with 305 additions and 513 deletions

View File

@@ -0,0 +1,28 @@
class PostgresSaver {
constructor(url) {
this.url = url;
}
connect() {
//TODO: It seems we never worry about open/close connection with Sequelize ?
//TODO: Check if postgres is ready
return true;
}
async save(results) {
let resultsForMongo = Object.keys(results).map(key => {
return results[key];
});
for (const doc of resultsForMongo) {
this.collection.update({ url: doc.url }, doc, { upsert: true });
}
}
close() {
//TODO: It seems we never worry about open/close connection with Sequelize ?
return true;
}
}
module.exports = PostgresSaver;