replace old crawler, without specific crawler and saver implementation
This commit is contained in:
28
app/crawler/savers/postgres.js
Normal file
28
app/crawler/savers/postgres.js
Normal file
@@ -0,0 +1,28 @@
|
||||
class PostgresSaver {
|
||||
constructor(url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
connect() {
|
||||
//TODO: It seems we never worry about open/close connection with Sequelize ?
|
||||
//TODO: Check if postgres is ready
|
||||
return true;
|
||||
}
|
||||
|
||||
async save(results) {
|
||||
let resultsForMongo = Object.keys(results).map(key => {
|
||||
return results[key];
|
||||
});
|
||||
|
||||
for (const doc of resultsForMongo) {
|
||||
this.collection.update({ url: doc.url }, doc, { upsert: true });
|
||||
}
|
||||
}
|
||||
|
||||
close() {
|
||||
//TODO: It seems we never worry about open/close connection with Sequelize ?
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = PostgresSaver;
|
||||
Reference in New Issue
Block a user