stop crawling when existing, non-renewed ad is found

This commit is contained in:
Bilal Catic
2019-09-24 23:23:09 +02:00
parent 746732f30b
commit 90bc57edb6
4 changed files with 155 additions and 61 deletions

View File

@@ -1,4 +1,9 @@
const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
const moment = require("moment");
const {
bulkUpsertRealEstates,
checkIfAlreadyExist
} = require("../../helpers/db/realEstate");
class PostgresSaver {
connect() {
@@ -7,9 +12,29 @@ class PostgresSaver {
return true;
}
async save(results, maxAge) {
async save(results) {
console.log("[POSTGRES] Saving...");
await bulkUpsertRealEstates(results, maxAge);
const resultsWithPublishedAndRenewedDateSame = results.filter(
realEstate => {
const { publishedDate, renewedDate } = realEstate;
const publishedMomentDate = moment.utc(publishedDate);
const renewedMomentDate = moment.utc(renewedDate);
return publishedMomentDate.isSame(renewedMomentDate, "minute");
}
);
const exist =
resultsWithPublishedAndRenewedDateSame.length > 0
? await checkIfAlreadyExist(resultsWithPublishedAndRenewedDateSame)
: false;
const savedRecords = await bulkUpsertRealEstates(results);
return {
exist,
savedRecords
};
}
close() {