senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

implement RENT option for Aktido; implement force crawl option

Browse Source
This commit is contained in:
Bilal Catic
2019-10-30 17:23:43 +01:00
parent fa712ce97d
commit ecc5b174a0
2 changed files with 9 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/crawler/specificConfigs/aktido.js
View File

@@ -29,5 +29,6 @@ module.exports = {
AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
AKTIDO_DELAY_BETWEEN_PAGES:
parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000
parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
};

26
app/crawler/specificCrawlers/aktido.js
View File

@@ -39,6 +39,8 @@ const AKTIDO_ENUMS = {
AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
};
const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
class AktidoCrawler {
constructor(
savers = [],
@@ -88,27 +90,13 @@ class AktidoCrawler {
newRealEstates.push(...newRecords);
if (Array.isArray(newRecords) && newRecords.length === 0) {
if (
Array.isArray(newRecords) &&
newRecords.length === 0 &&
!AKTIDO_FORCE_CRAWL
) {
generatorsToRemove[index] = true;
}
// for (const existingRecord of existingRecords) {
// const { publishedDate, renewedDate } = existingRecord;
//
// const publishedDateMoment = moment.utc(publishedDate);
// const renewedDateMoment = moment.utc(renewedDate);
//
// const stopCrawlingThisCategory = publishedDateMoment.isSame(
// renewedDateMoment,
// "minute"
// );
//
// if (stopCrawlingThisCategory) {
// generatorsToRemove[index] = true;
// // console.log("\tGenerator ", index + 1, "has no more new ads");
// break;
// }
// }
} else {
//Generator returned undefined, remove this generator from array
generatorsToRemove[index] = true;