use function generator to index pages; crawl in parallel

2019-09-23 10:46:31 +02:00
parent c4f6c6e1c3
commit 3140fdf0c0
6 changed files with 127 additions and 77 deletions
--- a/app/crawler/savers/postgres.js
+++ b/app/crawler/savers/postgres.js
@@ -7,9 +7,9 @@ class PostgresSaver {
    return true;
  }

-  async save(results) {
+  async save(results, maxAge) {
    console.log("[POSTGRES] Saving...");
-    await bulkUpsertRealEstates(results);
+    await bulkUpsertRealEstates(results, maxAge);
  }

  close() {