Merge pull request #4 from sts-ryan-holton/dev

Dev

README.md

@@ -59,6 +59,8 @@ $ git clone [email protected]:sts-ryan-holton/loan-risk-score.git
 $ npm install
 ```
 
+:warning: You will need to **enable** the scraper, and provide a URL to scrape. We've provided an example that you can copy.
+
 ### :wrench: Starting
 
 ``` bash

package.json

@@ -1,6 +1,6 @@
 {
   "name": "loan-risk-score",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Calculate loan affordability using machine learning.",
   "main": "scraper/scrape-jobs.js",
   "keywords": [

scraper/scrape-jobs.js

@@ -17,8 +17,8 @@ var searchRadius   = (args['searchRadius'] && args['searchRadius'] != '') ? pars
     scrapeInterval = (args['scrapeInterval'] && args['scrapeInterval'] != '') ? parseInt(args['scrapeInterval']) : 1000,
     pauseDelay     = (args['thresholdDelay'] && args['thresholdDelay'] != '') ? parseInt(args['thresholdDelay']) : 7500,
     pauseScraping  = false,
-    scrapeOnRun    = true,
-    urlToScrape    = 'https://www.indeed.co.uk/jobs?l=Bridgend',
+    scrapeOnRun    = false,
+    urlToScrape    = '', // example: https://www.indeed.co.uk/jobs?l=London
     jobCard        = '.jobsearch-SerpJobCard',
     jobCardSalary  = '.salarySnippet .salaryText'
 
@@ -165,7 +165,7 @@ async function scrapeWebsite(radius, pageNo) {
 
 
 // run the scraper on loop and increment details
-if (scrapeOnRun) {
+if (scrapeOnRun && urlToScrape != '') {
   setInterval(() => {
     if (!pauseScraping) {
       scrapeWebsite(searchRadius, pageNumber)
@@ -175,6 +175,9 @@ if (scrapeOnRun) {
 
 
 // run the scraper on start
-if (scrapeOnRun) {
+if (scrapeOnRun && urlToScrape != '') {
   scrapeWebsite(searchRadius, pageNumber)
 }
+
+if (urlToScrape == '') console.log('=== ERROR: You\'ll need to add a URL in: "scraper/scrape-jobs.js, line: 21" to scrape. ===')
+if (!scrapeOnRun) console.log('=== ERROR: You\'ll need to enable the scraper in: "scraper/scrape-jobs.js, line: 20". ===')