@@ -18,7 +18,7 @@ python3 -m pip install -r requirements.txt | |||
## Running | |||
TODO: Need to add section about url listing when finished doing that for multiple urls and not hardcoded. | |||
Add all required urls to scrape to the json file `urls.json` | |||
To run the script, make sure you are source'd into the virtual env, then run | |||
@@ -229,12 +229,8 @@ class Scraper: | |||
return "" | |||
urls = [ | |||
'https://landbot.io/u/H-62930-ZP46ZQSEK44QRYMQ/index.html', | |||
'https://landbot.io/u/H-352102-P3SL3H01U8XL08OV/index.html', | |||
'https://landbot.io/u/H-352137-OPZAVJDI1A0HTMJ1/index.html', | |||
'https://landbot.io/u/H-351906-X5HRGP1JXFC4WEFG/index.html', | |||
] | |||
with open("urls.json", "r") as fp: | |||
urls = json.loads(fp.read()) | |||
for url in urls: | |||
scraper = Scraper(url) |
@@ -0,0 +1,3 @@ | |||
[ | |||
"https://landbot.io/u/{ID}/index.html" | |||
] |