using selenium to get more accurate prices

batch_process.py

@@ -8,16 +8,21 @@ def get_prices(links):
         results = []
         for link in links:
             driver.get(link)
-            results.append(
-                price_finder(
-                    url = link,bs=BS(driver.page_source,'lxml')
+            try:
+                results.append(
+                    price_finder(
+                        url = link,bs=BS(driver.page_source,'lxml')
+                        )
                     )
-                )
+            except AttributeError:
+                results.append(price_finder(link))
         driver.quit()
         return results
     except Exception as excpt:
         driver.quit()
         raise excpt
-import saveto
-links  = saveto.load('quad_links')
-products = get_prices(links)
+if __name__ == "__main__":
+    
+    import saveto
+    links  = saveto.load('quad_links')
+    products = get_prices(links)

price_finder.py

@@ -6,6 +6,7 @@ import datetime
 
 user_agent = UserAgent().chrome
 re_words = lambda n: re.compile(r"( ?[^ ]+ ?)"+"{0,"+str(n-1)+"}"+r"[^ ]+")
+debug = None
 def get_page(url):
     page = None
     while not page:
@@ -56,9 +57,12 @@ class price_finder:
             
     def _get_product_info_(self):
         funcs = price_finder.page_funcs[self.info_url.netloc]
+        print(self.url)
+ 
         return {
             "product_name":self.words.match(
                 funcs["name"](self.bs)
                 ).group(0),
             "price":funcs["price"](self.bs).replace("$",""),
             }
+