From 90401abe8843848f5389cc98df274955823e25a9 Mon Sep 17 00:00:00 2001 From: Raphael Roberts Date: Mon, 8 Jul 2019 13:02:25 -0500 Subject: [PATCH] Moved functions from util.py to models.py because of circular import error --- restscrape/models.py | 27 +++++++++++++++++++++++++++ restscrape/utils.py | 31 ------------------------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/restscrape/models.py b/restscrape/models.py index 3feaf80..923a890 100644 --- a/restscrape/models.py +++ b/restscrape/models.py @@ -63,6 +63,33 @@ class Browser(models.Model): super().delete() +def get_tab(proxy, use_adblock=True): + try: + + browser_connection = Browser.objects.get( + proxy=proxy, use_adblock=use_adblock + ).connect() + except Browser.DoesNotExist: + browser_connection = start_browser(proxy=proxy, use_adblock=use_adblock) + created_browser = Browser( + address=browser_connection.address, proxy=proxy, use_adblock=use_adblock + ) + created_browser.save() + return browser_connection.create_tab() + + +def cleanup_browsers(max_browsers=5): + alive = 0 + for browser in Browser.objects.all(): + if alive < max_browsers: + try: + browser.connect() + alive += 1 + except Exception: + browser.delete() + else: + if len(browser.connect().tabs) == 1: + browser.delete() class PageRequestTask(models.Model): task_id = models.UUIDField(primary_key=True) _result = None diff --git a/restscrape/utils.py b/restscrape/utils.py index 17522bb..e69de29 100644 --- a/restscrape/utils.py +++ b/restscrape/utils.py @@ -1,31 +0,0 @@ -from restscrape.models import Browser -from restscrape.scraping.browser import start_browser - - -def get_tab(proxy, use_adblock=True): - try: - - browser_connection = Browser.objects.get( - proxy=proxy, use_adblock=use_adblock - ).connect() - except Browser.DoesNotExist: - browser_connection = start_browser(proxy=proxy, use_adblock=use_adblock) - created_browser = Browser( - address=browser_connection.address, proxy=proxy, use_adblock=use_adblock - ) - created_browser.save() - return browser_connection.create_tab() - - -def cleanup_browsers(max_browsers=5): - alive = 0 - for browser in Browser.objects.all(): - if alive < max_browsers: - try: - browser.connect() - alive += 1 - except Exception: - browser.delete() - else: - if len(browser.connect().tabs) == 1: - browser.delete()