Browse Source

Moved functions from util.py to models.py because of circular import error

master
Raphael Roberts 7 years ago
parent
commit
90401abe88
  1. 27
      restscrape/models.py
  2. 31
      restscrape/utils.py

27
restscrape/models.py

@ -63,6 +63,33 @@ class Browser(models.Model):
super().delete()
def get_tab(proxy, use_adblock=True):
try:
browser_connection = Browser.objects.get(
proxy=proxy, use_adblock=use_adblock
).connect()
except Browser.DoesNotExist:
browser_connection = start_browser(proxy=proxy, use_adblock=use_adblock)
created_browser = Browser(
address=browser_connection.address, proxy=proxy, use_adblock=use_adblock
)
created_browser.save()
return browser_connection.create_tab()
def cleanup_browsers(max_browsers=5):
alive = 0
for browser in Browser.objects.all():
if alive < max_browsers:
try:
browser.connect()
alive += 1
except Exception:
browser.delete()
else:
if len(browser.connect().tabs) == 1:
browser.delete()
class PageRequestTask(models.Model):
task_id = models.UUIDField(primary_key=True)
_result = None

31
restscrape/utils.py

@ -1,31 +0,0 @@
from restscrape.models import Browser
from restscrape.scraping.browser import start_browser
def get_tab(proxy, use_adblock=True):
try:
browser_connection = Browser.objects.get(
proxy=proxy, use_adblock=use_adblock
).connect()
except Browser.DoesNotExist:
browser_connection = start_browser(proxy=proxy, use_adblock=use_adblock)
created_browser = Browser(
address=browser_connection.address, proxy=proxy, use_adblock=use_adblock
)
created_browser.save()
return browser_connection.create_tab()
def cleanup_browsers(max_browsers=5):
alive = 0
for browser in Browser.objects.all():
if alive < max_browsers:
try:
browser.connect()
alive += 1
except Exception:
browser.delete()
else:
if len(browser.connect().tabs) == 1:
browser.delete()
Loading…
Cancel
Save