price_finder/get_link.py

import pyppeteer
import asyncio
import os
async def _get_link(browser,link):
    pages = await browser.pages()
    page = pages[0]
    await page.goto(link,timeout=60_000)
    webpage = None
    for i in range(20):
        try:
            webpage = await page.content()
            break
        except:
            time.sleep(1)
    return webpage
    
async def _single_link(browser,link):
    webpage = await _get_link(browser,link)
    await browser.close()
    return webpage

async def _multi_link(browser,links):
    results = {}
    for link in links:
        webpage = await _get_link(browser,link)
        results[link] = webpage
    await browser.close()
    return results

def get_link(links,headless = True,proxy = None):
    ext = os.path.join(os.path.dirname(__file__),'uBlock0.chromium')
    loop = asyncio.get_event_loop()
    run = loop.run_until_complete
    opts = {
        'headless':headless,
        }
    opts['args'] = [f'--disable-extensions-except={ext}', f'--load-extension={ext}']
    if proxy:
        opts['args'] += [f'--proxy-server={proxy}']
    # print(opts)
    browser = run(pyppeteer.launch(**opts))
    try:
        if isinstance(links,list):
            result = run(_multi_link(browser,links))
        else:
            result = run(_single_link(browser,links))
        return result
    except Exception as e:
        run(browser.close())
        raise e