You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
50 lines
1.4 KiB
50 lines
1.4 KiB
import pyppeteer
|
|
import asyncio
|
|
import os
|
|
async def _get_link(browser,link):
|
|
pages = await browser.pages()
|
|
page = pages[0]
|
|
await page.goto(link,timeout=60_000)
|
|
webpage = None
|
|
for i in range(20):
|
|
try:
|
|
webpage = await page.content()
|
|
break
|
|
except:
|
|
time.sleep(1)
|
|
return webpage
|
|
|
|
async def _single_link(browser,link):
|
|
webpage = await _get_link(browser,link)
|
|
await browser.close()
|
|
return webpage
|
|
|
|
async def _multi_link(browser,links):
|
|
results = {}
|
|
for link in links:
|
|
webpage = await _get_link(browser,link)
|
|
results[link] = webpage
|
|
await browser.close()
|
|
return results
|
|
|
|
def get_link(links,headless = True,proxy = None):
|
|
ext = os.path.join(os.path.dirname(__file__),'uBlock0.chromium')
|
|
loop = asyncio.get_event_loop()
|
|
run = loop.run_until_complete
|
|
opts = {
|
|
'headless':headless,
|
|
}
|
|
opts['args'] = [f'--disable-extensions-except={ext}', f'--load-extension={ext}']
|
|
if proxy:
|
|
opts['args'] += [f'--proxy-server={proxy}']
|
|
# print(opts)
|
|
browser = run(pyppeteer.launch(**opts))
|
|
try:
|
|
if isinstance(links,list):
|
|
result = run(_multi_link(browser,links))
|
|
else:
|
|
result = run(_single_link(browser,links))
|
|
return result
|
|
except Exception as e:
|
|
run(browser.close())
|
|
raise e
|