You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
42 lines
1.1 KiB
42 lines
1.1 KiB
import pyppeteer
|
|
import asyncio
|
|
async def _get_link(browser,link):
|
|
pages = await browser.pages()
|
|
page = pages[0]
|
|
await page.goto(link)
|
|
webpage = None
|
|
for i in range(20):
|
|
try:
|
|
webpage = await page.content()
|
|
break
|
|
except:
|
|
time.sleep(1)
|
|
return webpage
|
|
|
|
async def _single_link(brower,link):
|
|
webpage = await _get_link(brower,link)
|
|
await brower.close()
|
|
return webpage
|
|
|
|
async def _multi_link(brower,links):
|
|
results = {}
|
|
for link in links:
|
|
webpage = await _get_link(brower,link)
|
|
result[link] = webpage
|
|
await brower.close()
|
|
return results
|
|
|
|
def get_link(links,headless = False,proxy = None):
|
|
loop = asyncio.get_event_loop()
|
|
run = loop.run_until_complete
|
|
opts = {
|
|
'headless':headless,
|
|
}
|
|
if proxy:
|
|
opts['args'] = ['--proxy-server={}'.format(proxy)]
|
|
browser = run(pyppeteer.launch(**opts))
|
|
if isinstance(links,list):
|
|
result = run(_multi_link(brower,links))
|
|
else:
|
|
result = run(_single_link(browser,links))
|
|
return result
|