from pyppeteer import launch import asyncio import time import scraper set_semester = "document.getElementsByName('term_in')[0].selectedIndex = 0" xpaths = { 'tab':".//a[text()='Current Student']", 'schedule':".//a[text()='Student Detail Schedule']", 'submit':"//input[@value='Submit']", 'frame':"//frame[@src='/cp/ip/login?sys=sctssb&url=https://ssb.neiu.edu/mercury_neiuprod/bwskfshd.P_CrseSchdDetl']" } async def xpath_single_element(xpath,page): await page.waitForXPath(xpath) elements = await page.xpath(xpath) return elements[0] async def main_loop(login): browser = await launch(headless = False) page_list = await browser.pages() page = page_list[0] r = await page.goto('https://neiuport.neiu.edu/cp/home/displaylogin') await page.evaluate(login) await page.waitFor('#tab') student_tab = await xpath_single_element(xpaths['tab'],page) await student_tab.click() await page.waitForXPath(xpaths['schedule']) schedule = await xpath_single_element(xpaths['schedule'],page) await schedule.click() page.waitForXPath(xpaths['frame']) await asyncio.sleep(3) frame = page.frames[-1] submit= await xpath_single_element(xpaths['submit'],frame) await submit.click() await asyncio.sleep(1) content = await page.frames[-1].content() await browser.close() return scraper.get_classes(content) def get_classes(user,password): login = """document.getElementById('user').value='{}' document.getElementById('pass').value='{}' login()""".format(user,password) loop = asyncio.get_event_loop() r = loop.run_until_complete return r(main_loop(login)) if __name__ == "__main__": cl = get_classes('rlroberts5','YxmZZ905p0w6')