diff --git a/.gitignore b/.gitignore index 82d60ea..9f8f1bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*.pkl -*.json -__pycache__ -/api_info +*.pkl +*.json +__pycache__ +/api_info diff --git a/Student Detail Schedule.html b/Student Detail Schedule.html new file mode 100644 index 0000000..5f5865e --- /dev/null +++ b/Student Detail Schedule.html @@ -0,0 +1,603 @@ + + + + + + + + +Student Detail Schedule + + + + + +
+
+Go to Main Content +

SCT WWW Information System

+ + +

+Skip Module Navigation Links + + + + + +
+ + + + + + + + + + + +
+Personal Information + +Tab Corner Right + +Student + +Tab Corner Right + +Financial Aid + +Tab Corner Right + +Faculty Services + +Tab Corner Right +
+
Transparent Image
+ + + +

+ + + + + +
+
+
+Search + + + +
+
+

+ +RETURN TO MENU +| +SITE MAP +| +HELP +| +EXIT + +

+
+
+ + + + + + + + + +
+

Student Detail Schedule

+		 +  +

+

+000645225 Raphael L. Roberts
+Spring 2019
+Jan 06, 2019 07:50 pm
+
+
Transparent Image
+ +
+
+ +
Information Class Schedule
+The Schedule of Classes contains important information you should know including but not limited to: registering for classes, add/drop dates, semester calendars, final exam schedule, tuition/fees rates, and how to make payments or request a payment plan. If there are questions we can answer for you, please contact the Registration Office.

+Remember to review your class schedule prior to the first day of classes for possible changes in class location and times.

+Total Credit Hours: 15.000 +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Human Origins: Introduction To Biological Anthropology - ANTH 215 - 1
Associated Term:Spring 2019
CRN:24651
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Lesa C. DavisE-mail +
Grade Mode:Standard
Credits: 3.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class11:30 am - 12:45 pmMWBernard J Brommel Hall 156Jan 07, 2019 - May 07, 2019LectureLesa C Davis (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Discrete Structures - CS 201 - 2
Associated Term:Spring 2019
CRN:24775
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Cristina A. HaidauE-mail +
Grade Mode:Standard
Credits: 3.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class4:15 pm - 6:55 pmMLech Walesa Hall 3046Jan 07, 2019 - May 07, 2019LectureCristina A Haidau (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Introduction To Environmental Science - ENVI 101 - 2
Associated Term:Spring 2019
CRN:28223
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Pamela GeddesE-mail +
Grade Mode:Standard
Credits: 3.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class2:20 pm - 3:35 pmMBernard J Brommel Hall 135Jan 07, 2019 - May 07, 2019LecturePamela Geddes (P)E-mail
Class2:20 pm - 4:25 pmWBernard J Brommel Hall 135Jan 07, 2019 - May 07, 2019LabPamela Geddes (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Music Theory II - MUS 122 - 1
Associated Term:Spring 2019
CRN:25262
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Jeffrey F. KowalkowskiE-mail +
Grade Mode:Standard
Credits: 3.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class9:25 am - 10:40 amTRFine Arts Center 145Jan 07, 2019 - May 07, 2019LectureJeffrey F Kowalkowski (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Applied Music: Tuba - MUS 151K - 1
Associated Term:Spring 2019
CRN:27420
Status:**Web Registered** on Jan 06, 2019
Assigned Instructor: +William R. RussellE-mail +
Grade Mode:Standard
Credits: 1.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
ClassTBA TBAJan 07, 2019 - May 07, 2019LectureWilliam R Russell (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Band - MUS 231 - 1
Associated Term:Spring 2019
CRN:25277
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Travis M. HeathE-mail +
Grade Mode:Standard
Credits: 1.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class7:05 pm - 9:45 pmTRFine Arts Center 144Jan 07, 2019 - May 07, 2019LectureTravis M Heath (P)E-mail
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Instrumental Ensemble:Jazz Band - MUS 235A - 2
Associated Term:Spring 2019
CRN:25287
Status:**Web Registered** on Jan 04, 2019
Assigned Instructor: +Mayo TianaE-mail +
Grade Mode:Standard
Credits: 1.000
Level:Undergraduate
Campus:Main Campus
+ + + + + + + + + + + + + + + + + + + +
Scheduled Meeting Times
TypeTimeDaysWhereDate RangeSchedule TypeInstructors
Class1:40 pm - 2:55 pmTRFine Arts Center 144Jan 07, 2019 - May 07, 2019LectureMayo Tiana (P)E-mail
+
+Return to Previous + + +
Transparent Image
+Skip to top of page +
+
+ +
+
+ + +

+Skip Student Detail Schedule Links +

[ Show Registration History + | Add or Drop Classes + | Look Up Classes + ] + + + +

+
+ +
+
+ +
+
+ +
+
+Release: 8.7.1 +
+
+
+
+
+
+
+
+
+ + + diff --git a/body_create.py b/body_create.py index b198a7b..a73e472 100644 --- a/body_create.py +++ b/body_create.py @@ -1,87 +1,87 @@ -from dateutil import rrule -from gcalendar import dateTime -import datetime -import pickle -import json -import re -LOCATION = "5500 St Louis Ave, Chicago, IL 60625" -event = { - 'summary': 'Google I/O 2015', - 'location': '800 Howard St., San Francisco, CA 94103', - 'description': 'A chance to hear more about Google\'s developer products.', - 'start': { - 'dateTime': '2015-05-28T09:00:00-07:00', - 'timeZone': 'America/Los_Angeles', - }, - 'end': { - 'dateTime': '2015-05-28T17:00:00-07:00', - 'timeZone': 'America/Los_Angeles', - }, - 'recurrence': [ - 'RRULE:FREQ=DAILY;COUNT=2' - ], - 'attendees': [ - {'email': 'lpage@example.com'}, - {'email': 'sbrin@example.com'}, - ], - 'reminders': { - 'useDefault': False, - 'overrides': [ - {'method': 'email', 'minutes': 24 * 60}, - {'method': 'popup', 'minutes': 10}, - ], - }, -} - -def rrule_former(class_obj): - days = class_obj.days - start =datetime.datetime.combine(class_obj.date_range[0],class_obj.time_range[0]).astimezone() - end =datetime.datetime.combine(class_obj.date_range[1],class_obj.time_range[1]).astimezone() - - days = [ (day -1) % 7 for day in days] - ret = rrule.rrule(freq=rrule.WEEKLY,dtstart=start,wkst=rrule.SU,until=end,byweekday=days) - ret_str = str(ret).split('\n')[-1] - ret_str=re.sub(r'(UNTIL=[^;]+)',r'\1Z',ret_str) - return 'RRULE:{}'.format(ret_str) -def create_body(_class): - if _class.time_range: - body = { - # "kind": "calendar#event", - } - body['recurrence'] = [rrule_former(_class)] - body['start'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[0])) - body['end'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[1])) - body['summary'] = _class.title - body['description'] = 'location: {}'.format(_class.location) - body['location'] = LOCATION - body['reminders'] = {'useDefault':True} - return body -def json_dump(obj): - with open('classes.json','w') as file: - json.dump(obj,file) -def test_rrule(): - #test - now = datetime.datetime.now() - from munch import Munch - test_obj = Munch( - days=[1,3,5], - time_range=[ - now.time(), - (now+datetime.timedelta(seconds=50*60)).time() - ], - date_range=[ - now.date(), - (now+datetime.timedelta(days=20)).date() - ], - ) - test_result = rrule_former(test_obj) - return locals() - -def test_class2body(): - with open('classes.pkl','rb') as file: - classes = pickle.load(file) - test_result = list(filter(bool,map(create_body,classes))) - return test_result - -if __name__ == "__main__": +from dateutil import rrule +from gcalendar import dateTime +import datetime +import pickle +import json +import re +LOCATION = "5500 St Louis Ave, Chicago, IL 60625" +event = { + 'summary': 'Google I/O 2015', + 'location': '800 Howard St., San Francisco, CA 94103', + 'description': 'A chance to hear more about Google\'s developer products.', + 'start': { + 'dateTime': '2015-05-28T09:00:00-07:00', + 'timeZone': 'America/Los_Angeles', + }, + 'end': { + 'dateTime': '2015-05-28T17:00:00-07:00', + 'timeZone': 'America/Los_Angeles', + }, + 'recurrence': [ + 'RRULE:FREQ=DAILY;COUNT=2' + ], + 'attendees': [ + {'email': 'lpage@example.com'}, + {'email': 'sbrin@example.com'}, + ], + 'reminders': { + 'useDefault': False, + 'overrides': [ + {'method': 'email', 'minutes': 24 * 60}, + {'method': 'popup', 'minutes': 10}, + ], + }, +} + +def rrule_former(class_obj): + days = class_obj.days + start =datetime.datetime.combine(class_obj.date_range[0],class_obj.time_range[0]).astimezone() + end =datetime.datetime.combine(class_obj.date_range[1],class_obj.time_range[1]).astimezone() + + days = [ (day -1) % 7 for day in days] + ret = rrule.rrule(freq=rrule.WEEKLY,dtstart=start,wkst=rrule.SU,until=end,byweekday=days) + ret_str = str(ret).split('\n')[-1] + ret_str=re.sub(r'(UNTIL=[^;]+)',r'\1Z',ret_str) + return 'RRULE:{}'.format(ret_str) +def create_body(_class): + if _class.time_range: + body = { + # "kind": "calendar#event", + } + body['recurrence'] = [rrule_former(_class)] + body['start'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[0])) + body['end'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[1])) + body['summary'] = _class.title + body['description'] = 'location: {}'.format(_class.location) + body['location'] = LOCATION + body['reminders'] = {'useDefault':True} + return body +def json_dump(obj): + with open('classes.json','w') as file: + json.dump(obj,file) +def test_rrule(): + #test + now = datetime.datetime.now() + from munch import Munch + test_obj = Munch( + days=[1,3,5], + time_range=[ + now.time(), + (now+datetime.timedelta(seconds=50*60)).time() + ], + date_range=[ + now.date(), + (now+datetime.timedelta(days=20)).date() + ], + ) + test_result = rrule_former(test_obj) + return locals() + +def test_class2body(): + with open('classes.pkl','rb') as file: + classes = pickle.load(file) + test_result = list(filter(bool,map(create_body,classes))) + return test_result + +if __name__ == "__main__": json_dump(test_class2body()) \ No newline at end of file diff --git a/gcalendar.py b/gcalendar.py new file mode 100644 index 0000000..a1d9dca --- /dev/null +++ b/gcalendar.py @@ -0,0 +1,5 @@ +import os +import sys +parent = os.path.dirname(__file__) +sys.path.insert(0,os.path.join(parent,'google_api_wrapper')) +from gapi.calendar_api import * \ No newline at end of file diff --git a/get_classes.py b/get_classes.py index 2e84b0f..a8dd2f9 100644 --- a/get_classes.py +++ b/get_classes.py @@ -1,46 +1,46 @@ -from pyppeteer import launch -import asyncio -import time -import scraper -set_semester = "document.getElementsByName('term_in')[0].selectedIndex = 0" -xpaths = { - 'tab':".//a[text()='Current Student']", - 'schedule':".//a[text()='Student Detail Schedule']", - 'submit':"//input[@value='Submit']", - 'frame':"//frame[@src='/cp/ip/login?sys=sctssb&url=https://ssb.neiu.edu/mercury_neiuprod/bwskfshd.P_CrseSchdDetl']" - } -async def xpath_single_element(xpath,page): - await page.waitForXPath(xpath) - elements = await page.xpath(xpath) - return elements[0] -async def main_loop(login): - browser = await launch(headless = False) - page_list = await browser.pages() - page = page_list[0] - r = await page.goto('https://neiuport.neiu.edu/cp/home/displaylogin') - await page.evaluate(login) - await page.waitFor('#tab') - student_tab = await xpath_single_element(xpaths['tab'],page) - await student_tab.click() - await page.waitForXPath(xpaths['schedule']) - schedule = await xpath_single_element(xpaths['schedule'],page) - await schedule.click() - page.waitForXPath(xpaths['frame']) - await asyncio.sleep(3) - frame = page.frames[-1] - submit= await xpath_single_element(xpaths['submit'],frame) - await submit.click() - await asyncio.sleep(1) - content = await page.frames[-1].content() - await browser.close() - return scraper.get_classes(content) - -def get_classes(user,password): - login = """document.getElementById('user').value='{}' - document.getElementById('pass').value='{}' - login()""".format(user,password) - loop = asyncio.get_event_loop() - r = loop.run_until_complete - return r(main_loop(login)) -if __name__ == "__main__": - cl = get_classes('rlroberts5','YxmZZ905p0w6') +from pyppeteer import launch +import asyncio +import time +import scraper +set_semester = "document.getElementsByName('term_in')[0].selectedIndex = 0" +xpaths = { + 'tab':".//a[text()='Current Student']", + 'schedule':".//a[text()='Student Detail Schedule']", + 'submit':"//input[@value='Submit']", + 'frame':"//frame[@src='/cp/ip/login?sys=sctssb&url=https://ssb.neiu.edu/mercury_neiuprod/bwskfshd.P_CrseSchdDetl']" + } +async def xpath_single_element(xpath,page): + await page.waitForXPath(xpath) + elements = await page.xpath(xpath) + return elements[0] +async def main_loop(login): + browser = await launch(headless = False) + page_list = await browser.pages() + page = page_list[0] + r = await page.goto('https://neiuport.neiu.edu/cp/home/displaylogin') + await page.evaluate(login) + await page.waitFor('#tab') + student_tab = await xpath_single_element(xpaths['tab'],page) + await student_tab.click() + await page.waitForXPath(xpaths['schedule']) + schedule = await xpath_single_element(xpaths['schedule'],page) + await schedule.click() + page.waitForXPath(xpaths['frame']) + await asyncio.sleep(3) + frame = page.frames[-1] + submit= await xpath_single_element(xpaths['submit'],frame) + await submit.click() + await asyncio.sleep(1) + content = await page.frames[-1].content() + await browser.close() + return scraper.get_classes(content) + +def get_classes(user,password): + login = """document.getElementById('user').value='{}' + document.getElementById('pass').value='{}' + login()""".format(user,password) + loop = asyncio.get_event_loop() + r = loop.run_until_complete + return r(main_loop(login)) +if __name__ == "__main__": + cl = get_classes('rlroberts5','YxmZZ905p0w6') diff --git a/google_api_wrapper b/google_api_wrapper index 78ee774..de63a38 160000 --- a/google_api_wrapper +++ b/google_api_wrapper @@ -1 +1 @@ -Subproject commit 78ee774585aa6c8993d24a9fdcc4604dc5522da9 +Subproject commit de63a3871564e2b3f1faaa5f9c211e388358bc1f diff --git a/main.py b/main.py index 68a5862..ccb1ba5 100644 --- a/main.py +++ b/main.py @@ -1,13 +1,13 @@ -from gcalendar import api -import json -import pprint -api = api(r'api_info\client_secret.json','api_info') -cals = api.get_calendars() -cal = next(filter(lambda cal: cal['id'] == api.ids['school schedule'],cals)) -with open('classes.json') as file: - bodies = json.load(file) -for body in bodies: - # body['colorId'] = cal['colorId'] - # pprint.pprint(body) - # input() +from gcalendar import api +import json +import pprint +api = api(r'api_info\client_secret.json','api_info') +cals = api.get_calendars() +cal = next(filter(lambda cal: cal['id'] == api.ids['school schedule'],cals)) +with open('classes.json') as file: + bodies = json.load(file) +for body in bodies: + # body['colorId'] = cal['colorId'] + # pprint.pprint(body) + # input() api.create_event('school schedule',body) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ac6304e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pyppeteer +bs4 +lxml \ No newline at end of file diff --git a/scraper.py b/scraper.py index a0f95a4..803c14a 100644 --- a/scraper.py +++ b/scraper.py @@ -1,88 +1,108 @@ -from bs4 import BeautifulSoup as BS -import datetime -import re -from operator import sub -def dateparse(datetime_str): - date = '%b %d, %Y' - time = '%I:%M %p' - try: - return datetime.datetime.strptime(datetime_str,date) - except ValueError: - return datetime.datetime.strptime(datetime_str,time) -days = [None,'M','T','W','R','F',None] -simp_exceptions = ['Grade Mode'] -def datetime2date_time(dtime,mode): - if mode == 'date': - return datetime.date(dtime.year,dtime.month,dtime.day) - elif mode == 'time': - return datetime.time(dtime.hour,dtime.minute,dtime.second) -def seconds_from_midnight(t): - return t.hour*60**2+ t.minute*60+t.second -class Class: - def __init__(self,data): - info,times = data - #info - self.title,self.abrv,self.session = info.find('caption').text.split(' - ') - self.session = int(self.session) - rows = info.find_all('tr') - for row in rows: - name = row.find('th').text.rstrip(':') - data = re.sub(r'^ +|[\n\r\t]','',row.find('td').text) - - if name == 'Status': - type,date = data.split(' on ') - type = type.replace('*','') - self.type = type - self.registration_date = dateparse(date) - else: - if name in simp_exceptions: - name = name.lower().replace(' ','_') - else: - name = name.lower().split(' ')[-1] - if name != 'instructor': - data = data.lower() - try: - data = int(re.sub(r'\.\d+','',data)) - except: - - pass - self.__dict__[name] = data - - #time - headers,data = times.find_all('tr') - data = (col.text for col in data.find_all('td')) - headers = (header.text.lower() for header in headers.find_all('th')) - time_data = dict(zip(headers,data)) - if time_data['time'] == 'TBA': - self.time_range = None - else: - s,e = map(dateparse,time_data['time'].split(' - ')) - self.time_range = ( - datetime2date_time(s,'time'), - datetime2date_time(e,'time'), - ) - s,e = map(dateparse,time_data['date range'].split(' - ')) - self.date_range = ( - datetime2date_time(s,'date'), - datetime2date_time(e,'date'), - ) - time_data['days'] = re.sub('[^{}]'.format(''.join(filter(bool,days))),'',time_data['days']) - self.days = list(days.index(time_data['days'][i]) for i in range(len(time_data['days']))) - self.location = time_data['where'] - @property - def length(self): - return datetime.timedelta(seconds = sub( - seconds_from_midnight(self.time_range[1]), - seconds_from_midnight(self.time_range[0]), - )) -def get_classes(page): - if not isinstance(page,BS): - page = BS(page,'lxml') - tables = page.find_all('table',attrs= {'class':'datadisplaytable'}) - groups = ((tables[i],tables[i+1]) for i in range(0,len(tables),2)) - return list(map(Class,groups)) - -if __name__ == "__main__": - with open('schedule.html') as file: - page = BS(file.read(),'lxml') +from bs4 import BeautifulSoup as BS +import datetime +import re +from operator import sub +def dateparse(datetime_str): + date = '%b %d, %Y' + time = '%I:%M %p' + try: + return datetime.datetime.strptime(datetime_str,date) + except ValueError: + return datetime.datetime.strptime(datetime_str,time) +days = [None,'M','T','W','R','F',None] +simp_exceptions = ['Grade Mode'] +def datetime2date_time(dtime,mode): + if mode == 'date': + return datetime.date(dtime.year,dtime.month,dtime.day) + elif mode == 'time': + return datetime.time(dtime.hour,dtime.minute,dtime.second) +def seconds_from_midnight(t): + return t.hour*60**2+ t.minute*60+t.second +class Class: + def __init__(self,title,session,days,location,time_range): + self.title = title + self.session = session + self.days = days + self.location = location + self.time_range = time_range + self.lab = None + # data is a list of two html tables + def scrape(self,data): + info,times = data + # info + self.title,self.abrv,self.session = info.find('caption').text.split(' - ') + self.lab = None + self.session = int(self.session) + rows = info.find_all('tr') + for row in rows: + name = row.find('th').text.rstrip(':') + data = re.sub(r'^ +|[\n\r\t]','',row.find('td').text) + + if name == 'Status': + type,date = data.split(' on ') + type = type.replace('*','') + self.type = type + self.registration_date = dateparse(date) + else: + if name in simp_exceptions: + name = name.lower().replace(' ','_') + else: + name = name.lower().split(' ')[-1] + if name != 'instructor': + data = data.lower() + try: + data = int(re.sub(r'\.\d+','',data)) + except: + + pass + self.__dict__[name] = data + + # time + headers,*data = times.find_all('tr') + if len(data) > 1: + data,lab = data[:2] + else + lab = None + data = data[0] + data = (col.text for col in data.find_all('td')) + headers = (header.text.lower() for header in headers.find_all('th')) + + def parse_horz_row(headers,row): + ret = {} + time_data = dict(zip(headers,data)) + if time_data['time'] == 'TBA': + self.time_range = None + else: + s,e = map(dateparse,time_data['time'].split(' - ')) + self.time_range = ( + datetime2date_time(s,'time'), + datetime2date_time(e,'time'), + ) + s,e = map(dateparse,time_data['date range'].split(' - ')) + self.date_range = ( + datetime2date_time(s,'date'), + datetime2date_time(e,'date'), + ) + time_data['days'] = re.sub('[^{}]'.format(''.join(filter(bool,days))),'',time_data['days']) + self.days = list(days.index(time_data['days'][i]) for i in range(len(time_data['days']))) + self.location = time_data['where'] + + @property + def length(self): + return datetime.timedelta(seconds = sub( + seconds_from_midnight(self.time_range[1]), + seconds_from_midnight(self.time_range[0]), + )) + + +def get_classes(page): + if not isinstance(page,BS): + page = BS(page,'lxml') + tables = page.find_all('table',attrs= {'class':'datadisplaytable'}) + groups = ((tables[i],tables[i+1]) for i in range(0,len(tables),2)) + return list(map(Class.scrape,groups)) + +if __name__ == "__main__": + with open('schedule.html') as file: + page = BS(file.read(),'lxml') class1,*classes = get_classes(page) \ No newline at end of file