rlbr
/
class_sync
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

partial commit slight refactor of Class

partial
Raphael Roberts 7 years ago
parent
8be147a3f9
commit
8ef653634a
9 changed files with 867 additions and 236 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      .gitignore
  2. 603
      Student Detail Schedule.html
  3. 172
      body_create.py
  4. 5
      gcalendar.py
  5. 92
      get_classes.py
  6. 2
      google_api_wrapper
  7. 24
      main.py
  8. 3
      requirements.txt
  9. 194
      scraper.py

8
.gitignore
View File

@ -1,4 +1,4 @@
*.pkl
*.json
__pycache__
/api_info
*.pkl
*.json
__pycache__
/api_info

603
Student Detail Schedule.html
View File

@ -0,0 +1,603 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/transitional.dtd">
<HTML lang="en">
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta http-equiv="Pragma" name="Cache-Control" content="no-cache">
<meta http-equiv="Cache-Control" name="Cache-Control" content="no-cache">
<LINK REL="stylesheet" HREF="/css/web_defaultapp.css" TYPE="text/css">
<LINK REL="stylesheet" HREF="/css/web_defaultprint.css" TYPE="text/css" media="print">
<title>Student Detail Schedule</title>
<meta http-equiv="Content-Script-Type" name="Default_Script_Language" content="text/javascript">
<SCRIPT LANGUAGE="JavaScript" TYPE="text/javascript">
<!-- Hide JavaScript from older browsers
window.onunload = function() {submitcount=0;}
var submitcount=0;
function checkSubmit() {
if (submitcount == 0)
{
submitcount++;
return true;
}
else
{
alert("Your changes have already been submitted.");
return false;
}
}
// End script hiding -->
</SCRIPT>
<SCRIPT LANGUAGE="JavaScript" TYPE="text/javascript">
<!-- Hide JavaScript from older browsers
// Function to open a window
function windowOpen(window_url) {
helpWin = window.open(window_url,'','toolbar=yes,status=no,scrollbars=yes,menubar=yes,resizable=yes,directories=no,location=no,width=350,height=400');
if (document.images) {
if (helpWin) helpWin.focus()
}
}
// End script hiding -->
</SCRIPT>
</head>
<body>
<div class="headerwrapperdiv">
<div class="pageheaderdiv1">
<a href="#main_content" onMouseover="window.status='Go to Main Content'; return true" onMouseout="window.status=''; return true" OnFocus="window.status='Go to Main Content'; return true" onBlur="window.status=''; return true" class="skiplinks">Go to Main Content</a>
<h1>SCT WWW Information System</h1></DIV><div class="headerlinksdiv">
<SPAN class="pageheaderlinks2">
<map name="Module_Navigation_Links_H" title="Module Navigation Links">
<p>
<a href="#skip_Module_Navigation_Links_H" onMouseover="window.status='Skip Module Navigation Links'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Skip Module Navigation Links'; return true" onBlur="window.status=''; return true" class="skiplinks">Skip Module Navigation Links</a>
<table CLASS="plaintable" SUMMARY="This is main table for displaying Tab Items."
WIDTH="100%" cellSpacing=0 cellPadding=0 border=0>
<tr>
<TD CLASS="pldefault">
<table CLASS="plaintable" SUMMARY="This table displays Tab Items."
cellSpacing=0 cellPadding=0 border=0>
<tr>
<td class="taboff" height=22>
<a href="/neiuprod/twbkwbis.P_GenMenu?name=bmenu.P_GenMnu" onMouseover="window.status='Personal Information'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Personal Information'; return true" onBlur="window.status=''; return true" >Personal Information</a>
</TD>
<TD class="bgtaboff" height=22 vAlign="top" align="right">
<img src="/wtlgifs/web_tab_corner_right.gif" alt="Tab Corner Right" CLASS="headerImg" TITLE="Tab Corner Right" NAME="web_tab_corner_right" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=20 WIDTH=8 />
</TD>
<td class="tabon" height=22>
<a href="/neiuprod/twbkwbis.P_GenMenu?name=bmenu.P_StuMainMnu" onMouseover="window.status='Student'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Student'; return true" onBlur="window.status=''; return true" >Student</a>
</TD>
<TD class="bgtabon" height=22 vAlign="top" align="right">
<img src="/wtlgifs/web_tab_corner_right.gif" alt="Tab Corner Right" CLASS="headerImg" TITLE="Tab Corner Right" NAME="web_tab_corner_right" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=20 WIDTH=8 />
</TD>
<td class="taboff" height=22>
<a href="/neiuprod/twbkwbis.P_GenMenu?name=bmenu.P_FinAidMainMnu" onMouseover="window.status='Financial Aid'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Financial Aid'; return true" onBlur="window.status=''; return true" >Financial Aid</a>
</TD>
<TD class="bgtaboff" height=22 vAlign="top" align="right">
<img src="/wtlgifs/web_tab_corner_right.gif" alt="Tab Corner Right" CLASS="headerImg" TITLE="Tab Corner Right" NAME="web_tab_corner_right" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=20 WIDTH=8 />
</TD>
<td class="taboff" height=22>
<a href="/neiuprod/twbkwbis.P_GenMenu?name=bmenu.P_FacMainMnu" onMouseover="window.status='Faculty Services'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Faculty Services'; return true" onBlur="window.status=''; return true" >Faculty Services</a>
</TD>
<TD class="bgtaboff" height=22 vAlign="top" align="right">
<img src="/wtlgifs/web_tab_corner_right.gif" alt="Tab Corner Right" CLASS="headerImg" TITLE="Tab Corner Right" NAME="web_tab_corner_right" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=20 WIDTH=8 />
</TD>
</tr>
</table>
</TD>
</tr>
<tr>
<TD class="bgtabon" width="100%" colSpan=2><img src="/wtlgifs/web_transparent.gif" alt="Transparent Image" CLASS="headerImg" TITLE="Transparent Image" NAME="web_transparent" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=3 WIDTH=10 /></TD></tr></table>
</map>
</SPAN>
<a name="skip_Module_Navigation_Links_H"></a>
</DIV>
<table CLASS="plaintable" SUMMARY="This table displays Menu Items and Banner Search textbox."
WIDTH="100%">
<tr>
<TD CLASS="pldefault">
<div class="headerlinksdiv2">
<form action="/neiuprod/twbksrch.P_ShowResults" method="post">
Search
<SPAN class="fieldlabeltextinvisible"><LABEL for=keyword_in_id><SPAN class="fieldlabeltext">Search</SPAN></LABEL></SPAN>
<input type="text" name="KEYWRD_IN" size="20" maxlength="65" ID="keyword_in_id" />
<input type="submit" value="Go" />
</form>
</div>
</TD>
<TD CLASS="pldefault"><p class="rightaligntext" /p>
<SPAN class="pageheaderlinks">
<a href="/neiuprod/twbkwbis.P_GenMenu?name=bmenu.P_RegMnu" class="submenulinktext2" id="ssbbackurl">RETURN TO MENU</a>
|
<a href="/neiuprod/twbksite.P_DispSiteMap?menu_name_in=bmenu.P_MainMnu&amp;depth_in=2&amp;columns_in=3" accesskey="2" class="submenulinktext2">SITE MAP</a>
|
<a href="/neiuprod/twbkfrmt.P_DispHelp?pagename_in=bwskfshd.P_CrseSchdDetl" accesskey="H" onClick="popup = window.open('/neiuprod/twbkfrmt.P_DispHelp?pagename_in=bwskfshd.P_CrseSchdDetl', 'PopupPage','height=500,width=450,scrollbars=yes,resizable=yes'); return false" target="_blank" onMouseOver="window.status=''; return true" onMouseOut="window.status=''; return true"onFocus="window.status=''; return true" onBlur="window.status=''; return true" class="submenulinktext2">HELP</a>
|
<a href="twbkwbis.P_Logout" accesskey="3" class="submenulinktext2">EXIT</a>
</span>
</TD>
</tr>
</table>
</DIV>
<div class="pagetitlediv">
<table CLASS="plaintable" SUMMARY="This table displays title and static header displays."
WIDTH="100%">
<tr>
<TD CLASS="pldefault">
<h2>Student Detail Schedule</h2>
</TD>
<TD CLASS="pldefault">
&nbsp;
</TD>
<TD CLASS="pldefault"><p class="rightaligntext" /p>
<div class="staticheaders">
000645225 Raphael L. Roberts<br>
Spring 2019<br>
Jan 06, 2019 07:50 pm<br>
</div>
</TD>
</tr>
<tr>
<TD class="bg3" width="100%" colSpan=3><img src="/wtlgifs/web_transparent.gif" alt="Transparent Image" CLASS="headerImg" TITLE="Transparent Image" NAME="web_transparent" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=3 WIDTH=10 /></TD>
</tr>
</table>
<a name="main_content"></a>
</DIV>
<div class="pagebodydiv">
<!-- ** END OF twbkwbis.P_OpenDoc ** -->
<div class="infotextdiv"><table CLASS="infotexttable" SUMMARY="This layout table contains information that may be helpful in understanding the content and functionality of this page. It could be a brief set of instructions, a description of error messages, or other special information."><tr><td CLASS="indefault"><img src="/wtlgifs/web_info_cascade.png" alt="Information" CLASS="headerImg" TITLE="Information" NAME="web_info" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=12 WIDTH=14 /></td><td CLASS="indefault"><SPAN class="infotext"> <b>Class Schedule</b><br>
The <a href="https://www.neiu.edu/academics/registrar-services/class-schedules-and-registration"> Schedule of Classes </a> contains important information you should know including but not limited to: registering for classes, add/drop dates, semester calendars, final exam schedule, tuition/fees rates, and how to make payments or request a payment plan. If there are questions we can answer for you, please contact the <a href="mailto:registration@neiu.edu"> Registration Office.</a><br><br>
Remember to review your class schedule prior to the first day of classes for possible changes in class location and times.<br><br></SPAN></td></tr></table><p></DIV>
Total Credit Hours: 15.000
<br />
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Human Origins: Introduction To Biological Anthropology - ANTH 215 - 1</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">24651</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Lesa C. Davis<a href="mailto:lcdavis@neiu.edu" target="Lesa C. Davis" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 3.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">11:30 am - 12:45 pm</td>
<td CLASS="dddefault">MW</td>
<td CLASS="dddefault">Bernard J Brommel Hall 156</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Lesa C Davis (<ABBR title= "Primary">P</ABBR>)<a href="mailto:lcdavis@neiu.edu" target="Lesa C. Davis" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Discrete Structures - CS 201 - 2</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">24775</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Cristina A. Haidau<a href="mailto:C-Haidau@neiu.edu" target="Cristina A. Haidau" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 3.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">4:15 pm - 6:55 pm</td>
<td CLASS="dddefault">M</td>
<td CLASS="dddefault">Lech Walesa Hall 3046</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Cristina A Haidau (<ABBR title= "Primary">P</ABBR>)<a href="mailto:C-Haidau@neiu.edu" target="Cristina A. Haidau" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Introduction To Environmental Science - ENVI 101 - 2</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">28223</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Pamela Geddes<a href="mailto:P-Geddes@neiu.edu" target="Pamela Geddes" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 3.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">2:20 pm - 3:35 pm</td>
<td CLASS="dddefault">M</td>
<td CLASS="dddefault">Bernard J Brommel Hall 135</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Pamela Geddes (<ABBR title= "Primary">P</ABBR>)<a href="mailto:P-Geddes@neiu.edu" target="Pamela Geddes" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">2:20 pm - 4:25 pm</td>
<td CLASS="dddefault">W</td>
<td CLASS="dddefault">Bernard J Brommel Hall 135</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lab</td>
<td CLASS="dddefault">Pamela Geddes (<ABBR title= "Primary">P</ABBR>)<a href="mailto:P-Geddes@neiu.edu" target="Pamela Geddes" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Music Theory II - MUS 122 - 1</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">25262</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Jeffrey F. Kowalkowski<a href="mailto:J-Kowalkowski1@neiu.edu" target="Jeffrey F. Kowalkowski" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 3.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">9:25 am - 10:40 am</td>
<td CLASS="dddefault">TR</td>
<td CLASS="dddefault">Fine Arts Center 145</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Jeffrey F Kowalkowski (<ABBR title= "Primary">P</ABBR>)<a href="mailto:J-Kowalkowski1@neiu.edu" target="Jeffrey F. Kowalkowski" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Applied Music: Tuba - MUS 151K - 1</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">27420</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 06, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
William R. Russell<a href="mailto:W-Russell1@neiu.edu" target="William R. Russell" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 1.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault"><ABBR title = "To Be Announced">TBA</ABBR></td>
<td CLASS="dddefault">&nbsp;</td>
<td CLASS="dddefault"><ABBR title = "To Be Announced">TBA</ABBR></td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">William R Russell (<ABBR title= "Primary">P</ABBR>)<a href="mailto:W-Russell1@neiu.edu" target="William R. Russell" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Band - MUS 231 - 1</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">25277</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Travis M. Heath<a href="mailto:T-Heath@neiu.edu" target="Travis M. Heath" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 1.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">7:05 pm - 9:45 pm</td>
<td CLASS="dddefault">TR</td>
<td CLASS="dddefault">Fine Arts Center 144</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Travis M Heath (<ABBR title= "Primary">P</ABBR>)<a href="mailto:T-Heath@neiu.edu" target="Travis M. Heath" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<table CLASS="datadisplaytable" SUMMARY="This layout table is used to present the schedule course detail"><caption class="captiontext">Instrumental Ensemble:Jazz Band - MUS 235A - 2</caption>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Associated Term:</th>
<td CLASS="dddefault">Spring 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" ><ACRONYM title = "Course Reference Number">CRN</ACRONYM>:</th>
<td CLASS="dddefault">25287</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Status:</th>
<td CLASS="dddefault">**Web Registered** on Jan 04, 2019</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Assigned Instructor:</th>
<TD CLASS="dddefault">
Mayo Tiana<a href="mailto:M-Tiana@neiu.edu" target="Mayo Tiana" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a>
</TD>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Grade Mode:</th>
<td CLASS="dddefault">Standard</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Credits:</th>
<td CLASS="dddefault"> 1.000</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Level:</th>
<td CLASS="dddefault">Undergraduate</td>
</tr>
<tr>
<th colspan="2" CLASS="ddlabel" scope="row" >Campus:</th>
<td CLASS="dddefault">Main Campus</td>
</tr>
</table>
<table CLASS="datadisplaytable" SUMMARY="This table lists the scheduled meeting times and assigned instructors for this class.."><caption class="captiontext">Scheduled Meeting Times</caption>
<tr>
<th CLASS="ddheader" scope="col" >Type</th>
<th CLASS="ddheader" scope="col" >Time</th>
<th CLASS="ddheader" scope="col" >Days</th>
<th CLASS="ddheader" scope="col" >Where</th>
<th CLASS="ddheader" scope="col" >Date Range</th>
<th CLASS="ddheader" scope="col" >Schedule Type</th>
<th CLASS="ddheader" scope="col" >Instructors</th>
</tr>
<tr>
<td CLASS="dddefault">Class</td>
<td CLASS="dddefault">1:40 pm - 2:55 pm</td>
<td CLASS="dddefault">TR</td>
<td CLASS="dddefault">Fine Arts Center 144</td>
<td CLASS="dddefault">Jan 07, 2019 - May 07, 2019</td>
<td CLASS="dddefault">Lecture</td>
<td CLASS="dddefault">Mayo Tiana (<ABBR title= "Primary">P</ABBR>)<a href="mailto:M-Tiana@neiu.edu" target="Mayo Tiana" ><img src="/wtlgifs/web_email.gif" align="middle" alt="E-mail" CLASS="headerImg" TITLE="E-mail" NAME="web_email" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=28 WIDTH=28 /></a></td>
</tr>
</table>
<br />
<a href="javascript:history.go(-1)" onMouseOver="window.status='Return to Previous'; return true" onFocus="window.status='Return to Previous'; return true" onMouseOut="window.status=''; return true"onBlur="window.status=''; return true">Return to Previous</a>
<!-- ** START OF twbkwbis.P_CloseDoc ** -->
<table CLASS="plaintable" SUMMARY="This is table displays line separator at end of the page."
WIDTH="100%" cellSpacing=0 cellPadding=0 border=0><tr><TD class="bgtabon" width="100%" colSpan=2><img src="/wtlgifs/web_transparent.gif" alt="Transparent Image" CLASS="headerImg" TITLE="Transparent Image" NAME="web_transparent" HSPACE=0 VSPACE=0 BORDER=0 HEIGHT=3 WIDTH=10 /></TD></tr></table>
<a href="#top" onMouseover="window.status='Skip to top of page'; return true" onMouseout="window.status=''; return true" OnFocus="window.status='Skip to top of page'; return true" onBlur="window.status=''; return true" class="skiplinks">Skip to top of page</a>
</DIV>
<div class="footerbeforediv">
</DIV>
<div class="footerlinksdiv">
<SPAN class="pagefooterlinks">
<map name="Student_Detail_Schedule_Links_F" title="Student Detail Schedule Links">
<p>
<a href="#skip_Student_Detail_Schedule_Links_F" onMouseover="window.status='Skip Student Detail Schedule Links'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Skip Student Detail Schedule Links'; return true" onBlur="window.status=''; return true" class="skiplinks">Skip Student Detail Schedule Links</a>
<P>[ <a href="/neiuprod/bwskhreg.p_reg_hist" onMouseover="window.status='Show Registration History'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Show Registration History'; return true" onBlur="window.status=''; return true" >Show Registration History</a>
| <a href="/neiuprod/bwskfreg.P_AltPin" onMouseover="window.status='Add or Drop Classes'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Add or Drop Classes'; return true" onBlur="window.status=''; return true" >Add or Drop Classes</a>
| <a href="/neiuprod/bwskfcls.p_sel_crse_search" onMouseover="window.status='Look Up Classes'; return true" onMouseout="window.status=''; return true" onFocus="window.status='Look Up Classes'; return true" onBlur="window.status=''; return true" >Look Up Classes</a>
]
</map>
</SPAN>
<a name="skip_Student_Detail_Schedule_Links_F"></a>
</DIV>
<div class="footerafterdiv">
</DIV>
<div class="globalafterdiv">
</DIV>
<div class="globalfooterdiv">
</DIV>
<div class="pagefooterdiv">
<SPAN class="releasetext">Release: 8.7.1</SPAN>
</DIV>
<div class="poweredbydiv">
</DIV>
<DIV class="div1"></DIV>
<DIV class="div2"></DIV>
<DIV class="div3"></DIV>
<DIV class="div4"></DIV>
<DIV class="div5"></DIV>
<DIV class="div6"></DIV>
<div class="banner_copyright"> <br><h5>© 2019 Ellucian Company L.P. and its affiliates.<br></h5></div>
</body>
</html>

172
body_create.py
View File

@ -1,87 +1,87 @@
from dateutil import rrule
from gcalendar import dateTime
import datetime
import pickle
import json
import re
LOCATION = "5500 St Louis Ave, Chicago, IL 60625"
event = {
'summary': 'Google I/O 2015',
'location': '800 Howard St., San Francisco, CA 94103',
'description': 'A chance to hear more about Google\'s developer products.',
'start': {
'dateTime': '2015-05-28T09:00:00-07:00',
'timeZone': 'America/Los_Angeles',
},
'end': {
'dateTime': '2015-05-28T17:00:00-07:00',
'timeZone': 'America/Los_Angeles',
},
'recurrence': [
'RRULE:FREQ=DAILY;COUNT=2'
],
'attendees': [
{'email': 'lpage@example.com'},
{'email': 'sbrin@example.com'},
],
'reminders': {
'useDefault': False,
'overrides': [
{'method': 'email', 'minutes': 24 * 60},
{'method': 'popup', 'minutes': 10},
],
},
}
def rrule_former(class_obj):
days = class_obj.days
start =datetime.datetime.combine(class_obj.date_range[0],class_obj.time_range[0]).astimezone()
end =datetime.datetime.combine(class_obj.date_range[1],class_obj.time_range[1]).astimezone()
days = [ (day -1) % 7 for day in days]
ret = rrule.rrule(freq=rrule.WEEKLY,dtstart=start,wkst=rrule.SU,until=end,byweekday=days)
ret_str = str(ret).split('\n')[-1]
ret_str=re.sub(r'(UNTIL=[^;]+)',r'\1Z',ret_str)
return 'RRULE:{}'.format(ret_str)
def create_body(_class):
if _class.time_range:
body = {
# "kind": "calendar#event",
}
body['recurrence'] = [rrule_former(_class)]
body['start'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[0]))
body['end'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[1]))
body['summary'] = _class.title
body['description'] = 'location: {}'.format(_class.location)
body['location'] = LOCATION
body['reminders'] = {'useDefault':True}
return body
def json_dump(obj):
with open('classes.json','w') as file:
json.dump(obj,file)
def test_rrule():
#test
now = datetime.datetime.now()
from munch import Munch
test_obj = Munch(
days=[1,3,5],
time_range=[
now.time(),
(now+datetime.timedelta(seconds=50*60)).time()
],
date_range=[
now.date(),
(now+datetime.timedelta(days=20)).date()
],
)
test_result = rrule_former(test_obj)
return locals()
def test_class2body():
with open('classes.pkl','rb') as file:
classes = pickle.load(file)
test_result = list(filter(bool,map(create_body,classes)))
return test_result
if __name__ == "__main__":
from dateutil import rrule
from gcalendar import dateTime
import datetime
import pickle
import json
import re
LOCATION = "5500 St Louis Ave, Chicago, IL 60625"
event = {
'summary': 'Google I/O 2015',
'location': '800 Howard St., San Francisco, CA 94103',
'description': 'A chance to hear more about Google\'s developer products.',
'start': {
'dateTime': '2015-05-28T09:00:00-07:00',
'timeZone': 'America/Los_Angeles',
},
'end': {
'dateTime': '2015-05-28T17:00:00-07:00',
'timeZone': 'America/Los_Angeles',
},
'recurrence': [
'RRULE:FREQ=DAILY;COUNT=2'
],
'attendees': [
{'email': 'lpage@example.com'},
{'email': 'sbrin@example.com'},
],
'reminders': {
'useDefault': False,
'overrides': [
{'method': 'email', 'minutes': 24 * 60},
{'method': 'popup', 'minutes': 10},
],
},
}
def rrule_former(class_obj):
days = class_obj.days
start =datetime.datetime.combine(class_obj.date_range[0],class_obj.time_range[0]).astimezone()
end =datetime.datetime.combine(class_obj.date_range[1],class_obj.time_range[1]).astimezone()
days = [ (day -1) % 7 for day in days]
ret = rrule.rrule(freq=rrule.WEEKLY,dtstart=start,wkst=rrule.SU,until=end,byweekday=days)
ret_str = str(ret).split('\n')[-1]
ret_str=re.sub(r'(UNTIL=[^;]+)',r'\1Z',ret_str)
return 'RRULE:{}'.format(ret_str)
def create_body(_class):
if _class.time_range:
body = {
# "kind": "calendar#event",
}
body['recurrence'] = [rrule_former(_class)]
body['start'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[0]))
body['end'] = dateTime(datetime.datetime.combine(_class.date_range[0],_class.time_range[1]))
body['summary'] = _class.title
body['description'] = 'location: {}'.format(_class.location)
body['location'] = LOCATION
body['reminders'] = {'useDefault':True}
return body
def json_dump(obj):
with open('classes.json','w') as file:
json.dump(obj,file)
def test_rrule():
#test
now = datetime.datetime.now()
from munch import Munch
test_obj = Munch(
days=[1,3,5],
time_range=[
now.time(),
(now+datetime.timedelta(seconds=50*60)).time()
],
date_range=[
now.date(),
(now+datetime.timedelta(days=20)).date()
],
)
test_result = rrule_former(test_obj)
return locals()
def test_class2body():
with open('classes.pkl','rb') as file:
classes = pickle.load(file)
test_result = list(filter(bool,map(create_body,classes)))
return test_result
if __name__ == "__main__":
json_dump(test_class2body())

5
gcalendar.py
View File

@ -0,0 +1,5 @@
import os
import sys
parent = os.path.dirname(__file__)
sys.path.insert(0,os.path.join(parent,'google_api_wrapper'))
from gapi.calendar_api import *

92
get_classes.py
View File

@ -1,46 +1,46 @@
from pyppeteer import launch
import asyncio
import time
import scraper
set_semester = "document.getElementsByName('term_in')[0].selectedIndex = 0"
xpaths = {
'tab':".//a[text()='Current Student']",
'schedule':".//a[text()='Student Detail Schedule']",
'submit':"//input[@value='Submit']",
'frame':"//frame[@src='/cp/ip/login?sys=sctssb&url=https://ssb.neiu.edu/mercury_neiuprod/bwskfshd.P_CrseSchdDetl']"
}
async def xpath_single_element(xpath,page):
await page.waitForXPath(xpath)
elements = await page.xpath(xpath)
return elements[0]
async def main_loop(login):
browser = await launch(headless = False)
page_list = await browser.pages()
page = page_list[0]
r = await page.goto('https://neiuport.neiu.edu/cp/home/displaylogin')
await page.evaluate(login)
await page.waitFor('#tab')
student_tab = await xpath_single_element(xpaths['tab'],page)
await student_tab.click()
await page.waitForXPath(xpaths['schedule'])
schedule = await xpath_single_element(xpaths['schedule'],page)
await schedule.click()
page.waitForXPath(xpaths['frame'])
await asyncio.sleep(3)
frame = page.frames[-1]
submit= await xpath_single_element(xpaths['submit'],frame)
await submit.click()
await asyncio.sleep(1)
content = await page.frames[-1].content()
await browser.close()
return scraper.get_classes(content)
def get_classes(user,password):
login = """document.getElementById('user').value='{}'
document.getElementById('pass').value='{}'
login()""".format(user,password)
loop = asyncio.get_event_loop()
r = loop.run_until_complete
return r(main_loop(login))
if __name__ == "__main__":
cl = get_classes('rlroberts5','YxmZZ905p0w6')
from pyppeteer import launch
import asyncio
import time
import scraper
set_semester = "document.getElementsByName('term_in')[0].selectedIndex = 0"
xpaths = {
'tab':".//a[text()='Current Student']",
'schedule':".//a[text()='Student Detail Schedule']",
'submit':"//input[@value='Submit']",
'frame':"//frame[@src='/cp/ip/login?sys=sctssb&url=https://ssb.neiu.edu/mercury_neiuprod/bwskfshd.P_CrseSchdDetl']"
}
async def xpath_single_element(xpath,page):
await page.waitForXPath(xpath)
elements = await page.xpath(xpath)
return elements[0]
async def main_loop(login):
browser = await launch(headless = False)
page_list = await browser.pages()
page = page_list[0]
r = await page.goto('https://neiuport.neiu.edu/cp/home/displaylogin')
await page.evaluate(login)
await page.waitFor('#tab')
student_tab = await xpath_single_element(xpaths['tab'],page)
await student_tab.click()
await page.waitForXPath(xpaths['schedule'])
schedule = await xpath_single_element(xpaths['schedule'],page)
await schedule.click()
page.waitForXPath(xpaths['frame'])
await asyncio.sleep(3)
frame = page.frames[-1]
submit= await xpath_single_element(xpaths['submit'],frame)
await submit.click()
await asyncio.sleep(1)
content = await page.frames[-1].content()
await browser.close()
return scraper.get_classes(content)
def get_classes(user,password):
login = """document.getElementById('user').value='{}'
document.getElementById('pass').value='{}'
login()""".format(user,password)
loop = asyncio.get_event_loop()
r = loop.run_until_complete
return r(main_loop(login))
if __name__ == "__main__":
cl = get_classes('rlroberts5','YxmZZ905p0w6')

2
google_api_wrapper

@ -1 +1 @@
Subproject commit 78ee774585aa6c8993d24a9fdcc4604dc5522da9
Subproject commit de63a3871564e2b3f1faaa5f9c211e388358bc1f

24
main.py
View File

@ -1,13 +1,13 @@
from gcalendar import api
import json
import pprint
api = api(r'api_info\client_secret.json','api_info')
cals = api.get_calendars()
cal = next(filter(lambda cal: cal['id'] == api.ids['school schedule'],cals))
with open('classes.json') as file:
bodies = json.load(file)
for body in bodies:
# body['colorId'] = cal['colorId']
# pprint.pprint(body)
# input()
from gcalendar import api
import json
import pprint
api = api(r'api_info\client_secret.json','api_info')
cals = api.get_calendars()
cal = next(filter(lambda cal: cal['id'] == api.ids['school schedule'],cals))
with open('classes.json') as file:
bodies = json.load(file)
for body in bodies:
# body['colorId'] = cal['colorId']
# pprint.pprint(body)
# input()
api.create_event('school schedule',body)

3
requirements.txt
View File

@ -0,0 +1,3 @@
pyppeteer
bs4
lxml

194
scraper.py
View File

@ -1,88 +1,108 @@
from bs4 import BeautifulSoup as BS
import datetime
import re
from operator import sub
def dateparse(datetime_str):
date = '%b %d, %Y'
time = '%I:%M %p'
try:
return datetime.datetime.strptime(datetime_str,date)
except ValueError:
return datetime.datetime.strptime(datetime_str,time)
days = [None,'M','T','W','R','F',None]
simp_exceptions = ['Grade Mode']
def datetime2date_time(dtime,mode):
if mode == 'date':
return datetime.date(dtime.year,dtime.month,dtime.day)
elif mode == 'time':
return datetime.time(dtime.hour,dtime.minute,dtime.second)
def seconds_from_midnight(t):
return t.hour*60**2+ t.minute*60+t.second
class Class:
def __init__(self,data):
info,times = data
#info
self.title,self.abrv,self.session = info.find('caption').text.split(' - ')
self.session = int(self.session)
rows = info.find_all('tr')
for row in rows:
name = row.find('th').text.rstrip(':')
data = re.sub(r'^ +|[\n\r\t]','',row.find('td').text)
if name == 'Status':
type,date = data.split(' on ')
type = type.replace('*','')
self.type = type
self.registration_date = dateparse(date)
else:
if name in simp_exceptions:
name = name.lower().replace(' ','_')
else:
name = name.lower().split(' ')[-1]
if name != 'instructor':
data = data.lower()
try:
data = int(re.sub(r'\.\d+','',data))
except:
pass
self.__dict__[name] = data
#time
headers,data = times.find_all('tr')
data = (col.text for col in data.find_all('td'))
headers = (header.text.lower() for header in headers.find_all('th'))
time_data = dict(zip(headers,data))
if time_data['time'] == 'TBA':
self.time_range = None
else:
s,e = map(dateparse,time_data['time'].split(' - '))
self.time_range = (
datetime2date_time(s,'time'),
datetime2date_time(e,'time'),
)
s,e = map(dateparse,time_data['date range'].split(' - '))
self.date_range = (
datetime2date_time(s,'date'),
datetime2date_time(e,'date'),
)
time_data['days'] = re.sub('[^{}]'.format(''.join(filter(bool,days))),'',time_data['days'])
self.days = list(days.index(time_data['days'][i]) for i in range(len(time_data['days'])))
self.location = time_data['where']
@property
def length(self):
return datetime.timedelta(seconds = sub(
seconds_from_midnight(self.time_range[1]),
seconds_from_midnight(self.time_range[0]),
))
def get_classes(page):
if not isinstance(page,BS):
page = BS(page,'lxml')
tables = page.find_all('table',attrs= {'class':'datadisplaytable'})
groups = ((tables[i],tables[i+1]) for i in range(0,len(tables),2))
return list(map(Class,groups))
if __name__ == "__main__":
with open('schedule.html') as file:
page = BS(file.read(),'lxml')
from bs4 import BeautifulSoup as BS
import datetime
import re
from operator import sub
def dateparse(datetime_str):
date = '%b %d, %Y'
time = '%I:%M %p'
try:
return datetime.datetime.strptime(datetime_str,date)
except ValueError:
return datetime.datetime.strptime(datetime_str,time)
days = [None,'M','T','W','R','F',None]
simp_exceptions = ['Grade Mode']
def datetime2date_time(dtime,mode):
if mode == 'date':
return datetime.date(dtime.year,dtime.month,dtime.day)
elif mode == 'time':
return datetime.time(dtime.hour,dtime.minute,dtime.second)
def seconds_from_midnight(t):
return t.hour*60**2+ t.minute*60+t.second
class Class:
def __init__(self,title,session,days,location,time_range):
self.title = title
self.session = session
self.days = days
self.location = location
self.time_range = time_range
self.lab = None
# data is a list of two html tables
def scrape(self,data):
info,times = data
# info
self.title,self.abrv,self.session = info.find('caption').text.split(' - ')
self.lab = None
self.session = int(self.session)
rows = info.find_all('tr')
for row in rows:
name = row.find('th').text.rstrip(':')
data = re.sub(r'^ +|[\n\r\t]','',row.find('td').text)
if name == 'Status':
type,date = data.split(' on ')
type = type.replace('*','')
self.type = type
self.registration_date = dateparse(date)
else:
if name in simp_exceptions:
name = name.lower().replace(' ','_')
else:
name = name.lower().split(' ')[-1]
if name != 'instructor':
data = data.lower()
try:
data = int(re.sub(r'\.\d+','',data))
except:
pass
self.__dict__[name] = data
# time
headers,*data = times.find_all('tr')
if len(data) > 1:
data,lab = data[:2]
else
lab = None
data = data[0]
data = (col.text for col in data.find_all('td'))
headers = (header.text.lower() for header in headers.find_all('th'))
def parse_horz_row(headers,row):
ret = {}
time_data = dict(zip(headers,data))
if time_data['time'] == 'TBA':
self.time_range = None
else:
s,e = map(dateparse,time_data['time'].split(' - '))
self.time_range = (
datetime2date_time(s,'time'),
datetime2date_time(e,'time'),
)
s,e = map(dateparse,time_data['date range'].split(' - '))
self.date_range = (
datetime2date_time(s,'date'),
datetime2date_time(e,'date'),
)
time_data['days'] = re.sub('[^{}]'.format(''.join(filter(bool,days))),'',time_data['days'])
self.days = list(days.index(time_data['days'][i]) for i in range(len(time_data['days'])))
self.location = time_data['where']
@property
def length(self):
return datetime.timedelta(seconds = sub(
seconds_from_midnight(self.time_range[1]),
seconds_from_midnight(self.time_range[0]),
))
def get_classes(page):
if not isinstance(page,BS):
page = BS(page,'lxml')
tables = page.find_all('table',attrs= {'class':'datadisplaytable'})
groups = ((tables[i],tables[i+1]) for i in range(0,len(tables),2))
return list(map(Class.scrape,groups))
if __name__ == "__main__":
with open('schedule.html') as file:
page = BS(file.read(),'lxml')
class1,*classes = get_classes(page)
Write Preview
Loading…
Cancel
Save