Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Astra scraper #31

Merged
merged 11 commits into from
Oct 14, 2024
+205 −0
Merged

Astra scraper #31

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d6750ab
Log in to Astra
TyHil Sep 25, 2024
012f0de
Attempt to call backend Astra API
TyHil Sep 26, 2024
bf2503a
Uncomment cookie code
TyHil Sep 26, 2024
f1cccc9
Successful scrape!
TyHil Sep 26, 2024
b9b4bda
Request in loop
TyHil Sep 26, 2024
90a97e5
Scrape until 90 days of less than 10 events
TyHil Sep 27, 2024
5ed6907
Sort by start time
TyHil Sep 27, 2024
12cd58d
Check if max events exceeded
TyHil Sep 28, 2024
6b98eb5
Run close commands when not needed
TyHil Sep 28, 2024
e4d2501
Start on previous day
TyHil Sep 28, 2024
3579c4b
Merge branch 'develop' into astra-scraper
jpahm Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Uncomment cookie code
@TyHil
TyHil committed Sep 26, 2024
commit bf2503a0ffc87995f7ef53c9f02ea74ba1e5485c
14 changes: 6 additions & 8 deletions scrapers/astra.go
View file
Original file line number Diff line number Diff line change
@@ -25,28 +25,26 @@ func ScrapeAstra(outDir string) {
// Start chromedp
chromedpCtx, cancel := utils.InitChromeDp()
defer cancel()
fmt.Println("1")

// Init http client
tr := &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
}
fmt.Println("2")
cli := &http.Client{Transport: tr}
fmt.Println("3")

/*astraHeaders := */
utils.RefreshAstraToken(chromedpCtx)
fmt.Println("4")
astraHeaders := utils.RefreshAstraToken(chromedpCtx)
time.Sleep(500 * time.Millisecond)

url := fmt.Sprintf("https://www.aaiscloud.com/UTXDallas/~api/calendar/CalendarWeekGrid?_dc=%d&action=GET", time.Now().UnixMilli())
body := "start=0&limit=5000&isForWeekView=false&fields=ActivityId%2CActivityPk%2CActivityName%2CParentActivityId%2CParentActivityName%2CMeetingType%2CDescription%2CStartDate%2CEndDate%2CDayOfWeek%2CStartMinute%2CEndMinute%2CActivityTypeCode%2CResourceId%2CCampusName%2CBuildingCode%2CRoomNumber%2CRoomName%2CLocationName%2CInstitutionId%2CSectionId%2CSectionPk%2CIsExam%2CIsCrosslist%2CIsAllDay%2CIsPrivate%2CEventId%2CEventPk%2CCurrentState%2CNotAllowedUsageMask%2CUsageColor%2CUsageColorIsPrimary%2CEventTypeColor%2CMaxAttendance%2CActualAttendance%2CCapacity&filter=(((StartDate%3C%3D%222024-09-26T23%3A00%3A00%22)%26%26(EndDate%3E%3D%222024-09-26T00%3A00%3A00%22))%26%26((((((((Resource.Building.CampusId%20in%20(%2203c9d930-7343-11e9-8a0c-35dcbeb1edcd%22))%26%26(Resource.Regions.Id%20in%20(%223578b3b0-9dab-11e9-bb13-b5bc7e192516%22)))%26%26(Resource.RoomTypeId%20in%20(%22fe74a890-65f8-11e9-991a-ff0e0065dfaa%22)))%26%26(((EventMeetingByActivityId.Event.EventTypeId%20in%20(%221a7720e9-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ea-8d19-11e9-b19f-0556148ced27%22%2C%221a7720eb-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ec-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ed-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ee-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ef-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f0-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f1-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f2-8d19-11e9-b19f-0556148ced27%22%2C%22874f9347-10f4-4367-ab1e-d697b187e9cb%22%2C%221a7720f4-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f5-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f6-8d19-11e9-b19f-0556148ced27%22%2C%221a7720e8-8d19-11e9-b19f-0556148ced27%22%2C%220494ce20-15e1-11ee-9d2b-ff74be387a2d%22%2C%221a7720f8-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f9-8d19-11e9-b19f-0556148ced27%22))%26%26(CurrentState%20in%20(%22Incomplete%22%2C%22Requested%22%2C%22Scheduled%22)))%26%26(ActivityTypeCode%3D%3D2)))%7C%7C((((Resource.Building.CampusId%20in%20(%2203c9d930-7343-11e9-8a0c-35dcbeb1edcd%22))%26%26(Resource.Regions.Id%20in%20(%223578b3b0-9dab-11e9-bb13-b5bc7e192516%22)))%26%26(Resource.RoomTypeId%20in%20(%22fe74a890-65f8-11e9-991a-ff0e0065dfaa%22)))%26%26(ActivityTypeCode%3D%3D1)))%7C%7C(((((Resource.Building.CampusId%20in%20(%2203c9d930-7343-11e9-8a0c-35dcbeb1edcd%22))%26%26(Resource.Regions.Id%20in%20(%223578b3b0-9dab-11e9-bb13-b5bc7e192516%22)))%26%26(Resource.RoomTypeId%20in%20(%22fe74a890-65f8-11e9-991a-ff0e0065dfaa%22)))%26%26(((PrePostMeetingByActivityId.EventMeeting.Event.EventTypeId%20in%20(%221a7720e9-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ea-8d19-11e9-b19f-0556148ced27%22%2C%221a7720eb-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ec-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ed-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ee-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ef-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f0-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f1-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f2-8d19-11e9-b19f-0556148ced27%22%2C%22874f9347-10f4-4367-ab1e-d697b187e9cb%22%2C%221a7720f4-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f5-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f6-8d19-11e9-b19f-0556148ced27%22%2C%221a7720e8-8d19-11e9-b19f-0556148ced27%22%2C%220494ce20-15e1-11ee-9d2b-ff74be387a2d%22%2C%221a7720f8-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f9-8d19-11e9-b19f-0556148ced27%22))%26%26(CurrentState%20in%20(%22Incomplete%22%2C%22Requested%22%2C%22Scheduled%22)))%26%26(ActivityTypeCode%3D%3D252)))%7C%7C((((Resource.Building.CampusId%20in%20(%2203c9d930-7343-11e9-8a0c-35dcbeb1edcd%22))%26%26(Resource.Regions.Id%20in%20(%223578b3b0-9dab-11e9-bb13-b5bc7e192516%22)))%26%26(Resource.RoomTypeId%20in%20(%22fe74a890-65f8-11e9-991a-ff0e0065dfaa%22)))%26%26(((SetupTeardownWindowByActivityId.EventMeeting.Event.EventTypeId%20in%20(%221a7720e9-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ea-8d19-11e9-b19f-0556148ced27%22%2C%221a7720eb-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ec-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ed-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ee-8d19-11e9-b19f-0556148ced27%22%2C%221a7720ef-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f0-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f1-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f2-8d19-11e9-b19f-0556148ced27%22%2C%22874f9347-10f4-4367-ab1e-d697b187e9cb%22%2C%221a7720f4-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f5-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f6-8d19-11e9-b19f-0556148ced27%22%2C%221a7720e8-8d19-11e9-b19f-0556148ced27%22%2C%220494ce20-15e1-11ee-9d2b-ff74be387a2d%22%2C%221a7720f8-8d19-11e9-b19f-0556148ced27%22%2C%221a7720f9-8d19-11e9-b19f-0556148ced27%22))%26%26(CurrentState%20in%20(%22Incomplete%22%2C%22Requested%22%2C%22Scheduled%22)))%26%26(ActivityTypeCode%3D%3D251)))))%7C%7C(((((Resource.Building.CampusId%20in%20(%2203c9d930-7343-11e9-8a0c-35dcbeb1edcd%22))%26%26(Resource.Regions.Id%20in%20(%223578b3b0-9dab-11e9-bb13-b5bc7e192516%22)))%26%26(Resource.RoomTypeId%20in%20(%22fe74a890-65f8-11e9-991a-ff0e0065dfaa%22)))%26%26((ActivityTypeCode%3D%3D9)%26%26(ActivityId%3D%3Dnull)))%7C%7C((ActivityTypeCode%3D%3D356)%7C%7C(ActivityTypeCode%3D%3D357))))%7C%7C(ActivityTypeCode%3D%3D255)))&sortOrder=%2BStartDate%2C%2BStartMinute&page=1&group=%7B%22property%22%3A%22StartDate%22%2C%22direction%22%3A%22ASC%22%7D&sort=%5B%7B%22property%22%3A%22StartDate%22%2C%22direction%22%3A%22ASC%22%7D%2C%7B%22property%22%3A%22StartMinute%22%2C%22direction%22%3A%22ASC%22%7D%5D"
req, err := http.NewRequest("POST", url, strings.NewReader(body))
if err != nil {
panic(err)
}
fmt.Println("5")
//req.Header = astraHeaders

req.Header = astraHeaders
res, err := cli.Do(req)
if err != nil {
panic(err)
15 changes: 7 additions & 8 deletions utils/methods.go
View file
Original file line number Diff line number Diff line change
@@ -101,7 +101,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
}

// This function signs into Astra
func RefreshAstraToken(chromedpCtx context.Context) error /*map[string][]string*/ {
func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
// Get username and password
username, present := os.LookupEnv("LOGIN_ASTRA_USERNAME")
if !present {
@@ -120,27 +120,27 @@ func RefreshAstraToken(chromedpCtx context.Context) error /*map[string][]string*
return err
}),
chromedp.Navigate(`https://www.aaiscloud.com/UTXDallas/logon.aspx?ReturnUrl=%2futxdallas%2fcalendars%2fdailygridcalendar.aspx`),
//chromedp.Navigate(`https://www.aaiscloud.com/UTXDallas/logon.aspx`),
chromedp.WaitVisible(`input#userNameField-inputEl`),
chromedp.SendKeys(`input#userNameField-inputEl`, username),
chromedp.SendKeys(`input#textfield-1029-inputEl`, password),
chromedp.WaitVisible(`a#logonButton`),
chromedp.Click(`a#logonButton`),
chromedp.WaitVisible(`body`),
)
if err != nil {
panic(err)
}

/*var cookieStrs []string
var cookieStrs []string
_, err = chromedp.RunResponse(chromedpCtx,
chromedp.Navigate(`https://coursebook.utdallas.edu/`),
//chromedp.Navigate(`https://www.aaiscloud.com/UTXDallas/Calendars/DailyGridCalendar.aspx`),
chromedp.ActionFunc(func(ctx context.Context) error {
cookies, err := network.GetCookies().Do(ctx)
cookieStrs = make([]string, len(cookies))
gotToken := false
for i, cookie := range cookies {
cookieStrs[i] = fmt.Sprintf("%s=%s", cookie.Name, cookie.Value)
if cookie.Name == "PTGSESSID" {
if cookie.Name == "UTXDallas.ASPXFORMSAUTH" {
VPrintf("Got new token: PTGSESSID = %s", cookie.Value)
gotToken = true
}
@@ -158,14 +158,13 @@ func RefreshAstraToken(chromedpCtx context.Context) error /*map[string][]string*
return map[string][]string{
"Host": {"www.aaiscloud.com"},
"User-Agent": {"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0"},
"Accept": {"/*"}, // add back star
"Accept": {"*/*"},
"Accept-Encoding": {"gzip, deflate, br, zstd"},
"Accept-Language": {"en-US,en;q=0.5"},
"Content-Type": {"application/x-www-form-urlencoded; charset=UTF-8"},
"Cookie": cookieStrs,
"Connection": {"keep-alive"},
}*/
return nil
}
}

// Encodes and writes the given data as tab-indented JSON to the given filepath.