Update scraped course data with new CRN link (#209)

* Change url for course and term in scrapers * Add new URL to section parser * Update snapshot tests
sandboxnu · Nov 27, 2023 · cd9536f · cd9536f
1 parent cdce629
commit cd9536f
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 14 deletions.
diff --git a/scrapers/classes/parsersxe/classParser.ts b/scrapers/classes/parsersxe/classParser.ts
@@ -107,20 +107,19 @@ class ClassParser {
       nupath: this.nupath(attributes),
       desc: he.decode(description),
       prettyUrl:
-        "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?" +
+        "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" +
         `cat_term_in=${termId}&subj_code_in=${subjectCode}&crse_numb_in=${courseNumber}`,
       name: he.decode(SR.courseTitle),
       url:
-        "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?" +
-        `term_in=${termId}&subj_in=${subjectCode}&crse_in=${courseNumber}&schd_in=%`,
+        "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" +
+        `cat_term_in=${termId}&subj_code_in=${subjectCode}&crse_numb_in=${courseNumber}`,
       lastUpdateTime: Date.now(),
       maxCredits: SR.creditHourHigh || SR.creditHourLow,
       minCredits: SR.creditHourLow,
       college: collegeNames[termId.charAt(termId.length - 1)],
       feeAmount,
       feeDescription,
     };
-
     if (prereqs) {
       classDetails.prereqs = prereqs;
     }

diff --git a/scrapers/classes/parsersxe/sectionParser.ts b/scrapers/classes/parsersxe/sectionParser.ts
@@ -63,8 +63,8 @@ class SectionParser {
         return a.description === "Honors";
       }),
       url:
-        "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched" +
-        `?term_in=${SR.term}&crn_in=${SR.courseReferenceNumber}`,
+        "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" +
+        `cat_term_in=${SR.term}&subj_code_in=${SR.subject}&crse_numb_in=${SR.courseNumber}`,
       profs: SR.faculty.map(MeetingParser.profName),
       meetings: MeetingParser.parseMeetings(SR.meetingsFaculty),
     };

diff --git a/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap b/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap
@@ -35,10 +35,10 @@ Object {
       },
     ],
   },
-  "prettyUrl": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311",
+  "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311",
   "subject": "CHEM",
   "termId": "202010",
-  "url": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?term_in=202010&subj_in=CHEM&crse_in=2311&schd_in=%",
+  "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311",
 }
 `;
 
@@ -77,9 +77,9 @@ Object {
       },
     ],
   },
-  "prettyUrl": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500",
+  "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500",
   "subject": "CS",
   "termId": "202010",
-  "url": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?term_in=202010&subj_in=CS&crse_in=2500&schd_in=%",
+  "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500",
 }
 `;
diff --git a/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap b/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap
@@ -51,7 +51,7 @@ Object {
   "seatsRemaining": 36,
   "subject": "CHEM",
   "termId": "202010",
-  "url": "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched?term_in=202010&crn_in=10259",
+  "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311",
   "waitCapacity": 0,
   "waitRemaining": 0,
 }
@@ -101,8 +101,8 @@ Object {
   "seatsRemaining": 4,
   "subject": "CS",
   "termId": "201930",
-  "url": "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched?term_in=201930&crn_in=30340",
+  "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=201930&subj_code_in=CS&crse_numb_in=2500",
   "waitCapacity": 0,
   "waitRemaining": 0,
 }
-`;
+`;
diff --git a/scrapers/request.ts b/scrapers/request.ts
@@ -75,7 +75,7 @@ const separateReqPools: Record<string, RequestPool> = {
   // Took 1hr and 15 min with 500 sockets and RETRY_DELAY set to 20000 and delta set to 15000.
   // Usually takes just under 1 hr at 1k sockets and the same timeouts.
   // Took around 20 min with timeouts set to 100ms and 150ms and 100 sockets.
-  "wl11gp.neu.edu": {
+  "https://bnrordsp.neu.edu": {
     options: { maxSockets: 100, keepAlive: true, maxFreeSockets: 100 },
     agents: false,
   },