From f3eb7e7a71d2588aeaf189a16d03210bab58fa09 Mon Sep 17 00:00:00 2001 From: Dennis Wang <66754085+hellolol2016@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:52:48 -0500 Subject: [PATCH 1/5] minor endpoints folder, types, and missing data --- packages/api/src/minor/minor-collator.ts | 125 ++++++++++++++++++ packages/api/src/minor/minor.controller.ts | 32 +++++ packages/api/src/minor/minor.module.ts | 10 ++ packages/api/src/minor/minor.service.ts | 49 +++++++ .../api/src/minor/minors/missing-data.txt | 2 + packages/common/src/api-response-types.ts | 4 + packages/common/src/types.ts | 30 +++++ 7 files changed, 252 insertions(+) create mode 100644 packages/api/src/minor/minor-collator.ts create mode 100644 packages/api/src/minor/minor.controller.ts create mode 100644 packages/api/src/minor/minor.module.ts create mode 100644 packages/api/src/minor/minor.service.ts create mode 100644 packages/api/src/minor/minors/missing-data.txt diff --git a/packages/api/src/minor/minor-collator.ts b/packages/api/src/minor/minor-collator.ts new file mode 100644 index 000000000..19f696c21 --- /dev/null +++ b/packages/api/src/minor/minor-collator.ts @@ -0,0 +1,125 @@ +import { Minor } from "@graduate/common"; +// year minor name minor data +const MINORS: Record> = {}; +const MINOR_YEARS = new Set(); + +const rootDir = "./src/minor/minors"; + +interface YearData { + year: string; +} + +interface YearCollegeData { + year: string; + college: string; +} + +interface YearCollegeMinorData { + year: string; + college: string; + minor: string; +} + +async function fileExists( + fs: typeof import("fs/promises"), + path: string +): Promise { + return await fs.access(path, fs.constants.F_OK).then( + () => true, + () => false + ); +} + +// TODO: this code is quick and dirty but works. this should be replaced with some dry-er code later. +/** + * Iterates over the ./minors directory, collecting minors and adding them to + * the exported MINORS and MINOR_YEARS object/set respectively. It prioritizes + * parsed.commit.json files over parsed.initial.json files because _.commit._ + * files have been human-reviewed and _.initial._ files are raw scraper output. + */ +async function collateMinors() { + // TODO: determine why these needed to be runtime imports (normal import statements didn't work here). + const fs = await import("fs/promises"); + const path = await import("path"); + const years = ( + await fs.readdir(path.resolve(rootDir), { + withFileTypes: true, + }) + ) + .filter((dirent) => dirent.isDirectory()) + .map( + (dirent): YearData => ({ + year: dirent.name, + }) + ); + + const colleges = ( + await Promise.all( + years.map(async ({ year }) => { + const colleges = await fs.readdir(path.join(rootDir, year), { + withFileTypes: true, + }); + return colleges + .filter((dirent) => dirent.isDirectory()) + .map( + (college): YearCollegeData => ({ + year: year, + college: college.name, + }) + ); + }) + ) + ).flat(); + + const minors = ( + await Promise.all( + colleges.map(async ({ year, college }) => { + const minors = await fs.readdir(path.join(rootDir, year, college), { + withFileTypes: true, + }); + return minors + .filter((dirent) => dirent.isDirectory()) + .map( + (minor): YearCollegeMinorData => ({ + year: year, + college: college, + minor: minor.name, + }) + ); + }) + ) + ).flat(); + + years.forEach(({ year }) => { + MINOR_YEARS.add(year); + MINORS[year] = {}; + }); + + const done = await Promise.all( + minors.map(async ({ year, college, minor }) => { + const basePath = path.join(rootDir, year, college, minor); + const commitFile = path.join(basePath, "parsed.commit.json"); + const initialFile = path.join(basePath, "parsed.initial.json"); + + if (await fileExists(fs, commitFile)) { + const fileData = JSON.parse( + (await fs.readFile(commitFile)).toString() + ) as Minor; + MINORS[year][fileData.name] = fileData; + } else if (await fileExists(fs, initialFile)) { + const fileData = JSON.parse( + (await fs.readFile(initialFile)).toString() + ) as Minor; + if (MINORS[year]) MINORS[year][fileData.name] = fileData; + } + }) + ); + + console.log( + `Successfully loaded ${done.length} minors across ${MINOR_YEARS.size} years!` + ); +} + +collateMinors(); + +export { MINORS, MINOR_YEARS }; diff --git a/packages/api/src/minor/minor.controller.ts b/packages/api/src/minor/minor.controller.ts new file mode 100644 index 000000000..02ad778a5 --- /dev/null +++ b/packages/api/src/minor/minor.controller.ts @@ -0,0 +1,32 @@ +import { GetSupportedMinorsResponse, Minor } from "@graduate/common"; +import { MinorService } from "./minor.service"; +import { + Controller, + Get, + NotFoundException, + Param, + ParseIntPipe, +} from "@nestjs/common"; + +@Controller("minors") +export class MinorController { + constructor(private readonly minorService: MinorService) {} + @Get("/:catalogYear/:minorName") + getMinor( + @Param("catalogYear", ParseIntPipe) catalogYear: number, + @Param("minorName") minorName: string + ): Minor { + const minor = this.minorService.findByMinorAndYear(minorName, catalogYear); + if (!minor) { + throw new NotFoundException(); + } + + return minor; + } + + @Get("supportedMinors") + getSupportedMinors(): GetSupportedMinorsResponse { + const supportedMinors = this.minorService.getSupportedMinors(); + return { supportedMinors }; + } +} diff --git a/packages/api/src/minor/minor.module.ts b/packages/api/src/minor/minor.module.ts new file mode 100644 index 000000000..cfb7fabf0 --- /dev/null +++ b/packages/api/src/minor/minor.module.ts @@ -0,0 +1,10 @@ +import { Module } from "@nestjs/common"; +import { MinorService } from "./minor.service"; +import { MinorController } from "./minor.controller"; + +@Module({ + controllers: [MinorController], + providers: [MinorService], + exports: [MinorService], +}) +export class MinorModule {} diff --git a/packages/api/src/minor/minor.service.ts b/packages/api/src/minor/minor.service.ts new file mode 100644 index 000000000..0ee018f46 --- /dev/null +++ b/packages/api/src/minor/minor.service.ts @@ -0,0 +1,49 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { formatServiceCtx } from "src/utils"; +import { MINOR_YEARS, MINORS } from "./minor-collator"; +import { Minor, SupportedMinors } from "@graduate/common"; + +@Injectable() +export class MinorService { + private readonly logger: Logger = new Logger(); + findByMinorAndYear(minorName: string, catalogYear: number): Minor | null { + if (!MINOR_YEARS.has(String(catalogYear))) { + this.logger.debug( + { message: "Minor year not found", catalogYear }, + MinorService.formatMinorServiceCtx("findByMinorAndYear") + ); + return null; + } + + if (!MINORS[catalogYear][minorName]) { + this.logger.debug( + { message: "Minor within year not found", minorName, catalogYear }, + MinorService.formatMinorServiceCtx("findByMinorAndYear") + ); + return null; + } + return MINORS[catalogYear][minorName]; + } + + getSupportedMinors(): SupportedMinors { + //const supportedMinors: SupportedMinors = MINORS; + //MINOR_YEARS.forEach((year) => { + //const supportedMinorNames = Object.keys(MINORS[year]); + + //const supportedMinorForYear: SupportedMinorsForYear = {}; + //supportedMinorNames.forEach((name) => { + ////no concentration for minors??? + ////supportedMinorForYear[minorName] = this.getConcentrationsInfoForMinor( + ////minorName, + ////parseInt(year) + ////); + //}); + //supportedMinors[year] = supportedMinorForYear; + //}); + return MINORS; + } + + private static formatMinorServiceCtx(methodName: string): string { + return formatServiceCtx(MinorService.name, methodName); + } +} diff --git a/packages/api/src/minor/minors/missing-data.txt b/packages/api/src/minor/minors/missing-data.txt new file mode 100644 index 000000000..668770550 --- /dev/null +++ b/packages/api/src/minor/minors/missing-data.txt @@ -0,0 +1,2 @@ +:D +this should be json files of data for the minors \ No newline at end of file diff --git a/packages/common/src/api-response-types.ts b/packages/common/src/api-response-types.ts index 7cb2c1dee..4f92b036a 100644 --- a/packages/common/src/api-response-types.ts +++ b/packages/common/src/api-response-types.ts @@ -5,6 +5,7 @@ import { ScheduleCourse2, MetaInfo, Maybe, + SupportedMinors, } from "./types"; /** Types our API responds with. */ @@ -54,6 +55,9 @@ export class GetSupportedMajorsResponse { // { year => { majorName => {concentrations, minRequiredConcentrations} }} supportedMajors: SupportedMajors; } +export class GetSupportedMinorsResponse { + supportedMinors: SupportedMinors; +} export class GetMetaInfoResponse implements MetaInfo { commit: Maybe; diff --git a/packages/common/src/types.ts b/packages/common/src/types.ts index 79d200119..be6ff2c57 100644 --- a/packages/common/src/types.ts +++ b/packages/common/src/types.ts @@ -146,6 +146,34 @@ export interface Major2 { metadata?: MajorMetadata; } +/** + * A Minor, containing all the requirements. + * + * @param name The name of the minor. + * @param requirementSections A list of the sections of requirements. + * @param totalCreditsRequired Total credits required to graduate with this minor. + * @param yearVersion The catalog version year of this minor. + * @param metadata Metadata for the minor. + */ + +export interface Minor { + name: string; + requirementSections: Section[]; + totalCreditsRequired: number; + yearVersion: number; + metadata?: MinorMetaData; +} +/** + * Metadata for a minor. + * + * @param verified Whether the major has been manually verified. + * @param lastEdited The last time the major was edited MM/DD/YYYY. + */ +export interface MinorMetaData { + verified: boolean; + lastEdited: string; +} + /** * Metadata for a major. * @@ -476,6 +504,8 @@ export type SupportedMajorsForYear = Record; // { year => supported majors } export type SupportedMajors = Record; +export type SupportedMinors = Record>; + /** * Types for a some result from an algorithim. Currently used for the result of * the Major 2 validation algorithm. From 6ab4a80112e05f2d9fa6866b126082cad752410c Mon Sep 17 00:00:00 2001 From: Dennis Wang <66754085+hellolol2016@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:16:53 -0500 Subject: [PATCH 2/5] fake math minor requirements --- .../2022/mathematics/parsed.initial.json | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 packages/api/src/minor/minors/2022/mathematics/parsed.initial.json diff --git a/packages/api/src/minor/minors/2022/mathematics/parsed.initial.json b/packages/api/src/minor/minors/2022/mathematics/parsed.initial.json new file mode 100644 index 000000000..1c03cfa9c --- /dev/null +++ b/packages/api/src/minor/minors/2022/mathematics/parsed.initial.json @@ -0,0 +1,58 @@ +{ + "name": "Mathematics", + "totalCreditsRequired": 24, + "yearVersion": 2022, + "requirementSections": [ + { + "type": "SECTION", + "title": "Required Courses", + "requirements": [ + { + "type": "COURSE", + "classId": 1341, + "subject": "MATH" + }, + { + "type": "COURSE", + "classId": 1341, + "subject": "MATH" + } + ] + }, + { + "type": "SECTION", + "title": "Intermediate-Level Courses", + "requirements": [ + { + "type": "COURSE", + "classId": 2321, + "subject": "MATH" + }, + { + "type": "COURSE", + "classId": 2341, + "subject": "MATH" + }, + { + "type": "COURSE", + "classId": 2331, + "subject": "MATH" + } + ], + "minRequirementCount": 2 + }, + { + "type": "RANGE", + "subject": "MATH", + "idRangeStart": 3001, + "idRangeEnd": 4699, + "exceptions": [ + { + "subject": "MATH", + "classId": 4000, + "type": "COURSE" + } + ] + } + ] +} From f227ced54674d7a5f4dca50450349099c5879270 Mon Sep 17 00:00:00 2001 From: Dennis Wang <66754085+hellolol2016@users.noreply.github.com> Date: Mon, 18 Nov 2024 11:59:51 -0500 Subject: [PATCH 3/5] added minor module to app.module.ts not sure how to run the collator --- packages/api/src/app.module.ts | 2 ++ packages/api/src/minor/minors/missing-data.txt | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 packages/api/src/minor/minors/missing-data.txt diff --git a/packages/api/src/app.module.ts b/packages/api/src/app.module.ts index 6ca2109b1..955b0c0e1 100644 --- a/packages/api/src/app.module.ts +++ b/packages/api/src/app.module.ts @@ -11,6 +11,7 @@ import { MajorModule } from "./major/major.module"; import { EmailModule } from "./email/email.module"; import { ThrottlerGuard, ThrottlerModule } from "@nestjs/throttler"; import { MetaModule } from "./meta/meta.module"; +import { MinorModule } from "./minor/minor.module"; @Module({ imports: [ @@ -29,6 +30,7 @@ import { MetaModule } from "./meta/meta.module"; AuthModule, PlanModule, MajorModule, + MinorModule, EmailModule, MetaModule, ], diff --git a/packages/api/src/minor/minors/missing-data.txt b/packages/api/src/minor/minors/missing-data.txt deleted file mode 100644 index 668770550..000000000 --- a/packages/api/src/minor/minors/missing-data.txt +++ /dev/null @@ -1,2 +0,0 @@ -:D -this should be json files of data for the minors \ No newline at end of file From 45ec819f27230292d387ee88395729ce2de410b4 Mon Sep 17 00:00:00 2001 From: Dennis Wang <66754085+hellolol2016@users.noreply.github.com> Date: Sun, 24 Nov 2024 15:22:54 -0500 Subject: [PATCH 4/5] moved folder --- .../minors/2022/{ => science}/mathematics/parsed.initial.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/api/src/minor/minors/2022/{ => science}/mathematics/parsed.initial.json (100%) diff --git a/packages/api/src/minor/minors/2022/mathematics/parsed.initial.json b/packages/api/src/minor/minors/2022/science/mathematics/parsed.initial.json similarity index 100% rename from packages/api/src/minor/minors/2022/mathematics/parsed.initial.json rename to packages/api/src/minor/minors/2022/science/mathematics/parsed.initial.json From 902b80a09dc91a1d6d5ee03b38f340867e150664 Mon Sep 17 00:00:00 2001 From: Suraj Ramchandran <30753067+Suraj-Ram@users.noreply.github.com> Date: Sun, 8 Dec 2024 15:29:27 -0500 Subject: [PATCH 5/5] Addressed comments --- packages/api/src/minor/minor.service.ts | 14 -------------- packages/common/src/types.ts | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/api/src/minor/minor.service.ts b/packages/api/src/minor/minor.service.ts index 0ee018f46..8061b5bf3 100644 --- a/packages/api/src/minor/minor.service.ts +++ b/packages/api/src/minor/minor.service.ts @@ -26,20 +26,6 @@ export class MinorService { } getSupportedMinors(): SupportedMinors { - //const supportedMinors: SupportedMinors = MINORS; - //MINOR_YEARS.forEach((year) => { - //const supportedMinorNames = Object.keys(MINORS[year]); - - //const supportedMinorForYear: SupportedMinorsForYear = {}; - //supportedMinorNames.forEach((name) => { - ////no concentration for minors??? - ////supportedMinorForYear[minorName] = this.getConcentrationsInfoForMinor( - ////minorName, - ////parseInt(year) - ////); - //}); - //supportedMinors[year] = supportedMinorForYear; - //}); return MINORS; } diff --git a/packages/common/src/types.ts b/packages/common/src/types.ts index d317a1d4f..739771145 100644 --- a/packages/common/src/types.ts +++ b/packages/common/src/types.ts @@ -501,11 +501,11 @@ export type SupportedConcentrations = { // { majorName => { concentration, minRequiredConcentrations, verified} } export type SupportedMajorsForYear = Record; +export type SupportedMinorsForYear = Record; // { year => supported majors } export type SupportedMajors = Record; - -export type SupportedMinors = Record>; +export type SupportedMinors = Record; /** * Types for a some result from an algorithim. Currently used for the result of