Skip to content

Support scraping across multiple years #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fritzrehde wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Support scraping across multiple years #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bef6f0a
refactor class parsing error handling
fritzrehde Sep 10, 2025
7dfe2c4
implemented revised course class and time ids
fritzrehde Sep 11, 2025
beff9da
adjusted sql to support new ids and fields
fritzrehde Sep 19, 2025
bbd6609
fixed declaring correct data fields in hasuragres comms
fritzrehde Oct 1, 2025
39981a5
wip support hasuragres batch inserting
fritzrehde Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions sql/Classes/down_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DELETE FROM Classes
WHERE "year" = {year};
7 changes: 4 additions & 3 deletions sql/Classes/up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
CREATE TYPE status_enum AS ENUM ('Open', 'Closed', 'Full', 'On Hold');
CREATE TABLE Classes (
"class_id" VARCHAR(255) PRIMARY KEY,
"career" VARCHAR(255),
"course_id" VARCHAR(255) NOT NULL,
"career" VARCHAR(255) NOT NULL,
"course_id" VARCHAR(255) NOT NULL,
"class_nr" VARCHAR(255) NOT NULL,
"section" VARCHAR(255) NOT NULL,
"term" VARCHAR(50) NOT NULL,
"year" VARCHAR(4) NOT NULL,
"year" INT NOT NULL,
"activity" VARCHAR(255) NOT NULL,
"status" status_enum,
"course_enrolment" VARCHAR(255) NOT NULL,
Expand Down
21 changes: 21 additions & 0 deletions sql/Classes/up_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
SELECT 1;
-- CREATE TYPE IF NOT EXISTS status_enum AS ENUM ('Open', 'Closed', 'Full', 'On Hold');
-- CREATE TABLE IF NOT EXISTS Classes (
-- "class_id" VARCHAR(255) PRIMARY KEY,
-- "career" VARCHAR(255) NOT NULL,
-- "course_id" VARCHAR(255) NOT NULL,
-- "class_nr" VARCHAR(255) NOT NULL,
-- "section" VARCHAR(255) NOT NULL,
-- "term" VARCHAR(50) NOT NULL,
-- "year" INT NOT NULL,
-- "activity" VARCHAR(255) NOT NULL,
-- "status" status_enum,
-- "course_enrolment" VARCHAR(255) NOT NULL,
-- "offering_period" VARCHAR(255) NOT NULL,
-- "meeting_dates" VARCHAR(255) NOT NULL,
-- "census_date" VARCHAR(255) NOT NULL,
-- "consent" VARCHAR(255) NOT NULL,
-- "mode" VARCHAR(255) NOT NULL,
-- "class_notes" TEXT,
-- FOREIGN KEY ("course_id") REFERENCES Courses("course_id") ON DELETE CASCADE
-- );
2 changes: 2 additions & 0 deletions sql/Courses/down_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DELETE FROM Courses
WHERE "year" = {year};
9 changes: 5 additions & 4 deletions sql/Courses/up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
CREATE TABLE Courses (
"course_id" VARCHAR(255) PRIMARY KEY,
"course_code" VARCHAR(8), --id
"course_code" VARCHAR(8) NOT NULL,
"year" INT NOT NULL,
"course_name" VARCHAR(255) NOT NULL,
"uoc" INT NOT NULL,
"uoc" INT NOT NULL,
"faculty" VARCHAR(255),
"school" VARCHAR(255),
"campus" VARCHAR(255),
"career" VARCHAR(255),
"terms" TEXT,
"career" VARCHAR(255) NOT NULL,
"terms" VARCHAR(255)[],
"modes" VARCHAR(255)[]
);
14 changes: 14 additions & 0 deletions sql/Courses/up_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
SELECT 1;
-- CREATE TABLE IF NOT EXISTS Courses (
-- "course_id" VARCHAR(255) PRIMARY KEY,
-- "course_code" VARCHAR(8) NOT NULL,
-- "year" INT NOT NULL,
-- "course_name" VARCHAR(255) NOT NULL,
-- "uoc" INT NOT NULL,
-- "faculty" VARCHAR(255),
-- "school" VARCHAR(255),
-- "campus" VARCHAR(255),
-- "career" VARCHAR(255) NOT NULL,
-- "terms" VARCHAR(255)[],
-- "modes" VARCHAR(255)[]
-- );
2 changes: 2 additions & 0 deletions sql/Times/down_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DELETE FROM Times
WHERE "year" = {year};
9 changes: 5 additions & 4 deletions sql/Times/up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
CREATE TABLE Times (
"id" VARCHAR(500) PRIMARY KEY,
"time_id" VARCHAR(511) PRIMARY KEY,
"year" INT NOT NULL,
"class_id" VARCHAR(255) NOT NULL,
"day" VARCHAR(255) NOT NULL,
"instructor" VARCHAR(255),
"location" VARCHAR(255) NOT NULL,
"time" VARCHAR(100) NOT NULL,
"weeks" VARCHAR(100) NOT NULL,
"career" VARCHAR(255),
"time" VARCHAR(255) NOT NULL,
"weeks" VARCHAR(255) NOT NULL,
"career" VARCHAR(255) NOT NULL,
FOREIGN KEY ("class_id") REFERENCES Classes("class_id") ON DELETE CASCADE
);
13 changes: 13 additions & 0 deletions sql/Times/up_just_year.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
SELECT 1;
-- CREATE TABLE IF NOT EXISTS Times (
-- "time_id" VARCHAR(511) PRIMARY KEY,
-- "year" INT NOT NULL,
-- "class_id" VARCHAR(255) NOT NULL,
-- "day" VARCHAR(255) NOT NULL,
-- "instructor" VARCHAR(255),
-- "location" VARCHAR(255) NOT NULL,
-- "time" VARCHAR(255) NOT NULL,
-- "weeks" VARCHAR(255) NOT NULL,
-- "career" VARCHAR(255) NOT NULL,
-- FOREIGN KEY ("class_id") REFERENCES Classes("class_id") ON DELETE CASCADE
-- );
2 changes: 2 additions & 0 deletions src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ pub struct ScrapingEnv {
timetable_api_url: String,
}

#[derive(Debug)]
pub struct ScrapingConfig {
timetable_api_url: String,
}
Expand All @@ -29,6 +30,7 @@ impl ScrapingConfig {

/// Regexes that can be used to extract the year and course code from a UNSW
/// timetable url.
#[derive(Debug)]
pub struct TimetableUrlRegex {
year_only_regex: Regex,
course_url_regex: Regex,
Expand Down
Loading
Loading