From bef6f0af839643e0519bc80bc439450b57b8cba8 Mon Sep 17 00:00:00 2001
From: fritzrehde <fritzrehde.work@gmail.com>
Date: Wed, 10 Sep 2025 16:35:52 +1000
Subject: [PATCH 1/5] refactor class parsing error handling

---
 src/course_scraper.rs | 112 ++++++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 52 deletions(-)

diff --git a/src/course_scraper.rs b/src/course_scraper.rs
index a0e2b62..abfbc82 100644
--- a/src/course_scraper.rs
+++ b/src/course_scraper.rs
@@ -226,69 +226,77 @@ fn parse_class_info(
         map.insert(key, value);
         i += 2;
     }
-    let offering_period_str = map.get("Offering Period").unwrap_or(&"").to_string();
+
+    let missing_field_error = |missing_field_name: &str| {
+        anyhow::anyhow!(format!(
+            "{} for course {} is missing",
+            missing_field_name, course_id
+        ))
+    };
+    let get_expected_field = |field_name: &str| {
+        map.get(field_name)
+            .ok_or_else(|| missing_field_error(field_name))
+    };
+
+    let offering_period_str = get_expected_field("Offering Period")?;
     let mut split_offering_period_str = offering_period_str.split(" - ");
 
-    let error_msg = format!("failed to parse a class for course {}", course_id);
+    let section = get_expected_field("Section")?;
+
     let date = split_offering_period_str
         .next()
-        .ok_or_else(|| anyhow::anyhow!(error_msg.clone()))?;
+        .ok_or_else(|| missing_field_error("date"))?;
     let year = date
         .split("/")
         .nth(2)
-        .ok_or_else(|| anyhow::anyhow!(error_msg.clone()))?;
+        .ok_or_else(|| missing_field_error("year"))?;
+    let class_nr = get_expected_field("Class Nbr")?;
+    let term = get_expected_field("Teaching Period")?
+        .split(" - ")
+        .next()
+        .ok_or_else(|| {
+            anyhow::anyhow!(format!(
+                "failed to parse term from teaching period for course {}",
+                course_id
+            ))
+        })?;
+
+    let class_id = format!("{}-{}-{}-{}", course_id, class_nr, term, year);
+    let activity = get_expected_field("Activity")?;
+    let status = get_expected_field("Status")?;
+    let course_enrolment = get_expected_field("Enrols/Capacity")?.replace("*", "");
+    let offering_period = get_expected_field("Offering Period")?;
+    let meeting_dates = get_expected_field("Meeting Dates")?;
+    let census_date = get_expected_field("Census Date")?;
+    let mode = get_expected_field("Mode of Delivery")?;
+    let consent = get_expected_field("Consent")?;
+    let times = if !times_parsed.is_empty() {
+        Some(times_parsed)
+    } else {
+        None
+    };
+    let class_notes = map
+        .get("Class Notes")
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string());
+
     Ok(Class {
         course_id: course_id.to_string(),
-        class_id: format!(
-            "{}-{}-{}-{}",
-            course_id,
-            map.get("Class Nbr").unwrap_or(&""),
-            map.get("Teaching Period")
-                .unwrap_or(&"")
-                .split(" - ")
-                .next()
-                .ok_or_else(|| anyhow::anyhow!(format!(
-                    "{}: {}",
-                    &error_msg, "Could not split teaching periods properly!"
-                )))?,
-            year,
-        ),
-        section: map.get("Section").unwrap_or(&"").to_string(),
-        term: map
-            .get("Teaching Period")
-            .unwrap_or(&"")
-            .split(" - ")
-            .next()
-            .ok_or_else(|| {
-                anyhow::anyhow!(format!(
-                    "{}: {}",
-                    &error_msg, "Could not split teaching periods properly!"
-                ))
-            })?
-            .to_string(),
+        class_id,
+        section: section.to_string(),
+        term: term.to_string(),
         year: year.to_string(),
-        activity: map.get("Activity").unwrap_or(&"").to_string(),
-        status: map.get("Status").unwrap_or(&"").to_string(),
-        course_enrolment: map
-            .get("Enrols/Capacity")
-            .unwrap_or(&"")
-            .replace("*", "")
-            .to_string(),
-        offering_period: map.get("Offering Period").unwrap_or(&"").to_string(),
-        meeting_dates: map.get("Meeting Dates").unwrap_or(&"").to_string(),
-        census_date: map.get("Census Date").unwrap_or(&"").to_string(),
-        mode: map.get("Mode of Delivery").unwrap_or(&"").to_string(),
-        consent: map.get("Consent").unwrap_or(&"").to_string(),
+        activity: activity.to_string(),
+        status: status.to_string(),
+        course_enrolment,
+        offering_period: offering_period.to_string(),
+        meeting_dates: meeting_dates.to_string(),
+        census_date: census_date.to_string(),
+        mode: mode.to_string(),
+        consent: consent.to_string(),
         career: career.to_string(),
-        times: if times_parsed.is_empty() {
-            None
-        } else {
-            Some(times_parsed)
-        },
-        class_notes: map
-            .get("Class Notes")
-            .filter(|s| !s.is_empty())
-            .map(|s| s.to_string()),
+        times,
+        class_notes,
     })
 }
 

From 7dfe2c4c40cb9112f35cfa0798895ea175a550e9 Mon Sep 17 00:00:00 2001
From: fritzrehde <fritzrehde.work@gmail.com>
Date: Thu, 11 Sep 2025 14:26:51 +1000
Subject: [PATCH 2/5] implemented revised course class and time ids

---
 src/course_scraper.rs       | 137 +++++++++++++++++++++++++-----------
 src/subject_area_scraper.rs |   1 +
 2 files changed, 95 insertions(+), 43 deletions(-)

diff --git a/src/course_scraper.rs b/src/course_scraper.rs
index abfbc82..015b8f3 100644
--- a/src/course_scraper.rs
+++ b/src/course_scraper.rs
@@ -4,18 +4,18 @@ use scraper::Selector;
 use serde::Serialize;
 use std::collections::{HashMap, HashSet};
 
-use crate::{ScrapingContext, text_manipulators::extract_text};
+use crate::{ScrapingContext, Year, text_manipulators::extract_text};
 
 #[derive(Debug, Serialize)]
 pub struct Course {
     pub course_id: String,
     pub course_code: String,
+    pub year: Year,
     pub course_name: String,
     pub uoc: i32,
-    // TODO: try making non-optional.
     pub faculty: Option<String>,
     pub school: Option<String>,
-    pub career: Option<String>,
+    pub career: String,
     // Sorted ascendingly.
     pub modes: Vec<String>, // For Notangles.
     pub campus: Option<String>,
@@ -28,6 +28,7 @@ pub struct Class {
     pub course_id: String,
     pub career: String,
     pub class_id: String,
+    pub class_nr: String,
     pub section: String,
     pub term: String,
     pub year: String,
@@ -45,10 +46,20 @@ pub struct Class {
 
 #[derive(Debug, Serialize)]
 pub struct Time {
+    pub time_id: String,
     pub career: String,
+    pub location: String,
     pub day: String,
     pub time: String,
+    pub weeks: String,
+    pub instructor: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct PartialTime {
     pub location: String,
+    pub day: String,
+    pub time: String,
     pub weeks: String,
     pub instructor: Option<String>,
 }
@@ -60,6 +71,7 @@ pub struct PartialCourse {
     pub career: String,
     pub uoc: i32,
     pub url: String,
+    pub year: Year,
 }
 
 impl PartialCourse {
@@ -162,14 +174,18 @@ impl PartialCourse {
                 }
             }
 
-            let course_id = format!("{}{}", &self.course_code, career);
+            // The reason we aren't including the term in the course id is that the handbook only
+            // contains one page per course per year, which contains data for the course that year.
+            // We use the same format to reduce duplicated data.
+            let course_id = format!("{}-{}-{}", self.course_code, career, self.year);
+
             let course_code = self.course_code;
             let course_name = self.course_name;
             let uoc = self.uoc;
 
             let classes: Vec<Class> = class_activity_information
                 .into_par_iter()
-                .map(|class_data| parse_class_info(class_data, course_id.as_str(), career.as_ref()))
+                .map(|class_data| parse_class_info(class_data, &course_id, &course_code, &career))
                 .collect::<anyhow::Result<_>>()?;
 
             let unique_modes: HashSet<&String> = classes.iter().map(|class| &class.mode).collect();
@@ -180,12 +196,13 @@ impl PartialCourse {
             Ok(Course {
                 course_id,
                 course_code,
+                year: self.year,
                 course_name,
                 uoc,
                 faculty,
                 school,
                 campus,
-                career: Some(career),
+                career,
                 modes,
                 terms,
                 classes,
@@ -200,11 +217,12 @@ impl PartialCourse {
 fn parse_class_info(
     class_data: Vec<String>,
     course_id: &str,
-    career: &str,
+    course_code: &str,
+    course_career: &str,
 ) -> anyhow::Result<Class> {
     let mut map: HashMap<&str, &str> = HashMap::new();
     let mut i = 0;
-    let mut times_parsed = Vec::<Time>::new();
+    let mut partial_times_parsed = Vec::new();
 
     while i < class_data.len() {
         let key = &class_data[i];
@@ -213,7 +231,7 @@ fn parse_class_info(
             while j < class_data.len() && class_data[j] != "Class Notes" {
                 j += 1;
             }
-            times_parsed = parse_meeting_info(&class_data[i + 1..j], career);
+            partial_times_parsed = parse_meeting_info(&class_data[i + 1..j])?;
             i = j + 1;
             continue;
         }
@@ -229,8 +247,8 @@ fn parse_class_info(
 
     let missing_field_error = |missing_field_name: &str| {
         anyhow::anyhow!(format!(
-            "{} for course {} is missing",
-            missing_field_name, course_id
+            "{} field for course {} was missing while parsing a class",
+            missing_field_name, course_code
         ))
     };
     let get_expected_field = |field_name: &str| {
@@ -257,11 +275,14 @@ fn parse_class_info(
         .ok_or_else(|| {
             anyhow::anyhow!(format!(
                 "failed to parse term from teaching period for course {}",
-                course_id
+                course_code
             ))
         })?;
 
-    let class_id = format!("{}-{}-{}-{}", course_id, class_nr, term, year);
+    let class_id = format!(
+        "{}-{}-{}-{}-{}",
+        course_code, course_career, year, term, class_nr
+    );
     let activity = get_expected_field("Activity")?;
     let status = get_expected_field("Status")?;
     let course_enrolment = get_expected_field("Enrols/Capacity")?.replace("*", "");
@@ -270,8 +291,15 @@ fn parse_class_info(
     let census_date = get_expected_field("Census Date")?;
     let mode = get_expected_field("Mode of Delivery")?;
     let consent = get_expected_field("Consent")?;
-    let times = if !times_parsed.is_empty() {
-        Some(times_parsed)
+    let times = if !partial_times_parsed.is_empty() {
+        Some(
+            partial_times_parsed
+                .into_iter()
+                .map(|partial_time| {
+                    partial_time.complete(course_code, course_career, year, term, class_nr)
+                })
+                .collect(),
+        )
     } else {
         None
     };
@@ -283,6 +311,7 @@ fn parse_class_info(
     Ok(Class {
         course_id: course_id.to_string(),
         class_id,
+        class_nr: class_nr.to_string(),
         section: section.to_string(),
         term: term.to_string(),
         year: year.to_string(),
@@ -294,56 +323,78 @@ fn parse_class_info(
         census_date: census_date.to_string(),
         mode: mode.to_string(),
         consent: consent.to_string(),
-        career: career.to_string(),
+        career: course_career.to_string(),
         times,
         class_notes,
     })
 }
 
-fn parse_meeting_info(vec: &[String], career: &str) -> Vec<Time> {
+fn parse_meeting_info(vec: &[String]) -> anyhow::Result<Vec<PartialTime>> {
     let days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
     let mut meetings = Vec::new();
     let mut iter: Box<dyn Iterator<Item = &String>> = Box::new(vec.iter());
 
     while let Some(day) = iter.next() {
         if days.contains(&day.as_str()) {
-            let mut timeslot = get_blank_time_struct();
-            timeslot.day = day.clone();
-
-            // Safely unwrap time, location, and weeks
-            if let (Some(time), Some(location), Some(weeks)) =
-                (iter.next(), iter.next(), iter.next())
-            {
-                timeslot.time = time.clone();
-                timeslot.location = location.clone();
-                timeslot.weeks = weeks.clone();
-            } else {
+            let (Some(time), Some(location), Some(weeks)) = (iter.next(), iter.next(), iter.next())
+            else {
                 break;
-            }
+            };
 
             // Optional instructor parsing
-            if let Some(instructor) = iter.next() {
-                if !days.contains(&instructor.as_str()) {
-                    timeslot.instructor = Some(instructor.clone());
+            let mut instructor: Option<String> = None;
+            if let Some(some_instructor) = iter.next() {
+                if !days.contains(&some_instructor.as_str()) {
+                    instructor = Some(some_instructor.clone());
                 } else {
-                    iter = Box::new(std::iter::once(instructor).chain(iter));
+                    iter = Box::new(std::iter::once(some_instructor).chain(iter));
                 }
             }
-            timeslot.career = career.to_string();
+
+            let timeslot = PartialTime {
+                day: day.to_string(),
+                time: time.to_string(),
+                location: location.to_string(),
+                weeks: weeks.to_string(),
+                instructor,
+            };
+
             meetings.push(timeslot);
         }
     }
 
-    meetings
+    Ok(meetings)
 }
 
-fn get_blank_time_struct() -> Time {
-    Time {
-        career: "".to_string(),
-        day: "".to_string(),
-        time: "".to_string(),
-        location: "".to_string(),
-        weeks: "".to_string(),
-        instructor: None,
+impl PartialTime {
+    fn complete(
+        self,
+        course_code: &str,
+        course_career: &str,
+        year: &str,
+        term: &str,
+        class_nr: &str,
+    ) -> Time {
+        let time_id = format!(
+            "{}_{}_{}_{}_{}_{}_{}_{}_{}",
+            course_code,
+            course_career,
+            year,
+            term,
+            class_nr,
+            self.location,
+            self.day,
+            self.time,
+            self.weeks
+        );
+        Time {
+            time_id,
+            career: course_career.to_string(),
+            location: self.location,
+            day: self.day,
+            time: self.time,
+            weeks: self.weeks,
+            instructor: self.instructor,
+        }
     }
 }
diff --git a/src/subject_area_scraper.rs b/src/subject_area_scraper.rs
index 34a9392..2c51f70 100644
--- a/src/subject_area_scraper.rs
+++ b/src/subject_area_scraper.rs
@@ -129,6 +129,7 @@ impl SubjectArea {
                             career.trim().to_string(),
                             uoc,
                             url_to_scrape_further,
+                            year_to_scrape,
                         );
                         tx.send(course_scraper)?;
                     }

From beff9dae6732c08cb350fb72382517dad242ef2b Mon Sep 17 00:00:00 2001
From: fritzrehde <fritzrehde.work@gmail.com>
Date: Fri, 19 Sep 2025 18:45:06 +1000
Subject: [PATCH 3/5] adjusted sql to support new ids and fields

---
 sql/Classes/up.sql    |  7 ++++---
 sql/Courses/up.sql    |  9 +++++----
 sql/Times/up.sql      |  8 ++++----
 src/course_scraper.rs | 21 +++++++++++++++------
 src/main.rs           |  6 +++---
 5 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/sql/Classes/up.sql b/sql/Classes/up.sql
index d476f96..bd7248a 100644
--- a/sql/Classes/up.sql
+++ b/sql/Classes/up.sql
@@ -1,11 +1,12 @@
 CREATE TYPE status_enum AS ENUM ('Open', 'Closed', 'Full', 'On Hold');
 CREATE TABLE Classes (
     "class_id"                        VARCHAR(255) PRIMARY KEY,
-    "career"                          VARCHAR(255),
-    "course_id"                       VARCHAR(255)   NOT NULL,
+    "career"                          VARCHAR(255) NOT NULL,
+    "course_id"                       VARCHAR(255) NOT NULL,
+    "course_nr"                       VARCHAR(255) NOT NULL,
     "section"                         VARCHAR(255) NOT NULL,
     "term"                            VARCHAR(50)  NOT NULL,
-    "year"                            VARCHAR(4)   NOT NULL,
+    "year"                            INT          NOT NULL,
     "activity"                        VARCHAR(255) NOT NULL,
     "status"                          status_enum,
     "course_enrolment"                VARCHAR(255) NOT NULL,
diff --git a/sql/Courses/up.sql b/sql/Courses/up.sql
index f215c4a..56ae534 100644
--- a/sql/Courses/up.sql
+++ b/sql/Courses/up.sql
@@ -1,12 +1,13 @@
 CREATE TABLE Courses (
     "course_id"                       VARCHAR(255) PRIMARY KEY,
-    "course_code"                     VARCHAR(8), --id
+    "course_code"                     VARCHAR(8)   NOT NULL,
+    "year"                            INT          NOT NULL,
     "course_name"                     VARCHAR(255) NOT NULL,
-    "uoc"                             INT NOT NULL,
+    "uoc"                             INT          NOT NULL,
     "faculty"                         VARCHAR(255),
     "school"                          VARCHAR(255),
     "campus"                          VARCHAR(255),
-    "career"                          VARCHAR(255),
-    "terms"                           TEXT,
+    "career"                          VARCHAR(255) NOT NULL,
+    "terms"                           VARCHAR(255)[],
     "modes"                           VARCHAR(255)[]
 );
\ No newline at end of file
diff --git a/sql/Times/up.sql b/sql/Times/up.sql
index edd03e9..3d4d8f8 100644
--- a/sql/Times/up.sql
+++ b/sql/Times/up.sql
@@ -1,11 +1,11 @@
 CREATE TABLE Times (
-    "id"                              VARCHAR(500) PRIMARY KEY,
+    "time_id"                         VARCHAR(511) PRIMARY KEY,
     "class_id"                        VARCHAR(255) NOT NULL,
     "day"                             VARCHAR(255) NOT NULL,
     "instructor"                      VARCHAR(255),
     "location"                        VARCHAR(255) NOT NULL,
-    "time"                            VARCHAR(100) NOT NULL,
-    "weeks"                           VARCHAR(100) NOT NULL,
-    "career"                          VARCHAR(255),
+    "time"                            VARCHAR(255) NOT NULL,
+    "weeks"                           VARCHAR(255) NOT NULL,
+    "career"                          VARCHAR(255) NOT NULL,
     FOREIGN KEY ("class_id") REFERENCES Classes("class_id") ON DELETE CASCADE
 );
diff --git a/src/course_scraper.rs b/src/course_scraper.rs
index 015b8f3..78f16cc 100644
--- a/src/course_scraper.rs
+++ b/src/course_scraper.rs
@@ -31,7 +31,7 @@ pub struct Class {
     pub class_nr: String,
     pub section: String,
     pub term: String,
-    pub year: String,
+    pub year: Year,
     pub activity: String,
     pub status: String,
     pub course_enrolment: String,
@@ -185,7 +185,9 @@ impl PartialCourse {
 
             let classes: Vec<Class> = class_activity_information
                 .into_par_iter()
-                .map(|class_data| parse_class_info(class_data, &course_id, &course_code, &career))
+                .map(|class_data| {
+                    parse_class_info(class_data, &course_id, &course_code, &career, self.year)
+                })
                 .collect::<anyhow::Result<_>>()?;
 
             let unique_modes: HashSet<&String> = classes.iter().map(|class| &class.mode).collect();
@@ -219,6 +221,7 @@ fn parse_class_info(
     course_id: &str,
     course_code: &str,
     course_career: &str,
+    course_year: Year,
 ) -> anyhow::Result<Class> {
     let mut map: HashMap<&str, &str> = HashMap::new();
     let mut i = 0;
@@ -264,10 +267,16 @@ fn parse_class_info(
     let date = split_offering_period_str
         .next()
         .ok_or_else(|| missing_field_error("date"))?;
-    let year = date
+
+    let class_year: Year = date
         .split("/")
         .nth(2)
-        .ok_or_else(|| missing_field_error("year"))?;
+        .ok_or_else(|| missing_field_error("year"))?
+        .parse()?;
+    assert!(class_year == course_year || class_year == (course_year - 1));
+    // If the course is in the summer term, the offering period might start in the previous year, so we use the course year.
+    let year = course_year;
+
     let class_nr = get_expected_field("Class Nbr")?;
     let term = get_expected_field("Teaching Period")?
         .split(" - ")
@@ -314,7 +323,7 @@ fn parse_class_info(
         class_nr: class_nr.to_string(),
         section: section.to_string(),
         term: term.to_string(),
-        year: year.to_string(),
+        year,
         activity: activity.to_string(),
         status: status.to_string(),
         course_enrolment,
@@ -371,7 +380,7 @@ impl PartialTime {
         self,
         course_code: &str,
         course_career: &str,
-        year: &str,
+        year: Year,
         term: &str,
         class_nr: &str,
     ) -> Time {
diff --git a/src/main.rs b/src/main.rs
index 99c00c1..0840331 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -30,17 +30,17 @@ fn convert_courses_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
         json_courses.push(json!({
             "course_id": course.course_id,
             "course_code": course.course_code,
+            "year": course.year,
             "course_name": course.course_name,
             "uoc": course.uoc,
             "faculty": course.faculty,
             "school": course.school,
             "campus": course.campus,
             "career": course.career,
-            "terms": json![course.terms],
+            "terms": course.terms,
             "modes": course.modes,
         }));
     }
-
     json_courses
 }
 
@@ -58,7 +58,7 @@ fn convert_classes_times_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
             if let Some(times) = &class.times {
                 for time in times.iter() {
                     times_json.push(json!({
-                        "id": generate_time_id(class, time),
+                        "time_id": generate_time_id(class, time),
                         "class_id": class.class_id,
                         "day": time.day,
                         "career": time.career,

From bbd66099cc597ef61ba87a7d61ca6c2d6b7889ee Mon Sep 17 00:00:00 2001
From: fritzrehde <fritzrehde.work@gmail.com>
Date: Wed, 1 Oct 2025 18:38:37 +1000
Subject: [PATCH 4/5] fixed declaring correct data fields in hasuragres comms

---
 sql/Classes/up.sql         |  2 +-
 src/config.rs              |  2 ++
 src/hasuragres_b_insert.rs | 16 ++++++++++++++--
 src/main.rs                | 24 +++++++++---------------
 src/ratelimit.rs           |  5 ++++-
 src/requests.rs            |  1 +
 src/scraping_context.rs    |  1 +
 7 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/sql/Classes/up.sql b/sql/Classes/up.sql
index bd7248a..0df3159 100644
--- a/sql/Classes/up.sql
+++ b/sql/Classes/up.sql
@@ -3,7 +3,7 @@ CREATE TABLE Classes (
     "class_id"                        VARCHAR(255) PRIMARY KEY,
     "career"                          VARCHAR(255) NOT NULL,
     "course_id"                       VARCHAR(255) NOT NULL,
-    "course_nr"                       VARCHAR(255) NOT NULL,
+    "class_nr"                        VARCHAR(255) NOT NULL,
     "section"                         VARCHAR(255) NOT NULL,
     "term"                            VARCHAR(50)  NOT NULL,
     "year"                            INT          NOT NULL,
diff --git a/src/config.rs b/src/config.rs
index 41047dd..205e30e 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -10,6 +10,7 @@ pub struct ScrapingEnv {
     timetable_api_url: String,
 }
 
+#[derive(Debug)]
 pub struct ScrapingConfig {
     timetable_api_url: String,
 }
@@ -29,6 +30,7 @@ impl ScrapingConfig {
 
 /// Regexes that can be used to extract the year and course code from a UNSW
 /// timetable url.
+#[derive(Debug)]
 pub struct TimetableUrlRegex {
     year_only_regex: Regex,
     course_url_regex: Regex,
diff --git a/src/hasuragres_b_insert.rs b/src/hasuragres_b_insert.rs
index cc3f5d3..6f04201 100644
--- a/src/hasuragres_b_insert.rs
+++ b/src/hasuragres_b_insert.rs
@@ -95,6 +95,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
                     "career".to_string(),
                     "terms".to_string(),
                     "modes".to_string(),
+                    "year".to_string(),
                 ],
                 sql_up: read_sql_file("sql/Courses/up.sql")?,
                 sql_down: read_sql_file("sql/Courses/down.sql")?,
@@ -110,6 +111,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
                 table_name: "classes".to_string(),
                 columns: vec![
                     "class_id".to_string(),
+                    "class_nr".to_string(),
                     "career".to_string(),
                     "course_id".to_string(),
                     "section".to_string(),
@@ -138,7 +140,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
             metadata: Metadata {
                 table_name: "times".to_string(),
                 columns: vec![
-                    "id".to_string(),
+                    "time_id".to_string(),
                     "class_id".to_string(),
                     "career".to_string(),
                     "day".to_string(),
@@ -158,8 +160,12 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
         },
     ];
 
+    let url = format!(
+        "{}/batch_insert",
+        uploading_config.hasuragres_url.trim_end_matches('/')
+    );
     let response = client
-        .post(format!("{}/batch_insert", uploading_config.hasuragres_url))
+        .post(&url)
         .header("X-API-Key", uploading_config.hasuragres_api_key.clone())
         .json(&requests)
         .send()
@@ -167,6 +173,11 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
 
     match response {
         Ok(res) => {
+            log::info!(
+                "call to {} returned status code {}",
+                url,
+                res.status().as_u16()
+            );
             if res.status().as_u16() == 400 {
                 let error_body: Result<Value, reqwest::Error> = res.json().await;
 
@@ -183,6 +194,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
                 }
             } else {
                 let text = res.text().await?;
+                log::info!("hasuragres returned text: {}", text);
                 let data: Result<Value, serde_json::Error> = serde_json::from_str(&text);
                 match data {
                     Ok(_) => log::info!("Successfully inserted into Hasuragres"),
diff --git a/src/main.rs b/src/main.rs
index 0840331..b081705 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -5,8 +5,8 @@ use parse_display::FromStr;
 use serde::Serialize;
 use serde_json::{json, to_writer_pretty};
 use spooderman::{
-    Class, Course, SchoolArea, ScrapingContext, Time, Year, log_execution_time,
-    log_execution_time_async, send_batch_data, sort_by_key_ref,
+    Course, SchoolArea, ScrapingContext, Year, log_execution_time, log_execution_time_async,
+    send_batch_data, sort_by_key_ref,
 };
 use spooderman::{ReadFromFile, ReadFromMemory};
 use std::fs::File;
@@ -44,13 +44,6 @@ fn convert_courses_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
     json_courses
 }
 
-fn generate_time_id(class: &Class, time: &Time) -> String {
-    format!(
-        "{}{}{}{}{}",
-        &class.class_id, &time.day, &time.location, &time.time, &time.weeks
-    )
-}
-
 fn convert_classes_times_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
     let mut times_json = Vec::<serde_json::Value>::new();
     for course in courses.iter() {
@@ -58,7 +51,7 @@ fn convert_classes_times_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
             if let Some(times) = &class.times {
                 for time in times.iter() {
                     times_json.push(json!({
-                        "time_id": generate_time_id(class, time),
+                        "time_id": time.time_id,
                         "class_id": class.class_id,
                         "day": time.day,
                         "career": time.career,
@@ -81,6 +74,7 @@ fn convert_classes_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
             json_classes.push(json!({
                 "course_id": class.course_id,
                 "class_id": class.class_id,
+                "class_nr": class.class_nr,
                 "section": class.section,
                 "term": class.term,
                 "career": class.career,
@@ -188,7 +182,7 @@ trait Exec {
 }
 
 /// A tool for scraping UNSW course and class data.
-#[derive(FromArgs)]
+#[derive(Debug, FromArgs)]
 struct Cli {
     #[argh(subcommand)]
     command: Command,
@@ -268,7 +262,7 @@ impl YearToScrape {
     }
 }
 
-#[derive(FromArgs)]
+#[derive(Debug, FromArgs)]
 #[argh(subcommand)]
 #[enum_dispatch(Exec)]
 enum Command {
@@ -278,7 +272,7 @@ enum Command {
 }
 
 /// Perform scraping. Creates a JSON file to store the data.
-#[derive(FromArgs)]
+#[derive(Debug, FromArgs)]
 #[argh(subcommand, name = "scrape")]
 struct Scrape {
     /// the year for which data should be scraped: `latest-with-data` (the latest year with data available), or a calendar year, e.g. `2025`
@@ -313,7 +307,7 @@ impl Exec for Scrape {
 }
 
 /// Perform batch insert on JSON files created by `scrape`.
-#[derive(FromArgs)]
+#[derive(Debug, FromArgs)]
 #[argh(subcommand, name = "batch_insert")]
 struct BatchInsert {}
 
@@ -326,7 +320,7 @@ impl Exec for BatchInsert {
 }
 
 /// Perform scraping and batch insert. Does not create a JSON file to store the data.
-#[derive(FromArgs)]
+#[derive(Debug, FromArgs)]
 #[argh(subcommand, name = "scrape_n_batch_insert")]
 struct ScrapeAndBatchInsert {
     /// the year for which data should be scraped: `latest-with-data` (the latest year with data available), or a calendar year, e.g. `2025`.
diff --git a/src/ratelimit.rs b/src/ratelimit.rs
index eb7cbcf..f258ae1 100644
--- a/src/ratelimit.rs
+++ b/src/ratelimit.rs
@@ -34,7 +34,7 @@ const EXPONENTIAL_REQUEST_RATE_BACKOFF: f64 = 2.0 / 3.0;
 // The lower, the faster we restart after request rate change.
 const PAUSE_AFTER_REQ_RATE_CHANGE: Duration = Duration::from_secs(5);
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 pub struct RequestRate {
     req_per_sec: NonZeroU32,
     ms_between_req: Duration,
@@ -76,6 +76,7 @@ enum Gate {
 type SpecificGovernorRateLimiter =
     GovernorRateLimiter<NotKeyed, InMemoryState, QuantaClock, NoOpMiddleware<QuantaInstant>>;
 
+#[derive(Debug)]
 struct FixedRateLimiter {
     request_rate: RequestRate,
     req_per_sec_rate_limiter: SpecificGovernorRateLimiter,
@@ -130,6 +131,7 @@ impl Drop for FixedRateLimiter {
     }
 }
 
+#[derive(Debug)]
 struct RateLimiterGeneration {
     rate_limiter: FixedRateLimiter,
     cancel_token: CancellationToken,
@@ -144,6 +146,7 @@ impl RateLimiterGeneration {
     }
 }
 
+#[derive(Debug)]
 pub struct RateLimiter {
     // Hot-swappable rate limiter, wrapped in an Arc to avoid holding lock during waits.
     rate_limiter_generation: RwLock<Arc<RateLimiterGeneration>>,
diff --git a/src/requests.rs b/src/requests.rs
index 416c603..76efcff 100644
--- a/src/requests.rs
+++ b/src/requests.rs
@@ -11,6 +11,7 @@ use crate::{
 const GET_REQUEST_TIMEOUT: Duration = Duration::from_secs(3);
 const RESPONSE_BODY_TIMEOUT: Duration = Duration::from_secs(3);
 
+#[derive(Debug)]
 pub struct RequestClient {
     client: Client,
     rate_limiter: RateLimiter,
diff --git a/src/scraping_context.rs b/src/scraping_context.rs
index 88d36e1..9309d49 100644
--- a/src/scraping_context.rs
+++ b/src/scraping_context.rs
@@ -3,6 +3,7 @@ use crate::{
     requests::RequestClient,
 };
 
+#[derive(Debug)]
 pub struct ScrapingContext {
     pub scraping_config: ScrapingConfig,
     pub timetable_url_regex: TimetableUrlRegex,

From 39981a523facfdc3b5f434f4e97419f8ae3ccc19 Mon Sep 17 00:00:00 2001
From: fritzrehde <fritzrehde.work@gmail.com>
Date: Wed, 8 Oct 2025 20:01:09 +1100
Subject: [PATCH 5/5] wip support hasuragres batch inserting

---
 sql/Classes/down_just_year.sql |   2 +
 sql/Classes/up_just_year.sql   |  21 +++++
 sql/Courses/down_just_year.sql |   2 +
 sql/Courses/up_just_year.sql   |  14 ++++
 sql/Times/down_just_year.sql   |   2 +
 sql/Times/up.sql               |   1 +
 sql/Times/up_just_year.sql     |  13 +++
 src/course_scraper.rs          |   2 +
 src/hasuragres_b_insert.rs     | 145 ++++++++++++++++++++++++++++++++-
 src/lib.rs                     |   4 +-
 src/main.rs                    |  43 +++++++++-
 11 files changed, 246 insertions(+), 3 deletions(-)
 create mode 100644 sql/Classes/down_just_year.sql
 create mode 100644 sql/Classes/up_just_year.sql
 create mode 100644 sql/Courses/down_just_year.sql
 create mode 100644 sql/Courses/up_just_year.sql
 create mode 100644 sql/Times/down_just_year.sql
 create mode 100644 sql/Times/up_just_year.sql

diff --git a/sql/Classes/down_just_year.sql b/sql/Classes/down_just_year.sql
new file mode 100644
index 0000000..1fd6d08
--- /dev/null
+++ b/sql/Classes/down_just_year.sql
@@ -0,0 +1,2 @@
+DELETE FROM Classes
+WHERE "year" = {year};
\ No newline at end of file
diff --git a/sql/Classes/up_just_year.sql b/sql/Classes/up_just_year.sql
new file mode 100644
index 0000000..f26a647
--- /dev/null
+++ b/sql/Classes/up_just_year.sql
@@ -0,0 +1,21 @@
+SELECT 1;
+-- CREATE TYPE IF NOT EXISTS status_enum AS ENUM ('Open', 'Closed', 'Full', 'On Hold');
+-- CREATE TABLE IF NOT EXISTS Classes (
+--     "class_id"                        VARCHAR(255) PRIMARY KEY,
+--     "career"                          VARCHAR(255) NOT NULL,
+--     "course_id"                       VARCHAR(255) NOT NULL,
+--     "class_nr"                        VARCHAR(255) NOT NULL,
+--     "section"                         VARCHAR(255) NOT NULL,
+--     "term"                            VARCHAR(50)  NOT NULL,
+--     "year"                            INT          NOT NULL,
+--     "activity"                        VARCHAR(255) NOT NULL,
+--     "status"                          status_enum,
+--     "course_enrolment"                VARCHAR(255) NOT NULL,
+--     "offering_period"                 VARCHAR(255) NOT NULL,
+--     "meeting_dates"                   VARCHAR(255) NOT NULL,
+--     "census_date"                     VARCHAR(255) NOT NULL,
+--     "consent"                         VARCHAR(255) NOT NULL,
+--     "mode"                            VARCHAR(255) NOT NULL,
+--     "class_notes"                     TEXT,
+--     FOREIGN KEY ("course_id") REFERENCES Courses("course_id") ON DELETE CASCADE
+-- );
diff --git a/sql/Courses/down_just_year.sql b/sql/Courses/down_just_year.sql
new file mode 100644
index 0000000..cf6d058
--- /dev/null
+++ b/sql/Courses/down_just_year.sql
@@ -0,0 +1,2 @@
+DELETE FROM Courses
+WHERE "year" = {year};
\ No newline at end of file
diff --git a/sql/Courses/up_just_year.sql b/sql/Courses/up_just_year.sql
new file mode 100644
index 0000000..2d66dda
--- /dev/null
+++ b/sql/Courses/up_just_year.sql
@@ -0,0 +1,14 @@
+SELECT 1;
+-- CREATE TABLE IF NOT EXISTS Courses (
+--     "course_id"                       VARCHAR(255) PRIMARY KEY,
+--     "course_code"                     VARCHAR(8)   NOT NULL,
+--     "year"                            INT          NOT NULL,
+--     "course_name"                     VARCHAR(255) NOT NULL,
+--     "uoc"                             INT          NOT NULL,
+--     "faculty"                         VARCHAR(255),
+--     "school"                          VARCHAR(255),
+--     "campus"                          VARCHAR(255),
+--     "career"                          VARCHAR(255) NOT NULL,
+--     "terms"                           VARCHAR(255)[],
+--     "modes"                           VARCHAR(255)[]
+-- );
\ No newline at end of file
diff --git a/sql/Times/down_just_year.sql b/sql/Times/down_just_year.sql
new file mode 100644
index 0000000..b704212
--- /dev/null
+++ b/sql/Times/down_just_year.sql
@@ -0,0 +1,2 @@
+DELETE FROM Times
+WHERE "year" = {year};
\ No newline at end of file
diff --git a/sql/Times/up.sql b/sql/Times/up.sql
index 3d4d8f8..8842eee 100644
--- a/sql/Times/up.sql
+++ b/sql/Times/up.sql
@@ -1,5 +1,6 @@
 CREATE TABLE Times (
     "time_id"                         VARCHAR(511) PRIMARY KEY,
+    "year"                            INT          NOT NULL,
     "class_id"                        VARCHAR(255) NOT NULL,
     "day"                             VARCHAR(255) NOT NULL,
     "instructor"                      VARCHAR(255),
diff --git a/sql/Times/up_just_year.sql b/sql/Times/up_just_year.sql
new file mode 100644
index 0000000..d290066
--- /dev/null
+++ b/sql/Times/up_just_year.sql
@@ -0,0 +1,13 @@
+SELECT 1;
+-- CREATE TABLE IF NOT EXISTS Times (
+--     "time_id"                         VARCHAR(511) PRIMARY KEY,
+--     "year"                            INT          NOT NULL,
+--     "class_id"                        VARCHAR(255) NOT NULL,
+--     "day"                             VARCHAR(255) NOT NULL,
+--     "instructor"                      VARCHAR(255),
+--     "location"                        VARCHAR(255) NOT NULL,
+--     "time"                            VARCHAR(255) NOT NULL,
+--     "weeks"                           VARCHAR(255) NOT NULL,
+--     "career"                          VARCHAR(255) NOT NULL,
+--     FOREIGN KEY ("class_id") REFERENCES Classes("class_id") ON DELETE CASCADE
+-- );
diff --git a/src/course_scraper.rs b/src/course_scraper.rs
index 78f16cc..8e13f94 100644
--- a/src/course_scraper.rs
+++ b/src/course_scraper.rs
@@ -47,6 +47,7 @@ pub struct Class {
 #[derive(Debug, Serialize)]
 pub struct Time {
     pub time_id: String,
+    pub year: Year,
     pub career: String,
     pub location: String,
     pub day: String,
@@ -398,6 +399,7 @@ impl PartialTime {
         );
         Time {
             time_id,
+            year,
             career: course_career.to_string(),
             location: self.location,
             day: self.day,
diff --git a/src/hasuragres_b_insert.rs b/src/hasuragres_b_insert.rs
index 6f04201..21fb348 100644
--- a/src/hasuragres_b_insert.rs
+++ b/src/hasuragres_b_insert.rs
@@ -5,8 +5,8 @@ use std::fs::File;
 use std::io::Read;
 use std::vec;
 
-use crate::UploadingConfig;
 use crate::config::LoadFromEnv;
+use crate::{UploadingConfig, Year};
 
 #[derive(Serialize, Deserialize)]
 struct Metadata {
@@ -141,6 +141,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
                 table_name: "times".to_string(),
                 columns: vec![
                     "time_id".to_string(),
+                    "year".to_string(),
                     "class_id".to_string(),
                     "career".to_string(),
                     "day".to_string(),
@@ -206,3 +207,145 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> anyhow::Result<()>
     }
     Ok(())
 }
+
+pub async fn send_batch_data_for_year(
+    hdata: &impl HasuragresData,
+    year: Year,
+) -> anyhow::Result<()> {
+    // This should only be run if the schema hasn't changed since the last
+    // upload, otherwise all data should be removed (with `send_batch_data`) and
+    // re-uploaded (for every year we want data for, unfortunately).
+
+    let uploading_config = UploadingConfig::load_from_env()?;
+    let client = Client::new();
+    log::info!("Starting insert for year {} into Hasuragres!", year);
+    let requests = vec![
+        BatchInsertRequest {
+            metadata: Metadata {
+                table_name: "courses".to_string(),
+                columns: vec![
+                    "course_id".to_string(),
+                    "course_code".to_string(),
+                    "course_name".to_string(),
+                    "uoc".to_string(),
+                    "faculty".to_string(),
+                    "school".to_string(),
+                    "campus".to_string(),
+                    "career".to_string(),
+                    "terms".to_string(),
+                    "modes".to_string(),
+                    "year".to_string(),
+                ],
+                sql_up: read_sql_file("sql/Courses/up_just_year.sql")?,
+                sql_down: read_sql_file("sql/Courses/down_just_year.sql")?
+                    .replace("{year}", &year.to_string()),
+                write_mode: Some("overwrite".to_string()),
+                sql_before: None,
+                sql_after: None,
+                dryrun: Some(true),
+            },
+            payload: hdata.get_courses(),
+        },
+        BatchInsertRequest {
+            metadata: Metadata {
+                table_name: "classes".to_string(),
+                columns: vec![
+                    "class_id".to_string(),
+                    "class_nr".to_string(),
+                    "career".to_string(),
+                    "course_id".to_string(),
+                    "section".to_string(),
+                    "term".to_string(),
+                    "activity".to_string(),
+                    "year".to_string(),
+                    "status".to_string(),
+                    "course_enrolment".to_string(),
+                    "offering_period".to_string(),
+                    "meeting_dates".to_string(),
+                    "census_date".to_string(),
+                    "consent".to_string(),
+                    "mode".to_string(),
+                    "class_notes".to_string(),
+                ],
+                sql_up: read_sql_file("sql/Classes/up_just_year.sql")?,
+                sql_down: read_sql_file("sql/Classes/down_just_year.sql")?
+                    .replace("{year}", &year.to_string()),
+                write_mode: Some("overwrite".to_string()),
+                sql_before: None,
+                sql_after: None,
+                dryrun: Some(true),
+            },
+            payload: hdata.get_classes(),
+        },
+        BatchInsertRequest {
+            metadata: Metadata {
+                table_name: "times".to_string(),
+                columns: vec![
+                    "time_id".to_string(),
+                    "year".to_string(),
+                    "class_id".to_string(),
+                    "career".to_string(),
+                    "day".to_string(),
+                    "instructor".to_string(),
+                    "location".to_string(),
+                    "time".to_string(),
+                    "weeks".to_string(),
+                ],
+                sql_up: read_sql_file("sql/Times/up_just_year.sql")?,
+                sql_down: read_sql_file("sql/Times/down_just_year.sql")?
+                    .replace("{year}", &year.to_string()),
+                write_mode: Some("overwrite".to_string()),
+                sql_before: None,
+                sql_after: None,
+                dryrun: Some(true),
+            },
+            payload: hdata.get_times(),
+        },
+    ];
+
+    let url = format!(
+        "{}/batch_insert",
+        uploading_config.hasuragres_url.trim_end_matches('/')
+    );
+    let response = client
+        .post(&url)
+        .header("X-API-Key", uploading_config.hasuragres_api_key.clone())
+        .json(&requests)
+        .send()
+        .await;
+
+    match response {
+        Ok(res) => {
+            log::info!(
+                "call to {} returned status code {}",
+                url,
+                res.status().as_u16()
+            );
+            if res.status().as_u16() == 400 {
+                let error_body: Result<Value, reqwest::Error> = res.json().await;
+
+                match error_body {
+                    Ok(json) => {
+                        log::error!("Error occurred: {:?}", json);
+                        if let Some(error_message) = json.get("error") {
+                            log::error!("Error message: {}", error_message);
+                        }
+                    }
+                    Err(err) => {
+                        log::error!("Failed to parse error body: {:?}", err);
+                    }
+                }
+            } else {
+                let text = res.text().await?;
+                log::info!("hasuragres returned text: {}", text);
+                let data: Result<Value, serde_json::Error> = serde_json::from_str(&text);
+                match data {
+                    Ok(_) => log::info!("Successfully inserted into Hasuragres"),
+                    Err(err) => log::error!("Failed to parse response body: {:?}", err),
+                }
+            }
+        }
+        Err(e) => log::error!("Failed to insert batch data: {:?}", e),
+    }
+    Ok(())
+}
diff --git a/src/lib.rs b/src/lib.rs
index eceab97..ab656db 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,7 +12,9 @@ mod utils;
 
 pub use config::{ScrapingEnv, UploadingConfig};
 pub use course_scraper::{Class, Course, PartialCourse, Time};
-pub use hasuragres_b_insert::{ReadFromFile, ReadFromMemory, send_batch_data};
+pub use hasuragres_b_insert::{
+    ReadFromFile, ReadFromMemory, send_batch_data, send_batch_data_for_year,
+};
 pub use requests::{Request, RequestClient};
 pub use school_area_scraper::SchoolArea;
 pub use scraping_context::ScrapingContext;
diff --git a/src/main.rs b/src/main.rs
index b081705..9dc5fe4 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -6,7 +6,7 @@ use serde::Serialize;
 use serde_json::{json, to_writer_pretty};
 use spooderman::{
     Course, SchoolArea, ScrapingContext, Year, log_execution_time, log_execution_time_async,
-    send_batch_data, sort_by_key_ref,
+    send_batch_data, send_batch_data_for_year, sort_by_key_ref,
 };
 use spooderman::{ReadFromFile, ReadFromMemory};
 use std::fs::File;
@@ -52,6 +52,7 @@ fn convert_classes_times_to_json(courses: &[Course]) -> Vec<serde_json::Value> {
                 for time in times.iter() {
                     times_json.push(json!({
                         "time_id": time.time_id,
+                        "year": time.year,
                         "class_id": class.class_id,
                         "day": time.day,
                         "career": time.career,
@@ -176,6 +177,28 @@ async fn handle_batch_insert() -> anyhow::Result<()> {
     Ok(())
 }
 
+async fn handle_batch_insert_for_year(year: Year) -> anyhow::Result<()> {
+    log::info!("Handling batch insert...");
+    if !Path::new("courses.json").is_file() {
+        return Err(anyhow::anyhow!(
+            "courses.json doesn't exist, please run cargo r -- scrape"
+        ));
+    }
+    if !Path::new("classes.json").is_file() {
+        return Err(anyhow::anyhow!(
+            "classes.json doesn't exist, please run cargo r -- scrape"
+        ));
+    }
+    if !Path::new("times.json").is_file() {
+        return Err(anyhow::anyhow!(
+            "times.json doesn't exist, please run cargo r -- scrape"
+        ));
+    }
+
+    send_batch_data_for_year(&ReadFromFile, year).await?;
+    Ok(())
+}
+
 #[enum_dispatch]
 trait Exec {
     async fn exec(&self) -> anyhow::Result<()>;
@@ -268,6 +291,7 @@ impl YearToScrape {
 enum Command {
     Scrape(Scrape),
     BatchInsert(BatchInsert),
+    BatchInsertForYear(BatchInsertForYear),
     ScrapeAndBatchInsert(ScrapeAndBatchInsert),
 }
 
@@ -319,6 +343,23 @@ impl Exec for BatchInsert {
     }
 }
 
+/// Perform batch insert for a given year on JSON files created by `scrape`.
+#[derive(Debug, FromArgs)]
+#[argh(subcommand, name = "batch_insert_for_year")]
+struct BatchInsertForYear {
+    /// the year for which data should be batch inserted, e.g. `2025`.
+    #[argh(option, long = "year", short = 'y')]
+    year: Year,
+}
+
+impl Exec for BatchInsertForYear {
+    async fn exec(&self) -> anyhow::Result<()> {
+        log::info!("Handling batch insert...");
+        handle_batch_insert_for_year(self.year).await?;
+        Ok(())
+    }
+}
+
 /// Perform scraping and batch insert. Does not create a JSON file to store the data.
 #[derive(Debug, FromArgs)]
 #[argh(subcommand, name = "scrape_n_batch_insert")]