scraping.scrape_registration_sections

@file scrape_registration_sections.py @brief Parses a locally saved HTML file of QCC's registration page into structured JSON.

@details This module reads a saved HTML snapshot of The Q portal's course registration page and extracts all course section data into a list of structured dictionaries. The parsed data is saved to registration_sections.json.

This script does not perform live web requests — it parses a pre-saved HTML file. For live scraping with automatic refresh, see refresh_registration.py.

@date 2026

@par Input saved_pages/registration_page.html — saved HTML snapshot of The Q portal

@par Output registration_sections.json — list of course section dictionaries

@par Dependencies - beautifulsoup4 - json (stdlib) - re (stdlib) - datetime (stdlib)

@par Usage python scrape_registration_sections.py

View Source

  1# =============================================================================
  2#
  3# Copyright 2026 
  4#
  5# Author:   Noel Mensah
  6# GitHub:   https://github.com/LSilver17/CSC212---AI-Agent
  7# 
  8# =============================================================================
  9
 10"""
 11@file scrape_registration_sections.py
 12@brief Parses a locally saved HTML file of QCC's registration page into structured JSON.
 13
 14@details
 15This module reads a saved HTML snapshot of The Q portal's course registration
 16page and extracts all course section data into a list of structured dictionaries.
 17The parsed data is saved to registration_sections.json.
 18
 19This script does not perform live web requests — it parses a pre-saved HTML file.
 20For live scraping with automatic refresh, see refresh_registration.py.
 21
 22@date 2026
 23
 24@par Input
 25    saved_pages/registration_page.html — saved HTML snapshot of The Q portal
 26
 27@par Output
 28    registration_sections.json — list of course section dictionaries
 29
 30@par Dependencies
 31    - beautifulsoup4
 32    - json (stdlib)
 33    - re (stdlib)
 34    - datetime (stdlib)
 35
 36@par Usage
 37    python scrape_registration_sections.py
 38"""
 39
 40from bs4 import BeautifulSoup
 41import json
 42import re
 43from datetime import datetime
 44
 45## @brief Path to the locally saved HTML registration page.
 46HTML_FILE = "saved_pages/registration_page.html"
 47
 48
 49def clean_text(text: str) -> str:
 50    """
 51    @brief Removes excess whitespace from a string.
 52
 53    @details
 54    Splits the input string on any whitespace, then rejoins with single spaces
 55    and strips leading/trailing whitespace. Handles newlines, tabs, and
 56    multiple consecutive spaces.
 57
 58    @param text The raw string to clean.
 59    @return A normalized single-line string with no extra whitespace.
 60
 61    @par Example
 62    @code
 63    clean_text("  ACC   101  ") -> "ACC 101"
 64    @endcode
 65    """
 66    return " ".join(text.split()).strip()
 67
 68
 69def parse_seats(seats_str: str) -> dict:
 70    """
 71    @brief Parses a seats string into open and total seat counts.
 72
 73    @details
 74    The Q portal displays seat availability in the format "1 / 24" or "1 ∕ 24"
 75    (using either a standard slash or a Unicode division slash). This function
 76    extracts both numbers using a regex pattern that handles both slash variants.
 77
 78    @param seats_str The raw seats string from the registration table (e.g. "1 / 24").
 79    @return A dictionary with keys:
 80            - "open" (int or None): number of open seats
 81            - "total" (int or None): total seats in the section
 82
 83    @par Example
 84    @code
 85    parse_seats("1 ∕ 24") -> {"open": 1, "total": 24}
 86    parse_seats("")        -> {"open": None, "total": None}
 87    @endcode
 88    """
 89    match = re.search(r"(\d+)\s*[/∕]\s*(\d+)", seats_str)
 90    if match:
 91        return {"open": int(match.group(1)), "total": int(match.group(2))}
 92    return {"open": None, "total": None}
 93
 94
 95def parse_details(details_str: str) -> dict:
 96    """
 97    @brief Parses the details column into instructor, schedule, location, and method.
 98
 99    @details
100    The details column in The Q portal combines multiple fields into a single
101    slash-delimited string with semicolons for sub-fields. The expected format is:
102        "Instructor Name / Days Time; Location / Method"
103
104    Parsing rules:
105    - Part 0 (before first /): instructor name
106    - Part 1 (between first and second /): days/time before semicolon, location after semicolon
107    - Part 2 (after second /): delivery method (e.g. "Lecture", "Online")
108
109    @param details_str The raw details string from the registration table.
110    @return A dictionary with keys:
111            - "instructor" (str or None)
112            - "days_time" (str or None): e.g. "MW 08:00-09:15AM"
113            - "location" (str or None): e.g. "MAIN Campus, Surprenant Hall, 312"
114            - "method" (str or None): e.g. "Lecture"
115
116    @par Example
117    @code
118    parse_details("De Silva, Damindi / MW 08:00-09:15AM; MAIN Campus, Room 312 / Lecture")
119    # -> {"instructor": "De Silva, Damindi", "days_time": "MW 08:00-09:15AM",
120    #     "location": "MAIN Campus, Room 312", "method": "Lecture"}
121    @endcode
122    """
123    result = {"instructor": None, "days_time": None, "location": None, "method": None}
124    parts = [p.strip() for p in details_str.split("/")]
125    if len(parts) >= 1:
126        result["instructor"] = clean_text(parts[0])
127    if len(parts) >= 2:
128        days_time_raw = parts[1].split(";")[0]
129        result["days_time"] = clean_text(days_time_raw)
130        if ";" in parts[1]:
131            result["location"] = clean_text(parts[1].split(";")[1])
132    if len(parts) >= 3:
133        result["method"] = clean_text(parts[2])
134    return result
135
136
137def parse_course_code(code: str) -> dict:
138    """
139    @brief Splits a full course code string into department, number, and section.
140
141    @details
142    QCC course codes follow the format "DEPT NUM-SECTION" (e.g. "ACC 101-01").
143    This function extracts each component using a regex match.
144
145    @param code The full course code string (e.g. "ACC 101-01").
146    @return A dictionary with keys:
147            - "department" (str or None): e.g. "ACC"
148            - "number" (str or None): e.g. "101"
149            - "section" (str or None): e.g. "01"
150
151    @par Example
152    @code
153    parse_course_code("ACC 101-01") -> {"department": "ACC", "number": "101", "section": "01"}
154    parse_course_code("invalid")    -> {"department": None, "number": None, "section": None}
155    @endcode
156    """
157    match = re.match(r"([A-Z]+)\s+(\d+)-(\w+)", code)
158    if match:
159        return {"department": match.group(1), "number": match.group(2), "section": match.group(3)}
160    return {"department": None, "number": None, "section": None}
161
162
163def parse_registration_html(file_path: str) -> list:
164    """
165    @brief Parses a saved HTML registration page and returns a list of course section dicts.
166
167    @details
168    Reads the HTML file at the given path, finds all table rows, and extracts
169    course section data from rows that contain a valid course code (matching
170    the pattern "DEPT NUM-SECTION"). Skips header rows and empty rows.
171
172    The column layout detected from The Q portal is:
173    - [idx+0] course_code   e.g. "ACC 101-01"
174    - [idx+1] name          e.g. "Financial Accounting I"
175    - [idx+2] req           (empty, skipped)
176    - [idx+3] note          (empty, skipped)
177    - [idx+4] seats         e.g. "1 ∕ 24"
178    - [idx+5] status        e.g. "Reopened"
179    - [idx+6] details       e.g. "Instructor / Days;Location / Method"
180    - [idx+7] credits       e.g. "3.00"
181    - [idx+8] begin_date    e.g. "01/26/2026"
182    - [idx+9] end_date      e.g. "05/19/2026"
183
184    @param file_path Path to the locally saved HTML file.
185    @return A list of dictionaries, each representing one course section with keys:
186            course_code, department, course_number, section, name, status,
187            seats_open, seats_total, credits, instructor, days_time, location,
188            method, begin_date, end_date, scraped_at.
189
190    @throws FileNotFoundError if the HTML file does not exist at the given path.
191    """
192    with open(file_path, "r", encoding="utf-8") as f:
193        soup = BeautifulSoup(f, "html.parser")
194
195    rows = soup.select("table tr")
196    courses = []
197
198    for row in rows:
199        cols = row.find_all(["td", "th"])
200        if len(cols) < 4:
201            continue
202
203        values = [clean_text(col.get_text(" ", strip=True)) for col in cols]
204
205        # Skip header and empty rows
206        joined = " | ".join(values).lower()
207        if "course code" in joined and "name" in joined:
208            continue
209        if all(not v for v in values):
210            continue
211
212        # Find the course code column (e.g. "ACC 101-01")
213        idx = None
214        for i, v in enumerate(values):
215            if re.match(r"[A-Z]{2,4}\s+\d{3}-\w+", v):
216                idx = i
217                break
218
219        if idx is None:
220            continue
221
222        try:
223            code_str   = values[idx]
224            name       = values[idx + 1]
225            seats_str  = values[idx + 4]
226            status     = values[idx + 5]
227            details    = values[idx + 6]
228            credits    = values[idx + 7]
229            begin_date = values[idx + 8] if len(values) > idx + 8 else None
230            end_date   = values[idx + 9] if len(values) > idx + 9 else None
231        except IndexError:
232            continue
233
234        seats      = parse_seats(seats_str)
235        detail_map = parse_details(details)
236        code_map   = parse_course_code(code_str)
237
238        courses.append({
239            "course_code":   code_str,
240            "department":    code_map["department"],
241            "course_number": code_map["number"],
242            "section":       code_map["section"],
243            "name":          name,
244            "status":        status,
245            "seats_open":    seats["open"],
246            "seats_total":   seats["total"],
247            "credits":       credits,
248            "instructor":    detail_map["instructor"],
249            "days_time":     detail_map["days_time"],
250            "location":      detail_map["location"],
251            "method":        detail_map["method"],
252            "begin_date":    begin_date,
253            "end_date":      end_date,
254            "scraped_at":    datetime.now().isoformat(),
255        })
256
257    return courses
258
259
260if __name__ == "__main__":
261    data = parse_registration_html(HTML_FILE)
262
263    print(f"Parsed {len(data)} course sections.\n")
264    for course in data[:3]:
265        print(json.dumps(course, indent=2))
266        print("---")
267
268    with open("registration_sections.json", "w", encoding="utf-8") as f:
269        json.dump(data, f, indent=2)
270
271    print(f"\nSaved to registration_sections.json")

HTML_FILE = 'saved_pages/registration_page.html'

def clean_text(text: str) -> str: View Source

50def clean_text(text: str) -> str:
51    """
52    @brief Removes excess whitespace from a string.
53
54    @details
55    Splits the input string on any whitespace, then rejoins with single spaces
56    and strips leading/trailing whitespace. Handles newlines, tabs, and
57    multiple consecutive spaces.
58
59    @param text The raw string to clean.
60    @return A normalized single-line string with no extra whitespace.
61
62    @par Example
63    @code
64    clean_text("  ACC   101  ") -> "ACC 101"
65    @endcode
66    """
67    return " ".join(text.split()).strip()

@brief Removes excess whitespace from a string.

@details Splits the input string on any whitespace, then rejoins with single spaces and strips leading/trailing whitespace. Handles newlines, tabs, and multiple consecutive spaces.

@param text The raw string to clean. @return A normalized single-line string with no extra whitespace.

@par Example @code clean_text(" ACC 101 ") -> "ACC 101" @endcode

def parse_seats(seats_str: str) -> dict: View Source

70def parse_seats(seats_str: str) -> dict:
71    """
72    @brief Parses a seats string into open and total seat counts.
73
74    @details
75    The Q portal displays seat availability in the format "1 / 24" or "1 ∕ 24"
76    (using either a standard slash or a Unicode division slash). This function
77    extracts both numbers using a regex pattern that handles both slash variants.
78
79    @param seats_str The raw seats string from the registration table (e.g. "1 / 24").
80    @return A dictionary with keys:
81            - "open" (int or None): number of open seats
82            - "total" (int or None): total seats in the section
83
84    @par Example
85    @code
86    parse_seats("1 ∕ 24") -> {"open": 1, "total": 24}
87    parse_seats("")        -> {"open": None, "total": None}
88    @endcode
89    """
90    match = re.search(r"(\d+)\s*[/∕]\s*(\d+)", seats_str)
91    if match:
92        return {"open": int(match.group(1)), "total": int(match.group(2))}
93    return {"open": None, "total": None}

@brief Parses a seats string into open and total seat counts.

@details The Q portal displays seat availability in the format "1 / 24" or "1 ∕ 24" (using either a standard slash or a Unicode division slash). This function extracts both numbers using a regex pattern that handles both slash variants.

@param seats_str The raw seats string from the registration table (e.g. "1 / 24"). @return A dictionary with keys: - "open" (int or None): number of open seats - "total" (int or None): total seats in the section

@par Example @code parse_seats("1 ∕ 24") -> {"open": 1, "total": 24} parse_seats("") -> {"open": None, "total": None} @endcode

def parse_details(details_str: str) -> dict: View Source

 96def parse_details(details_str: str) -> dict:
 97    """
 98    @brief Parses the details column into instructor, schedule, location, and method.
 99
100    @details
101    The details column in The Q portal combines multiple fields into a single
102    slash-delimited string with semicolons for sub-fields. The expected format is:
103        "Instructor Name / Days Time; Location / Method"
104
105    Parsing rules:
106    - Part 0 (before first /): instructor name
107    - Part 1 (between first and second /): days/time before semicolon, location after semicolon
108    - Part 2 (after second /): delivery method (e.g. "Lecture", "Online")
109
110    @param details_str The raw details string from the registration table.
111    @return A dictionary with keys:
112            - "instructor" (str or None)
113            - "days_time" (str or None): e.g. "MW 08:00-09:15AM"
114            - "location" (str or None): e.g. "MAIN Campus, Surprenant Hall, 312"
115            - "method" (str or None): e.g. "Lecture"
116
117    @par Example
118    @code
119    parse_details("De Silva, Damindi / MW 08:00-09:15AM; MAIN Campus, Room 312 / Lecture")
120    # -> {"instructor": "De Silva, Damindi", "days_time": "MW 08:00-09:15AM",
121    #     "location": "MAIN Campus, Room 312", "method": "Lecture"}
122    @endcode
123    """
124    result = {"instructor": None, "days_time": None, "location": None, "method": None}
125    parts = [p.strip() for p in details_str.split("/")]
126    if len(parts) >= 1:
127        result["instructor"] = clean_text(parts[0])
128    if len(parts) >= 2:
129        days_time_raw = parts[1].split(";")[0]
130        result["days_time"] = clean_text(days_time_raw)
131        if ";" in parts[1]:
132            result["location"] = clean_text(parts[1].split(";")[1])
133    if len(parts) >= 3:
134        result["method"] = clean_text(parts[2])
135    return result

@brief Parses the details column into instructor, schedule, location, and method.

@details The details column in The Q portal combines multiple fields into a single slash-delimited string with semicolons for sub-fields. The expected format is: "Instructor Name / Days Time; Location / Method"

Parsing rules:

Part 0 (before first /): instructor name
Part 1 (between first and second /): days/time before semicolon, location after semicolon
Part 2 (after second /): delivery method (e.g. "Lecture", "Online")

@param details_str The raw details string from the registration table. @return A dictionary with keys: - "instructor" (str or None) - "days_time" (str or None): e.g. "MW 08:00-09:15AM" - "location" (str or None): e.g. "MAIN Campus, Surprenant Hall, 312" - "method" (str or None): e.g. "Lecture"

@par Example @code parse_details("De Silva, Damindi / MW 08:00-09:15AM; MAIN Campus, Room 312 / Lecture")

-> {"instructor": "De Silva, Damindi", "days_time": "MW 08:00-09:15AM",

"location": "MAIN Campus, Room 312", "method": "Lecture"}

@endcode

def parse_course_code(code: str) -> dict: View Source

138def parse_course_code(code: str) -> dict:
139    """
140    @brief Splits a full course code string into department, number, and section.
141
142    @details
143    QCC course codes follow the format "DEPT NUM-SECTION" (e.g. "ACC 101-01").
144    This function extracts each component using a regex match.
145
146    @param code The full course code string (e.g. "ACC 101-01").
147    @return A dictionary with keys:
148            - "department" (str or None): e.g. "ACC"
149            - "number" (str or None): e.g. "101"
150            - "section" (str or None): e.g. "01"
151
152    @par Example
153    @code
154    parse_course_code("ACC 101-01") -> {"department": "ACC", "number": "101", "section": "01"}
155    parse_course_code("invalid")    -> {"department": None, "number": None, "section": None}
156    @endcode
157    """
158    match = re.match(r"([A-Z]+)\s+(\d+)-(\w+)", code)
159    if match:
160        return {"department": match.group(1), "number": match.group(2), "section": match.group(3)}
161    return {"department": None, "number": None, "section": None}

@brief Splits a full course code string into department, number, and section.

@details QCC course codes follow the format "DEPT NUM-SECTION" (e.g. "ACC 101-01"). This function extracts each component using a regex match.

@param code The full course code string (e.g. "ACC 101-01"). @return A dictionary with keys: - "department" (str or None): e.g. "ACC" - "number" (str or None): e.g. "101" - "section" (str or None): e.g. "01"

@par Example @code parse_course_code("ACC 101-01") -> {"department": "ACC", "number": "101", "section": "01"} parse_course_code("invalid") -> {"department": None, "number": None, "section": None} @endcode

def parse_registration_html(file_path: str) -> list: View Source

164def parse_registration_html(file_path: str) -> list:
165    """
166    @brief Parses a saved HTML registration page and returns a list of course section dicts.
167
168    @details
169    Reads the HTML file at the given path, finds all table rows, and extracts
170    course section data from rows that contain a valid course code (matching
171    the pattern "DEPT NUM-SECTION"). Skips header rows and empty rows.
172
173    The column layout detected from The Q portal is:
174    - [idx+0] course_code   e.g. "ACC 101-01"
175    - [idx+1] name          e.g. "Financial Accounting I"
176    - [idx+2] req           (empty, skipped)
177    - [idx+3] note          (empty, skipped)
178    - [idx+4] seats         e.g. "1 ∕ 24"
179    - [idx+5] status        e.g. "Reopened"
180    - [idx+6] details       e.g. "Instructor / Days;Location / Method"
181    - [idx+7] credits       e.g. "3.00"
182    - [idx+8] begin_date    e.g. "01/26/2026"
183    - [idx+9] end_date      e.g. "05/19/2026"
184
185    @param file_path Path to the locally saved HTML file.
186    @return A list of dictionaries, each representing one course section with keys:
187            course_code, department, course_number, section, name, status,
188            seats_open, seats_total, credits, instructor, days_time, location,
189            method, begin_date, end_date, scraped_at.
190
191    @throws FileNotFoundError if the HTML file does not exist at the given path.
192    """
193    with open(file_path, "r", encoding="utf-8") as f:
194        soup = BeautifulSoup(f, "html.parser")
195
196    rows = soup.select("table tr")
197    courses = []
198
199    for row in rows:
200        cols = row.find_all(["td", "th"])
201        if len(cols) < 4:
202            continue
203
204        values = [clean_text(col.get_text(" ", strip=True)) for col in cols]
205
206        # Skip header and empty rows
207        joined = " | ".join(values).lower()
208        if "course code" in joined and "name" in joined:
209            continue
210        if all(not v for v in values):
211            continue
212
213        # Find the course code column (e.g. "ACC 101-01")
214        idx = None
215        for i, v in enumerate(values):
216            if re.match(r"[A-Z]{2,4}\s+\d{3}-\w+", v):
217                idx = i
218                break
219
220        if idx is None:
221            continue
222
223        try:
224            code_str   = values[idx]
225            name       = values[idx + 1]
226            seats_str  = values[idx + 4]
227            status     = values[idx + 5]
228            details    = values[idx + 6]
229            credits    = values[idx + 7]
230            begin_date = values[idx + 8] if len(values) > idx + 8 else None
231            end_date   = values[idx + 9] if len(values) > idx + 9 else None
232        except IndexError:
233            continue
234
235        seats      = parse_seats(seats_str)
236        detail_map = parse_details(details)
237        code_map   = parse_course_code(code_str)
238
239        courses.append({
240            "course_code":   code_str,
241            "department":    code_map["department"],
242            "course_number": code_map["number"],
243            "section":       code_map["section"],
244            "name":          name,
245            "status":        status,
246            "seats_open":    seats["open"],
247            "seats_total":   seats["total"],
248            "credits":       credits,
249            "instructor":    detail_map["instructor"],
250            "days_time":     detail_map["days_time"],
251            "location":      detail_map["location"],
252            "method":        detail_map["method"],
253            "begin_date":    begin_date,
254            "end_date":      end_date,
255            "scraped_at":    datetime.now().isoformat(),
256        })
257
258    return courses

@brief Parses a saved HTML registration page and returns a list of course section dicts.

@details Reads the HTML file at the given path, finds all table rows, and extracts course section data from rows that contain a valid course code (matching the pattern "DEPT NUM-SECTION"). Skips header rows and empty rows.

The column layout detected from The Q portal is:

[idx+0] course_code e.g. "ACC 101-01"
[idx+1] name e.g. "Financial Accounting I"
[idx+2] req (empty, skipped)
[idx+3] note (empty, skipped)
[idx+4] seats e.g. "1 ∕ 24"
[idx+5] status e.g. "Reopened"
[idx+6] details e.g. "Instructor / Days;Location / Method"
[idx+7] credits e.g. "3.00"
[idx+8] begin_date e.g. "01/26/2026"
[idx+9] end_date e.g. "05/19/2026"

@param file_path Path to the locally saved HTML file. @return A list of dictionaries, each representing one course section with keys: course_code, department, course_number, section, name, status, seats_open, seats_total, credits, instructor, days_time, location, method, begin_date, end_date, scraped_at.

@throws FileNotFoundError if the HTML file does not exist at the given path.