scraping.scrape_catalog

  1# =============================================================================
  2#
  3# Copyright 2026 
  4#
  5# Author:   Noel Mensah
  6# GitHub:   https://github.com/LSilver17/CSC212---AI-Agent
  7# 
  8# =============================================================================
  9
 10"""
 11@file scrape_catalog.py
 12@brief Scrapes the QCC course catalog from The Q portal for unique course details.
 13
 14@details
 15This module uses Playwright to automate a headless Chromium browser and navigate
 16The Q portal's course search page. For each unique course code discovered across
 17all departments, it clicks into the course detail page and extracts:
 18    - Course name
 19    - Course description (with SQL apostrophe artifacts cleaned)
 20    - Credits
 21    - Prerequisites
 22    - Semesters offered
 23
 24Only one section per unique base course code is scraped (e.g. ACC 101, not ACC 101-01
 25and ACC 101-02 separately), making this a catalog scraper rather than a section scraper.
 26
 27@date 2026
 28
 29@par Input
 30    The Q portal — live web scraping via Playwright
 31
 32@par Output
 33    course_catalog.json — list of unique course dictionaries
 34
 35@par Dependencies
 36    - playwright (pip install playwright && playwright install chromium)
 37    - beautifulsoup4
 38    - json (stdlib)
 39    - re (stdlib)
 40    - asyncio (stdlib)
 41
 42@par Usage
 43    python scrape_catalog.py
 44"""
 45
 46import json
 47import re
 48import asyncio
 49from datetime import datetime
 50from bs4 import BeautifulSoup
 51from playwright.async_api import async_playwright
 52
 53## @brief URL of The Q portal's advanced course search page.
 54SEARCH_URL = "https://theq.qcc.edu/ICS/Course_Offerings_and_Schedule.jnz?portlet=AddDrop_Courses&screen=Advanced+Course+Search&screenType=next"
 55
 56## @brief Academic term label to select from the portal dropdown.
 57TERM       = "Spring 2026"
 58
 59## @brief Output path for the scraped catalog JSON file.
 60OUTPUT     = "course_catalog.json"
 61
 62## @brief Whether to run the browser in headless (no UI) mode.
 63HEADLESS   = True
 64
 65
 66def clean_text(text):
 67    """
 68    @brief Removes excess whitespace from a string.
 69
 70    @param text The raw string to clean.
 71    @return A normalized single-line string with no extra whitespace.
 72    """
 73    return " ".join(text.split()).strip()
 74
 75
 76def fix_apostrophes(text):
 77    """
 78    @brief Fixes SQL-escaped double apostrophes from The Q portal database.
 79
 80    @details
 81    The Q portal stores course descriptions in a Jenzabar SQL database that
 82    escapes apostrophes as double apostrophes ('').  These artifacts bleed
 83    through into the scraped text (e.g. "company''s assets"). This function
 84    replaces all occurrences of '' with a single apostrophe.
 85
 86    @param text The raw string potentially containing double apostrophes.
 87    @return The cleaned string with proper single apostrophes, or None if input is None.
 88
 89    @par Example
 90    @code
 91    fix_apostrophes("company''s assets") -> "company's assets"
 92    fix_apostrophes(None)                -> None
 93    @endcode
 94    """
 95    if text:
 96        return text.replace("''", "'")
 97    return text
 98
 99
100def parse_detail_page(html):
101    """
102    @brief Parses a course detail page from The Q portal and extracts course fields.
103
104    @details
105    The Q portal renders course details inside a span with id "pg0_V_lblCourseDescValue".
106    The content is a mix of text nodes and <br> tags forming a block of labeled fields.
107
108    Extraction strategy:
109    - Course name: from the <h5> tag (format: "Course Name (DEPT NUM-SECTION)")
110    - Description: all text lines before the "Credits:" label
111    - Credits: regex match on "Credits: N" or "N Credits"
112    - Prerequisites: regex match on "Prerequisites: ..." stopping before "Semester Offered"
113    - Semesters offered: regex match on "Semester Offered: ..."
114
115    Double apostrophes in description and prerequisites are cleaned via fix_apostrophes().
116
117    @param html Raw HTML string of the course detail page.
118    @return A dictionary with keys:
119            - "name" (str or None)
120            - "description" (str or None)
121            - "credits" (str or None)
122            - "prerequisites" (str or None)
123            - "semesters_offered" (str or None)
124    """
125    soup = BeautifulSoup(html, "html.parser")
126    result = {
127        "name":              None,
128        "description":       None,
129        "credits":           None,
130        "prerequisites":     None,
131        "semesters_offered": None,
132    }
133
134    # Course name from the h5 tag (e.g. "Financial Accounting I (ACC 101-01)")
135    h5 = soup.find("h5")
136    if h5:
137        name_text = clean_text(h5.get_text())
138        name_match = re.match(r"^(.+?)\s*\([A-Z]", name_text)
139        if name_match:
140            result["name"] = name_match.group(1).strip()
141
142    # Description span
143    desc_span = soup.find("span", id="pg0_V_lblCourseDescValue")
144    if not desc_span:
145        return result
146
147    # Get text lines split by <br> tags
148    lines = []
149    for item in desc_span.children:
150        if hasattr(item, "get_text"):
151            text = clean_text(item.get_text())
152        else:
153            text = clean_text(str(item))
154        if text:
155            lines.append(text)
156
157    full = " ".join(lines)
158
159    # Description = all lines before "Credits:"
160    desc_lines = []
161    for line in lines:
162        if re.match(r"credits:", line, re.IGNORECASE):
163            break
164        desc_lines.append(line)
165    if desc_lines:
166        result["description"] = fix_apostrophes(" ".join(desc_lines))
167
168    # Credits — match "Credits: 3" or "3 Credits" or "3 credit hours"
169    credit_match = re.search(
170        r"Credits?:\s*(\d+(?:\.\d+)?)|(\d+(?:\.\d+)?)\s*[Cc]redit",
171        full
172    )
173    if credit_match:
174        result["credits"] = credit_match.group(1) or credit_match.group(2)
175
176    # Prerequisites — stop at Semester Offered or Credits
177    prereq_match = re.search(
178        r"Prerequisite[s]?:\s*(.+?)(?=\s*Semester Offered|\s*Credits?:|$)",
179        full, re.IGNORECASE
180    )
181    if prereq_match:
182        prereq = clean_text(prereq_match.group(1))
183        if "semester offered" not in prereq.lower():
184            result["prerequisites"] = fix_apostrophes(prereq)
185
186    # Semesters offered — allow upper and lowercase, stop at Credits
187    sem_match = re.search(
188        r"Semester[s]? Offered:\s*([A-Za-z/,\s]+?)(?=\s*Credits?:|$)",
189        full, re.IGNORECASE
190    )
191    if sem_match:
192        result["semesters_offered"] = clean_text(sem_match.group(1))
193
194    return result
195
196
197async def get_search_results(page, dept_value, dept_label):
198    """
199    @brief Loads the search results page for a department and returns course section links.
200
201    @details
202    Navigates to the course search page, selects the given term and department,
203    submits the search, and collects all course section links from the results table.
204    Retries up to 3 times on failure with a 2-second delay between attempts.
205
206    @param page The Playwright page object.
207    @param dept_value The dropdown option value for the department.
208    @param dept_label The human-readable department label (used for logging).
209    @return A list of (section_code, href) tuples for all course links found,
210            or an empty list if all attempts fail.
211    """
212    for attempt in range(3):
213        try:
214            await page.goto(SEARCH_URL, timeout=120000)
215            await page.wait_for_load_state("networkidle", timeout=120000)
216            await page.wait_for_selector("#pg0_V_ddlDept", timeout=60000)
217            await page.select_option("#pg0_V_ddlTerm", label=TERM)
218            await page.select_option("#pg0_V_ddlDept", value=dept_value)
219            await page.click("#pg0_V_btnSearch")
220            await page.wait_for_load_state("networkidle", timeout=120000)
221            await page.wait_for_timeout(1500)
222
223            course_links = await page.query_selector_all("table a")
224            results = []
225            for link in course_links:
226                text = (await link.inner_text()).strip()
227                href = await link.get_attribute("href") or ""
228                if re.match(r"[A-Z]{2,4}\s+\d{3}-\w+", text) and "lnkCourse" in href:
229                    results.append((text, href))
230            return results
231        except Exception as e:
232            if attempt < 2:
233                print(f"  ⚠ Retry {attempt+1}...")
234                await asyncio.sleep(2)
235            else:
236                print(f"  ✗ Failed: {e}")
237                return []
238
239
240async def scrape_catalog():
241    """
242    @brief Asynchronously scrapes all unique courses from The Q portal catalog.
243
244    @details
245    Iterates through all departments on The Q portal. For each department,
246    retrieves the list of course sections and identifies unique base course codes
247    (e.g. "ACC 101" from "ACC 101-01"). For each unique code not yet scraped,
248    navigates to the course detail page using a JavaScript postback and extracts
249    course details via parse_detail_page().
250
251    Courses already scraped in a previous department are skipped to avoid
252    duplicates in the output catalog.
253
254    @return A list of unique course dictionaries with fields:
255            course_code, department, course_number, name, description,
256            credits, prerequisites, semesters_offered, scraped_at.
257    """
258    catalog = {}
259
260    async with async_playwright() as p:
261        browser = await p.chromium.launch(headless=HEADLESS)
262        page = await browser.new_page()
263
264        print("Loading QCC course search page...")
265        await page.goto(SEARCH_URL, timeout=120000)
266        await page.wait_for_load_state("networkidle", timeout=120000)
267
268        # Get departments
269        dept_select = await page.query_selector("#pg0_V_ddlDept")
270        dept_options = await dept_select.query_selector_all("option")
271        departments = []
272        for opt in dept_options:
273            value = await opt.get_attribute("value")
274            label = (await opt.inner_text()).strip()
275            if label.lower() != "all" and label:
276                departments.append((value, label))
277
278        print(f"Found {len(departments)} departments.\n")
279
280        for dept_idx, (dept_value, dept_label) in enumerate(departments):
281            print(f"[{dept_idx+1}/{len(departments)}] {dept_label}")
282
283            course_links = await get_search_results(page, dept_value, dept_label)
284            print(f"  Found {len(course_links)} sections")
285
286            scraped_base = set()
287
288            for section_code, href in course_links:
289                base_match = re.match(r"([A-Z]{2,4}\s+\d+)", section_code)
290                if not base_match:
291                    continue
292                base_code = base_match.group(1)
293
294                if base_code in scraped_base or base_code in catalog:
295                    scraped_base.add(base_code)
296                    continue
297                scraped_base.add(base_code)
298
299                # Re-load search results before each course click
300                course_links_fresh = await get_search_results(page, dept_value, dept_label)
301
302                # Find this course's link in fresh results
303                target_href = None
304                for sc, href2 in course_links_fresh:
305                    bm = re.match(r"([A-Z]{2,4}\s+\d+)", sc)
306                    if bm and bm.group(1) == base_code:
307                        target_href = href2
308                        break
309
310                if not target_href:
311                    continue
312
313                try:
314                    postback_match = re.search(r"__doPostBack\('([^']+)'", target_href)
315                    if not postback_match:
316                        continue
317                    target = postback_match.group(1)
318
319                    await page.evaluate(f"__doPostBack('{target}', '')")
320                    await page.wait_for_load_state("networkidle", timeout=30000)
321                    try:
322                        await page.wait_for_selector("#pg0_V_lblCourseDescValue", timeout=8000)
323                    except Exception:
324                        pass
325
326                    html = await page.content()
327                    detail = parse_detail_page(html)
328
329                    dept_match = re.match(r"([A-Z]+)", base_code)
330                    dept_code = dept_match.group(1) if dept_match else None
331                    num_match = re.search(r"\d+", base_code)
332                    course_num = num_match.group(0) if num_match else None
333
334                    catalog[base_code] = {
335                        "course_code":       base_code,
336                        "department":        dept_code,
337                        "course_number":     course_num,
338                        "name":              detail["name"],
339                        "description":       detail["description"],
340                        "credits":           detail["credits"],
341                        "prerequisites":     detail["prerequisites"],
342                        "semesters_offered": detail["semesters_offered"],
343                        "scraped_at":        datetime.now().isoformat(),
344                    }
345
346                    desc_preview = detail["description"][:60] if detail["description"] else "no description"
347                    print(f"    ✓ {base_code} — {desc_preview}...")
348
349                except Exception as e:
350                    print(f"    ✗ Error on {base_code}: {e}")
351                    continue
352
353        await browser.close()
354
355    return list(catalog.values())
356
357
358async def main():
359    """
360    @brief Entry point — runs the catalog scraper and saves results to JSON.
361
362    @details
363    Calls scrape_catalog() to collect all unique course data, writes the
364    results to course_catalog.json, and prints a summary including total
365    courses scraped and elapsed time.
366    """
367    start = datetime.now()
368    print("=" * 50)
369    print("  QCC Course Catalog Scraper")
370    print(f"  {start.strftime('%Y-%m-%d %H:%M:%S')}")
371    print("=" * 50)
372
373    courses = await scrape_catalog()
374
375    with open(OUTPUT, "w", encoding="utf-8") as f:
376        json.dump(courses, f, indent=2)
377
378    elapsed = (datetime.now() - start).seconds
379    print(f"\n✅ Done in {elapsed}s")
380    print(f"   Scraped {len(courses)} unique courses")
381    print(f"   Saved to {OUTPUT}")
382    print("=" * 50)
383
384
385if __name__ == "__main__":
386    asyncio.run(main())