2022ST-project-Phikipathia/cgi-bin/scraper.py

#!/usr/bin/env python3
"""
Phikipathia Scraper Engine (cgi-bin/scraper.py)

Finds the semantic path between Wikipedia articles leading to a target article
(defaulting to "Philosophy" / "Filosofie" / "Philosophie") following the
'Getting to Philosophy' rule: the first valid non-parenthetical link in the main
body text of each article, with smart re-routing to bypass loops.
"""

import json
import os
import sys
from typing import List, Optional, Tuple
from urllib.parse import quote, unquote, urlparse
from bs4 import BeautifulSoup, NavigableString, Tag
import requests

# Default HTTP User-Agent adhering to Wikimedia API policy
DEFAULT_USER_AGENT = (
    "Phikipathia/1.0 (https://git.depeuter.dev/tdpeuter/2022ST-project-Phikipathia; tibo@depeuter.dev)"
)

# Per-request HTTP timeout in seconds
REQUEST_TIMEOUT = 15

# Maximum allowed hops per path search to prevent infinite exploration
MAX_HOPS = 100

# Blacklisted non-article MediaWiki namespaces (case-insensitive prefixes)
EXCLUDED_NAMESPACES = (
    # English
    "file:", "help:", "special:", "wikipedia:", "talk:", "category:",
    "portal:", "template:", "template_talk:", "user:", "user_talk:",
    "media:", "draft:", "draft_talk:", "module:", "module_talk:",
    "mediawiki:", "mediawiki_talk:", "timedtext:", "topic:",
    "portal_talk:", "category_talk:", "help_talk:", "wikipedia_talk:",
    # Dutch (nl)
    "bestand:", "hulp:", "speciaal:", "overleg:", "categorie:",
    "portaal:", "sjabloon:", "sjabloon_overleg:", "gebruiker:", "gebruiker_overleg:",
    "overleg_bestand:", "overleg_hulp:", "overleg_categorie:", "overleg_portaal:",
    "overleg_sjabloon:", "overleg_gebruiker:", "overleg_module:",
    # French (fr)
    "fichier:", "aide:", "spécial:", "discussion:", "catégorie:",
    "portail:", "modèle:", "discussion_modèle:", "utilisateur:", "discussion_utilisateur:",
    "discussion_fichier:", "discussion_aide:", "discussion_catégorie:", "discussion_portail:",
    "discussion_projet:", "projet:", "référence:", "discussion_référence:",
    # German (de)
    "datei:", "hilfe:", "spezial:", "diskussion:", "kategorie:",
    "vorlage:", "vorlagendiskussion:", "benutzer:", "benutzer_diskussion:",
    "datei_diskussion:", "hilfe_diskussion:", "kategorie_diskussion:", "portal_diskussion:",
    "portal:", "medium:",
    # Interwiki prefixes & Wikimedia projects
    "w:", "s:", "b:", "q:", "n:", "v:", "voy:", "d:", "m:", "meta:",
    "commons:", "wikidata:", "wikinews:", "wikiquote:", "wikibooks:",
    "wikisource:", "wikiversity:", "wikivoyage:", "species:", "wikt:",
    "wiktionary:", "incubator:", "foundation:", "phabricator:", "phab:",
)

# CSS selectors for elements that must be stripped before link extraction
BOILERPLATE_SELECTORS = (
    # Infoboxes & sidebars
    "table.infobox", "div.infobox", "table.sidebar", "div.sidebar",
    "div.vertical-navbox", "div.navbox", "table.vertical-navbox", "table.navbox",
    ".infobox", ".infobox_v3", ".ib-country",
    # Hatnotes & disambiguation notices
    "div.hatnote", "div.dablink", "div.rellink", 'div[role="note"]',
    "div.shortdescription", ".hatnote", ".dablink", ".rellink",
    # Maintenance message boxes & stub templates
    "table.ambox", "table.cmbox", "table.tmbox", "table.fmbox", "div.asbox",
    "div.metadata", "div.boilerplate_metadata",
    # Table of contents
    "nav#toc", "div#toc", "div.toc", ".toc", ".mw-table-of-contents-container",
    # Coordinates
    "span#coordinates", "div#coordinates", "span.coordinates", "div.coordinates",
    "span.geo-default", "span.geo-dec", "span.geo-dms", "span.geo",
    "#mw-indicator-coordinates", ".mw-indicator-coordinates",
    # References & citations & footnotes
    "sup.reference", "ol.references", "div.reflist", "div.references",
    "span.reference-text", "cite.citation", ".reflist", ".references",
    # Audio, IPA & pronunciation annotations
    "span.haudio", "span.IPA", "span.unicode", "div.audioplayer",
    "span.audiolink", "span.audio-button", "span.audio", ".phonetics",
    # Thumbnails, images & media containers
    "div.thumb", "figure", "div.floatright", "div.floatleft",
    "div.tright", "div.tleft", "span.mw-image-border",
    # UI controls & print artifacts
    "span.mw-editsection", "div.mw-jump-link", "div.noprint", "div.printfooter",
    # Empty placeholder elements
    "p.mw-empty-elt",
)


def get_user_agent() -> str:
    """
    Returns the configured User-Agent string with USER_AGENT environment variable override.
    """
    env_ua = os.environ.get("USER_AGENT", "").strip()
    return env_ua if env_ua else DEFAULT_USER_AGENT


def converteer(tekst: Optional[str], gebruiker: bool = True) -> Optional[str]:
    """
    Converts between URL-encoded slugs and display article titles.

    >>> converteer('Northwestern Europe')
    'Northwestern Europe'
    >>> converteer('Northwestern_Europe', False)
    'Northwestern_Europe'
    >>> converteer('Medchal%E2%80%93Malkajgiri_district')
    'Medchal–Malkajgiri district'
    >>> converteer('Medchal–Malkajgiri district', False)
    'Medchal%E2%80%93Malkajgiri_district'
    >>> converteer('C%23_(programming_language)')
    'C# (programming language)'
    >>> converteer('Caf%C3%A9#Etymology')
    'Café'
    >>> converteer(None) is None
    True
    """
    if tekst is None:
        return None

    if gebruiker:
        return unquote(str(tekst).split("#")[0]).replace("_", " ")
    else:
        return quote(str(tekst).replace(" ", "_"))


def extract_wiki_target(href: Optional[str]) -> Optional[str]:
    """
    Extracts the article slug from a wiki href (supports both /wiki/Slug and https://xx.wikipedia.org/wiki/Slug).
    """
    if not href or not isinstance(href, str):
        return None

    if "/wiki/" not in href:
        return None

    parts = href.split("/wiki/", 1)
    prefix, target = parts[0], parts[1]

    if prefix:
        if prefix.startswith("http://") or prefix.startswith("https://") or prefix.startswith("//"):
            parsed = urlparse(prefix if not prefix.startswith("//") else "https:" + prefix)
            if not parsed.netloc.endswith("wikipedia.org"):
                return None
        elif prefix not in ("", "."):
            return None

    target = target.split("#")[0].split("?")[0].strip()
    return target if target else None


def is_valid_link(a_tag: Tag, current_title: Optional[str] = None) -> bool:
    """
    Determines if an <a> tag is a valid main-namespace Wikipedia article link.
    """
    if not isinstance(a_tag, Tag) or a_tag.name != "a":
        return False

    href = a_tag.get("href")
    target = extract_wiki_target(href)
    if not target:
        return False

    # Filter out non-article namespaces and interwiki prefixes
    target_lower = target.lower()
    if any(target_lower.startswith(ns) for ns in EXCLUDED_NAMESPACES):
        return False

    if ":" in target:
        prefix_part = target.split(":", 1)[0]
        if (prefix_part.lower() + ":") in EXCLUDED_NAMESPACES:
            return False
        if len(prefix_part) in (2, 3) and prefix_part.isalpha():
            return False

    # Skip red links (non-existent pages) and self-links
    classes = a_tag.get("class", [])
    if isinstance(classes, str):
        classes = classes.split()
    if "new" in classes or "mw-selflink" in classes or "selflink" in classes or "external" in classes:
        return False

    # Skip image and file links
    if "image" in classes or "mw-file-description" in classes:
        return False
    if a_tag.find("img") is not None:
        return False

    # Skip italicized links (e.g. hatnotes or foreign translations)
    if a_tag.find_parent(["i", "em"]) is not None:
        return False

    # Check for inline style font-style: italic on parent hierarchy
    curr = a_tag.parent
    while curr and hasattr(curr, "get"):
        style = curr.get("style", "")
        if "italic" in style:
            return False
        if curr.name in ("p", "div", "body", "[document]"):
            break
        curr = curr.parent

    # Skip self-links matching current article title
    if current_title:
        clean_target = converteer(target, True).strip().lower()
        clean_current = converteer(current_title, True).strip().lower()
        if clean_target == clean_current:
            return False

    return True


def is_excluded_inline(tag: Tag) -> bool:
    """
    Checks if an inline tag should be skipped during parenthesis traversal.
    """
    if not isinstance(tag, Tag):
        return False
    if tag.name in ("sup", "table", "figure", "style", "script", "nav", "math"):
        return True
    classes = tag.get("class", [])
    if isinstance(classes, str):
        classes = classes.split()
    excluded_classes = {
        "infobox", "sidebar", "hatnote", "toc", "reference", "reflist",
        "haudio", "audiolink", "IPA", "mw-empty-elt", "navbox",
        "vertical-navbox", "shortdescription", "coordinates", "geo",
        "audioplayer", "mw-editsection", "noprint", "mw-jump-link",
    }
    if any(c in excluded_classes for c in classes):
        return True
    if tag.get("id") in ("coordinates", "toc", "mw-indicator-coordinates"):
        return True
    return False


def extract_all_links_from_block(block: Tag, current_title: Optional[str] = None) -> List[str]:
    """
    Traverses a content block in document order, tracking parenthesis depth
    strictly across text nodes (NavigableString) to find all valid
    non-parenthetical links.
    """
    paren_depth = 0
    links: List[str] = []

    def walk(node):
        nonlocal paren_depth

        if isinstance(node, NavigableString):
            for ch in str(node):
                if ch == "(":
                    paren_depth += 1
                elif ch == ")":
                    paren_depth = max(0, paren_depth - 1)
            return

        if isinstance(node, Tag):
            if is_excluded_inline(node):
                return

            if node.name == "a":
                if paren_depth == 0 and is_valid_link(node, current_title):
                    target = extract_wiki_target(node.get("href", ""))
                    if target and target not in links:
                        links.append(target)

                # Traverse anchor children to ensure any parens inside text update depth
                for child in node.children:
                    walk(child)
                return

            for child in node.children:
                walk(child)

    walk(block)
    return links


def clean_soup(soup: BeautifulSoup) -> Tag:
    """
    Finds the main content container and decomposes all boilerplate elements.
    Returns the cleaned content root Tag.
    """
    content_div = soup.find("div", id="mw-content-text")
    if content_div:
        p_out = content_div.find("div", class_="mw-parser-output")
        root = p_out if p_out else content_div
    else:
        p_out = soup.find("div", class_="mw-parser-output")
        if p_out:
            root = p_out
        else:
            body_content = soup.find("div", id="bodyContent")
            if body_content:
                root = body_content
            else:
                root = soup.body if soup.body else soup

    for selector in BOILERPLATE_SELECTORS:
        for el in root.select(selector):
            el.decompose()

    return root


def zoek_kandidaat_links(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> List[str]:
    """
    Extracts all candidate valid Wikipedia links from the article in document order.
    """
    if soep is None:
        return []

    content_root = clean_soup(soep)
    all_links: List[str] = []

    content_blocks = content_root.find_all(["p", "ul", "ol"])
    for block in content_blocks:
        classes = block.get("class", [])
        if isinstance(classes, str):
            classes = classes.split()
        if "mw-empty-elt" in classes or not block.get_text(strip=True):
            continue

        block_links = extract_all_links_from_block(block, current_title)
        for link in block_links:
            if link not in all_links:
                all_links.append(link)

    return all_links


def zoek_link(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> Optional[str]:
    """
    Searches the provided BeautifulSoup document for the first valid Wikipedia link.

    >>> html_basic = '''
    ... <div id="mw-content-text">
    ...   <div class="mw-parser-output">
    ...     <p class="mw-empty-elt"></p>
    ...     <div class="hatnote">For other uses, see <a href="/wiki/Belgium_(disambiguation)">Belgium (disambiguation)</a>.</div>
    ...     <table class="infobox"><tr><td><a href="/wiki/Infobox_Link">Capital</a></td></tr></table>
    ...     <p>
    ...       <b>Belgium</b> (<span class="IPA">/ˈbɛldʒəm/</span>; Dutch: <i lang="nl">België</i>, <a href="/wiki/Help:IPA">help</a>; <a href="/wiki/File:Audio.ogg">listen</a>)
    ...       is a country in <a href="/wiki/Northwestern_Europe">Northwestern Europe</a>.
    ...     </p>
    ...   </div>
    ... </div>
    ... '''
    >>> zoek_link(BeautifulSoup(html_basic, 'html.parser'))
    'Northwestern_Europe'
    """
    links = zoek_kandidaat_links(soep, current_title)
    return links[0] if links else None


def tussen_haakjes(zin: str, omgeving: str) -> bool:
    """
    Legacy helper checking if a phrase is inside parentheses.
    Preserved for backward compatibility.
    """
    s_omg = str(omgeving)
    s_zin = str(zin)
    idx = s_omg.find(s_zin)
    if idx == -1:
        return False
    open_count = s_omg.count("(", 0, idx)
    close_count = s_omg.count(")", 0, idx)
    return open_count > close_count


def fetch_wikipedia_page(
    session: requests.Session,
    url: str,
    headers: dict
) -> Tuple[Optional[BeautifulSoup], Optional[str], Optional[str]]:
    """
    Fetches a Wikipedia page following redirects.
    Returns (soup, canonical_resolved_title, error_type).
    """
    try:
        response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT, allow_redirects=True)
    except (requests.exceptions.ConnectionError, requests.exceptions.InvalidURL):
        return None, None, "dns_error"
    except requests.exceptions.Timeout:
        return None, None, "timeout_error"
    except Exception:
        return None, None, "request_error"

    if response.status_code != 200:
        return None, None, f"http_{response.status_code}"

    # Extract canonical title from final redirected URL
    parsed_path = urlparse(response.url).path
    canonical_title = None
    if parsed_path.startswith("/wiki/"):
        canonical_title = parsed_path[6:].split("#")[0].split("?")[0]

    soup = BeautifulSoup(response.content, "html.parser")
    return soup, canonical_title, None


import re

def is_valid_lang_code(taal: str) -> bool:
    """
    Validates Wikipedia language subdomain string to prevent SSRF or injection.
    Allows 2-10 lowercase alphanumeric characters/hyphens (e.g. 'en', 'nl', 'zh-min-nan', 'simple').
    """
    if not taal or not isinstance(taal, str):
        return False
    return bool(re.match(r"^[a-z]{2,10}(-[a-z0-9]+)?$", taal.strip().lower()))


def run(taal: str, start: str, stop: Optional[str] = "Philosophy", max_hops: int = MAX_HOPS) -> dict:
    """
    Executes the Wikipedia path traversal from start to stop in the specified language,
    with smart re-routing if the primary link creates a loop.
    """
    if not is_valid_lang_code(taal):
        return {
            "pad": [],
            "status": "error",
            "stappen": 0,
            "herleid": 0,
            "error": f"Ongeldige taalcode '{taal}'. Gebruik een geldige Wikipedia taalcode (bijv. 'en', 'nl', 'fr')."
        }

    if not stop:
        stop = "Filosofie" if taal == "nl" else ("Philosophie" if taal == "fr" else "Philosophy")

    session = requests.Session()
    headers = {"User-Agent": get_user_agent()}
    base_url = f"https://{taal}.wikipedia.org/wiki/"

    pad: List[str] = []
    visited: set[str] = set()
    current_title = start
    herleid_count = 0

    for hop in range(max_hops + 1):
        if hop >= max_hops:
            return {
                "pad": pad,
                "status": "max_hops",
                "stappen": len(pad) - 1,
                "herleid": herleid_count,
                "error": f"Maximum aantal stappen ({max_hops}) bereikt voor '{pad[0]}'."
            }

        target_url = base_url + converteer(current_title, False)
        soup, canonical_slug, err = fetch_wikipedia_page(session, target_url, headers)

        if err is not None or soup is None:
            if len(pad) == 0:
                err_msg = f"Er ging iets fout bij het inladen van '{start}'. Bestaat de taalcode of website?"
            else:
                err_msg = f"Er ging iets fout bij het inladen van '{pad[-1]}', vertrekkende vanaf '{pad[0]}'."
            return {
                "pad": pad,
                "status": "error",
                "stappen": len(pad) - 1,
                "herleid": herleid_count,
                "error": err_msg
            }

        resolved_title = converteer(canonical_slug if canonical_slug else current_title, True)

        if len(pad) == 0:
            pad.append(resolved_title)
            visited.add(resolved_title.lower())
        elif pad[-1] != resolved_title:
            if resolved_title.lower() in visited:
                return {
                    "pad": pad,
                    "status": "cyclus",
                    "cyclus_op": resolved_title,
                    "stappen": len(pad) - 1,
                    "herleid": herleid_count,
                    "error": f"Cyclus gedetecteerd op '{resolved_title}', startende vanaf '{pad[0]}'."
                }
            pad.append(resolved_title)
            visited.add(resolved_title.lower())

        # Check if destination article reached
        if resolved_title.lower() == stop.lower():
            return {
                "pad": pad,
                "status": "success",
                "stappen": len(pad) - 1,
                "herleid": herleid_count
            }

        # Option 2: Smart Re-routing among candidate links on current page
        candidate_slugs = zoek_kandidaat_links(soup, resolved_title)
        if not candidate_slugs:
            return {
                "pad": pad,
                "status": "doodlopend",
                "stappen": len(pad) - 1,
                "herleid": herleid_count,
                "error": f"Er konden geen nieuwe links meer gevonden worden op '{pad[-1]}'."
            }

        # Find first candidate link that does NOT lead into an already visited page
        selected_slug = None
        for idx, cand_slug in enumerate(candidate_slugs):
            cand_title = converteer(cand_slug, True)
            if cand_title.lower() not in visited and cand_title not in pad:
                selected_slug = cand_slug
                if idx > 0:
                    herleid_count += 1
                break

        if selected_slug is None:
            # All candidate links on this page lead to visited pages -> cycle
            primary_title = converteer(candidate_slugs[0], True)
            return {
                "pad": pad,
                "status": "cyclus",
                "cyclus_op": primary_title,
                "stappen": len(pad) - 1,
                "herleid": herleid_count,
                "error": f"Cyclus gedetecteerd op '{primary_title}', startende vanaf '{pad[0]}'."
            }

        current_title = selected_slug

    return {
        "pad": pad,
        "status": "max_hops",
        "stappen": len(pad) - 1,
        "herleid": herleid_count,
        "error": f"Maximum aantal stappen ({max_hops}) bereikt voor '{pad[0]}'."
    }


def main():
    if len(sys.argv) > 1 and sys.argv[1] == "--test":
        import doctest
        results = doctest.testmod()
        print(f"Doctests: {results}")
        sys.exit(0 if results.failed == 0 else 1)
    elif len(sys.argv) > 1:
        taal = sys.argv[1] if len(sys.argv) > 1 else "en"
        start = sys.argv[2] if len(sys.argv) > 2 else "Special:Random"
        stop = sys.argv[3] if len(sys.argv) > 3 else "Philosophy"
        antwoord = run(taal, start, stop)
        print(json.dumps(antwoord, indent=2, ensure_ascii=False))
    else:
        import doctest
        results = doctest.testmod()
        print(f"Doctests: {results}")
        sys.exit(0 if results.failed == 0 else 1)


if __name__ == "__main__":
    main()