diff --git a/cgi-bin/scraper.py b/cgi-bin/scraper.py index 793c81c..c97c49e 100644 --- a/cgi-bin/scraper.py +++ b/cgi-bin/scraper.py @@ -1,18 +1,107 @@ #!/usr/bin/env python3 - """ -Dit is het 'CGI-script'. -De bestandsextensie werd aangepast om te kunnen debuggen op zowel Windows als Linux. +Phikipathia Scraper Engine (cgi-bin/scraper.py) + +Finds the semantic path between Wikipedia articles leading to a target article +(defaulting to "Philosophy" / "Filosofie" / "Philosophie") following the +'Getting to Philosophy' rule: the first valid non-parenthetical link in the main +body text of each article, with smart re-routing to bypass loops. """ -from bs4 import BeautifulSoup +import json +import os +import sys +from typing import List, Optional, Tuple +from urllib.parse import quote, unquote, urlparse +from bs4 import BeautifulSoup, NavigableString, Tag import requests +# Default HTTP User-Agent adhering to Wikimedia API policy +DEFAULT_USER_AGENT = ( + "Phikipathia/1.0 (https://git.depeuter.dev/tdpeuter/2022ST-project-Phikipathia; tibo@depeuter.dev)" +) -def converteer(tekst, gebruiker=True): +# Per-request HTTP timeout in seconds +REQUEST_TIMEOUT = 15 + +# Maximum allowed hops per path search to prevent infinite exploration +MAX_HOPS = 100 + +# Blacklisted non-article MediaWiki namespaces (case-insensitive prefixes) +EXCLUDED_NAMESPACES = ( + # English + "file:", "help:", "special:", "wikipedia:", "talk:", "category:", + "portal:", "template:", "template_talk:", "user:", "user_talk:", + "media:", "draft:", "draft_talk:", "module:", "module_talk:", + "mediawiki:", "mediawiki_talk:", "timedtext:", "topic:", + "portal_talk:", "category_talk:", "help_talk:", "wikipedia_talk:", + # Dutch (nl) + "bestand:", "hulp:", "speciaal:", "overleg:", "categorie:", + "portaal:", "sjabloon:", "sjabloon_overleg:", "gebruiker:", "gebruiker_overleg:", + "overleg_bestand:", "overleg_hulp:", "overleg_categorie:", "overleg_portaal:", + "overleg_sjabloon:", "overleg_gebruiker:", "overleg_module:", + # French (fr) + "fichier:", "aide:", "spécial:", "discussion:", "catégorie:", + "portail:", "modèle:", "discussion_modèle:", "utilisateur:", "discussion_utilisateur:", + "discussion_fichier:", "discussion_aide:", "discussion_catégorie:", "discussion_portail:", + "discussion_projet:", "projet:", "référence:", "discussion_référence:", + # German (de) + "datei:", "hilfe:", "spezial:", "diskussion:", "kategorie:", + "vorlage:", "vorlagendiskussion:", "benutzer:", "benutzer_diskussion:", + "datei_diskussion:", "hilfe_diskussion:", "kategorie_diskussion:", "portal_diskussion:", + "portal:", "medium:", + # Interwiki prefixes & Wikimedia projects + "w:", "s:", "b:", "q:", "n:", "v:", "voy:", "d:", "m:", "meta:", + "commons:", "wikidata:", "wikinews:", "wikiquote:", "wikibooks:", + "wikisource:", "wikiversity:", "wikivoyage:", "species:", "wikt:", + "wiktionary:", "incubator:", "foundation:", "phabricator:", "phab:", +) + +# CSS selectors for elements that must be stripped before link extraction +BOILERPLATE_SELECTORS = ( + # Infoboxes & sidebars + "table.infobox", "div.infobox", "table.sidebar", "div.sidebar", + "div.vertical-navbox", "div.navbox", "table.vertical-navbox", "table.navbox", + ".infobox", ".infobox_v3", ".ib-country", + # Hatnotes & disambiguation notices + "div.hatnote", "div.dablink", "div.rellink", 'div[role="note"]', + "div.shortdescription", ".hatnote", ".dablink", ".rellink", + # Maintenance message boxes & stub templates + "table.ambox", "table.cmbox", "table.tmbox", "table.fmbox", "div.asbox", + "div.metadata", "div.boilerplate_metadata", + # Table of contents + "nav#toc", "div#toc", "div.toc", ".toc", ".mw-table-of-contents-container", + # Coordinates + "span#coordinates", "div#coordinates", "span.coordinates", "div.coordinates", + "span.geo-default", "span.geo-dec", "span.geo-dms", "span.geo", + "#mw-indicator-coordinates", ".mw-indicator-coordinates", + # References & citations & footnotes + "sup.reference", "ol.references", "div.reflist", "div.references", + "span.reference-text", "cite.citation", ".reflist", ".references", + # Audio, IPA & pronunciation annotations + "span.haudio", "span.IPA", "span.unicode", "div.audioplayer", + "span.audiolink", "span.audio-button", "span.audio", ".phonetics", + # Thumbnails, images & media containers + "div.thumb", "figure", "div.floatright", "div.floatleft", + "div.tright", "div.tleft", "span.mw-image-border", + # UI controls & print artifacts + "span.mw-editsection", "div.mw-jump-link", "div.noprint", "div.printfooter", + # Empty placeholder elements + "p.mw-empty-elt", +) + + +def get_user_agent() -> str: """ - Ik blijf bewust case sensitive werken omdat dit héél soms wel nog belangrijk is om een verschl - te maken tussen twee linkjes. + Returns the configured User-Agent string with USER_AGENT environment variable override. + """ + env_ua = os.environ.get("USER_AGENT", "").strip() + return env_ua if env_ua else DEFAULT_USER_AGENT + + +def converteer(tekst: Optional[str], gebruiker: bool = True) -> Optional[str]: + """ + Converts between URL-encoded slugs and display article titles. >>> converteer('Northwestern Europe') 'Northwestern Europe' @@ -22,180 +111,450 @@ def converteer(tekst, gebruiker=True): 'Medchal–Malkajgiri district' >>> converteer('Medchal–Malkajgiri district', False) 'Medchal%E2%80%93Malkajgiri_district' + >>> converteer('C%23_(programming_language)') + 'C# (programming language)' + >>> converteer('Caf%C3%A9#Etymology') + 'Café' + >>> converteer(None) is None + True """ - - from urllib import parse - if tekst is None: return None if gebruiker: - return parse.unquote(tekst).replace('_', ' ').split('#')[0] + return unquote(str(tekst).split("#")[0]).replace("_", " ") else: - return parse.quote(tekst.replace(' ', '_')) + return quote(str(tekst).replace(" ", "_")) -def zoek_link(soep): +def extract_wiki_target(href: Optional[str]) -> Optional[str]: """ - Doorzoek de meegeleverde soep naar een geldige Wikipedia-link. - :return: een geldige link indien beschikbaar, anders - - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Belgium').content, 'html.parser')) - 'Northwestern_Europe' - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Glossary_of_engineering').content, 'html.parser')) - 'Glossary_of_civil_engineering' - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Engineering').content, 'html.parser')) - 'Scientific_method' - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Python_(programming_language)').content, 'html.parser')) - 'High-level_programming_language' - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Department_of_Standards_Malaysia').content, 'html.parser')) - 'Ministry_of_International_Trade_and_Industry_(Malaysia)' - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Malawi').content, 'html.parser')) - 'Tumbuka_language' - - Maar eventueel als alternatief 'Landlocked_country' ?? - >>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/%27s-Gravenweg_168,_Kralingen').content, 'html.parser')) - 'Kralingen' - >>> zoek_link(BeautifulSoup(requests.get('https://fr.wikipedia.org/wiki/Langue').content, 'html.parser')) - 'Syst%C3%A8me' + Extracts the article slug from a wiki href (supports both /wiki/Slug and https://xx.wikipedia.org/wiki/Slug). """ + if not href or not isinstance(href, str): + return None - # Ik heb besloten om ook lijsttypes te kunnen doorzoeken op nuttige linkjes. Dit is strikt genomen tegen de - # beschrijving van de opgave, maar volgens https://en.wikipedia.org/wiki/Wikipedia:Getting_to_Philosophy lijkt dit - # de logische methode, dus ik besloot ook lijsten te ondersteunen, als extra uitdaging. Ik vermoedde dat de opgave - # van het project eerder
specifieerde om ons wat hoofdbreuk te besparen.
- ondersteuning = ['p', 'ul', 'ol']
- link = None
+ if "/wiki/" not in href:
+ return None
- # Navigeer naar het niveau op de pagina waarop we zullen zoeken.
- soep = soep.find('div', {'id': 'bodyContent'}) \
- .find('div', {'class': 'mw-parser-output'})
+ parts = href.split("/wiki/", 1)
+ prefix, target = parts[0], parts[1]
- # Niet recursief zoeken om niet in een tabel of iets dergelijke vast te komen.
- # Het niveau onder 'mw-parser-output' is wat we nodig hebben en waar alle nodige alinea's in staan.
- alinea = soep.find(ondersteuning, recursive=False)
-
- # Ga steeds verder omlaag (niet dieper) totdat er een geldige link is gevonden.
- while link is None:
-
- # Navigeer naar het volgende stuk tekst (alinea of lijst) met een link.
- # Negeer linkjes in superscript, dus zoek enkel op het huidige niveau.
- potentieel = alinea.find('a')
-
- # Spans negeren is redelijk enkel en alleen om coördinaat stukjes te vermijden.
- while potentieel is None or alinea.findChild().name == 'span':
-
- alinea = alinea.findNextSibling(ondersteuning, recursive=False)
-
- # Het zou kunnen dat de bewerking hierboven ervoor zorgt dat we op het einde van de pagina zijn
- if alinea is None:
+ if prefix:
+ if prefix.startswith("http://") or prefix.startswith("https://") or prefix.startswith("//"):
+ parsed = urlparse(prefix if not prefix.startswith("//") else "https:" + prefix)
+ if not parsed.netloc.endswith("wikipedia.org"):
return None
+ elif prefix not in ("", "."):
+ return None
- potentieel = alinea.find('a')
-
- # Blijf zoeken naar linkjes in dezelfde alinea totdat deze voorwaarden voldaan zijn, maar stop van zodra er geen
- # naasten meer gevonden kunnen worden.
- while potentieel is not None and (
- tussen_haakjes(potentieel, alinea)
- # Soms kunnen hier 'interne'/externe linkjes of linkjes die niet werken (rode) tussen zitten.
- or not potentieel['href'].startswith('/wiki')
- or not potentieel['href'].count(':') == 0 # Bijzondere Wikipedia links negeren.
- ):
- potentieel = potentieel.findNext('a')
-
- if potentieel is not None:
- link = potentieel['href'].split('/')[2]
-
- return link
+ target = target.split("#")[0].split("?")[0].strip()
+ return target if target else None
-def tussen_haakjes(zin, omgeving):
+def is_valid_link(a_tag: Tag, current_title: Optional[str] = None) -> bool:
"""
- Controleer de geldigheid van de link door het aantal openende haakjes te vergelijken met het aantal sluitende
- haakjes die VOOR de onderzochte link voorkomen.
- Deze methode is meer waterdicht dan het vergelijken van indexen van haakjes.
- Strikt genomen zou het kunnen voorkomen dat een paragraaf tussen haken staat, die we dus niet kunnen meerekenen,
- maar dit voelt grammaticaal onjuist en we zullen er hier dan ook vanuit gaan dat dit niet het geval is.
-
- >>> tussen_haakjes('woord', 'dit is een heel belangrijk woord dat we nu zoeken')
- False
- >>> tussen_haakjes('woord', 'dit is een (heel) belangrijk woord dat we nu zoeken')
- False
- >>> tussen_haakjes('woord', 'dit (is een heel belangrijk woord dat we nu) zoeken')
- True
- >>> tussen_haakjes('woord', 'dit (is (een heel) belangrijk woord) dat we nu zoeken')
- True
+ Determines if an tag is a valid main-namespace Wikipedia article link.
"""
+ if not isinstance(a_tag, Tag) or a_tag.name != "a":
+ return False
- openende_haakjes = str(omgeving).count("(", 0, str(omgeving).find(str(zin)))
- sluitende_haakjes = str(omgeving).count(")", 0, str(omgeving).find(str(zin)))
- return openende_haakjes != sluitende_haakjes
+ href = a_tag.get("href")
+ target = extract_wiki_target(href)
+ if not target:
+ return False
+
+ # Filter out non-article namespaces and interwiki prefixes
+ target_lower = target.lower()
+ if any(target_lower.startswith(ns) for ns in EXCLUDED_NAMESPACES):
+ return False
+
+ if ":" in target:
+ prefix_part = target.split(":", 1)[0]
+ if (prefix_part.lower() + ":") in EXCLUDED_NAMESPACES:
+ return False
+ if len(prefix_part) in (2, 3) and prefix_part.isalpha():
+ return False
+
+ # Skip red links (non-existent pages) and self-links
+ classes = a_tag.get("class", [])
+ if isinstance(classes, str):
+ classes = classes.split()
+ if "new" in classes or "mw-selflink" in classes or "selflink" in classes or "external" in classes:
+ return False
+
+ # Skip image and file links
+ if "image" in classes or "mw-file-description" in classes:
+ return False
+ if a_tag.find("img") is not None:
+ return False
+
+ # Skip italicized links (e.g. hatnotes or foreign translations)
+ if a_tag.find_parent(["i", "em"]) is not None:
+ return False
+
+ # Check for inline style font-style: italic on parent hierarchy
+ curr = a_tag.parent
+ while curr and hasattr(curr, "get"):
+ style = curr.get("style", "")
+ if "italic" in style:
+ return False
+ if curr.name in ("p", "div", "body", "[document]"):
+ break
+ curr = curr.parent
+
+ # Skip self-links matching current article title
+ if current_title:
+ clean_target = converteer(target, True).strip().lower()
+ clean_current = converteer(current_title, True).strip().lower()
+ if clean_target == clean_current:
+ return False
+
+ return True
-def run(taal, start, stop):
+def is_excluded_inline(tag: Tag) -> bool:
"""
- Paden kunnen gecontroleerd worden met https://www.xefer.com/wikipedia, al doet dit niets met 'speciale' Wikipedia-
- links.
-
- Gewone controles
- >>> run('en', "Belgium", "Philosophy")
- {'pad': ['Belgium', 'Northwestern Europe', 'Subregion', 'Region', 'Geography', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
- >>> run('en', 'Department of Standards Malaysia', 'Philosophy')
- {'pad': ['Department of Standards Malaysia', 'Ministry of International Trade and Industry (Malaysia)', 'Ministry (government department)', 'Executive (government)', 'Government', 'State (polity)', 'Germans', 'Germany', 'Central Europe', 'Europe', 'Continent', 'Landmass', 'Region', 'Geography', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
- >>> run('en', 'Malkajgiri mandal', 'Philosophy')
- {'pad': ['Malkajgiri mandal', 'Medchal–Malkajgiri district', 'District', 'Administrative division', 'Sovereign state', 'Polity', 'Politics', 'Decision-making', 'Psychology', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
- >>> run('fr', 'Langue', 'Philosophie')
- {'pad': ['Langue', 'Système', 'Ensemble', 'Mathématiques', 'Connaissance', 'Notion', 'Connaissance (philosophie)', 'Philosophie']}
-
- Loops
- >>> run('nl', 'België', 'Philosophy')
- {'error': "Cyclus gedetecteerd op 'Wetenschap', startende vanaf 'België'. Het onafgewerkte pad zal niet worden toegevoegd aan het overzicht. "}
- >>> run('en', 'Tom Inglesby', 'Contract failure')
- {'pad': ['Tom Inglesby', 'Johns Hopkins Center for Health Security', 'Nonprofit organization', 'Contract failure']}
-
- Doodlopende eindes
- >>> run('en', 'Gaumee Film Awards', 'Philosophy')
- {'error': "Er konden geen nieuwe links meer gevonden worden op 'Gaumee Film Awards'"}
-
- Foute linkjes
- >>> run('en', 'Deze pagina bestaat niet', 'Philosophy')
- {'error': "Er ging iets fout bij het inladen van 'Deze pagina bestaat niet'. Bestaat de website?"}
+ Checks if an inline tag should be skipped during parenthesis traversal.
"""
+ if not isinstance(tag, Tag):
+ return False
+ if tag.name in ("sup", "table", "figure", "style", "script", "nav", "math"):
+ return True
+ classes = tag.get("class", [])
+ if isinstance(classes, str):
+ classes = classes.split()
+ excluded_classes = {
+ "infobox", "sidebar", "hatnote", "toc", "reference", "reflist",
+ "haudio", "audiolink", "IPA", "mw-empty-elt", "navbox",
+ "vertical-navbox", "shortdescription", "coordinates", "geo",
+ "audioplayer", "mw-editsection", "noprint", "mw-jump-link",
+ }
+ if any(c in excluded_classes for c in classes):
+ return True
+ if tag.get("id") in ("coordinates", "toc", "mw-indicator-coordinates"):
+ return True
+ return False
- # Negeer de 'random' waarde als er op deze manier gestart wordt.
- lijst = [converteer(start, True)] if start != 'Special:Random' else []
- base = f"https://{taal}.wikipedia.org/wiki/"
- while start != stop:
+def extract_all_links_from_block(block: Tag, current_title: Optional[str] = None) -> List[str]:
+ """
+ Traverses a content block in document order, tracking parenthesis depth
+ strictly across text nodes (NavigableString) to find all valid
+ non-parenthetical links.
+ """
+ paren_depth = 0
+ links: List[str] = []
- # Pagina inladen.
- try:
- pagina = requests.get(base + converteer(start, False))
- except:
- return {'error': f"Er ging iets fout bij het inladen van '{start}'. Bestaat de taalcode?"}
+ def walk(node):
+ nonlocal paren_depth
- # Stoppen indien het inladen van de pagina niet lukte.
- if pagina.status_code != 200:
- if len(lijst) == 0:
- return {'error': f"Er ging iets fout bij het inladen van '{start}'. Bestaat de website?"}
+ if isinstance(node, NavigableString):
+ for ch in str(node):
+ if ch == "(":
+ paren_depth += 1
+ elif ch == ")":
+ paren_depth = max(0, paren_depth - 1)
+ return
+
+ if isinstance(node, Tag):
+ if is_excluded_inline(node):
+ return
+
+ if node.name == "a":
+ if paren_depth == 0 and is_valid_link(node, current_title):
+ target = extract_wiki_target(node.get("href", ""))
+ if target and target not in links:
+ links.append(target)
+
+ # Traverse anchor children to ensure any parens inside text update depth
+ for child in node.children:
+ walk(child)
+ return
+
+ for child in node.children:
+ walk(child)
+
+ walk(block)
+ return links
+
+
+def clean_soup(soup: BeautifulSoup) -> Tag:
+ """
+ Finds the main content container and decomposes all boilerplate elements.
+ Returns the cleaned content root Tag.
+ """
+ content_div = soup.find("div", id="mw-content-text")
+ if content_div:
+ p_out = content_div.find("div", class_="mw-parser-output")
+ root = p_out if p_out else content_div
+ else:
+ p_out = soup.find("div", class_="mw-parser-output")
+ if p_out:
+ root = p_out
+ else:
+ body_content = soup.find("div", id="bodyContent")
+ if body_content:
+ root = body_content
+ else:
+ root = soup.body if soup.body else soup
+
+ for selector in BOILERPLATE_SELECTORS:
+ for el in root.select(selector):
+ el.decompose()
+
+ return root
+
+
+def zoek_kandidaat_links(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> List[str]:
+ """
+ Extracts all candidate valid Wikipedia links from the article in document order.
+ """
+ if soep is None:
+ return []
+
+ content_root = clean_soup(soep)
+ all_links: List[str] = []
+
+ content_blocks = content_root.find_all(["p", "ul", "ol"])
+ for block in content_blocks:
+ classes = block.get("class", [])
+ if isinstance(classes, str):
+ classes = classes.split()
+ if "mw-empty-elt" in classes or not block.get_text(strip=True):
+ continue
+
+ block_links = extract_all_links_from_block(block, current_title)
+ for link in block_links:
+ if link not in all_links:
+ all_links.append(link)
+
+ return all_links
+
+
+def zoek_link(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> Optional[str]:
+ """
+ Searches the provided BeautifulSoup document for the first valid Wikipedia link.
+
+ >>> html_basic = '''
+ ...
+ ... Belgium (/ˈbɛldʒəm/; Dutch: België, help; listen)
+ ... is a country in Northwestern Europe.
+ ...
+ ... Capital