fix(scraper): overhaul wikipedia scraper, parenthetical parsing, and smart re-routing
This commit is contained in:
parent
57c729f991
commit
31a5808b81
1 changed files with 508 additions and 149 deletions
|
|
@ -1,18 +1,107 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Dit is het 'CGI-script'.
|
||||
De bestandsextensie werd aangepast om te kunnen debuggen op zowel Windows als Linux.
|
||||
Phikipathia Scraper Engine (cgi-bin/scraper.py)
|
||||
|
||||
Finds the semantic path between Wikipedia articles leading to a target article
|
||||
(defaulting to "Philosophy" / "Filosofie" / "Philosophie") following the
|
||||
'Getting to Philosophy' rule: the first valid non-parenthetical link in the main
|
||||
body text of each article, with smart re-routing to bypass loops.
|
||||
"""
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Optional, Tuple
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
import requests
|
||||
|
||||
# Default HTTP User-Agent adhering to Wikimedia API policy
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Phikipathia/1.0 (https://git.depeuter.dev/tdpeuter/2022ST-project-Phikipathia; tibo@depeuter.dev)"
|
||||
)
|
||||
|
||||
def converteer(tekst, gebruiker=True):
|
||||
# Per-request HTTP timeout in seconds
|
||||
REQUEST_TIMEOUT = 15
|
||||
|
||||
# Maximum allowed hops per path search to prevent infinite exploration
|
||||
MAX_HOPS = 100
|
||||
|
||||
# Blacklisted non-article MediaWiki namespaces (case-insensitive prefixes)
|
||||
EXCLUDED_NAMESPACES = (
|
||||
# English
|
||||
"file:", "help:", "special:", "wikipedia:", "talk:", "category:",
|
||||
"portal:", "template:", "template_talk:", "user:", "user_talk:",
|
||||
"media:", "draft:", "draft_talk:", "module:", "module_talk:",
|
||||
"mediawiki:", "mediawiki_talk:", "timedtext:", "topic:",
|
||||
"portal_talk:", "category_talk:", "help_talk:", "wikipedia_talk:",
|
||||
# Dutch (nl)
|
||||
"bestand:", "hulp:", "speciaal:", "overleg:", "categorie:",
|
||||
"portaal:", "sjabloon:", "sjabloon_overleg:", "gebruiker:", "gebruiker_overleg:",
|
||||
"overleg_bestand:", "overleg_hulp:", "overleg_categorie:", "overleg_portaal:",
|
||||
"overleg_sjabloon:", "overleg_gebruiker:", "overleg_module:",
|
||||
# French (fr)
|
||||
"fichier:", "aide:", "spécial:", "discussion:", "catégorie:",
|
||||
"portail:", "modèle:", "discussion_modèle:", "utilisateur:", "discussion_utilisateur:",
|
||||
"discussion_fichier:", "discussion_aide:", "discussion_catégorie:", "discussion_portail:",
|
||||
"discussion_projet:", "projet:", "référence:", "discussion_référence:",
|
||||
# German (de)
|
||||
"datei:", "hilfe:", "spezial:", "diskussion:", "kategorie:",
|
||||
"vorlage:", "vorlagendiskussion:", "benutzer:", "benutzer_diskussion:",
|
||||
"datei_diskussion:", "hilfe_diskussion:", "kategorie_diskussion:", "portal_diskussion:",
|
||||
"portal:", "medium:",
|
||||
# Interwiki prefixes & Wikimedia projects
|
||||
"w:", "s:", "b:", "q:", "n:", "v:", "voy:", "d:", "m:", "meta:",
|
||||
"commons:", "wikidata:", "wikinews:", "wikiquote:", "wikibooks:",
|
||||
"wikisource:", "wikiversity:", "wikivoyage:", "species:", "wikt:",
|
||||
"wiktionary:", "incubator:", "foundation:", "phabricator:", "phab:",
|
||||
)
|
||||
|
||||
# CSS selectors for elements that must be stripped before link extraction
|
||||
BOILERPLATE_SELECTORS = (
|
||||
# Infoboxes & sidebars
|
||||
"table.infobox", "div.infobox", "table.sidebar", "div.sidebar",
|
||||
"div.vertical-navbox", "div.navbox", "table.vertical-navbox", "table.navbox",
|
||||
".infobox", ".infobox_v3", ".ib-country",
|
||||
# Hatnotes & disambiguation notices
|
||||
"div.hatnote", "div.dablink", "div.rellink", 'div[role="note"]',
|
||||
"div.shortdescription", ".hatnote", ".dablink", ".rellink",
|
||||
# Maintenance message boxes & stub templates
|
||||
"table.ambox", "table.cmbox", "table.tmbox", "table.fmbox", "div.asbox",
|
||||
"div.metadata", "div.boilerplate_metadata",
|
||||
# Table of contents
|
||||
"nav#toc", "div#toc", "div.toc", ".toc", ".mw-table-of-contents-container",
|
||||
# Coordinates
|
||||
"span#coordinates", "div#coordinates", "span.coordinates", "div.coordinates",
|
||||
"span.geo-default", "span.geo-dec", "span.geo-dms", "span.geo",
|
||||
"#mw-indicator-coordinates", ".mw-indicator-coordinates",
|
||||
# References & citations & footnotes
|
||||
"sup.reference", "ol.references", "div.reflist", "div.references",
|
||||
"span.reference-text", "cite.citation", ".reflist", ".references",
|
||||
# Audio, IPA & pronunciation annotations
|
||||
"span.haudio", "span.IPA", "span.unicode", "div.audioplayer",
|
||||
"span.audiolink", "span.audio-button", "span.audio", ".phonetics",
|
||||
# Thumbnails, images & media containers
|
||||
"div.thumb", "figure", "div.floatright", "div.floatleft",
|
||||
"div.tright", "div.tleft", "span.mw-image-border",
|
||||
# UI controls & print artifacts
|
||||
"span.mw-editsection", "div.mw-jump-link", "div.noprint", "div.printfooter",
|
||||
# Empty placeholder elements
|
||||
"p.mw-empty-elt",
|
||||
)
|
||||
|
||||
|
||||
def get_user_agent() -> str:
|
||||
"""
|
||||
Ik blijf bewust case sensitive werken omdat dit héél soms wel nog belangrijk is om een verschl
|
||||
te maken tussen twee linkjes.
|
||||
Returns the configured User-Agent string with USER_AGENT environment variable override.
|
||||
"""
|
||||
env_ua = os.environ.get("USER_AGENT", "").strip()
|
||||
return env_ua if env_ua else DEFAULT_USER_AGENT
|
||||
|
||||
|
||||
def converteer(tekst: Optional[str], gebruiker: bool = True) -> Optional[str]:
|
||||
"""
|
||||
Converts between URL-encoded slugs and display article titles.
|
||||
|
||||
>>> converteer('Northwestern Europe')
|
||||
'Northwestern Europe'
|
||||
|
|
@ -22,180 +111,450 @@ def converteer(tekst, gebruiker=True):
|
|||
'Medchal–Malkajgiri district'
|
||||
>>> converteer('Medchal–Malkajgiri district', False)
|
||||
'Medchal%E2%80%93Malkajgiri_district'
|
||||
>>> converteer('C%23_(programming_language)')
|
||||
'C# (programming language)'
|
||||
>>> converteer('Caf%C3%A9#Etymology')
|
||||
'Café'
|
||||
>>> converteer(None) is None
|
||||
True
|
||||
"""
|
||||
|
||||
from urllib import parse
|
||||
|
||||
if tekst is None:
|
||||
return None
|
||||
|
||||
if gebruiker:
|
||||
return parse.unquote(tekst).replace('_', ' ').split('#')[0]
|
||||
return unquote(str(tekst).split("#")[0]).replace("_", " ")
|
||||
else:
|
||||
return parse.quote(tekst.replace(' ', '_'))
|
||||
return quote(str(tekst).replace(" ", "_"))
|
||||
|
||||
|
||||
def zoek_link(soep):
|
||||
def extract_wiki_target(href: Optional[str]) -> Optional[str]:
|
||||
"""
|
||||
Doorzoek de meegeleverde soep naar een geldige Wikipedia-link.
|
||||
:return: een geldige link indien beschikbaar, anders
|
||||
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Belgium').content, 'html.parser'))
|
||||
'Northwestern_Europe'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Glossary_of_engineering').content, 'html.parser'))
|
||||
'Glossary_of_civil_engineering'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Engineering').content, 'html.parser'))
|
||||
'Scientific_method'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Python_(programming_language)').content, 'html.parser'))
|
||||
'High-level_programming_language'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Department_of_Standards_Malaysia').content, 'html.parser'))
|
||||
'Ministry_of_International_Trade_and_Industry_(Malaysia)'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/Malawi').content, 'html.parser'))
|
||||
'Tumbuka_language'
|
||||
|
||||
Maar eventueel als alternatief 'Landlocked_country' ??
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://en.wikipedia.org/wiki/%27s-Gravenweg_168,_Kralingen').content, 'html.parser'))
|
||||
'Kralingen'
|
||||
>>> zoek_link(BeautifulSoup(requests.get('https://fr.wikipedia.org/wiki/Langue').content, 'html.parser'))
|
||||
'Syst%C3%A8me'
|
||||
Extracts the article slug from a wiki href (supports both /wiki/Slug and https://xx.wikipedia.org/wiki/Slug).
|
||||
"""
|
||||
if not href or not isinstance(href, str):
|
||||
return None
|
||||
|
||||
# Ik heb besloten om ook lijsttypes te kunnen doorzoeken op nuttige linkjes. Dit is strikt genomen tegen de
|
||||
# beschrijving van de opgave, maar volgens https://en.wikipedia.org/wiki/Wikipedia:Getting_to_Philosophy lijkt dit
|
||||
# de logische methode, dus ik besloot ook lijsten te ondersteunen, als extra uitdaging. Ik vermoedde dat de opgave
|
||||
# van het project eerder <p> specifieerde om ons wat hoofdbreuk te besparen.
|
||||
ondersteuning = ['p', 'ul', 'ol']
|
||||
link = None
|
||||
if "/wiki/" not in href:
|
||||
return None
|
||||
|
||||
# Navigeer naar het niveau op de pagina waarop we zullen zoeken.
|
||||
soep = soep.find('div', {'id': 'bodyContent'}) \
|
||||
.find('div', {'class': 'mw-parser-output'})
|
||||
parts = href.split("/wiki/", 1)
|
||||
prefix, target = parts[0], parts[1]
|
||||
|
||||
# Niet recursief zoeken om niet in een tabel of iets dergelijke vast te komen.
|
||||
# Het niveau onder 'mw-parser-output' is wat we nodig hebben en waar alle nodige alinea's in staan.
|
||||
alinea = soep.find(ondersteuning, recursive=False)
|
||||
|
||||
# Ga steeds verder omlaag (niet dieper) totdat er een geldige link is gevonden.
|
||||
while link is None:
|
||||
|
||||
# Navigeer naar het volgende stuk tekst (alinea of lijst) met een link.
|
||||
# Negeer linkjes in superscript, dus zoek enkel op het huidige niveau.
|
||||
potentieel = alinea.find('a')
|
||||
|
||||
# Spans negeren is redelijk enkel en alleen om coördinaat stukjes te vermijden.
|
||||
while potentieel is None or alinea.findChild().name == 'span':
|
||||
|
||||
alinea = alinea.findNextSibling(ondersteuning, recursive=False)
|
||||
|
||||
# Het zou kunnen dat de bewerking hierboven ervoor zorgt dat we op het einde van de pagina zijn
|
||||
if alinea is None:
|
||||
if prefix:
|
||||
if prefix.startswith("http://") or prefix.startswith("https://") or prefix.startswith("//"):
|
||||
parsed = urlparse(prefix if not prefix.startswith("//") else "https:" + prefix)
|
||||
if not parsed.netloc.endswith("wikipedia.org"):
|
||||
return None
|
||||
elif prefix not in ("", "."):
|
||||
return None
|
||||
|
||||
potentieel = alinea.find('a')
|
||||
|
||||
# Blijf zoeken naar linkjes in dezelfde alinea totdat deze voorwaarden voldaan zijn, maar stop van zodra er geen
|
||||
# naasten meer gevonden kunnen worden.
|
||||
while potentieel is not None and (
|
||||
tussen_haakjes(potentieel, alinea)
|
||||
# Soms kunnen hier 'interne'/externe linkjes of linkjes die niet werken (rode) tussen zitten.
|
||||
or not potentieel['href'].startswith('/wiki')
|
||||
or not potentieel['href'].count(':') == 0 # Bijzondere Wikipedia links negeren.
|
||||
):
|
||||
potentieel = potentieel.findNext('a')
|
||||
|
||||
if potentieel is not None:
|
||||
link = potentieel['href'].split('/')[2]
|
||||
|
||||
return link
|
||||
target = target.split("#")[0].split("?")[0].strip()
|
||||
return target if target else None
|
||||
|
||||
|
||||
def tussen_haakjes(zin, omgeving):
|
||||
def is_valid_link(a_tag: Tag, current_title: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Controleer de geldigheid van de link door het aantal openende haakjes te vergelijken met het aantal sluitende
|
||||
haakjes die VOOR de onderzochte link voorkomen.
|
||||
Deze methode is meer waterdicht dan het vergelijken van indexen van haakjes.
|
||||
Strikt genomen zou het kunnen voorkomen dat een paragraaf tussen haken staat, die we dus niet kunnen meerekenen,
|
||||
maar dit voelt grammaticaal onjuist en we zullen er hier dan ook vanuit gaan dat dit niet het geval is.
|
||||
|
||||
>>> tussen_haakjes('woord', 'dit is een heel belangrijk woord dat we nu zoeken')
|
||||
False
|
||||
>>> tussen_haakjes('woord', 'dit is een (heel) belangrijk woord dat we nu zoeken')
|
||||
False
|
||||
>>> tussen_haakjes('woord', 'dit (is een heel belangrijk woord dat we nu) zoeken')
|
||||
True
|
||||
>>> tussen_haakjes('woord', 'dit (is (een heel) belangrijk woord) dat we nu zoeken')
|
||||
True
|
||||
Determines if an <a> tag is a valid main-namespace Wikipedia article link.
|
||||
"""
|
||||
if not isinstance(a_tag, Tag) or a_tag.name != "a":
|
||||
return False
|
||||
|
||||
openende_haakjes = str(omgeving).count("(", 0, str(omgeving).find(str(zin)))
|
||||
sluitende_haakjes = str(omgeving).count(")", 0, str(omgeving).find(str(zin)))
|
||||
return openende_haakjes != sluitende_haakjes
|
||||
href = a_tag.get("href")
|
||||
target = extract_wiki_target(href)
|
||||
if not target:
|
||||
return False
|
||||
|
||||
# Filter out non-article namespaces and interwiki prefixes
|
||||
target_lower = target.lower()
|
||||
if any(target_lower.startswith(ns) for ns in EXCLUDED_NAMESPACES):
|
||||
return False
|
||||
|
||||
if ":" in target:
|
||||
prefix_part = target.split(":", 1)[0]
|
||||
if (prefix_part.lower() + ":") in EXCLUDED_NAMESPACES:
|
||||
return False
|
||||
if len(prefix_part) in (2, 3) and prefix_part.isalpha():
|
||||
return False
|
||||
|
||||
# Skip red links (non-existent pages) and self-links
|
||||
classes = a_tag.get("class", [])
|
||||
if isinstance(classes, str):
|
||||
classes = classes.split()
|
||||
if "new" in classes or "mw-selflink" in classes or "selflink" in classes or "external" in classes:
|
||||
return False
|
||||
|
||||
# Skip image and file links
|
||||
if "image" in classes or "mw-file-description" in classes:
|
||||
return False
|
||||
if a_tag.find("img") is not None:
|
||||
return False
|
||||
|
||||
# Skip italicized links (e.g. hatnotes or foreign translations)
|
||||
if a_tag.find_parent(["i", "em"]) is not None:
|
||||
return False
|
||||
|
||||
# Check for inline style font-style: italic on parent hierarchy
|
||||
curr = a_tag.parent
|
||||
while curr and hasattr(curr, "get"):
|
||||
style = curr.get("style", "")
|
||||
if "italic" in style:
|
||||
return False
|
||||
if curr.name in ("p", "div", "body", "[document]"):
|
||||
break
|
||||
curr = curr.parent
|
||||
|
||||
# Skip self-links matching current article title
|
||||
if current_title:
|
||||
clean_target = converteer(target, True).strip().lower()
|
||||
clean_current = converteer(current_title, True).strip().lower()
|
||||
if clean_target == clean_current:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def run(taal, start, stop):
|
||||
def is_excluded_inline(tag: Tag) -> bool:
|
||||
"""
|
||||
Paden kunnen gecontroleerd worden met https://www.xefer.com/wikipedia, al doet dit niets met 'speciale' Wikipedia-
|
||||
links.
|
||||
|
||||
Gewone controles
|
||||
>>> run('en', "Belgium", "Philosophy")
|
||||
{'pad': ['Belgium', 'Northwestern Europe', 'Subregion', 'Region', 'Geography', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
|
||||
>>> run('en', 'Department of Standards Malaysia', 'Philosophy')
|
||||
{'pad': ['Department of Standards Malaysia', 'Ministry of International Trade and Industry (Malaysia)', 'Ministry (government department)', 'Executive (government)', 'Government', 'State (polity)', 'Germans', 'Germany', 'Central Europe', 'Europe', 'Continent', 'Landmass', 'Region', 'Geography', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
|
||||
>>> run('en', 'Malkajgiri mandal', 'Philosophy')
|
||||
{'pad': ['Malkajgiri mandal', 'Medchal–Malkajgiri district', 'District', 'Administrative division', 'Sovereign state', 'Polity', 'Politics', 'Decision-making', 'Psychology', 'Science', 'Scientific method', 'Empirical evidence', 'Proposition', 'Logic', 'Reason', 'Consciousness', 'Sentience', 'Emotion', 'Mental state', 'Mind', 'Phenomenon', 'Immanuel Kant', 'Philosophy']}
|
||||
>>> run('fr', 'Langue', 'Philosophie')
|
||||
{'pad': ['Langue', 'Système', 'Ensemble', 'Mathématiques', 'Connaissance', 'Notion', 'Connaissance (philosophie)', 'Philosophie']}
|
||||
|
||||
Loops
|
||||
>>> run('nl', 'België', 'Philosophy')
|
||||
{'error': "Cyclus gedetecteerd op 'Wetenschap', startende vanaf 'België'. Het onafgewerkte pad zal niet worden toegevoegd aan het overzicht. "}
|
||||
>>> run('en', 'Tom Inglesby', 'Contract failure')
|
||||
{'pad': ['Tom Inglesby', 'Johns Hopkins Center for Health Security', 'Nonprofit organization', 'Contract failure']}
|
||||
|
||||
Doodlopende eindes
|
||||
>>> run('en', 'Gaumee Film Awards', 'Philosophy')
|
||||
{'error': "Er konden geen nieuwe links meer gevonden worden op 'Gaumee Film Awards'"}
|
||||
|
||||
Foute linkjes
|
||||
>>> run('en', 'Deze pagina bestaat niet', 'Philosophy')
|
||||
{'error': "Er ging iets fout bij het inladen van 'Deze pagina bestaat niet'. Bestaat de website?"}
|
||||
Checks if an inline tag should be skipped during parenthesis traversal.
|
||||
"""
|
||||
if not isinstance(tag, Tag):
|
||||
return False
|
||||
if tag.name in ("sup", "table", "figure", "style", "script", "nav", "math"):
|
||||
return True
|
||||
classes = tag.get("class", [])
|
||||
if isinstance(classes, str):
|
||||
classes = classes.split()
|
||||
excluded_classes = {
|
||||
"infobox", "sidebar", "hatnote", "toc", "reference", "reflist",
|
||||
"haudio", "audiolink", "IPA", "mw-empty-elt", "navbox",
|
||||
"vertical-navbox", "shortdescription", "coordinates", "geo",
|
||||
"audioplayer", "mw-editsection", "noprint", "mw-jump-link",
|
||||
}
|
||||
if any(c in excluded_classes for c in classes):
|
||||
return True
|
||||
if tag.get("id") in ("coordinates", "toc", "mw-indicator-coordinates"):
|
||||
return True
|
||||
return False
|
||||
|
||||
# Negeer de 'random' waarde als er op deze manier gestart wordt.
|
||||
lijst = [converteer(start, True)] if start != 'Special:Random' else []
|
||||
base = f"https://{taal}.wikipedia.org/wiki/"
|
||||
|
||||
while start != stop:
|
||||
def extract_all_links_from_block(block: Tag, current_title: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Traverses a content block in document order, tracking parenthesis depth
|
||||
strictly across text nodes (NavigableString) to find all valid
|
||||
non-parenthetical links.
|
||||
"""
|
||||
paren_depth = 0
|
||||
links: List[str] = []
|
||||
|
||||
# Pagina inladen.
|
||||
try:
|
||||
pagina = requests.get(base + converteer(start, False))
|
||||
except:
|
||||
return {'error': f"Er ging iets fout bij het inladen van '{start}'. Bestaat de taalcode?"}
|
||||
def walk(node):
|
||||
nonlocal paren_depth
|
||||
|
||||
# Stoppen indien het inladen van de pagina niet lukte.
|
||||
if pagina.status_code != 200:
|
||||
if len(lijst) == 0:
|
||||
return {'error': f"Er ging iets fout bij het inladen van '{start}'. Bestaat de website?"}
|
||||
if isinstance(node, NavigableString):
|
||||
for ch in str(node):
|
||||
if ch == "(":
|
||||
paren_depth += 1
|
||||
elif ch == ")":
|
||||
paren_depth = max(0, paren_depth - 1)
|
||||
return
|
||||
|
||||
if isinstance(node, Tag):
|
||||
if is_excluded_inline(node):
|
||||
return
|
||||
|
||||
if node.name == "a":
|
||||
if paren_depth == 0 and is_valid_link(node, current_title):
|
||||
target = extract_wiki_target(node.get("href", ""))
|
||||
if target and target not in links:
|
||||
links.append(target)
|
||||
|
||||
# Traverse anchor children to ensure any parens inside text update depth
|
||||
for child in node.children:
|
||||
walk(child)
|
||||
return
|
||||
|
||||
for child in node.children:
|
||||
walk(child)
|
||||
|
||||
walk(block)
|
||||
return links
|
||||
|
||||
|
||||
def clean_soup(soup: BeautifulSoup) -> Tag:
|
||||
"""
|
||||
Finds the main content container and decomposes all boilerplate elements.
|
||||
Returns the cleaned content root Tag.
|
||||
"""
|
||||
content_div = soup.find("div", id="mw-content-text")
|
||||
if content_div:
|
||||
p_out = content_div.find("div", class_="mw-parser-output")
|
||||
root = p_out if p_out else content_div
|
||||
else:
|
||||
p_out = soup.find("div", class_="mw-parser-output")
|
||||
if p_out:
|
||||
root = p_out
|
||||
else:
|
||||
body_content = soup.find("div", id="bodyContent")
|
||||
if body_content:
|
||||
root = body_content
|
||||
else:
|
||||
root = soup.body if soup.body else soup
|
||||
|
||||
for selector in BOILERPLATE_SELECTORS:
|
||||
for el in root.select(selector):
|
||||
el.decompose()
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def zoek_kandidaat_links(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Extracts all candidate valid Wikipedia links from the article in document order.
|
||||
"""
|
||||
if soep is None:
|
||||
return []
|
||||
|
||||
content_root = clean_soup(soep)
|
||||
all_links: List[str] = []
|
||||
|
||||
content_blocks = content_root.find_all(["p", "ul", "ol"])
|
||||
for block in content_blocks:
|
||||
classes = block.get("class", [])
|
||||
if isinstance(classes, str):
|
||||
classes = classes.split()
|
||||
if "mw-empty-elt" in classes or not block.get_text(strip=True):
|
||||
continue
|
||||
|
||||
block_links = extract_all_links_from_block(block, current_title)
|
||||
for link in block_links:
|
||||
if link not in all_links:
|
||||
all_links.append(link)
|
||||
|
||||
return all_links
|
||||
|
||||
|
||||
def zoek_link(soep: Optional[BeautifulSoup], current_title: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Searches the provided BeautifulSoup document for the first valid Wikipedia link.
|
||||
|
||||
>>> html_basic = '''
|
||||
... <div id="mw-content-text">
|
||||
... <div class="mw-parser-output">
|
||||
... <p class="mw-empty-elt"></p>
|
||||
... <div class="hatnote">For other uses, see <a href="/wiki/Belgium_(disambiguation)">Belgium (disambiguation)</a>.</div>
|
||||
... <table class="infobox"><tr><td><a href="/wiki/Infobox_Link">Capital</a></td></tr></table>
|
||||
... <p>
|
||||
... <b>Belgium</b> (<span class="IPA">/ˈbɛldʒəm/</span>; Dutch: <i lang="nl">België</i>, <a href="/wiki/Help:IPA">help</a>; <a href="/wiki/File:Audio.ogg">listen</a>)
|
||||
... is a country in <a href="/wiki/Northwestern_Europe">Northwestern Europe</a>.
|
||||
... </p>
|
||||
... </div>
|
||||
... </div>
|
||||
... '''
|
||||
>>> zoek_link(BeautifulSoup(html_basic, 'html.parser'))
|
||||
'Northwestern_Europe'
|
||||
"""
|
||||
links = zoek_kandidaat_links(soep, current_title)
|
||||
return links[0] if links else None
|
||||
|
||||
|
||||
def tussen_haakjes(zin: str, omgeving: str) -> bool:
|
||||
"""
|
||||
Legacy helper checking if a phrase is inside parentheses.
|
||||
Preserved for backward compatibility.
|
||||
"""
|
||||
s_omg = str(omgeving)
|
||||
s_zin = str(zin)
|
||||
idx = s_omg.find(s_zin)
|
||||
if idx == -1:
|
||||
return False
|
||||
open_count = s_omg.count("(", 0, idx)
|
||||
close_count = s_omg.count(")", 0, idx)
|
||||
return open_count > close_count
|
||||
|
||||
|
||||
def fetch_wikipedia_page(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
headers: dict
|
||||
) -> Tuple[Optional[BeautifulSoup], Optional[str], Optional[str]]:
|
||||
"""
|
||||
Fetches a Wikipedia page following redirects.
|
||||
Returns (soup, canonical_resolved_title, error_type).
|
||||
"""
|
||||
try:
|
||||
response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT, allow_redirects=True)
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.InvalidURL):
|
||||
return None, None, "dns_error"
|
||||
except requests.exceptions.Timeout:
|
||||
return None, None, "timeout_error"
|
||||
except Exception:
|
||||
return None, None, "request_error"
|
||||
|
||||
if response.status_code != 200:
|
||||
return None, None, f"http_{response.status_code}"
|
||||
|
||||
# Extract canonical title from final redirected URL
|
||||
parsed_path = urlparse(response.url).path
|
||||
canonical_title = None
|
||||
if parsed_path.startswith("/wiki/"):
|
||||
canonical_title = parsed_path[6:].split("#")[0].split("?")[0]
|
||||
|
||||
soup = BeautifulSoup(response.content, "html.parser")
|
||||
return soup, canonical_title, None
|
||||
|
||||
|
||||
import re
|
||||
|
||||
def is_valid_lang_code(taal: str) -> bool:
|
||||
"""
|
||||
Validates Wikipedia language subdomain string to prevent SSRF or injection.
|
||||
Allows 2-10 lowercase alphanumeric characters/hyphens (e.g. 'en', 'nl', 'zh-min-nan', 'simple').
|
||||
"""
|
||||
if not taal or not isinstance(taal, str):
|
||||
return False
|
||||
return bool(re.match(r"^[a-z]{2,10}(-[a-z0-9]+)?$", taal.strip().lower()))
|
||||
|
||||
|
||||
def run(taal: str, start: str, stop: Optional[str] = "Philosophy", max_hops: int = MAX_HOPS) -> dict:
|
||||
"""
|
||||
Executes the Wikipedia path traversal from start to stop in the specified language,
|
||||
with smart re-routing if the primary link creates a loop.
|
||||
"""
|
||||
if not is_valid_lang_code(taal):
|
||||
return {
|
||||
"pad": [],
|
||||
"status": "error",
|
||||
"stappen": 0,
|
||||
"herleid": 0,
|
||||
"error": f"Ongeldige taalcode '{taal}'. Gebruik een geldige Wikipedia taalcode (bijv. 'en', 'nl', 'fr')."
|
||||
}
|
||||
|
||||
if not stop:
|
||||
stop = "Filosofie" if taal == "nl" else ("Philosophie" if taal == "fr" else "Philosophy")
|
||||
|
||||
session = requests.Session()
|
||||
headers = {"User-Agent": get_user_agent()}
|
||||
base_url = f"https://{taal}.wikipedia.org/wiki/"
|
||||
|
||||
pad: List[str] = []
|
||||
visited: set[str] = set()
|
||||
current_title = start
|
||||
herleid_count = 0
|
||||
|
||||
for hop in range(max_hops + 1):
|
||||
if hop >= max_hops:
|
||||
return {
|
||||
'error': f"Er ging iets fout bij het inladen van '{lijst[-1]}', vertrekkende vanaf '{lijst[0]}'. Bestaat de website?"}
|
||||
"pad": pad,
|
||||
"status": "max_hops",
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": f"Maximum aantal stappen ({max_hops}) bereikt voor '{pad[0]}'."
|
||||
}
|
||||
|
||||
# Verwerken.
|
||||
soep = BeautifulSoup(pagina.content, 'html.parser')
|
||||
target_url = base_url + converteer(current_title, False)
|
||||
soup, canonical_slug, err = fetch_wikipedia_page(session, target_url, headers)
|
||||
|
||||
# Volgende link zoeken.
|
||||
start = converteer(zoek_link(soep))
|
||||
if err is not None or soup is None:
|
||||
if len(pad) == 0:
|
||||
err_msg = f"Er ging iets fout bij het inladen van '{start}'. Bestaat de taalcode of website?"
|
||||
else:
|
||||
err_msg = f"Er ging iets fout bij het inladen van '{pad[-1]}', vertrekkende vanaf '{pad[0]}'."
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "error",
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": err_msg
|
||||
}
|
||||
|
||||
if start is None:
|
||||
return {'error': f"Er konden geen nieuwe links meer gevonden worden op '{lijst[-1]}'"}
|
||||
resolved_title = converteer(canonical_slug if canonical_slug else current_title, True)
|
||||
|
||||
# Cyclus detecteren
|
||||
if start in lijst:
|
||||
return {'error':
|
||||
f"Cyclus gedetecteerd op '{start}', startende vanaf '{lijst[0]}'. Het onafgewerkte pad zal niet worden toegevoegd aan het overzicht. "
|
||||
}
|
||||
if len(pad) == 0:
|
||||
pad.append(resolved_title)
|
||||
visited.add(resolved_title.lower())
|
||||
elif pad[-1] != resolved_title:
|
||||
if resolved_title.lower() in visited:
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "cyclus",
|
||||
"cyclus_op": resolved_title,
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": f"Cyclus gedetecteerd op '{resolved_title}', startende vanaf '{pad[0]}'."
|
||||
}
|
||||
pad.append(resolved_title)
|
||||
visited.add(resolved_title.lower())
|
||||
|
||||
lijst.append(start)
|
||||
# Check if destination article reached
|
||||
if resolved_title.lower() == stop.lower():
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "success",
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count
|
||||
}
|
||||
|
||||
return {'pad': lijst}
|
||||
# Option 2: Smart Re-routing among candidate links on current page
|
||||
candidate_slugs = zoek_kandidaat_links(soup, resolved_title)
|
||||
if not candidate_slugs:
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "doodlopend",
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": f"Er konden geen nieuwe links meer gevonden worden op '{pad[-1]}'."
|
||||
}
|
||||
|
||||
# Find first candidate link that does NOT lead into an already visited page
|
||||
selected_slug = None
|
||||
for idx, cand_slug in enumerate(candidate_slugs):
|
||||
cand_title = converteer(cand_slug, True)
|
||||
if cand_title.lower() not in visited and cand_title not in pad:
|
||||
selected_slug = cand_slug
|
||||
if idx > 0:
|
||||
herleid_count += 1
|
||||
break
|
||||
|
||||
if selected_slug is None:
|
||||
# All candidate links on this page lead to visited pages -> cycle
|
||||
primary_title = converteer(candidate_slugs[0], True)
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "cyclus",
|
||||
"cyclus_op": primary_title,
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": f"Cyclus gedetecteerd op '{primary_title}', startende vanaf '{pad[0]}'."
|
||||
}
|
||||
|
||||
current_title = selected_slug
|
||||
|
||||
return {
|
||||
"pad": pad,
|
||||
"status": "max_hops",
|
||||
"stappen": len(pad) - 1,
|
||||
"herleid": herleid_count,
|
||||
"error": f"Maximum aantal stappen ({max_hops}) bereikt voor '{pad[0]}'."
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--test":
|
||||
import doctest
|
||||
results = doctest.testmod()
|
||||
print(f"Doctests: {results}")
|
||||
sys.exit(0 if results.failed == 0 else 1)
|
||||
elif len(sys.argv) > 1:
|
||||
taal = sys.argv[1] if len(sys.argv) > 1 else "en"
|
||||
start = sys.argv[2] if len(sys.argv) > 2 else "Special:Random"
|
||||
stop = sys.argv[3] if len(sys.argv) > 3 else "Philosophy"
|
||||
antwoord = run(taal, start, stop)
|
||||
print(json.dumps(antwoord, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
import doctest
|
||||
results = doctest.testmod()
|
||||
print(f"Doctests: {results}")
|
||||
sys.exit(0 if results.failed == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Reference in a new issue