From daccbbd9c73a5292ea9d6746560d7009e5aa666d Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Wed, 7 Sep 2022 11:37:37 +1000 Subject: Add python type annotations Also use consistent quote symbols Also use 'is None' instead of '== None' Also use 'if list1' instead of 'if len(list1) > 0' --- backend/server.py | 27 +- backend/tilo.py | 330 ++++++++++++----------- backend/tolData/dbpedia/genDescData.py | 104 +++---- backend/tolData/enwiki/downloadImgLicenseInfo.py | 136 +++++----- backend/tolData/enwiki/downloadImgs.py | 50 ++-- backend/tolData/enwiki/genDescData.py | 100 +++---- backend/tolData/enwiki/genDumpIndexDb.py | 39 +-- backend/tolData/enwiki/genImgData.py | 118 ++++---- backend/tolData/enwiki/genPageviewData.py | 10 +- backend/tolData/enwiki/lookupPage.py | 34 +-- backend/tolData/eol/downloadImgs.py | 76 +++--- backend/tolData/eol/genImagesListDb.py | 28 +- backend/tolData/eol/reviewImgs.py | 92 +++---- backend/tolData/genDescData.py | 19 +- backend/tolData/genImgs.py | 154 ++++++----- backend/tolData/genLinkedImgs.py | 102 +++---- backend/tolData/genMappingData.py | 60 ++--- backend/tolData/genNameData.py | 28 +- backend/tolData/genOtolData.py | 133 +++++---- backend/tolData/genPopData.py | 9 +- backend/tolData/genReducedTrees.py | 270 ++++++++++--------- backend/tolData/reviewImgsToGen.py | 118 ++++---- backend/tolData/wikidata/genTaxonSrcData.py | 64 +++-- 23 files changed, 1072 insertions(+), 1029 deletions(-) (limited to 'backend') diff --git a/backend/server.py b/backend/server.py index 48d6c3f..5b0d26b 100755 --- a/backend/server.py +++ b/backend/server.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 -import sys, os +from typing import Iterable +import os from wsgiref import simple_server, util import mimetypes from tilo import application @@ -11,26 +12,26 @@ Runs a basic dev server that serves a WSGI script and image files """) parser.parse_args() -# WSGI handler that uses 'application', but also serves image files -def wrappingApp(environ, start_response): - urlPath = environ["PATH_INFO"] - if urlPath.startswith("/data/"): +def wrappingApp(environ: dict[str, str], start_response) -> Iterable[bytes]: + """ WSGI handler that uses 'application', but also serves image files """ + urlPath = environ['PATH_INFO'] + if urlPath.startswith('/data/'): # Run WSGI script return application(environ, start_response) - elif urlPath.startswith("/tolData/img/"): + elif urlPath.startswith('/tolData/img/'): # Serve image file imgPath = os.path.join(os.getcwd(), urlPath[1:]) if os.path.exists(imgPath): imgType = mimetypes.guess_type(imgPath)[0] - start_response("200 OK", [("Content-type", imgType)]) - return util.FileWrapper(open(imgPath, "rb")) + start_response('200 OK', [('Content-type', imgType)]) + return util.FileWrapper(open(imgPath, 'rb')) else: - start_response("404 Not Found", [("Content-type", "text/plain")]) - return [b"No image found"] + start_response('404 Not Found', [('Content-type', 'text/plain')]) + return [b'No image found'] else: - start_response("404 Not Found", [("Content-type", "text/plain")]) - return [b"Unrecognised path"] + start_response('404 Not Found', [('Content-type', 'text/plain')]) + return [b'Unrecognised path'] # Start server with simple_server.make_server('', 8000, wrappingApp) as httpd: - print("Serving HTTP on port 8000...") + print('Serving HTTP on port 8000...') httpd.serve_forever() diff --git a/backend/tilo.py b/backend/tilo.py index 00b3fc9..c1ecc34 100755 --- a/backend/tilo.py +++ b/backend/tilo.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys, os.path, re, time -import urllib.parse -import sqlite3 +from typing import Iterable, cast +import sys, re +import urllib.parse, sqlite3 import gzip, jsonpickle HELP_INFO = """ @@ -28,144 +28,152 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description=HELP_INFO, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -DB_FILE = "tolData/data.db" +DB_FILE = 'tolData/data.db' DEFAULT_SUGG_LIM = 5 MAX_SUGG_LIM = 50 -ROOT_NAME = "cellular organisms" +ROOT_NAME = 'cellular organisms' # Classes for objects sent as responses (matches lib.ts types in client-side code) class TolNode: - " Used when responding to 'node' and 'chain' requests " - def __init__(self, otolId, children, parent=None, tips=0, pSupport=False, commonName=None, imgName=None, iucn=None): - self.otolId = otolId # string | null - self.children = children # string[] - self.parent = parent # string | null - self.tips = tips # number - self.pSupport = pSupport # boolean - self.commonName = commonName # null | string - self.imgName = imgName # null | string | [string,string] | [null, string] | [string, null] - self.iucn = iucn # null | string + """ Used when responding to 'node' and 'chain' requests """ + def __init__( + self, + otolId: str | None, + children: list[str], + parent: str | None = None, + tips=0, + pSupport=False, + commonName: str | None = None, + imgName: None | str | tuple[str, str] | tuple[None, str] | tuple[str, None] = None, + iucn: str = None): + self.otolId = otolId + self.children = children + self.parent = parent + self.tips = tips + self.pSupport = pSupport + self.commonName = commonName + self.imgName = imgName + self.iucn = iucn class SearchSugg: - " Represents a search suggestion " - def __init__(self, name, canonicalName=None, pop=0): - self.name = name # string - self.canonicalName = canonicalName # string | null - self.pop = pop if pop != None else 0 # number + """ Represents a search suggestion """ + def __init__(self, name: str, canonicalName: str | None = None, pop=0): + self.name = name + self.canonicalName = canonicalName + self.pop = pop if pop is not None else 0 class SearchSuggResponse: - " Sent as responses to 'sugg' requests " - def __init__(self, searchSuggs, hasMore): - self.suggs = searchSuggs # SearchSugg[] - self.hasMore = hasMore # boolean + """ Sent as responses to 'sugg' requests """ + def __init__(self, searchSuggs: list[SearchSugg], hasMore: bool): + self.suggs = searchSuggs + self.hasMore = hasMore class DescInfo: - " Represents a node's associated description " - def __init__(self, text, wikiId, fromDbp): - self.text = text # string - self.wikiId = wikiId # number - self.fromDbp = fromDbp # boolean + """ Represents a node's associated description """ + def __init__(self, text: str, wikiId: int, fromDbp: bool): + self.text = text + self.wikiId = wikiId + self.fromDbp = fromDbp class ImgInfo: - " Represents a node's associated image " - def __init__(self, id, src, url, license, artist, credit): - self.id = id # number - self.src = src # string - self.url = url # string - self.license = license # string - self.artist = artist # string - self.credit = credit # string + """ Represents a node's associated image """ + def __init__(self, id: int, src: str, url: str, license: str, artist: str, credit: str): + self.id = id + self.src = src + self.url = url + self.license = license + self.artist = artist + self.credit = credit class NodeInfo: - " Represents info about a node " - def __init__(self, tolNode, descInfo, imgInfo): - self.tolNode = tolNode # TolNode - self.descInfo = descInfo # null | DescInfo - self.imgInfo = imgInfo # null | ImgInfo + """ Represents info about a node """ + def __init__(self, tolNode: TolNode, descInfo: DescInfo | None, imgInfo: ImgInfo | None): + self.tolNode = tolNode + self.descInfo = descInfo + self.imgInfo = imgInfo class InfoResponse: - " Sent as responses to 'info' requests " - def __init__(self, nodeInfo, subNodesInfo): - self.nodeInfo = nodeInfo # NodeInfo - self.subNodesInfo = subNodesInfo # [] | [NodeInfo | null, NodeInfo | null] + """ Sent as responses to 'info' requests """ + def __init__(self, nodeInfo: NodeInfo, subNodesInfo: tuple[()] | tuple[NodeInfo | None, NodeInfo | None]): + self.nodeInfo = nodeInfo + self.subNodesInfo = subNodesInfo # For data lookup -def lookupNodes(names, tree, dbCur): - " For a set of node names, returns a name-to-TolNode map that describes those nodes " +def lookupNodes(names: list[str], tree: str, dbCur: sqlite3.Cursor) -> dict[str, TolNode]: + """ For a set of node names, returns a name-to-TolNode map that describes those nodes """ # Get node info - nameToNodes = {} + nameToNodes: dict[str, TolNode] = {} tblSuffix = getTableSuffix(tree) - nodesTable = f"nodes_{tblSuffix}" - edgesTable = f"edges_{tblSuffix}" - queryParamStr = ",".join(["?"] * len(names)) - query = f"SELECT name, id, tips FROM {nodesTable} WHERE name IN ({queryParamStr})" - for (nodeName, otolId, tips) in dbCur.execute(query, names): + nodesTable = f'nodes_{tblSuffix}' + edgesTable = f'edges_{tblSuffix}' + queryParamStr = ','.join(['?'] * len(names)) + query = f'SELECT name, id, tips FROM {nodesTable} WHERE name IN ({queryParamStr})' + for nodeName, otolId, tips in dbCur.execute(query, names): nameToNodes[nodeName] = TolNode(otolId, [], tips=tips) # Get child info - query = f"SELECT parent, child FROM {edgesTable} WHERE parent IN ({queryParamStr})" - for (nodeName, childName) in dbCur.execute(query, names): + query = f'SELECT parent, child FROM {edgesTable} WHERE parent IN ({queryParamStr})' + for nodeName, childName in dbCur.execute(query, names): nameToNodes[nodeName].children.append(childName) # Order children by tips - for (nodeName, node) in nameToNodes.items(): - childToTips = {} - query = "SELECT name, tips FROM {} WHERE name IN ({})" - query = query.format(nodesTable, ",".join(["?"] * len(node.children))) - for (n, tips) in dbCur.execute(query, node.children): + for nodeName, node in nameToNodes.items(): + childToTips: dict[str, int] = {} + query = 'SELECT name, tips FROM {} WHERE name IN ({})' + query = query.format(nodesTable, ','.join(['?'] * len(node.children))) + for n, tips in dbCur.execute(query, node.children): childToTips[n] = tips node.children.sort(key=lambda n: childToTips[n], reverse=True) # Get parent info - query = f"SELECT parent, child, p_support FROM {edgesTable} WHERE child IN ({queryParamStr})" - for (nodeName, childName, pSupport) in dbCur.execute(query, names): + query = f'SELECT parent, child, p_support FROM {edgesTable} WHERE child IN ({queryParamStr})' + for nodeName, childName, pSupport in dbCur.execute(query, names): nameToNodes[childName].parent = nodeName - nameToNodes[childName].pSupport = (pSupport == 1) + nameToNodes[childName].pSupport = pSupport == 1 # Get image names idsToNames = {nameToNodes[n].otolId: n for n in nameToNodes.keys()} - query = "SELECT nodes.id from nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name" \ - " WHERE nodes.id IN ({})".format(",".join(["?"] * len(idsToNames))) + query = 'SELECT nodes.id from nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name' \ + ' WHERE nodes.id IN ({})'.format(','.join(['?'] * len(idsToNames))) for (otolId,) in dbCur.execute(query, list(idsToNames.keys())): - nameToNodes[idsToNames[otolId]].imgName = otolId + ".jpg" + nameToNodes[idsToNames[otolId]].imgName = otolId + '.jpg' # Get 'linked' images for unresolved names - unresolvedNames = [n for n in nameToNodes if nameToNodes[n].imgName == None] - query = "SELECT name, otol_ids from linked_imgs WHERE name IN ({})" - query = query.format(",".join(["?"] * len(unresolvedNames))) - for (name, otolIds) in dbCur.execute(query, unresolvedNames): - if "," not in otolIds: - nameToNodes[name].imgName = otolIds + ".jpg" + unresolvedNames = [n for n in nameToNodes if nameToNodes[n].imgName is None] + query = 'SELECT name, otol_ids from linked_imgs WHERE name IN ({})' + query = query.format(','.join(['?'] * len(unresolvedNames))) + for name, otolIds in dbCur.execute(query, unresolvedNames): + if ',' not in otolIds: + nameToNodes[name].imgName = otolIds + '.jpg' else: - id1, id2 = otolIds.split(",") - nameToNodes[name].imgName = [ - id1 + ".jpg" if id1 != "" else None, - id2 + ".jpg" if id2 != "" else None, - ] + id1, id2 = otolIds.split(',') + nameToNodes[name].imgName = ( + id1 + '.jpg' if id1 != '' else None, + id2 + '.jpg' if id2 != '' else None, + ) # Get preferred-name info - query = f"SELECT name, alt_name FROM names WHERE pref_alt = 1 AND name IN ({queryParamStr})" - for (name, altName) in dbCur.execute(query, names): + query = f'SELECT name, alt_name FROM names WHERE pref_alt = 1 AND name IN ({queryParamStr})' + for name, altName in dbCur.execute(query, names): nameToNodes[name].commonName = altName # Get IUCN status - query = f"SELECT name, iucn FROM node_iucn WHERE name IN ({queryParamStr})" - for (name, iucn) in dbCur.execute(query, names): + query = f'SELECT name, iucn FROM node_iucn WHERE name IN ({queryParamStr})' + for name, iucn in dbCur.execute(query, names): nameToNodes[name].iucn = iucn # return nameToNodes -def lookupSuggs(searchStr, suggLimit, tree, dbCur): - " For a search string, returns a SearchSuggResponse describing search suggestions " - results = [] +def lookupSuggs(searchStr: str, suggLimit: int, tree: str, dbCur: sqlite3.Cursor) -> SearchSuggResponse: + """ For a search string, returns a SearchSuggResponse describing search suggestions """ hasMore = False # Get node names and alt-names, ordering by popularity - nodesTable = f"nodes_{getTableSuffix(tree)}" - nameQuery = f"SELECT {nodesTable}.name, node_pop.pop FROM {nodesTable}" \ - f" LEFT JOIN node_pop ON {nodesTable}.name = node_pop.name" \ - f" WHERE node_pop.name LIKE ? AND node_pop.name NOT LIKE '[%'" \ - f" ORDER BY node_pop.pop DESC" - altNameQuery = f"SELECT alt_name, names.name, pref_alt, node_pop.pop FROM" \ - f" names INNER JOIN {nodesTable} ON names.name = {nodesTable}.name" \ - f" LEFT JOIN node_pop ON {nodesTable}.name = node_pop.name" \ - f" WHERE alt_name LIKE ? ORDER BY node_pop.pop DESC" - suggs = {} + nodesTable = f'nodes_{getTableSuffix(tree)}' + nameQuery = f'SELECT {nodesTable}.name, node_pop.pop FROM {nodesTable}' \ + f' LEFT JOIN node_pop ON {nodesTable}.name = node_pop.name' \ + f' WHERE node_pop.name LIKE ? AND node_pop.name NOT LIKE "[%"' \ + f' ORDER BY node_pop.pop DESC' + altNameQuery = f'SELECT alt_name, names.name, pref_alt, node_pop.pop FROM' \ + f' names INNER JOIN {nodesTable} ON names.name = {nodesTable}.name' \ + f' LEFT JOIN node_pop ON {nodesTable}.name = node_pop.name' \ + f' WHERE alt_name LIKE ? ORDER BY node_pop.pop DESC' + suggs: dict[str, SearchSugg] = {} tempLimit = suggLimit + 1 # For determining if 'more suggestions exist' # Prefix search - for altName, nodeName, prefAlt, pop in dbCur.execute(altNameQuery, (searchStr + "%",)): - if nodeName not in suggs or prefAlt == 1 and suggs[nodeName].canonicalName != None: + for altName, nodeName, prefAlt, pop in dbCur.execute(altNameQuery, (searchStr + '%',)): + if nodeName not in suggs or prefAlt == 1 and suggs[nodeName].canonicalName is not None: suggs[nodeName] = SearchSugg(altName, nodeName, pop) if len(suggs) == tempLimit: break if len(suggs) < tempLimit: # Prefix search of canonical names - for nodeName, pop in dbCur.execute(nameQuery, (searchStr + "%",)): + for nodeName, pop in dbCur.execute(nameQuery, (searchStr + '%',)): if nodeName not in suggs: suggs[nodeName] = SearchSugg(nodeName, pop=pop) if len(suggs) == tempLimit: @@ -173,17 +181,17 @@ def lookupSuggs(searchStr, suggLimit, tree, dbCur): suggList = sorted(suggs.values(), key=lambda x: x.pop, reverse=True) # If insufficient results, try substring-search if len(suggs) < tempLimit: - newNames = set() + newNames: set[str] = set() oldNames = suggs.keys() - for altName, nodeName, prefAlt, pop in dbCur.execute(altNameQuery, ("%" + searchStr + "%",)): + for altName, nodeName, prefAlt, pop in dbCur.execute(altNameQuery, ('%' + searchStr + '%',)): if nodeName not in suggs or \ - nodeName not in oldNames and prefAlt == 1 and suggs[nodeName].canonicalName != None: + nodeName not in oldNames and prefAlt == 1 and suggs[nodeName].canonicalName is not None: suggs[nodeName] = SearchSugg(altName, nodeName, pop) newNames.add(nodeName) if len(suggs) == tempLimit: break if len(suggs) < tempLimit: - for nodeName, pop in dbCur.execute(nameQuery, ("%" + searchStr + "%",)): + for nodeName, pop in dbCur.execute(nameQuery, ('%' + searchStr + '%',)): if nodeName not in suggs: suggs[nodeName] = SearchSugg(nodeName, pop=pop) newNames.add(nodeName) @@ -194,38 +202,39 @@ def lookupSuggs(searchStr, suggLimit, tree, dbCur): if len(suggList) > suggLimit: hasMore = True return SearchSuggResponse(suggList[:suggLimit], hasMore) -def lookupInfo(name, tree, dbCur): - " For a node name, returns an InfoResponse, or None " +def lookupInfo(name: str, tree: str, dbCur: sqlite3.Cursor) -> InfoResponse | None: + """ For a node name, returns a descriptive InfoResponse, or None """ # Get node info nameToNodes = lookupNodes([name], tree, dbCur) tolNode = nameToNodes[name] if name in nameToNodes else None - if tolNode == None: + if tolNode is None: return None # Check for compound node - match = re.fullmatch(r"\[(.+) \+ (.+)]", name) - subNames = [match.group(1), match.group(2)] if match != None else [] - if len(subNames) > 0: + match = re.fullmatch(r'\[(.+) \+ (.+)]', name) + subNames = [match.group(1), match.group(2)] if match is not None else [] + if subNames: nameToSubNodes = lookupNodes(subNames, tree, dbCur) if len(nameToSubNodes) < 2: # Possible when a subname-denoted node has been trimmed away subNames = [n if n in nameToSubNodes else None for n in subNames] nameToNodes.update(nameToSubNodes) - namesToLookup = [name] if len(subNames) == 0 else [n for n in subNames if n != None] + namesToLookup = [name] if not subNames else [n for n in subNames if n is not None] # Get desc info - nameToDescInfo = {} - query = "SELECT name, desc, wiki_id, from_dbp FROM" \ - " wiki_ids INNER JOIN descs ON wiki_ids.id = descs.wiki_id" \ - " WHERE wiki_ids.name IN ({})".format(",".join(["?"] * len(namesToLookup))) - for (nodeName, desc, wikiId, fromDbp) in dbCur.execute(query, namesToLookup): + nameToDescInfo: dict[str, DescInfo] = {} + query = 'SELECT name, desc, wiki_id, from_dbp FROM' \ + ' wiki_ids INNER JOIN descs ON wiki_ids.id = descs.wiki_id' \ + ' WHERE wiki_ids.name IN ({})'.format(','.join(['?'] * len(namesToLookup))) + for nodeName, desc, wikiId, fromDbp in dbCur.execute(query, namesToLookup): nameToDescInfo[nodeName] = DescInfo(desc, wikiId, fromDbp == 1) # Get image info - nameToImgInfo = {} - idsToNames = {nameToNodes[n].imgName[:-4]: n for n in namesToLookup if nameToNodes[n].imgName != None} + nameToImgInfo: dict[str, ImgInfo] = {} + idsToNames = {cast(str, nameToNodes[n].imgName)[:-4]: n + for n in namesToLookup if nameToNodes[n].imgName is not None} idsToLookup = list(idsToNames.keys()) # Lookup using IDs avoids having to check linked_imgs - query = "SELECT nodes.id, images.id, images.src, url, license, artist, credit FROM" \ - " nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name" \ - " INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src" \ - " WHERE nodes.id IN ({})".format(",".join(["?"] * len(idsToLookup))) - for (id, imgId, imgSrc, url, license, artist, credit) in dbCur.execute(query, idsToLookup): + query = 'SELECT nodes.id, images.id, images.src, url, license, artist, credit FROM' \ + ' nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name' \ + ' INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src' \ + ' WHERE nodes.id IN ({})'.format(','.join(['?'] * len(idsToLookup))) + for id, imgId, imgSrc, url, license, artist, credit in dbCur.execute(query, idsToLookup): nameToImgInfo[idsToNames[id]] = ImgInfo(imgId, imgSrc, url, license, artist, credit) # Construct response nodeInfoObjs = [ @@ -233,61 +242,66 @@ def lookupInfo(name, tree, dbCur): nameToNodes[n], nameToDescInfo[n] if n in nameToDescInfo else None, nameToImgInfo[n] if n in nameToImgInfo else None - ) if n != None else None for n in [name] + subNames + ) if n is not None else None for n in [name] + subNames ] - return InfoResponse(nodeInfoObjs[0], nodeInfoObjs[1:]) -def getTableSuffix(tree): - return "t" if tree == "trimmed" else "i" if tree == "images" else "p" + return InfoResponse( + nodeInfoObjs[0], + cast(tuple[()] | tuple[NodeInfo | None, NodeInfo | None], nodeInfoObjs[1:])) +def getTableSuffix(tree: str) -> str: + """ converts a reduced-tree descriptor into a sql-table-suffix """ + return 't' if tree == 'trimmed' else 'i' if tree == 'images' else 'p' -# Queries the database, and constructs a response object -def handleReq(dbCur, environ): +def handleReq( + dbCur: sqlite3.Cursor, + environ: dict[str, str]) -> None | dict[str, TolNode] | SearchSuggResponse | InfoResponse: + """ Queries the database, and constructs a response object """ # Get query params - queryStr = environ["QUERY_STRING"] if "QUERY_STRING" in environ else "" + queryStr = environ['QUERY_STRING'] if 'QUERY_STRING' in environ else '' queryDict = urllib.parse.parse_qs(queryStr) # Set vars from params - name = queryDict["name"][0] if "name" in queryDict else None - if name == None: # Get root node + name = queryDict['name'][0] if 'name' in queryDict else None + if name is None: # Get root node name = ROOT_NAME # Hard-coding this is significantly faster (in testing, querying could take 0.5 seconds) - #query = "SELECT name FROM nodes LEFT JOIN edges ON nodes.name = edges.child WHERE edges.parent IS NULL LIMIT 1" + #query = 'SELECT name FROM nodes LEFT JOIN edges ON nodes.name = edges.child WHERE edges.parent IS NULL LIMIT 1' #(name,) = dbCur.execute(query).fetchone() - reqType = queryDict["type"][0] if "type" in queryDict else None - tree = queryDict["tree"][0] if "tree" in queryDict else "images" + reqType = queryDict['type'][0] if 'type' in queryDict else None + tree = queryDict['tree'][0] if 'tree' in queryDict else 'images' # Check for valid 'tree' - if tree != None and re.fullmatch(r"trimmed|images|picked", tree) == None: + if tree is not None and re.fullmatch(r'trimmed|images|picked', tree) is None: return None # Get data of requested type - if reqType == "node": - toroot = queryDict["toroot"][0] == '1' if "toroot" in queryDict else False + if reqType == 'node': + toroot = queryDict['toroot'][0] == '1' if 'toroot' in queryDict else False if not toroot: tolNodes = lookupNodes([name], tree, dbCur) - if len(tolNodes) > 0: + if tolNodes: tolNode = tolNodes[name] childNodeObjs = lookupNodes(tolNode.children, tree, dbCur) childNodeObjs[name] = tolNode return childNodeObjs else: # Get ancestors to skip inclusion of - nodesToSkip = set() - nodeName = queryDict["excl"][0] if "excl" in queryDict else None - if nodeName != None: - edgesTable = f"edges_{getTableSuffix(tree)}" + nodesToSkip: set[str] = set() + nodeName = queryDict['excl'][0] if 'excl' in queryDict else None + if nodeName is not None: + edgesTable = f'edges_{getTableSuffix(tree)}' while True: - row = dbCur.execute(f"SELECT parent FROM {edgesTable} WHERE child = ?", (nodeName,)).fetchone() - if row == None: + row = dbCur.execute(f'SELECT parent FROM {edgesTable} WHERE child = ?', (nodeName,)).fetchone() + if row is None: break parent = row[0] nodesToSkip.add(parent) nodeName = parent # - results = {} + results: dict[str, TolNode] = {} ranOnce = False while True: # Get node tolNodes = lookupNodes([name], tree, dbCur) - if len(tolNodes) == 0: + if not tolNodes: if not ranOnce: return results - print(f"ERROR: Parent-chain node {name} not found", file=sys.stderr) + print(f'ERROR: Parent-chain node {name} not found', file=sys.stderr) break tolNode = tolNodes[name] results[name] = tolNode @@ -302,32 +316,32 @@ def handleReq(dbCur, environ): childNodeObjs = lookupNodes(childNamesToAdd, tree, dbCur) results.update(childNodeObjs) # Check if root - if tolNode.parent == None or tolNode.parent in nodesToSkip: + if tolNode.parent is None or tolNode.parent in nodesToSkip: return results else: name = tolNode.parent - elif reqType == "sugg": + elif reqType == 'sugg': # Check for suggestion-limit - suggLimit = None + suggLimit: int invalidLimit = False try: - suggLimit = int(queryDict["limit"][0]) if "limit" in queryDict else DEFAULT_SUGG_LIM + suggLimit = int(queryDict['limit'][0]) if 'limit' in queryDict else DEFAULT_SUGG_LIM if suggLimit <= 0 or suggLimit > MAX_SUGG_LIM: invalidLimit = True except ValueError: invalidLimit = True - print(f"INFO: Invalid limit {suggLimit}", file=sys.stderr) + print(f'INFO: Invalid limit {suggLimit}', file=sys.stderr) # Get search suggestions if not invalidLimit: return lookupSuggs(name, suggLimit, tree, dbCur) - elif reqType == "info": + elif reqType == 'info': infoResponse = lookupInfo(name, tree, dbCur) - if infoResponse != None: + if infoResponse is not None: return infoResponse # On failure, provide empty response return None -# Entry point for the WSGI script -def application(environ, start_response): +def application(environ: dict[str, str], start_response) -> Iterable[bytes]: + """ Entry point for the WSGI script """ # Open db dbCon = sqlite3.connect(DB_FILE) dbCur = dbCon.cursor() @@ -335,11 +349,11 @@ def application(environ, start_response): val = handleReq(dbCur, environ) # Construct response data = jsonpickle.encode(val, unpicklable=False).encode() - headers = [("Content-type", "application/json")] - if "HTTP_ACCEPT_ENCODING" in environ and "gzip" in environ["HTTP_ACCEPT_ENCODING"]: + headers = [('Content-type', 'application/json')] + if 'HTTP_ACCEPT_ENCODING' in environ and 'gzip' in environ['HTTP_ACCEPT_ENCODING']: if len(data) > 100: data = gzip.compress(data, compresslevel=5) - headers.append(("Content-encoding", "gzip")) + headers.append(('Content-encoding', 'gzip')) headers.append(('Content-Length', str(len(data)))) start_response('200 OK', headers) return [data] diff --git a/backend/tolData/dbpedia/genDescData.py b/backend/tolData/dbpedia/genDescData.py index 8756a40..43ed815 100755 --- a/backend/tolData/dbpedia/genDescData.py +++ b/backend/tolData/dbpedia/genDescData.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import sys, re +import re import bz2, sqlite3 import argparse @@ -9,120 +9,120 @@ Adds DBpedia labels/types/abstracts/etc data into a database """, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -labelsFile = "labels_lang=en.ttl.bz2" # Had about 16e6 entries -idsFile = "page_lang=en_ids.ttl.bz2" -redirectsFile = "redirects_lang=en_transitive.ttl.bz2" -disambigFile = "disambiguations_lang=en.ttl.bz2" -typesFile = "instance-types_lang=en_specific.ttl.bz2" -abstractsFile = "short-abstracts_lang=en.ttl.bz2" -dbFile = "descData.db" +labelsFile = 'labels_lang=en.ttl.bz2' # Had about 16e6 entries +idsFile = 'page_lang=en_ids.ttl.bz2' +redirectsFile = 'redirects_lang=en_transitive.ttl.bz2' +disambigFile = 'disambiguations_lang=en.ttl.bz2' +typesFile = 'instance-types_lang=en_specific.ttl.bz2' +abstractsFile = 'short-abstracts_lang=en.ttl.bz2' +dbFile = 'descData.db' # In testing, this script took a few hours to run, and generated about 10GB -print("Creating database") +print('Creating database') dbCon = sqlite3.connect(dbFile) dbCur = dbCon.cursor() -print("Reading/storing label data") -dbCur.execute("CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)") -dbCur.execute("CREATE INDEX labels_idx ON labels(label)") -dbCur.execute("CREATE INDEX labels_idx_nc ON labels(label COLLATE NOCASE)") +print('Reading/storing label data') +dbCur.execute('CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)') +dbCur.execute('CREATE INDEX labels_idx ON labels(label)') +dbCur.execute('CREATE INDEX labels_idx_nc ON labels(label COLLATE NOCASE)') labelLineRegex = re.compile(r'<([^>]+)> <[^>]+> "((?:[^"]|\\")+)"@en \.\n') lineNum = 0 with bz2.open(labelsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = labelLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") - dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') + dbCur.execute('INSERT INTO labels VALUES (?, ?)', (match.group(1), match.group(2))) -print("Reading/storing wiki page ids") -dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)") -dbCur.execute("CREATE INDEX ids_idx ON ids(id)") +print('Reading/storing wiki page ids') +dbCur.execute('CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)') +dbCur.execute('CREATE INDEX ids_idx ON ids(id)') idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n') lineNum = 0 with bz2.open(idsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = idLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') try: - dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) + dbCur.execute('INSERT INTO ids VALUES (?, ?)', (match.group(1), int(match.group(2)))) except sqlite3.IntegrityError as e: # Accounts for certain lines that have the same IRI - print(f"WARNING: Failed to add entry with IRI \"{match.group(1)}\": {e}") + print(f'WARNING: Failed to add entry with IRI "{match.group(1)}": {e}') -print("Reading/storing redirection data") -dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") +print('Reading/storing redirection data') +dbCur.execute('CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)') redirLineRegex = re.compile(r'<([^>]+)> <[^>]+> <([^>]+)> \.\n') lineNum = 0 with bz2.open(redirectsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = redirLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") - dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') + dbCur.execute('INSERT INTO redirects VALUES (?, ?)', (match.group(1), match.group(2))) -print("Reading/storing diambiguation-page data") -dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") +print('Reading/storing diambiguation-page data') +dbCur.execute('CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)') disambigLineRegex = redirLineRegex lineNum = 0 with bz2.open(disambigFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = disambigLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") - dbCur.execute("INSERT OR IGNORE INTO disambiguations VALUES (?)", (match.group(1),)) + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') + dbCur.execute('INSERT OR IGNORE INTO disambiguations VALUES (?)', (match.group(1),)) -print("Reading/storing instance-type data") -dbCur.execute("CREATE TABLE types (iri TEXT, type TEXT)") -dbCur.execute("CREATE INDEX types_iri_idx ON types(iri)") +print('Reading/storing instance-type data') +dbCur.execute('CREATE TABLE types (iri TEXT, type TEXT)') +dbCur.execute('CREATE INDEX types_iri_idx ON types(iri)') typeLineRegex = redirLineRegex lineNum = 0 with bz2.open(typesFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = typeLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") - dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') + dbCur.execute('INSERT INTO types VALUES (?, ?)', (match.group(1), match.group(2))) -print("Reading/storing abstracts") -dbCur.execute("CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)") +print('Reading/storing abstracts') +dbCur.execute('CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)') descLineRegex = labelLineRegex lineNum = 0 with bz2.open(abstractsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # - if line[0] == "#": + if line[0] == '#': continue match = descLineRegex.fullmatch(line) - if match == None: - raise Exception(f"ERROR: Line {lineNum} has unexpected format") - dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", + if match is None: + raise Exception(f'ERROR: Line {lineNum} has unexpected format') + dbCur.execute('INSERT INTO abstracts VALUES (?, ?)', (match.group(1), match.group(2).replace(r'\"', '"'))) -print("Closing database") +print('Closing database') dbCon.commit() dbCon.close() diff --git a/backend/tolData/enwiki/downloadImgLicenseInfo.py b/backend/tolData/enwiki/downloadImgLicenseInfo.py index dd39d54..ba6317e 100755 --- a/backend/tolData/enwiki/downloadImgLicenseInfo.py +++ b/backend/tolData/enwiki/downloadImgLicenseInfo.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import sys, re +import re import sqlite3, urllib.parse, html import requests import time, signal @@ -16,33 +16,33 @@ at already-processed names to decide what to skip. """, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -imgDb = "imgData.db" -apiUrl = "https://en.wikipedia.org/w/api.php" -userAgent = "terryt.dev (terry06890@gmail.com)" +imgDb = 'imgData.db' +apiUrl = 'https://en.wikipedia.org/w/api.php' +userAgent = 'terryt.dev (terry06890@gmail.com)' batchSz = 50 # Max 50 -tagRegex = re.compile(r"<[^<]+>") -whitespaceRegex = re.compile(r"\s+") +tagRegex = re.compile(r'<[^<]+>') +whitespaceRegex = re.compile(r'\s+') -print("Opening database") +print('Opening database') dbCon = sqlite3.connect(imgDb) dbCur = dbCon.cursor() dbCur2 = dbCon.cursor() -print("Checking for table") -if dbCur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='imgs'").fetchone() == None: - dbCur.execute("CREATE TABLE imgs(" \ - "name TEXT PRIMARY KEY, license TEXT, artist TEXT, credit TEXT, restrictions TEXT, url TEXT)") +print('Checking for table') +if dbCur.execute('SELECT name FROM sqlite_master WHERE type="table" AND name="imgs"').fetchone() is None: + dbCur.execute('CREATE TABLE imgs(' \ + 'name TEXT PRIMARY KEY, license TEXT, artist TEXT, credit TEXT, restrictions TEXT, url TEXT)') -print("Reading image names") -imgNames = set() -for (imgName,) in dbCur.execute("SELECT DISTINCT img_name FROM page_imgs WHERE img_name NOT NULL"): +print('Reading image names') +imgNames: set[str] = set() +for (imgName,) in dbCur.execute('SELECT DISTINCT img_name FROM page_imgs WHERE img_name NOT NULL'): imgNames.add(imgName) -print(f"Found {len(imgNames)}") +print(f'Found {len(imgNames)}') -print("Checking for already-processed images") +print('Checking for already-processed images') oldSz = len(imgNames) -for (imgName,) in dbCur.execute("SELECT name FROM imgs"): +for (imgName,) in dbCur.execute('SELECT name FROM imgs'): imgNames.discard(imgName) -print(f"Found {oldSz - len(imgNames)}") +print(f'Found {oldSz - len(imgNames)}') # Set SIGINT handler interrupted = False @@ -53,95 +53,95 @@ def onSigint(sig, frame): signal.signal(signal.SIGINT, oldHandler) oldHandler = signal.signal(signal.SIGINT, onSigint) -print("Iterating through image names") -imgNames = list(imgNames) +print('Iterating through image names') +imgNameList = list(imgNames) iterNum = 0 -for i in range(0, len(imgNames), batchSz): +for i in range(0, len(imgNameList), batchSz): iterNum += 1 if iterNum % 1 == 0: - print(f"At iteration {iterNum} (after {(iterNum - 1) * batchSz} images)") + print(f'At iteration {iterNum} (after {(iterNum - 1) * batchSz} images)') if interrupted: - print(f"Exiting loop at iteration {iterNum}") + print(f'Exiting loop at iteration {iterNum}') break # Get batch - imgBatch = imgNames[i:i+batchSz] - imgBatch = ["File:" + x for x in imgBatch] + imgBatch = imgNameList[i:i+batchSz] + imgBatch = ['File:' + x for x in imgBatch] # Make request headers = { - "user-agent": userAgent, - "accept-encoding": "gzip", + 'user-agent': userAgent, + 'accept-encoding': 'gzip', } params = { - "action": "query", - "format": "json", - "prop": "imageinfo", - "iiprop": "extmetadata|url", - "maxlag": "5", - "titles": "|".join(imgBatch), - "iiextmetadatafilter": "Artist|Credit|LicenseShortName|Restrictions", + 'action': 'query', + 'format': 'json', + 'prop': 'imageinfo', + 'iiprop': 'extmetadata|url', + 'maxlag': '5', + 'titles': '|'.join(imgBatch), + 'iiextmetadatafilter': 'Artist|Credit|LicenseShortName|Restrictions', } responseObj = None try: response = requests.get(apiUrl, params=params, headers=headers) responseObj = response.json() except Exception as e: - print(f"ERROR: Exception while downloading info: {e}") - print(f"\tImage batch: " + "|".join(imgBatch)) + print(f'ERROR: Exception while downloading info: {e}') + print('\tImage batch: ' + '|'.join(imgBatch)) continue # Parse response-object - if "query" not in responseObj or "pages" not in responseObj["query"]: - print("WARNING: Response object for doesn't have page data") - print("\tImage batch: " + "|".join(imgBatch)) - if "error" in responseObj: - errorCode = responseObj["error"]["code"] - print(f"\tError code: {errorCode}") - if errorCode == "maxlag": + if 'query' not in responseObj or 'pages' not in responseObj['query']: + print('WARNING: Response object for doesn\'t have page data') + print('\tImage batch: ' + '|'.join(imgBatch)) + if 'error' in responseObj: + errorCode = responseObj['error']['code'] + print(f'\tError code: {errorCode}') + if errorCode == 'maxlag': time.sleep(5) continue - pages = responseObj["query"]["pages"] - normalisedToInput = {} - if "normalized" in responseObj["query"]: - for entry in responseObj["query"]["normalized"]: - normalisedToInput[entry["to"]] = entry["from"] - for (_, page) in pages.items(): + pages = responseObj['query']['pages'] + normalisedToInput: dict[str, str] = {} + if 'normalized' in responseObj['query']: + for entry in responseObj['query']['normalized']: + normalisedToInput[entry['to']] = entry['from'] + for _, page in pages.items(): # Some fields // More info at https://www.mediawiki.org/wiki/Extension:CommonsMetadata#Returned_data # LicenseShortName: short human-readable license name, apparently more reliable than 'License', # Artist: author name (might contain complex html, multiple authors, etc) # Credit: 'source' # For image-map-like images, can be quite large/complex html, creditng each sub-image - # May be text2, where the text2 might be non-indicative + # May be text2, where the text2 might be non-indicative # Restrictions: specifies non-copyright legal restrictions - title = page["title"] + title: str = page['title'] if title in normalisedToInput: title = normalisedToInput[title] title = title[5:] # Remove 'File:' if title not in imgNames: - print(f"WARNING: Got title \"{title}\" not in image-name list") + print(f'WARNING: Got title "{title}" not in image-name list') continue - if "imageinfo" not in page: - print(f"WARNING: No imageinfo section for page \"{title}\"") + if 'imageinfo' not in page: + print(f'WARNING: No imageinfo section for page "{title}"') continue - metadata = page["imageinfo"][0]["extmetadata"] - url = page["imageinfo"][0]["url"] - license = metadata['LicenseShortName']['value'] if 'LicenseShortName' in metadata else None - artist = metadata['Artist']['value'] if 'Artist' in metadata else None - credit = metadata['Credit']['value'] if 'Credit' in metadata else None - restrictions = metadata['Restrictions']['value'] if 'Restrictions' in metadata else None + metadata = page['imageinfo'][0]['extmetadata'] + url: str = page['imageinfo'][0]['url'] + license: str | None = metadata['LicenseShortName']['value'] if 'LicenseShortName' in metadata else None + artist: str | None = metadata['Artist']['value'] if 'Artist' in metadata else None + credit: str | None = metadata['Credit']['value'] if 'Credit' in metadata else None + restrictions: str | None = metadata['Restrictions']['value'] if 'Restrictions' in metadata else None # Remove markup - if artist != None: - artist = tagRegex.sub(" ", artist) - artist = whitespaceRegex.sub(" ", artist) + if artist is not None: + artist = tagRegex.sub(' ', artist) + artist = whitespaceRegex.sub(' ', artist) artist = html.unescape(artist) artist = urllib.parse.unquote(artist) - if credit != None: - credit = tagRegex.sub(" ", credit) - credit = whitespaceRegex.sub(" ", credit) + if credit is not None: + credit = tagRegex.sub(' ', credit) + credit = whitespaceRegex.sub(' ', credit) credit = html.unescape(credit) credit = urllib.parse.unquote(credit) # Add to db - dbCur2.execute("INSERT INTO imgs VALUES (?, ?, ?, ?, ?, ?)", + dbCur2.execute('INSERT INTO imgs VALUES (?, ?, ?, ?, ?, ?)', (title, license, artist, credit, restrictions, url)) -print("Closing database") +print('Closing database') dbCon.commit() dbCon.close() diff --git a/backend/tolData/enwiki/downloadImgs.py b/backend/tolData/enwiki/downloadImgs.py index 520677f..def4714 100755 --- a/backend/tolData/enwiki/downloadImgs.py +++ b/backend/tolData/enwiki/downloadImgs.py @@ -16,20 +16,20 @@ in the output directory do decide what to skip. """, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -imgDb = "imgData.db" # About 130k image names -outDir = "imgs" -licenseRegex = re.compile(r"cc0|cc([ -]by)?([ -]sa)?([ -][1234]\.[05])?( \w\w\w?)?", flags=re.IGNORECASE) +imgDb = 'imgData.db' # About 130k image names +outDir = 'imgs' +licenseRegex = re.compile(r'cc0|cc([ -]by)?([ -]sa)?([ -][1234]\.[05])?( \w\w\w?)?', flags=re.IGNORECASE) # In testing, this downloaded about 100k images, over several days if not os.path.exists(outDir): os.mkdir(outDir) -print("Checking for already-downloaded images") +print('Checking for already-downloaded images') fileList = os.listdir(outDir) -pageIdsDone = set() +pageIdsDone: set[int] = set() for filename in fileList: - (basename, extension) = os.path.splitext(filename) + basename, extension = os.path.splitext(filename) pageIdsDone.add(int(basename)) -print(f"Found {len(pageIdsDone)}") +print(f'Found {len(pageIdsDone)}') # Set SIGINT handler interrupted = False @@ -40,49 +40,49 @@ def onSigint(sig, frame): signal.signal(signal.SIGINT, oldHandler) oldHandler = signal.signal(signal.SIGINT, onSigint) -print("Opening database") +print('Opening database') dbCon = sqlite3.connect(imgDb) dbCur = dbCon.cursor() -print("Starting downloads") +print('Starting downloads') iterNum = 0 -query = "SELECT page_id, license, artist, credit, restrictions, url FROM" \ - " imgs INNER JOIN page_imgs ON imgs.name = page_imgs.img_name" -for (pageId, license, artist, credit, restrictions, url) in dbCur.execute(query): +query = 'SELECT page_id, license, artist, credit, restrictions, url FROM' \ + ' imgs INNER JOIN page_imgs ON imgs.name = page_imgs.img_name' +for pageId, license, artist, credit, restrictions, url in dbCur.execute(query): if pageId in pageIdsDone: continue if interrupted: - print(f"Exiting loop") + print('Exiting loop') break # Check for problematic attributes - if license == None or licenseRegex.fullmatch(license) == None: + if license is None or licenseRegex.fullmatch(license) is None: continue - if artist == None or artist == "" or len(artist) > 100 or re.match(r"(\d\. )?File:", artist) != None: + if artist is None or artist == '' or len(artist) > 100 or re.match(r'(\d\. )?File:', artist) is not None: continue - if credit == None or len(credit) > 300 or re.match(r"File:", credit) != None: + if credit is None or len(credit) > 300 or re.match(r'File:', credit) is not None: continue - if restrictions != None and restrictions != "": + if restrictions is not None and restrictions != '': continue # Download image iterNum += 1 - print(f"Iteration {iterNum}: Downloading for page-id {pageId}") + print(f'Iteration {iterNum}: Downloading for page-id {pageId}') urlParts = urllib.parse.urlparse(url) extension = os.path.splitext(urlParts.path)[1] if len(extension) <= 1: - print(f"WARNING: No filename extension found in URL {url}") + print(f'WARNING: No filename extension found in URL {url}') sys.exit(1) - outFile = f"{outDir}/{pageId}{extension}" + outFile = f'{outDir}/{pageId}{extension}' headers = { - "user-agent": "terryt.dev (terry06890@gmail.com)", - "accept-encoding": "gzip", + 'user-agent': 'terryt.dev (terry06890@gmail.com)', + 'accept-encoding': 'gzip', } try: response = requests.get(url, headers=headers) with open(outFile, 'wb') as file: file.write(response.content) time.sleep(1) - # https://en.wikipedia.org/wiki/Wikipedia:Database_download says to "throttle self to 1 cache miss per sec" + # https://en.wikipedia.org/wiki/Wikipedia:Database_download says to 'throttle self to 1 cache miss per sec' # It's unclear how to properly check for cache misses, so this just aims for 1 per sec except Exception as e: - print(f"Error while downloading to {outFile}: {e}") -print("Closing database") + print(f'Error while downloading to {outFile}: {e}') +print('Closing database') dbCon.close() diff --git a/backend/tolData/enwiki/genDescData.py b/backend/tolData/enwiki/genDescData.py index 0085d70..1698f5c 100755 --- a/backend/tolData/enwiki/genDescData.py +++ b/backend/tolData/enwiki/genDescData.py @@ -12,46 +12,46 @@ and add them to a database """, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -dumpFile = "enwiki-20220501-pages-articles-multistream.xml.bz2" # Had about 22e6 pages -enwikiDb = "descData.db" +dumpFile = 'enwiki-20220501-pages-articles-multistream.xml.bz2' # Had about 22e6 pages +enwikiDb = 'descData.db' # In testing, this script took over 10 hours to run, and generated about 5GB -descLineRegex = re.compile("^ *[A-Z'\"]") -embeddedHtmlRegex = re.compile(r"<[^<]+/>||<[^([^<]*|[^<]*<[^<]+>[^<]*)|<[^<]+$") +descLineRegex = re.compile('^ *[A-Z\'"]') +embeddedHtmlRegex = re.compile(r'<[^<]+/>||<[^([^<]*|[^<]*<[^<]+>[^<]*)|<[^<]+$') # Recognises a self-closing HTML tag, a tag with 0 children, tag with 1 child with 0 children, or unclosed tag -convertTemplateRegex = re.compile(r"{{convert\|(\d[^|]*)\|(?:(to|-)\|(\d[^|]*)\|)?([a-z][^|}]*)[^}]*}}") +convertTemplateRegex = re.compile(r'{{convert\|(\d[^|]*)\|(?:(to|-)\|(\d[^|]*)\|)?([a-z][^|}]*)[^}]*}}') def convertTemplateReplace(match): - if match.group(2) == None: - return f"{match.group(1)} {match.group(4)}" + if match.group(2) is None: + return f'{match.group(1)} {match.group(4)}' else: - return f"{match.group(1)} {match.group(2)} {match.group(3)} {match.group(4)}" -parensGroupRegex = re.compile(r" \([^()]*\)") -leftoverBraceRegex = re.compile(r"(?:{\||{{).*") + return f'{match.group(1)} {match.group(2)} {match.group(3)} {match.group(4)}' +parensGroupRegex = re.compile(r' \([^()]*\)') +leftoverBraceRegex = re.compile(r'(?:{\||{{).*') -def parseDesc(text): +def parseDesc(text: str) -> str | None: # Find first matching line outside {{...}}, [[...]], and block-html-comment constructs, # and then accumulate lines until a blank one. # Some cases not accounted for include: disambiguation pages, abstracts with sentences split-across-lines, # nested embedded html, 'content significant' embedded-html, markup not removable with mwparsefromhell, - lines = [] + lines: list[str] = [] openBraceCount = 0 openBracketCount = 0 inComment = False skip = False for line in text.splitlines(): line = line.strip() - if len(lines) == 0: - if len(line) > 0: - if openBraceCount > 0 or line[0] == "{": - openBraceCount += line.count("{") - openBraceCount -= line.count("}") + if not lines: + if line: + if openBraceCount > 0 or line[0] == '{': + openBraceCount += line.count('{') + openBraceCount -= line.count('}') skip = True - if openBracketCount > 0 or line[0] == "[": - openBracketCount += line.count("[") - openBracketCount -= line.count("]") + if openBracketCount > 0 or line[0] == '[': + openBracketCount += line.count('[') + openBracketCount -= line.count(']') skip = True - if inComment or line.find("") != -1: + if inComment or line.find('') != -1: if inComment: inComment = False skip = True @@ -61,64 +61,64 @@ def parseDesc(text): if skip: skip = False continue - if line[-1] == ":": # Seems to help avoid disambiguation pages + if line[-1] == ':': # Seems to help avoid disambiguation pages return None - if descLineRegex.match(line) != None: + if descLineRegex.match(line) is not None: lines.append(line) else: - if len(line) == 0: - return removeMarkup(" ".join(lines)) + if not line: + return removeMarkup(' '.join(lines)) lines.append(line) - if len(lines) > 0: - return removeMarkup(" ".join(lines)) + if lines: + return removeMarkup(' '.join(lines)) return None -def removeMarkup(content): - content = embeddedHtmlRegex.sub("", content) +def removeMarkup(content: str) -> str: + content = embeddedHtmlRegex.sub('', content) content = convertTemplateRegex.sub(convertTemplateReplace, content) content = mwparserfromhell.parse(content).strip_code() # Remove wikitext markup - content = parensGroupRegex.sub("", content) - content = leftoverBraceRegex.sub("", content) + content = parensGroupRegex.sub('', content) + content = leftoverBraceRegex.sub('', content) return content -def convertTitle(title): - return html.unescape(title).replace("_", " ") +def convertTitle(title: str) -> str: + return html.unescape(title).replace('_', ' ') -print("Creating database") +print('Creating database') if os.path.exists(enwikiDb): - raise Exception(f"ERROR: Existing {enwikiDb}") + raise Exception(f'ERROR: Existing {enwikiDb}') dbCon = sqlite3.connect(enwikiDb) dbCur = dbCon.cursor() -dbCur.execute("CREATE TABLE pages (id INT PRIMARY KEY, title TEXT UNIQUE)") -dbCur.execute("CREATE INDEX pages_title_idx ON pages(title COLLATE NOCASE)") -dbCur.execute("CREATE TABLE redirects (id INT PRIMARY KEY, target TEXT)") -dbCur.execute("CREATE INDEX redirects_idx ON redirects(target)") -dbCur.execute("CREATE TABLE descs (id INT PRIMARY KEY, desc TEXT)") +dbCur.execute('CREATE TABLE pages (id INT PRIMARY KEY, title TEXT UNIQUE)') +dbCur.execute('CREATE INDEX pages_title_idx ON pages(title COLLATE NOCASE)') +dbCur.execute('CREATE TABLE redirects (id INT PRIMARY KEY, target TEXT)') +dbCur.execute('CREATE INDEX redirects_idx ON redirects(target)') +dbCur.execute('CREATE TABLE descs (id INT PRIMARY KEY, desc TEXT)') -print("Iterating through dump file") +print('Iterating through dump file') with bz2.open(dumpFile, mode='rt') as file: dump = mwxml.Dump.from_file(file) pageNum = 0 for page in dump: pageNum += 1 if pageNum % 1e4 == 0: - print(f"At page {pageNum}") + print(f'At page {pageNum}') if pageNum > 3e4: break # Parse page if page.namespace == 0: try: - dbCur.execute("INSERT INTO pages VALUES (?, ?)", (page.id, convertTitle(page.title))) + dbCur.execute('INSERT INTO pages VALUES (?, ?)', (page.id, convertTitle(page.title))) except sqlite3.IntegrityError as e: # Accounts for certain pages that have the same title - print(f"Failed to add page with title \"{page.title}\": {e}", file=sys.stderr) + print(f'Failed to add page with title "{page.title}": {e}', file=sys.stderr) continue - if page.redirect != None: - dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (page.id, convertTitle(page.redirect))) + if page.redirect is not None: + dbCur.execute('INSERT INTO redirects VALUES (?, ?)', (page.id, convertTitle(page.redirect))) else: revision = next(page) desc = parseDesc(revision.text) - if desc != None: - dbCur.execute("INSERT INTO descs VALUES (?, ?)", (page.id, desc)) + if desc is not None: + dbCur.execute('INSERT INTO descs VALUES (?, ?)', (page.id, desc)) -print("Closing database") +print('Closing database') dbCon.commit() dbCon.close() diff --git a/backend/tolData/enwiki/genDumpIndexDb.py b/backend/tolData/enwiki/genDumpIndexDb.py index 1bffb27..3bd129f 100755 --- a/backend/tolData/enwiki/genDumpIndexDb.py +++ b/backend/tolData/enwiki/genDumpIndexDb.py @@ -10,46 +10,47 @@ Adds data from the wiki dump index-file into a database """, formatter_class=argparse.RawDescriptionHelpFormatter) parser.parse_args() -indexFile = "enwiki-20220501-pages-articles-multistream-index.txt.bz2" # Had about 22e6 lines -indexDb = "dumpIndex.db" +indexFile = 'enwiki-20220501-pages-articles-multistream-index.txt.bz2' # Had about 22e6 lines +indexDb = 'dumpIndex.db' if os.path.exists(indexDb): - raise Exception(f"ERROR: Existing {indexDb}") -print("Creating database") + raise Exception(f'ERROR: Existing {indexDb}') +print('Creating database') dbCon = sqlite3.connect(indexDb) dbCur = dbCon.cursor() -dbCur.execute("CREATE TABLE offsets (title TEXT PRIMARY KEY, id INT UNIQUE, offset INT, next_offset INT)") +dbCur.execute('CREATE TABLE offsets (title TEXT PRIMARY KEY, id INT UNIQUE, offset INT, next_offset INT)') -print("Iterating through index file") -lineRegex = re.compile(r"([^:]+):([^:]+):(.*)") +print('Iterating through index file') +lineRegex = re.compile(r'([^:]+):([^:]+):(.*)') lastOffset = 0 lineNum = 0 -entriesToAdd = [] +entriesToAdd: list[tuple[str, str]] = [] with bz2.open(indexFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"At line {lineNum}") + print(f'At line {lineNum}') # match = lineRegex.fullmatch(line.rstrip()) - (offset, pageId, title) = match.group(1,2,3) - offset = int(offset) + assert match is not None + offsetStr, pageId, title = match.group(1,2,3) + offset = int(offsetStr) if offset > lastOffset: - for (t, p) in entriesToAdd: + for t, p in entriesToAdd: try: - dbCur.execute("INSERT INTO offsets VALUES (?, ?, ?, ?)", (t, p, lastOffset, offset)) + dbCur.execute('INSERT INTO offsets VALUES (?, ?, ?, ?)', (t, int(p), lastOffset, offset)) except sqlite3.IntegrityError as e: # Accounts for certain entries in the file that have the same title - print(f"Failed on title \"{t}\": {e}", file=sys.stderr) + print(f'Failed on title "{t}": {e}', file=sys.stderr) entriesToAdd = [] lastOffset = offset - entriesToAdd.append([title, pageId]) -for (title, pageId) in entriesToAdd: + entriesToAdd.append((title, pageId)) +for title, pageId in entriesToAdd: try: - dbCur.execute("INSERT INTO offsets VALUES (?, ?, ?, ?)", (title, pageId, lastOffset, -1)) + dbCur.execute('INSERT INTO offsets VALUES (?, ?, ?, ?)', (title, int(pageId), lastOffset, -1)) except sqlite3.IntegrityError as e: - print(f"Failed on title \"{t}\": {e}", file=sys.stderr) + print(f'Failed on title "{t}": {e}', file=sys.stderr) -print("Closing database") +print('Closing database') dbCon.commit() dbCon.close() diff --git a/backend/tolData/enwiki/genImgData.py b/backend/tolData/enwiki/genImgData.py index b5d546d..00140f6 100755 --- a/backend/tolData/enwiki/genImgData.py +++ b/backend/tolData/enwiki/genImgData.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import sys, re +import re import bz2, html, urllib.parse import sqlite3 @@ -15,117 +15,117 @@ will skip already-processed page IDs. parser.parse_args() def getInputPageIds(): - pageIds = set() - dbCon = sqlite3.connect("../data.db") + pageIds: set[int] = set() + dbCon = sqlite3.connect('../data.db') dbCur = dbCon.cursor() - for (pageId,) in dbCur.execute("SELECT id from wiki_ids"): + for (pageId,) in dbCur.execute('SELECT id from wiki_ids'): pageIds.add(pageId) dbCon.close() return pageIds -dumpFile = "enwiki-20220501-pages-articles-multistream.xml.bz2" -indexDb = "dumpIndex.db" -imgDb = "imgData.db" # The database to create -idLineRegex = re.compile(r"(.*)") -imageLineRegex = re.compile(r".*\| *image *= *([^|]*)") -bracketImageRegex = re.compile(r"\[\[(File:[^|]*).*]]") -imageNameRegex = re.compile(r".*\.(jpg|jpeg|png|gif|tiff|tif)", flags=re.IGNORECASE) -cssImgCropRegex = re.compile(r"{{css image crop\|image *= *(.*)", flags=re.IGNORECASE) +dumpFile = 'enwiki-20220501-pages-articles-multistream.xml.bz2' +indexDb = 'dumpIndex.db' +imgDb = 'imgData.db' # The database to create +idLineRegex = re.compile(r'(.*)') +imageLineRegex = re.compile(r'.*\| *image *= *([^|]*)') +bracketImageRegex = re.compile(r'\[\[(File:[^|]*).*]]') +imageNameRegex = re.compile(r'.*\.(jpg|jpeg|png|gif|tiff|tif)', flags=re.IGNORECASE) +cssImgCropRegex = re.compile(r'{{css image crop\|image *= *(.*)', flags=re.IGNORECASE) -print("Getting input page-ids") +print('Getting input page-ids') pageIds = getInputPageIds() -print(f"Found {len(pageIds)}") +print(f'Found {len(pageIds)}') -print("Opening databases") +print('Opening databases') indexDbCon = sqlite3.connect(indexDb) indexDbCur = indexDbCon.cursor() imgDbCon = sqlite3.connect(imgDb) imgDbCur = imgDbCon.cursor() -print("Checking tables") -if imgDbCur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='page_imgs'").fetchone() == None: +print('Checking tables') +if imgDbCur.execute('SELECT name FROM sqlite_master WHERE type="table" AND name="page_imgs"').fetchone() is None: # Create tables if not present - imgDbCur.execute("CREATE TABLE page_imgs (page_id INT PRIMARY KEY, img_name TEXT)") # img_name may be NULL - imgDbCur.execute("CREATE INDEX page_imgs_idx ON page_imgs(img_name)") + imgDbCur.execute('CREATE TABLE page_imgs (page_id INT PRIMARY KEY, img_name TEXT)') # img_name may be NULL + imgDbCur.execute('CREATE INDEX page_imgs_idx ON page_imgs(img_name)') else: # Check for already-processed page IDs numSkipped = 0 - for (pid,) in imgDbCur.execute("SELECT page_id FROM page_imgs"): + for (pid,) in imgDbCur.execute('SELECT page_id FROM page_imgs'): if pid in pageIds: pageIds.remove(pid) numSkipped += 1 else: - print(f"WARNING: Found already-processed page ID {pid} which was not in input set") - print(f"Will skip {numSkipped} already-processed page IDs") + print(f'WARNING: Found already-processed page ID {pid} which was not in input set') + print(f'Will skip {numSkipped} already-processed page IDs') -print("Getting dump-file offsets") -offsetToPageids = {} -offsetToEnd = {} # Maps chunk-start offsets to their chunk-end offsets +print('Getting dump-file offsets') +offsetToPageids: dict[int, list[int]] = {} +offsetToEnd: dict[int, int] = {} # Maps chunk-start offsets to their chunk-end offsets iterNum = 0 for pageId in pageIds: iterNum += 1 if iterNum % 1e4 == 0: - print(f"At iteration {iterNum}") + print(f'At iteration {iterNum}') # - query = "SELECT offset, next_offset FROM offsets WHERE id = ?" - row = indexDbCur.execute(query, (pageId,)).fetchone() - if row == None: - print(f"WARNING: Page ID {pageId} not found") + query = 'SELECT offset, next_offset FROM offsets WHERE id = ?' + row: tuple[int, int] | None = indexDbCur.execute(query, (pageId,)).fetchone() + if row is None: + print(f'WARNING: Page ID {pageId} not found') continue - (chunkOffset, endOffset) = row + chunkOffset, endOffset = row offsetToEnd[chunkOffset] = endOffset if chunkOffset not in offsetToPageids: offsetToPageids[chunkOffset] = [] offsetToPageids[chunkOffset].append(pageId) -print(f"Found {len(offsetToEnd)} chunks to check") +print(f'Found {len(offsetToEnd)} chunks to check') -print("Iterating through chunks in dump file") -def getImageName(content): - " Given an array of text-content lines, tries to return an infoxbox image name, or None " +print('Iterating through chunks in dump file') +def getImageName(content: list[str]) -> str | None: + """ Given an array of text-content lines, tries to return an infoxbox image name, or None """ # Doesn't try and find images in outside-infobox [[File:...]] and sections for line in content: match = imageLineRegex.match(line) - if match != None: + if match is not None: imageName = match.group(1).strip() - if imageName == "": + if imageName == '': return None imageName = html.unescape(imageName) # Account for {{... - if imageName.startswith("{"): + if imageName.startswith('{'): match = cssImgCropRegex.match(imageName) - if match == None: + if match is None: return None imageName = match.group(1) # Account for [[File:...|...]] - if imageName.startswith("["): + if imageName.startswith('['): match = bracketImageRegex.match(imageName) - if match == None: + if match is None: return None imageName = match.group(1) # Account for