From 9a7bb3db01fe2e99ccc12285c63323bc67c278e8 Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Mon, 20 Jun 2022 19:50:32 +1000 Subject: Increase type-consistency via server-classes and client-types --- backend/server.py | 202 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 116 insertions(+), 86 deletions(-) (limited to 'backend') diff --git a/backend/server.py b/backend/server.py index 5525eb5..888f73a 100755 --- a/backend/server.py +++ b/backend/server.py @@ -1,10 +1,10 @@ #!/usr/bin/python3 -import sys, re, sqlite3, json +import sys, re, sqlite3 import os.path from http.server import HTTPServer, BaseHTTPRequestHandler import urllib.parse -import gzip +import gzip, jsonpickle hostname = "localhost" port = 8000 @@ -26,60 +26,96 @@ if len(sys.argv) > 1: print(usageInfo, file=sys.stderr) sys.exit(1) +# Classes for objects sent as responses (matches lib.ts types in client-side code) +class TolNode: + """ Used when responding to 'node' and 'chain' requests """ + def __init__(self, otolId, children, parent=None, tips=0, pSupport=False, commonName=None, imgName=None): + self.otolId = otolId # string | null + self.children = children # string[] + self.parent = parent # string | null + self.tips = tips # number + self.pSupport = pSupport # boolean + self.commonName = commonName # null | string + self.imgName = imgName # null | string | [string,string] | [null, string] | [string, null] +class SearchSugg: + """ Represents a search suggestion """ + def __init__(self, name, canonicalName=None): + self.name = name # string + self.canonicalName = canonicalName # string | null +class SearchSuggResponse: + """ Sent as responses to 'search' requests """ + def __init__(self, searchSuggs, hasMore): + self.suggs = searchSuggs # SearchSugg[] + self.hasMore = hasMore # boolean +class DescInfo: + """ Represents a tol-node's associated description """ + def __init__(self, text, wikiId, fromRedirect, fromDbp): + self.text = text # string + self.wikiId = wikiId # number + self.fromRedirect = fromRedirect # boolean + self.fromDbp = fromDbp # boolean +class ImgInfo: + """ Represents a tol-node's associated image """ + def __init__(self, id, src, url, license, artist, credit): + self.id = id # number + self.src = src # string + self.url = url # string + self.license = license # string + self.artist = artist # string + self.credit = credit # string +class InfoResponse: + """ Sent as responses to 'info' requests """ + def __init__(self, tolNode, descData, imgData): + self.tolNode = tolNode # null | TolNode + self.descData = descData # null | DescInfo | [DescInfo, DescInfo] + self.imgData = imgData # null | ImgInfo | [ImgInfo, ImgInfo] + # Connect to db dbCon = sqlite3.connect(dbFile) # Some functions def lookupNodes(names, useReducedTree): # Get node info - nodeObjs = {} + nameToNodes = {} cur = dbCon.cursor() nodesTable = "nodes" if not useReducedTree else "r_nodes" edgesTable = "edges" if not useReducedTree else "r_edges" queryParamStr = ",".join(["?"] * len(names)) query = f"SELECT name, id, tips FROM {nodesTable} WHERE name IN ({queryParamStr})" for (nodeName, otolId, tips) in cur.execute(query, names): - nodeObjs[nodeName] = { - "otolId": otolId, - "children": [], - "parent": None, - "tips": tips, - "pSupport": False, - "commonName": None, - "imgName": None, - } + nameToNodes[nodeName] = TolNode(otolId, [], tips=tips) # Get child info query = f"SELECT node, child FROM {edgesTable} WHERE node IN ({queryParamStr})" for (nodeName, childName) in cur.execute(query, names): - nodeObjs[nodeName]["children"].append(childName) + nameToNodes[nodeName].children.append(childName) # Order children by tips - for (nodeName, nodeObj) in nodeObjs.items(): - childList = nodeObj["children"] + for (nodeName, node) in nameToNodes.items(): childToTips = {} - query = "SELECT name, tips FROM {} WHERE name IN ({})".format(nodesTable, ",".join(["?"] * len(childList))) - for (n, tips) in cur.execute(query, childList): + query = "SELECT name, tips FROM {} WHERE name IN ({})" + query = query.format(nodesTable, ",".join(["?"] * len(node.children))) + for (n, tips) in cur.execute(query, node.children): childToTips[n] = tips - childList.sort(key=lambda n: childToTips[n], reverse=True) + node.children.sort(key=lambda n: childToTips[n], reverse=True) # Get parent info query = f"SELECT node, child, p_support FROM {edgesTable} WHERE child IN ({queryParamStr})" for (nodeName, childName, pSupport) in cur.execute(query, names): - nodeObjs[childName]["parent"] = None if nodeName == "" else nodeName - nodeObjs[childName]["pSupport"] = (pSupport == 1) + nameToNodes[childName].parent = nodeName + nameToNodes[childName].pSupport = (pSupport == 1) # Get image names - idsToNames = {nodeObjs[n]["otolId"]: n for n in nodeObjs.keys()} + idsToNames = {nameToNodes[n].otolId: n for n in nameToNodes.keys()} query = "SELECT nodes.id from nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name" \ " WHERE nodes.id IN ({})".format(",".join(["?"] * len(idsToNames))) for (otolId,) in cur.execute(query, list(idsToNames.keys())): - nodeObjs[idsToNames[otolId]]["imgName"] = otolId + ".jpg" + nameToNodes[idsToNames[otolId]].imgName = otolId + ".jpg" # Get 'linked' images for unresolved names - unresolvedNames = [n for n in nodeObjs if nodeObjs[n]["imgName"] == None] + unresolvedNames = [n for n in nameToNodes if nameToNodes[n].imgName == None] query = "SELECT name, otol_ids from linked_imgs WHERE name IN ({})" query = query.format(",".join(["?"] * len(unresolvedNames))) for (name, otolIds) in cur.execute(query, unresolvedNames): if "," not in otolIds: - nodeObjs[name]["imgName"] = otolIds + ".jpg" + nameToNodes[name].imgName = otolIds + ".jpg" else: id1, id2 = otolIds.split(",") - nodeObjs[name]["imgName"] = [ + nameToNodes[name].imgName = [ id1 + ".jpg" if id1 != "" else None, id2 + ".jpg" if id2 != "" else None, ] @@ -87,9 +123,9 @@ def lookupNodes(names, useReducedTree): query = f"SELECT name, alt_name FROM names WHERE pref_alt = 1 AND name IN ({queryParamStr})" for (name, altName) in cur.execute(query, names): if altName != name: - nodeObjs[name]["commonName"] = altName + nameToNodes[name].commonName = altName # - return nodeObjs + return nameToNodes def lookupName(name, useReducedTree): cur = dbCon.cursor() results = [] @@ -108,86 +144,80 @@ def lookupName(name, useReducedTree): " names INNER JOIN r_nodes ON names.name = r_nodes.name" \ " WHERE alt_name LIKE ? ORDER BY length(alt_name) LIMIT ?" # Join results, and get shortest - temp = [] - for row in cur.execute(query1, (name + "%", SEARCH_SUGG_LIMIT + 1)): - temp.append({"name": row[0], "canonicalName": None}) - for row in cur.execute(query2, (name + "%", SEARCH_SUGG_LIMIT + 1)): - temp.append({"name": row[0], "canonicalName": row[1]}) + suggs = [] + for (nodeName,) in cur.execute(query1, (name + "%", SEARCH_SUGG_LIMIT + 1)): + suggs.append(SearchSugg(nodeName)) + for (altName, nodeName) in cur.execute(query2, (name + "%", SEARCH_SUGG_LIMIT + 1)): + suggs.append(SearchSugg(altName, nodeName)) # If insufficient results, try substring-search - foundNames = {n["name"] for n in temp} - if len(temp) < SEARCH_SUGG_LIMIT: - newLim = SEARCH_SUGG_LIMIT + 1 - len(temp) - for (altName,) in cur.execute(query1, ("%" + name + "%", newLim)): - if altName not in foundNames: - temp.append({"name": altName, "canonicalName": None}) - foundNames.add(altName) - if len(temp) < SEARCH_SUGG_LIMIT: - newLim = SEARCH_SUGG_LIMIT + 1 - len(temp) - for (altName, cName) in cur.execute(query2, ("%" + name + "%", SEARCH_SUGG_LIMIT + 1)): + foundNames = {n.name for n in suggs} + if len(suggs) < SEARCH_SUGG_LIMIT: + newLim = SEARCH_SUGG_LIMIT + 1 - len(suggs) + for (nodeName,) in cur.execute(query1, ("%" + name + "%", newLim)): + if nodeName not in foundNames: + suggs.append(SearchSugg(nodeName)) + foundNames.add(nodeName) + if len(suggs) < SEARCH_SUGG_LIMIT: + newLim = SEARCH_SUGG_LIMIT + 1 - len(suggs) + for (altName, nodeName) in cur.execute(query2, ("%" + name + "%", SEARCH_SUGG_LIMIT + 1)): if altName not in foundNames: - temp.append({"name": altName, "canonicalName": cName}) + suggs.append(SearchSugg(altName, nodeName)) foundNames.add(altName) # - temp.sort(key=lambda x: x["name"]) - temp.sort(key=lambda x: len(x["name"])) - results = temp[:SEARCH_SUGG_LIMIT] - if len(temp) > SEARCH_SUGG_LIMIT: + suggs.sort(key=lambda x: x.name) + suggs.sort(key=lambda x: len(x.name)) + results = suggs[:SEARCH_SUGG_LIMIT] + if len(suggs) > SEARCH_SUGG_LIMIT: hasMore = True - return [results, hasMore] + return SearchSuggResponse(results, hasMore) def lookupNodeInfo(name, useReducedTree): cur = dbCon.cursor() # Get node-object info - temp = lookupNodes([name], useReducedTree) - nodeObj = temp[name] if name in temp else None + nameToNodes = lookupNodes([name], useReducedTree) + tolNode = nameToNodes[name] if name in nameToNodes else None # Get node desc descData = None match = re.fullmatch(r"\[(.+) \+ (.+)]", name) if match == None: - query = "SELECT wiki_id, redirected, desc, from_dbp FROM" \ + query = "SELECT desc, wiki_id, redirected, from_dbp FROM" \ " wiki_ids INNER JOIN descs ON wiki_ids.id = descs.wiki_id WHERE wiki_ids.name = ?" row = cur.execute(query, (name,)).fetchone() if row != None: - descData = {"wikiId": row[0], "fromRedirect": row[1] == 1, "text": row[2], "fromDbp": row[3] == 1} + (desc, wikiId, redirected, fromDbp) = row + descData = DescInfo(desc, wikiId, redirected == 1, fromDbp == 1) else: # Get descs for compound-node element descData = [None, None] - query = "SELECT name, wiki_id, redirected, desc, from_dbp FROM" \ + query = "SELECT name, desc, wiki_id, redirected, from_dbp FROM" \ " wiki_ids INNER JOIN descs ON wiki_ids.id = descs.wiki_id WHERE wiki_ids.name IN (?, ?)" - for row in cur.execute(query, match.group(1,2)): - if row[0] == match.group(1): - descData[0] = {"wikiId": row[1], "fromRedirect": row[2] == 1, "text": row[3], "fromDbp": row[4] == 1} - else: - descData[1] = {"wikiId": row[1], "fromRedirect": row[2] == 1, "text": row[3], "fromDbp": row[4] == 1} + for (nodeName, desc, wikiId, redirected, fromDbp) in cur.execute(query, match.group(1,2)): + idx = 0 if nodeName == match.group(1) else 1 + descData[idx] = DescInfo(desc, wikiId, redirected == 1, fromDbp == 1) # Get img info imgData = None - if nodeObj != None: - if isinstance(nodeObj["imgName"], str): - otolId = nodeObj["imgName"][:-4] # Convert filename excluding .jpg suffix + if tolNode != None: + if isinstance(tolNode.imgName, str): + otolId = tolNode.imgName[:-4] # Convert filename excluding .jpg suffix query = "SELECT images.id, images.src, url, license, artist, credit FROM" \ " nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name" \ " INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src" \ " WHERE nodes.id = ?" (imgId, imgSrc, url, license, artist, credit) = cur.execute(query, (otolId,)).fetchone() - imgData = {"imgId": imgId, "imgSrc": imgSrc, - "url": url, "license": license, "artist": artist, "credit": credit} - elif isinstance(nodeObj["imgName"], list): + imgData = ImgInfo(imgId, imgSrc, url, license, artist, credit) + elif isinstance(tolNode.imgName, list): # Get info for compound-image parts imgData = [None, None] - idsToLookup = [n[:-4] for n in nodeObj["imgName"] if n != None] + idsToLookup = [n[:-4] for n in tolNode.imgName if n != None] query = "SELECT nodes.id, images.id, images.src, url, license, artist, credit FROM" \ " nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name" \ " INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src" \ " WHERE nodes.id IN ({})".format(",".join(["?"] * len(idsToLookup))) for (imgOtolId, imgId, imgSrc, url, license, artist, credit) in cur.execute(query, idsToLookup): - imgDataVal = {"imgId": imgId, "imgSrc": imgSrc, - "url": url, "license": license, "artist": artist, "credit": credit} - imgName1 = nodeObj["imgName"][0] - if imgName1 != None and imgOtolId == imgName1[:-4]: - imgData[0] = imgDataVal - else: - imgData[1] = imgDataVal + imgName1 = tolNode.imgName[0] + idx = 0 if (imgName1 != None and imgOtolId == imgName1[:-4]) else 1 + imgData[idx] = ImgInfo(imgId, imgSrc, url, license, artist, credit) # - return {"descData": descData, "imgData": imgData, "nodeObj": nodeObj} + return InfoResponse(tolNode, descData, imgData) class DbServer(BaseHTTPRequestHandler): def do_GET(self): @@ -204,11 +234,11 @@ class DbServer(BaseHTTPRequestHandler): useReducedTree = "tree" in queryDict # Check query string if reqType == "node": - nodeObjs = lookupNodes([name], useReducedTree) - if len(nodeObjs) > 0: - nodeObj = nodeObjs[name] - childNodeObjs = lookupNodes(nodeObj["children"], useReducedTree) - childNodeObjs[name] = nodeObj + tolNodes = lookupNodes([name], useReducedTree) + if len(tolNodes) > 0: + tolNode = tolNodes[name] + childNodeObjs = lookupNodes(tolNode.children, useReducedTree) + childNodeObjs[name] = tolNode self.respondJson(childNodeObjs) return elif reqType == "chain": @@ -216,31 +246,31 @@ class DbServer(BaseHTTPRequestHandler): ranOnce = False while True: # Get node - nodeObjs = lookupNodes([name], useReducedTree) - if len(nodeObjs) == 0: + tolNodes = lookupNodes([name], useReducedTree) + if len(tolNodes) == 0: if not ranOnce: self.respondJson(results) return print(f"ERROR: Parent-chain node {name} not found", file=sys.stderr) break - nodeObj = nodeObjs[name] - results[name] = nodeObj + tolNode = tolNodes[name] + results[name] = tolNode # Conditionally add children if not ranOnce: ranOnce = True else: childNamesToAdd = [] - for childName in nodeObj["children"]: + for childName in tolNode.children: if childName not in results: childNamesToAdd.append(childName) childNodeObjs = lookupNodes(childNamesToAdd, useReducedTree) results.update(childNodeObjs) # Check if root - if nodeObj["parent"] == None: + if tolNode.parent == None: self.respondJson(results) return else: - name = nodeObj["parent"] + name = tolNode.parent elif reqType == "search": self.respondJson(lookupName(name, useReducedTree)) return @@ -249,7 +279,7 @@ class DbServer(BaseHTTPRequestHandler): return self.send_response(404) def respondJson(self, val): - content = json.dumps(val).encode("utf-8") + content = jsonpickle.encode(val, unpicklable=False).encode("utf-8") self.send_response(200) self.send_header("Content-type", "application/json") if "accept-encoding" in self.headers and "gzip" in self.headers["accept-encoding"]: -- cgit v1.2.3