Add backend unit tests

- Add unit testing code in backend/tests/
- Change to snake-case for script/file/directory names
- Use os.path.join() instead of '/'
- Refactor script code into function defs and a main-guard
- Make global vars all-caps

Some fixes:
- For getting descriptions, some wiki redirects weren't properly resolved
- Linked images were sub-optimally propagated
- Generation of reduced trees assumed a wiki-id association implied a description
- Tilo.py had potential null dereferences by not always using a reduced node set
- EOL image downloading didn't properly wait for all threads to end when finishing

1 files changed, 68 insertions, 18 deletions

diff --git a/backend/tilo.py b/backend/tilo.py
index c1ecc34..dfefab1 100755
--- a/backend/tilo.py
+++ b/backend/tilo.py
@@ -28,7 +28,7 @@ if __name__ == '__main__':
 	parser = argparse.ArgumentParser(description=HELP_INFO, formatter_class=argparse.RawDescriptionHelpFormatter)
 	parser.parse_args()
 
-DB_FILE = 'tolData/data.db'
+DB_FILE = 'tol_data/data.db'
 DEFAULT_SUGG_LIM = 5
 MAX_SUGG_LIM = 50
 ROOT_NAME = 'cellular organisms'
@@ -45,7 +45,7 @@ class TolNode:
 			pSupport=False,
 			commonName: str | None = None,
 			imgName: None | str | tuple[str, str] | tuple[None, str] | tuple[str, None] = None,
-			iucn: str = None):
+			iucn: str | None = None):
 		self.otolId = otolId
 		self.children = children
 		self.parent = parent
@@ -54,23 +54,52 @@ class TolNode:
 		self.commonName = commonName
 		self.imgName = imgName
 		self.iucn = iucn
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, TolNode) and \
+			(self.otolId, set(self.children), self.parent, self.tips, \
+				self.pSupport, self.commonName, self.imgName, self.iucn) == \
+			(other.otolId, set(other.children), other.parent, other.tips, \
+				other.pSupport, other.commonName, other.imgName, other.iucn)
+	def __repr__(self):
+		return str(self.__dict__)
 class SearchSugg:
 	""" Represents a search suggestion """
 	def __init__(self, name: str, canonicalName: str | None = None, pop=0):
 		self.name = name
 		self.canonicalName = canonicalName
 		self.pop = pop if pop is not None else 0
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, SearchSugg) and \
+			(self.name, self.canonicalName, self.pop) == (other.name, other.canonicalName, other.pop)
+	def __repr__(self):
+		return str(self.__dict__)
+	def __hash__(self):
+		return (self.name, self.canonicalName, self.pop).__hash__()
 class SearchSuggResponse:
 	""" Sent as responses to 'sugg' requests """
 	def __init__(self, searchSuggs: list[SearchSugg], hasMore: bool):
 		self.suggs = searchSuggs
 		self.hasMore = hasMore
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, SearchSuggResponse) and \
+			(set(self.suggs), self.hasMore) == (set(other.suggs), other.hasMore)
+	def __repr__(self):
+		return str(self.__dict__)
 class DescInfo:
 	""" Represents a node's associated description """
 	def __init__(self, text: str, wikiId: int, fromDbp: bool):
 		self.text = text
 		self.wikiId = wikiId
 		self.fromDbp = fromDbp
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, DescInfo) and \
+			(self.text, self.wikiId, self.fromDbp) == (other.text, other.wikiId, other.fromDbp)
+	def __repr__(self):
+		return str(self.__dict__)
 class ImgInfo:
 	""" Represents a node's associated image """
 	def __init__(self, id: int, src: str, url: str, license: str, artist: str, credit: str):
@@ -80,17 +109,36 @@ class ImgInfo:
 		self.license = license
 		self.artist = artist
 		self.credit = credit
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, ImgInfo) and \
+			(self.id, self.src, self.url, self.license, self.artist, self.credit) == \
+			(other.id, other.src, other.url, other.license, other.artist, other.credit)
+	def __repr__(self):
+		return str(self.__dict__)
 class NodeInfo:
 	""" Represents info about a node """
 	def __init__(self, tolNode: TolNode, descInfo: DescInfo | None, imgInfo: ImgInfo | None):
 		self.tolNode = tolNode
 		self.descInfo = descInfo
 		self.imgInfo = imgInfo
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, NodeInfo) and \
+			(self.tolNode, self.descInfo, self.imgInfo) == (other.tolNode, other.descInfo, other.imgInfo)
+	def __repr__(self):
+		return str(self.__dict__)
 class InfoResponse:
 	""" Sent as responses to 'info' requests """
 	def __init__(self, nodeInfo: NodeInfo, subNodesInfo: tuple[()] | tuple[NodeInfo | None, NodeInfo | None]):
 		self.nodeInfo = nodeInfo
 		self.subNodesInfo = subNodesInfo
+	# Used in unit testing
+	def __eq__(self, other):
+		return isinstance(other, InfoResponse) and \
+			(self.nodeInfo, self.subNodesInfo) == (other.nodeInfo, other.subNodesInfo)
+	def __repr__(self):
+		return str(self.__dict__)
 
 # For data lookup
 def lookupNodes(names: list[str], tree: str, dbCur: sqlite3.Cursor) -> dict[str, TolNode]:
@@ -123,8 +171,9 @@ def lookupNodes(names: list[str], tree: str, dbCur: sqlite3.Cursor) -> dict[str,
 		nameToNodes[childName].pSupport = pSupport == 1
 	# Get image names
 	idsToNames = {nameToNodes[n].otolId: n for n in nameToNodes.keys()}
-	query = 'SELECT nodes.id from nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name' \
-		' WHERE nodes.id IN ({})'.format(','.join(['?'] * len(idsToNames)))
+	query = f'SELECT {nodesTable}.id from {nodesTable}' \
+		f' INNER JOIN node_imgs ON {nodesTable}.name = node_imgs.name' \
+		f' WHERE {nodesTable}.id IN ' '({})'.format(','.join(['?'] * len(idsToNames)))
 	for (otolId,) in dbCur.execute(query, list(idsToNames.keys())):
 		nameToNodes[idsToNames[otolId]].imgName = otolId + '.jpg'
 	# Get 'linked' images for unresolved names
@@ -143,11 +192,13 @@ def lookupNodes(names: list[str], tree: str, dbCur: sqlite3.Cursor) -> dict[str,
 	# Get preferred-name info
 	query = f'SELECT name, alt_name FROM names WHERE pref_alt = 1 AND name IN ({queryParamStr})'
 	for name, altName in dbCur.execute(query, names):
-		nameToNodes[name].commonName = altName
+		if name in nameToNodes:
+			nameToNodes[name].commonName = altName
 	# Get IUCN status
 	query = f'SELECT name, iucn FROM node_iucn WHERE name IN ({queryParamStr})'
 	for name, iucn in dbCur.execute(query, names):
-		nameToNodes[name].iucn = iucn
+		if name in nameToNodes:
+			nameToNodes[name].iucn = iucn
 	#
 	return nameToNodes
 def lookupSuggs(searchStr: str, suggLimit: int, tree: str, dbCur: sqlite3.Cursor) -> SearchSuggResponse:
@@ -157,7 +208,7 @@ def lookupSuggs(searchStr: str, suggLimit: int, tree: str, dbCur: sqlite3.Cursor
 	nodesTable = f'nodes_{getTableSuffix(tree)}'
 	nameQuery = f'SELECT {nodesTable}.name, node_pop.pop FROM {nodesTable}' \
 		f' LEFT JOIN node_pop ON {nodesTable}.name = node_pop.name' \
-		f' WHERE node_pop.name LIKE ? AND node_pop.name NOT LIKE "[%"' \
+		f' WHERE {nodesTable}.name LIKE ? AND {nodesTable}.name NOT LIKE "[%"' \
 		f' ORDER BY node_pop.pop DESC'
 	altNameQuery = f'SELECT alt_name, names.name, pref_alt, node_pop.pop FROM' \
 		f' names INNER JOIN {nodesTable} ON names.name = {nodesTable}.name' \
@@ -204,6 +255,7 @@ def lookupSuggs(searchStr: str, suggLimit: int, tree: str, dbCur: sqlite3.Cursor
 	return SearchSuggResponse(suggList[:suggLimit], hasMore)
 def lookupInfo(name: str, tree: str, dbCur: sqlite3.Cursor) -> InfoResponse | None:
 	""" For a node name, returns a descriptive InfoResponse, or None """
+	nodesTable = f'nodes_{getTableSuffix(tree)}'
 	# Get node info
 	nameToNodes = lookupNodes([name], tree, dbCur)
 	tolNode = nameToNodes[name] if name in nameToNodes else None
@@ -230,10 +282,10 @@ def lookupInfo(name: str, tree: str, dbCur: sqlite3.Cursor) -> InfoResponse | No
 	idsToNames = {cast(str, nameToNodes[n].imgName)[:-4]: n
 		for n in namesToLookup if nameToNodes[n].imgName is not None}
 	idsToLookup = list(idsToNames.keys()) # Lookup using IDs avoids having to check linked_imgs
-	query = 'SELECT nodes.id, images.id, images.src, url, license, artist, credit FROM' \
-		' nodes INNER JOIN node_imgs ON nodes.name = node_imgs.name' \
-		' INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src' \
-		' WHERE nodes.id IN ({})'.format(','.join(['?'] * len(idsToLookup)))
+	query = f'SELECT {nodesTable}.id, images.id, images.src, url, license, artist, credit FROM' \
+		f' {nodesTable} INNER JOIN node_imgs ON {nodesTable}.name = node_imgs.name' \
+		f' INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src' \
+		f' WHERE {nodesTable}.id IN ' '({})'.format(','.join(['?'] * len(idsToLookup)))
 	for id, imgId, imgSrc, url, license, artist, credit in dbCur.execute(query, idsToLookup):
 		nameToImgInfo[idsToNames[id]] = ImgInfo(imgId, imgSrc, url, license, artist, credit)
 	# Construct response
@@ -251,10 +303,11 @@ def getTableSuffix(tree: str) -> str:
 	""" converts a reduced-tree descriptor into a sql-table-suffix """
 	return 't' if tree == 'trimmed' else 'i' if tree == 'images' else 'p'
 
-def handleReq(
-	dbCur: sqlite3.Cursor,
-	environ: dict[str, str]) -> None | dict[str, TolNode] | SearchSuggResponse | InfoResponse:
+def handleReq(dbFile: str, environ: dict[str, str]) -> None | dict[str, TolNode] | SearchSuggResponse | InfoResponse:
 	""" Queries the database, and constructs a response object """
+	# Open db
+	dbCon = sqlite3.connect(dbFile)
+	dbCur = dbCon.cursor()
 	# Get query params
 	queryStr = environ['QUERY_STRING'] if 'QUERY_STRING' in environ else ''
 	queryDict = urllib.parse.parse_qs(queryStr)
@@ -342,11 +395,8 @@ def handleReq(
 	return None
 def application(environ: dict[str, str], start_response) -> Iterable[bytes]:
 	""" Entry point for the WSGI script """
-	# Open db
-	dbCon = sqlite3.connect(DB_FILE)
-	dbCur = dbCon.cursor()
 	# Get response object
-	val = handleReq(dbCur, environ)
+	val = handleReq(DB_FILE, environ)
 	# Construct response
 	data = jsonpickle.encode(val, unpicklable=False).encode()
 	headers = [('Content-type', 'application/json')]