Add reduced-tree data generation+serving+querying+setting

Add genReducedTreeData.py, which generates a reduced_nodes table.
Adjust server to serve that data for queries with a tree=reduced query param.
Adjust client to query for that data depending on a useReducedTree variable.
Add a SettingsPane setting to change that useReducedTree variable.

8 files changed, 249 insertions, 37 deletions

diff --git a/backend/data/README.md b/backend/data/README.md
index 27619de..c4c46ba 100644
--- a/backend/data/README.md
+++ b/backend/data/README.md
@@ -7,7 +7,7 @@ File Generation Process
         table using data in otol/*.
 2   Name Data for Search
     1   Obtain data in eol/, as specified in it's README.
-    2   Run genEolNameData.py, which adds 'names' and 'eol\_ids' tables to data.db, 
+    2   Run genEolNameData.py, which adds 'names' and 'eol\_ids' tables to data.db,
         using data in eol/vernacularNames.csv and the 'nodes' table.
 3   Image Data
     1   Use downloadImgsForReview.py to download EOL images into imgsForReview/.
@@ -20,6 +20,9 @@ File Generation Process
     1   Obtain data in enwiki/, as specified in it's README.
     2   Run genEnwikiData.py, which adds a 'descs' table to data.db,
         using data in enwiki/enwikiData.db, and the 'nodes' table.
+5   Reduced Tree Structure Data
+    1   Run genReducedTreeData.py, which adds a 'reduced_nodes' table to data.db,
+        using reducedTol/names.txt, and the 'nodes' and 'names' tables.
 
 data.db tables
 ==============
@@ -33,3 +36,5 @@ data.db tables
     eol\_id INT PRIMARY KEY, source\_url TEXT, license TEXT, copyright\_owner TEXT
 -   descs <br>
     name TEXT PRIMARY KEY, desc TEXT, redirected INT
+-   reduced\_nodes <br>
+    name TEXT PRIMARY KEY, children TEXT, parent TEXT, tips INT, p_support INT
diff --git a/backend/data/genReducedTreeData.py b/backend/data/genReducedTreeData.py
new file mode 100755
index 0000000..ed8fae9
--- /dev/null
+++ b/backend/data/genReducedTreeData.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+
+import sys, os.path, re
+import json, sqlite3
+
+usageInfo =  f"usage: {sys.argv[0]}\n"
+usageInfo += "Reads \n"
+if len(sys.argv) > 1:
+	print(usageInfo, file=sys.stderr)
+	sys.exit(1)
+
+dbFile = "data.db"
+nodeNamesFile = "reducedTol/names.txt"
+minimalNames = set()
+nodeMap = {} # Maps node names to node objects
+PREF_NUM_CHILDREN = 3 # Attempt inclusion of children up to this limit
+compNameRegex = re.compile(r"\[.+ \+ .+]")
+
+# Connect to db
+dbCon = sqlite3.connect(dbFile)
+dbCur = dbCon.cursor()
+# Read in minimal set of node names
+print("Getting minimal name set")
+iterNum = 0
+with open(nodeNamesFile) as file:
+	for line in file:
+		iterNum += 1
+		if iterNum % 100 == 0:
+			print("Iteration {}".format(iterNum))
+		#
+		row = dbCur.execute("SELECT name, alt_name from names WHERE alt_name = ?", (line.rstrip(),)).fetchone()
+		if row != None:
+			minimalNames.add(row[0])
+if len(minimalNames) == 0:
+	print("ERROR: No names found", file=sys.stderr)
+	sys.exit(1)
+print("Name set has {} names".format(len(minimalNames)))
+# Add nodes that connect up to root
+print("Getting connected nodes set")
+iterNum = 0
+rootName = None
+for name in minimalNames:
+	iterNum += 1
+	if iterNum % 100 == 0:
+		print("Iteration {}".format(iterNum))
+	#
+	prevName = None
+	while name != None:
+		if name not in nodeMap:
+			(parent, tips, p_support) = dbCur.execute(
+				"SELECT parent, tips, p_support from nodes WHERE name = ?", (name,)).fetchone()
+			parent = None if parent == "" else parent
+			nodeMap[name] = {
+				"children": [] if prevName == None else [prevName],
+				"parent": parent,
+				"tips": 0,
+				"pSupport": p_support == 1,
+			}
+			prevName = name
+			name = parent
+		else:
+			if prevName != None:
+				nodeMap[name]["children"].append(prevName)
+			break
+	if name == None:
+		rootName = prevName
+print("New node set has {} nodes".format(len(nodeMap)))
+# Remove certain 'chain collapsible' nodes
+print("Removing 'chain collapsible' nodes")
+namesToRemove = set()
+for (name, nodeObj) in nodeMap.items():
+	if name not in minimalNames and len(nodeObj["children"]) == 1:
+		parentName = nodeObj["parent"]
+		childName = nodeObj["children"][0]
+		# Connect parent and child
+		nodeMap[parentName]["children"].remove(name)
+		nodeMap[parentName]["children"].append(childName)
+		nodeMap[childName]["parent"] = parentName
+		# Adjust child pSupport
+		nodeMap[childName]["pSupport"] &= nodeObj["pSupport"]
+		# Remember for removal
+		namesToRemove.add(name)
+for name in namesToRemove:
+	del nodeMap[name]
+print("New node set has {} nodes".format(len(nodeMap)))
+# Merge-upward compsite-named nodes
+print("Merging-upward composite-named nodes")
+namesToRemove2 = set()
+for (name, nodeObj) in nodeMap.items():
+	parent = nodeObj["parent"]
+	if parent != None and compNameRegex.fullmatch(name) != None:
+		# Connect children to parent
+		nodeMap[parent]["children"].remove(name)
+		nodeMap[parent]["children"].extend(nodeObj["children"])
+		for n in nodeObj["children"]:
+			nodeMap[n]["parent"] = parent
+			nodeMap[n]["pSupport"] &= nodeObj["pSupport"]
+		# Remember for removal
+		namesToRemove2.add(name)
+for name in namesToRemove2:
+	del nodeMap[name]
+	namesToRemove.add(name)
+print("New node set has {} nodes".format(len(nodeMap)))
+# Add some connected children
+print("Adding additional nearby children")
+namesToAdd = []
+iterNum = 0
+for (name, nodeObj) in nodeMap.items():
+	iterNum += 1
+	if iterNum % 100 == 0:
+		print("Iteration {}".format(iterNum))
+	#
+	numChildren = len(nodeObj["children"])
+	if numChildren < PREF_NUM_CHILDREN:
+		row = dbCur.execute("SELECT children from nodes WHERE name = ?", (name,)).fetchone()
+		newChildren = [n for n in json.loads(row[0]) if
+			not (n in nodeMap or n in namesToRemove) and
+			compNameRegex.fullmatch(n) == None]
+		newChildNames = newChildren[:max(0, PREF_NUM_CHILDREN - numChildren)]
+		nodeObj["children"].extend(newChildNames)
+		namesToAdd.extend(newChildNames)
+for name in namesToAdd:
+	(parent, pSupport) = dbCur.execute("SELECT parent, p_support from nodes WHERE name = ?", (name,)).fetchone()
+	nodeMap[name] = {
+		"children": [],
+		"parent": parent,
+		"tips": 0,
+		"pSupport": pSupport,
+	}
+print("New node set has {} nodes".format(len(nodeMap)))
+# set tips vals
+print("Setting tips vals")
+def setTips(nodeName):
+	nodeObj = nodeMap[nodeName]
+	if len(nodeObj["children"]) == 0:
+		nodeObj["tips"] = 1
+		return 1
+	tips = sum([setTips(childName) for childName in nodeObj["children"]])
+	nodeObj["tips"] = tips
+	return tips
+setTips(rootName)
+# Add new nodes to db
+print("Adding to db")
+dbCur.execute(
+	"CREATE TABLE reduced_nodes (name TEXT PRIMARY KEY, children TEXT, parent TEXT, tips INT, p_support INT)")
+for (name, nodeObj) in nodeMap.items():
+	parentName = "" if nodeObj["parent"] == None else nodeObj["parent"]
+	dbCur.execute("INSERT INTO reduced_nodes VALUES (?, ?, ?, ?, ?)",
+		(name, json.dumps(nodeObj["children"]), parentName, nodeObj["tips"], 1 if nodeObj["pSupport"] else 0))
+# Close db
+dbCon.commit()
+dbCon.close()
diff --git a/backend/data/reducedTol/README.md b/backend/data/reducedTol/README.md
new file mode 100644
index 0000000..103bffc
--- /dev/null
+++ b/backend/data/reducedTol/README.md
@@ -0,0 +1,4 @@
+Files
+=====
+-   names.txt <br>
+	Contains names of nodes to be kept in a reduced Tree of Life.
diff --git a/backend/server.py b/backend/server.py
index 9c9764b..374fb53 100755
--- a/backend/server.py
+++ b/backend/server.py
@@ -14,6 +14,7 @@ SEARCH_SUGG_LIMIT = 5
 usageInfo =  f"usage: {sys.argv[0]}\n"
 usageInfo += "Starts a server that listens for GET requests to http://" + hostname + ":" + str(port) + ".\n"
 usageInfo += "Responds to path+query /data/type1?name=name1 with JSON data.\n"
+usageInfo += "An additional query parameter tree=reduced is usable to get reduced-tree data\n"
 usageInfo += "\n"
 usageInfo += "If type1 is 'node': Responds with map from names to objects representing node name1 and it's children.\n"
 usageInfo += "If type1 is 'chain': Like 'node', but gets nodes from name1 up to the root, and their direct children.\n"
@@ -25,12 +26,13 @@ if len(sys.argv) > 1:
 # Connect to db, and load spellfix extension
 dbCon = sqlite3.connect(dbFile)
 # Some functions
-def lookupNodes(names):
+def lookupNodes(names, useReducedTree):
 	nodeObjs = {}
 	cur = dbCon.cursor()
 	# Get node info
-	query = "SELECT name, children, parent, tips, p_support FROM nodes WHERE" \
-		" name IN ({})".format(",".join(["?"] * len(names)))
+	nodesTable = "nodes" if not useReducedTree else "reduced_nodes"
+	query = "SELECT name, children, parent, tips, p_support FROM {} WHERE" \
+		" name IN ({})".format(nodesTable, ",".join(["?"] * len(names)))
 	namesForImgs = []
 	firstSubnames = {}
 	secondSubnames = {}
@@ -89,13 +91,19 @@ def getNodeImg(name):
 		if os.path.exists(imgDir + filename):
 			return filename
 	return None
-def lookupName(name):
+def lookupName(name, useReducedTree):
 	cur = dbCon.cursor()
 	results = []
 	hasMore = False
-	for row in cur.execute(
-		"SELECT DISTINCT name, alt_name FROM names WHERE alt_name LIKE ? ORDER BY length(alt_name) LIMIT ?",
-		(name + "%", SEARCH_SUGG_LIMIT)):
+	query = None
+	if not useReducedTree:
+		query = "SELECT DISTINCT name, alt_name FROM names" \
+			" WHERE alt_name LIKE ? ORDER BY length(alt_name) LIMIT ?"
+	else:
+		query = "SELECT DISTINCT names.name, alt_name FROM" \
+			" names INNER JOIN reduced_nodes ON names.name = reduced_nodes.name" \
+			" WHERE alt_name LIKE ? ORDER BY length(alt_name) LIMIT ?"
+	for row in cur.execute(query, (name + "%", SEARCH_SUGG_LIMIT)):
 		results.append({"name": row[0], "altName": row[1]})
 	if len(results) > SEARCH_SUGG_LIMIT:
 		hasMore = True
@@ -124,15 +132,17 @@ class DbServer(BaseHTTPRequestHandler):
 		queryDict = urllib.parse.parse_qs(urlParts.query)
 		# Check first element of path
 		match = re.match(r"/([^/]+)/(.+)", path)
-		if match != None and match.group(1) == "data" and "name" in queryDict:
+		if match != None and match.group(1) == "data" and "name" in queryDict and \
+			("tree" not in queryDict or queryDict["tree"][0] == "reduced"):
 			reqType = match.group(2)
 			name = queryDict["name"][0]
+			useReducedTree = "tree" in queryDict
 			# Check query string
 			if reqType == "node":
-				nodeObjs = lookupNodes([name])
+				nodeObjs = lookupNodes([name], useReducedTree)
 				if len(nodeObjs) > 0:
 					nodeObj = nodeObjs[name]
-					childNodeObjs = lookupNodes(nodeObj["children"])
+					childNodeObjs = lookupNodes(nodeObj["children"], useReducedTree)
 					childNodeObjs[name] = nodeObj
 					self.respondJson(childNodeObjs)
 					return
@@ -141,7 +151,7 @@ class DbServer(BaseHTTPRequestHandler):
 				ranOnce = False
 				while True:
 					# Get node
-					nodeObjs = lookupNodes([name])
+					nodeObjs = lookupNodes([name], useReducedTree)
 					if len(nodeObjs) == 0:
 						if not ranOnce:
 							self.respondJson(results)
@@ -158,7 +168,7 @@ class DbServer(BaseHTTPRequestHandler):
 						for childName in nodeObj["children"]:
 							if childName not in results:
 								childNamesToAdd.append(childName)
-						childNodeObjs = lookupNodes(childNamesToAdd)
+						childNodeObjs = lookupNodes(childNamesToAdd, useReducedTree)
 						results.update(childNodeObjs)
 					# Check if root
 					if nodeObj["parent"] == None:
@@ -167,7 +177,7 @@ class DbServer(BaseHTTPRequestHandler):
 					else:
 						name = nodeObj["parent"]
 			elif reqType == "search":
-				self.respondJson(lookupName(name))
+				self.respondJson(lookupName(name, useReducedTree))
 				return
 			elif reqType == "info":
 				self.respondJson(lookupNodeInfo(name))
diff --git a/src/App.vue b/src/App.vue
index caf5aa9..55dc271 100644
--- a/src/App.vue
+++ b/src/App.vue
@@ -39,9 +39,9 @@ function getReverseAction(action: Action): Action | null {
 }
 
 // Initialise tree-of-life data
-const rootName = "cellular organisms";
-const tolMap: TolMap = new Map();
-tolMap.set(rootName, new TolNode());
+const ROOT_NAME = "cellular organisms";
+const initialTolMap: TolMap = new Map();
+initialTolMap.set(ROOT_NAME, new TolNode());
 
 // Configurable options
 const defaultLytOpts: LayoutOptions = {
@@ -86,13 +86,15 @@ const defaultUiOpts = {
 	// Timing related
 	tileChgDuration: 300, //ms (for tile move/expand/collapse)
 	clickHoldDuration: 400, //ms (duration after mousedown when a click-and-hold is recognised)
+	// Other
+	useReducedTree: false,
 };
 
 export default defineComponent({
 	data(){
-		let layoutTree = initLayoutTree(tolMap, rootName, 0);
+		let layoutTree = initLayoutTree(initialTolMap, ROOT_NAME, 0);
 		return {
-			tolMap: tolMap,
+			tolMap: initialTolMap,
 			layoutTree: layoutTree,
 			activeRoot: layoutTree, // Differs from layoutTree root when expand-to-view is used
 			layoutMap: initLayoutMap(layoutTree), // Maps names to LayoutNode objects
@@ -207,7 +209,9 @@ export default defineComponent({
 			// Check if data for node-to-expand exists, getting from server if needed
 			let tolNode = this.tolMap.get(layoutNode.name)!;
 			if (!this.tolMap.has(tolNode.children[0])){
-				return fetch('/data/node?name=' + encodeURIComponent(layoutNode.name))
+				let urlPath = '/data/node?name=' + encodeURIComponent(layoutNode.name)
+				urlPath += this.uiOpts.useReducedTree ? '&tree=reduced' : '';
+				return fetch(urlPath)
 					.then(response => response.json())
 					.then(obj => {
 						Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap.set(key, obj[key])});
@@ -284,7 +288,9 @@ export default defineComponent({
 			// Check if data for node-to-expand exists, getting from server if needed
 			let tolNode = this.tolMap.get(layoutNode.name)!;
 			if (!this.tolMap.has(tolNode.children[0])){
-				return fetch('/data/node?name=' + encodeURIComponent(layoutNode.name))
+				let urlPath = '/data/node?name=' + encodeURIComponent(layoutNode.name)
+				urlPath += this.uiOpts.useReducedTree ? '&tree=reduced' : '';
+				return fetch(urlPath)
 					.then(response => response.json())
 					.then(obj => {
 						Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap.set(key, obj[key])});
@@ -513,6 +519,15 @@ export default defineComponent({
 			tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
 				{allowCollapse: true, layoutMap: this.layoutMap});
 		},
+		onTreeChange(){
+			// Collapse tree to root
+			if (this.activeRoot != this.layoutTree){
+				this.onDetachedAncestorClick(this.layoutTree);
+			}
+			this.onNonleafClick(this.layoutTree);
+			// Re-initialise tree
+			this.initTreeFromServer();
+		},
 		// For other events
 		onResize(){
 			if (!this.resizeThrottled){
@@ -540,6 +555,24 @@ export default defineComponent({
 			}
 		},
 		// Helper methods
+		initTreeFromServer(){
+			let urlPath = '/data/node?name=' + encodeURIComponent(ROOT_NAME);
+			urlPath += this.uiOpts.useReducedTree ? '&tree=reduced' : '';
+			fetch(urlPath)
+				.then(response => response.json())
+				.then(obj => {
+					this.tolMap.clear();
+					Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap.set(key, obj[key])});
+					this.layoutTree = initLayoutTree(this.tolMap, this.layoutTree.name, 0);
+					this.activeRoot = this.layoutTree;
+					this.layoutMap = initLayoutMap(this.layoutTree);
+					tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
+						{allowCollapse: true, layoutMap: this.layoutMap});
+				})
+				.catch(error => {
+					console.log('ERROR loading initial tolnode data', error);
+				});
+		},
 		resetMode(){
 			this.infoModalNode = null;
 			this.searchOpen = false;
@@ -563,20 +596,7 @@ export default defineComponent({
 		window.addEventListener('keyup', this.onKeyUp);
 		tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
 			{allowCollapse: true, layoutMap: this.layoutMap});
-		// Get initial tol node data
-		fetch('/data/node?name=' + encodeURIComponent(rootName))
-			.then(response => response.json())
-			.then(obj => {
-				Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap.set(key, obj[key])});
-				this.layoutTree = initLayoutTree(this.tolMap, this.layoutTree.name, 0);
-				this.activeRoot = this.layoutTree;
-				this.layoutMap = initLayoutMap(this.layoutTree);
-				tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
-					{allowCollapse: true, layoutMap: this.layoutMap});
-			})
-			.catch(error => {
-				console.log('ERROR loading initial tolnode data', error);
-			});
+		this.initTreeFromServer();
 	},
 	unmounted(){
 		window.removeEventListener('resize', this.onResize);
@@ -627,7 +647,8 @@ export default defineComponent({
 	<!-- Settings -->
 	<transition name="slide-bottom-right">
 		<settings-pane v-if="settingsOpen" :lytOpts="lytOpts" :uiOpts="uiOpts"
-			@settings-close="settingsOpen = false" @layout-option-change="onLayoutOptionChange"/>
+			@settings-close="settingsOpen = false"
+			@layout-option-change="onLayoutOptionChange" @tree-change="onTreeChange"/>
 		<div v-else class="absolute bottom-0 right-0 w-[100px] h-[100px] invisible">
 			<!-- Note: Above enclosing div prevents transition interference with inner rotate -->
 			<div class="absolute bottom-[-50px] right-[-50px] w-[100px] h-[100px] visible -rotate-45
diff --git a/src/components/SearchModal.vue b/src/components/SearchModal.vue
index dbe47af..eccc685 100644
--- a/src/components/SearchModal.vue
+++ b/src/components/SearchModal.vue
@@ -41,6 +41,7 @@ export default defineComponent({
 			let url = new URL(window.location.href);
 			url.pathname = '/data/chain';
 			url.search = '?name=' + encodeURIComponent(tolNodeName);
+			url.search += (this.uiOpts.useReducedTree ? '&tree=reduced' : '');
 			fetch(url.toString())
 				.then(response => response.json())
 				.then(obj => {
@@ -80,6 +81,7 @@ export default defineComponent({
 			let url = new URL(window.location.href);
 			url.pathname = '/data/search';
 			url.search = '?name=' + encodeURIComponent(input.value);
+			url.search += (this.uiOpts.useReducedTree ? '&tree=reduced' : '');
 			this.lastSuggReqId += 1;
 			let suggsId = this.lastSuggReqId;
 			let reqDelay = 0;
diff --git a/src/components/SettingsPane.vue b/src/components/SettingsPane.vue
index 13a7f26..cf046c3 100644
--- a/src/components/SettingsPane.vue
+++ b/src/components/SettingsPane.vue
@@ -34,9 +34,12 @@ export default defineComponent({
 			}
 			this.onLytOptChg();
 		},
+		onTreeChg(){
+			this.$emit('tree-change');
+		},
 	},
 	components: {CloseIcon, },
-	emits: ['settings-close', 'layout-option-change', ],
+	emits: ['settings-close', 'layout-option-change', 'tree-change', ],
 });
 </script>
 
@@ -115,5 +118,19 @@ export default defineComponent({
 		<label>Animation Duration <input type="range" min="0" max="3000" class="mx-2 w-[3cm]"
 			v-model.number="uiOpts.tileChgDuration"/></label>
 	</div>
+	<hr class="border-stone-400"/>
+	<div>
+		Tree
+		<ul>
+			<li>
+				<label> <input type="radio" v-model="uiOpts.useReducedTree" :value="false"
+					@change="onTreeChg"/> Default </label>
+			</li>
+			<li>
+				<label> <input type="radio" v-model="uiOpts.useReducedTree" :value="true"
+					@change="onTreeChg"/> Reduced </label>
+			</li>
+		</ul>
+	</div>
 </div>
 </template>
diff --git a/src/components/TileInfoModal.vue b/src/components/TileInfoModal.vue
index 72515d3..6701f1f 100644
--- a/src/components/TileInfoModal.vue
+++ b/src/components/TileInfoModal.vue
@@ -82,6 +82,7 @@ export default defineComponent({
 		<h1 class="text-center text-xl font-bold mb-2">
 			{{displayName}}
 			<div v-if="tolNode.children.length > 0">({{tolNode.children.length}} children)</div>
+			<div>({{tolNode.tips}} tips)</div>
 		</h1>
 		<hr class="mb-4 border-stone-400"/>
 		<div class="flex">