Have tol data in sqlite db, and add server script that accesses it

Adapt otol-data-converting script to generate otol.db, add server.py
script that provides access to that db, and adapt the app to query the
server for tol data when needed.

11 files changed, 343 insertions, 567 deletions

diff --git a/.gitignore b/.gitignore
index 7f8c994..2357fe7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,8 +2,7 @@ rem.not
 package-lock.json
 node_modules/
 dist/
-src/tolData.json
 public/img/
-data_tol_old/img/
-data_tol_old/tolData.json
-data_otol/
+backend/data/otol
+backend/data/otol.db
+backend/data/eol
diff --git a/backend/data/eol/README.md b/backend/data/eol/README.md
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/backend/data/eol/README.md
diff --git a/backend/data/otol/README.md b/backend/data/otol/README.md
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/backend/data/otol/README.md
diff --git a/data_otol/namedTreeToJSON.py b/backend/data/otolToSqlite.py
index 30b8033..187e224 100755
--- a/data_otol/namedTreeToJSON.py
+++ b/backend/data/otolToSqlite.py
@@ -1,11 +1,13 @@
 #!/usr/bin/python3
 
-import sys, re, json
+import sys, re, json, sqlite3
+import os.path
 
 usageInfo =  f"usage: {sys.argv[0]}\n"
 usageInfo += "Reads labelled_supertree_ottnames.tre & annotations.json (from an Open Tree of Life release), \n"
-usageInfo += "and prints a JSON object, which maps node names to objects of the form \n"
-usageInfo += "{\"children\": [name1, ...], \"parent\": name1, \"tips\": int1, \"pSupport\": bool1}, which holds \n"
+usageInfo += "and creates an sqlite database otol.db, which holds entries of the form (name text, data text).\n"
+usageInfo += "Each row holds a tree-of-life node name, and a JSON string with the form \n"
+usageInfo += "{\"children\": [name1, ...], \"parent\": name1, \"tips\": int1, \"pSupport\": bool1}, holding \n"
 usageInfo += "child names, a parent name or null, descendant 'tips', and a phylogeny-support indicator\n"
 usageInfo += "\n"
 usageInfo += "This script was adapted to handle Open Tree of Life version 13.4.\n"
@@ -25,12 +27,20 @@ if len(sys.argv) > 1:
 	print(usageInfo, file=sys.stderr)
 	sys.exit(1)
 
-nodeMap = {} # The JSON object to output
+treeFile = "otol/labelled_supertree_ottnames.tre"
+annFile = "otol/annotations.json"
+dbFile = "otol.db"
+nodeMap = {} # Maps node names to node objects
 idToName = {} # Maps node IDs to names
 
-# Parse labelled_supertree_ottnames.tre
+# Check for existing db
+if os.path.exists(dbFile):
+	print("ERROR: Existing {} file".format(dbFile), file=sys.stderr)
+	sys.exit(1)
+
+# Parse treeFile
 data = None
-with open("labelled_supertree_ottnames.tre") as file:
+with open(treeFile) as file:
 	data = file.read()
 dataIdx = 0
 def parseNewick():
@@ -67,6 +77,13 @@ def parseNewick():
 				for childName in childNames:
 					tips += nodeMap[childName]["tips"]
 				# Add node to nodeMap
+				if name in nodeMap: # Turns out the names might not actually be unique
+					count = 2
+					name2 = name + " [" + str(count) + "]"
+					while name2 in nodeMap:
+						count += 1
+						name2 = name + " [" + str(count) + "]"
+					name = name2
 				nodeMap[name] = {
 					"n": name, "id": id, "children": childNames, "parent": None, "tips": tips, "pSupport": False
 				}
@@ -112,6 +129,7 @@ def parseNewickName():
 			name = name[:-1]
 		dataIdx = end
 	# Convert to [name, id]
+	name = name.lower()
 	if name.startswith("mrca"):
 		return [name, name]
 	elif name[0] == "'":
@@ -127,29 +145,63 @@ def parseNewickName():
 		return [match.group(1).replace("_", " "), match.group(2)]
 rootName = parseNewick()
 
-# Parse annotations.json
+# Parse annFile
 data = None
-with open("annotations.json") as file:
+with open(annFile) as file:
 	data = file.read()
 obj = json.loads(data)
 nodeAnnsMap = obj['nodes']
 
-# Do some more postprocessing on each node
-def convertMrcaName(name):
-	"""Given an mrca* name, returns an expanded version with the form [name1 + name2]"""
-	match = re.fullmatch(r"mrca(ott\d+)(ott\d+)", name)
-	if match == None:
-		print("ERROR: Invalid name \"{}\"".format(name), file=sys.stderr)
-	else:
-		subName1 = match.group(1)
-		subName2 = match.group(2)
-		if subName1 not in idToName:
-			print("ERROR: MRCA name \"{}\" sub-name \"{}\" not found".format(subName1), file=sys.stderr)
-		elif subName2 not in idToName:
-			print("ERROR: MRCA name \"{}\" sub-name \"{}\" not found".format(subName2), file=sys.stderr)
-		else:
-			return "[{} + {}]".format(idToName[subName1], idToName[subName2])
-namesToSwap = [] # Will hold [oldName, newName] pairs, for renaming nodes in nodeMap
+# Change mrca* names
+def applyMrcaNameConvert(name, namesToSwap):
+	"""
+	Given an mrca* name, makes namesToSwap map it to an expanded version with the form [childName1 + childName2].
+	May recurse on child nodes with mrca* names.
+	Also returns the name of the highest-tips child (used when recursing).
+	"""
+	node = nodeMap[name]
+	childNames = node["children"]
+	if len(childNames) < 2:
+		print("WARNING: MRCA node \"{}\" has less than 2 children".format(name), file=sys.stderr)
+		return name
+	# Get 2 children with most tips
+	childTips = []
+	for n in childNames:
+		childTips.append(nodeMap[n]["tips"])
+	maxTips = max(childTips)
+	maxIdx = childTips.index(maxTips)
+	childTips[maxIdx] = 0
+	maxTips2 = max(childTips)
+	maxIdx2 = childTips.index(maxTips2)
+	#
+	childName1 = node["children"][maxIdx]
+	childName2 = node["children"][maxIdx2]
+	if childName1.startswith("mrca"):
+		childName1 = applyMrcaNameConvert(childName1, namesToSwap)
+	if childName2.startswith("mrca"):
+		childName2 = applyMrcaNameConvert(childName2, namesToSwap)
+	# Create composite name
+	namesToSwap[name] = "[{} + {}]".format(childName1, childName2)
+	return childName1
+namesToSwap = {} # Maps mrca* names to replacement names
+for node in nodeMap.values():
+	name = node["n"]
+	if (name.startswith("mrca") and name not in namesToSwap):
+		applyMrcaNameConvert(name, namesToSwap)
+for [oldName, newName] in namesToSwap.items():
+	nodeMap[newName] = nodeMap[oldName]
+	del nodeMap[oldName]
+for node in nodeMap.values():
+	parentName = node["parent"]
+	if (parentName in namesToSwap):
+		node["parent"] = namesToSwap[parentName]
+	childNames = node["children"]
+	for i in range(len(childNames)):
+		childName = childNames[i]
+		if (childName in namesToSwap):
+			childNames[i] = namesToSwap[childName]
+
+# Add annotations data, and delete certain fields
 for node in nodeMap.values():
 	# Set has-support value using annotations
 	id = node["id"]
@@ -158,24 +210,19 @@ for node in nodeMap.values():
 		supportQty = len(nodeAnns["supported_by"]) if "supported_by" in nodeAnns else 0
 		conflictQty = len(nodeAnns["conflicts_with"]) if "conflicts_with" in nodeAnns else 0
 		node["pSupport"] = supportQty > 0 and conflictQty == 0
-	# Change mrca* names
-	name = node["n"]
-	if (name.startswith("mrca")):
-		namesToSwap.append([name, convertMrcaName(name)])
-	parentName = node["parent"]
-	if (parentName != None and parentName.startswith("mrca")):
-		node["parent"] = convertMrcaName(parentName)
-	childNames = node["children"]
-	for i in range(len(childNames)):
-		if (childNames[i].startswith("mrca")):
-			childNames[i] = convertMrcaName(childNames[i])
+	# Root node gets support
+	if node["parent"] == None:
+		node["pSupport"] = True
 	# Delete some no-longer-needed fields
 	del node["n"]
 	del node["id"]
-# Finish mrca* renamings
-for [oldName, newName] in namesToSwap:
-	nodeMap[newName] = nodeMap[oldName]
-	del nodeMap[oldName]
 
-# Output JSON
-print(json.dumps(nodeMap))
+# Create db
+con = sqlite3.connect(dbFile)
+cur = con.cursor()
+cur.execute("CREATE TABLE nodes (name TEXT PRIMARY KEY, data TEXT)")
+for name in nodeMap.keys():
+	cur.execute("INSERT INTO nodes VALUES (?, ?)", (name, json.dumps(nodeMap[name])))
+cur.execute("CREATE UNIQUE INDEX nodes_idx on nodes(name)")
+con.commit()
+con.close()
diff --git a/backend/server.py b/backend/server.py
new file mode 100755
index 0000000..2ff7e74
--- /dev/null
+++ b/backend/server.py
@@ -0,0 +1,142 @@
+#!/usr/bin/python3
+
+import sys, re, sqlite3, json
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import urllib.parse
+
+hostname = "localhost"
+port = 8000
+dbFile = "data/otol.db"
+tolnodeReqDepth = 2
+	# For a /tolnode/name1 request, respond with name1's node, and descendent nodes in a subtree to some depth
+	# A depth of 0 means only respond with one node
+
+usageInfo =  f"usage: {sys.argv[0]}\n"
+usageInfo += "Starts a server that listens for GET requests to " + hostname + ":" + str(port) + "/tolnode/name1,\n"
+usageInfo += "and responds with JSON representing an object mapping names to node objects.\n"
+usageInfo += "Normally, the response includes node name1, and child nodes up to depth " + str(tolnodeReqDepth) + ".\n"
+usageInfo += "\n"
+usageInfo += "A query section ?type=type1 can be used to affect the result.\n"
+usageInfo += "If type1 is 'children', the operation acts on node name1's children, and the results combined.\n"
+usageInfo += "If type1 is 'chain', the response includes nodes from name1 up to the root, and their direct children.\n"
+
+dbCon = sqlite3.connect(dbFile)
+def lookupName(name):
+	cur = dbCon.cursor()
+	cur.execute("SELECT name, data FROM nodes WHERE name = ?", (name,))
+	row = cur.fetchone()
+	return row[1] if row != None else None
+#def lookupNameLike(name):
+#	cur = dbCon.cursor()
+#	found = False
+#	cur.execute("SELECT name, data FROM nodes WHERE name like ?", ("%{}%".format(name),))
+#	rows = cur.fetchall()
+#	if len(rows) == 0:
+#		return None
+#	else:
+#		jsonStr = "{"
+#		for i in range(len(rows)):
+#			jsonStr += json.dumps(rows[i][0]) + ":" + rows[i][1]
+#			if i < len(njList) - 1:
+#				jsonStr += ","
+#		jsonStr += "}"
+#		return jsonFromNameJsonList(rows)
+
+class DbServer(BaseHTTPRequestHandler):
+	def do_GET(self):
+		# Parse URL
+		urlParts = urllib.parse.urlparse(self.path)
+		path = urllib.parse.unquote(urlParts.path)
+		queryDict = urllib.parse.parse_qs(urlParts.query)
+		# Check first element of path
+		match = re.match(r"/([^/]+)/(.+)", path)
+		if match != None:
+			reqType = match.group(1)
+			if reqType == "tolnode":
+				name = match.group(2)
+				# Check query string
+				if "type" not in queryDict:
+					nodeJson = lookupName(name)
+					if nodeJson != None:
+						results = []
+						getResultsUntilDepth(name, nodeJson, tolnodeReqDepth, results)
+						self.respondJson(nodeResultsToJSON(results))
+						return
+				elif queryDict["type"][0] == "children":
+					nodeJson = lookupName(name)
+					if nodeJson != None:
+						obj = json.loads(nodeJson)
+						results = []
+						for childName in obj["children"]:
+							nodeJson = lookupName(childName)
+							if nodeJson != None:
+								getResultsUntilDepth(childName, nodeJson, tolnodeReqDepth, results)
+						self.respondJson(nodeResultsToJSON(results))
+						return
+				elif queryDict["type"][0] == "chain":
+					results = []
+					ranOnce = False
+					while True:
+						jsonResult = lookupName(name)
+						if jsonResult == None:
+							if ranOnce:
+								print("ERROR: Parent-chain node {} not found".format(name), file=sys.stderr)
+							break
+						results.append([name, jsonResult])
+						obj = json.loads(jsonResult)
+						# Add children
+						if not ranOnce:
+							ranOnce = True
+						else:
+							internalFail = False
+							for childName in obj["children"]:
+								jsonResult = lookupName(childName)
+								if jsonResult == None:
+									print("ERROR: Parent-chain-child node {} not found".format(name), file=sys.stderr)
+									internalFail = True
+									break
+								results.append([childName, jsonResult])
+							if internalFail:
+								break
+						# Check if root
+						if obj["parent"] == None:
+							self.respondJson(nodeResultsToJSON(results))
+							return
+						else:
+							name = obj["parent"]
+		self.send_response(404)
+		self.end_headers()
+		self.end_headers()
+	def respondJson(self, jsonStr):
+		self.send_response(200)
+		self.send_header("Content-type", "application/json")
+		self.end_headers()
+		self.wfile.write(jsonStr.encode("utf-8"))
+def getResultsUntilDepth(name, nodeJson, depth, results):
+	"""Given a node [name, nodeJson] pair, adds child node pairs to 'results', up until 'depth'"""
+	results.append([name, nodeJson])
+	if depth > 0:
+		obj = json.loads(nodeJson)
+		for childName in obj["children"]:
+			childJson = lookupName(childName)
+			if childJson != None:
+				getResultsUntilDepth(childName, childJson, depth-1, results)
+def nodeResultsToJSON(results):
+	"""Given a list of [name, nodeJson] pairs, returns a representative JSON string"""
+	jsonStr = "{"
+	for i in range(len(results)):
+		jsonStr += json.dumps(results[i][0]) + ":" + results[i][1]
+		if i < len(results) - 1:
+			jsonStr += ","
+	jsonStr += "}"
+	return jsonStr
+
+server = HTTPServer((hostname, port), DbServer)
+print("Server started at http://{}:{}".format(hostname, port))
+try:
+	server.serve_forever()
+except KeyboardInterrupt:
+	pass
+server.server_close()
+dbCon.close()
+print("Server stopped")
diff --git a/data_tol_old/genTestImgs.sh b/data_tol_old/genTestImgs.sh
deleted file mode 100755
index 21b001b..0000000
--- a/data_tol_old/genTestImgs.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-set -e
-
-#generate tol.json from tol.txt
-cat tolData.txt | ./txtTreeToJSON.py > tolData.json
-
-#reads through tolData.json, gets names, and generates image for each name
-cat tolData.json | \
-	gawk 'match ($0, /"name"\s*:\s*"([^"]*)"/, arr) {print arr[1]}' | \
-	while read; do
-		convert -size 200x200 xc:khaki +repage \
-			-size 150x150  -fill black -background None  \
-			-font Ubuntu-Mono -gravity center caption:"$REPLY" +repage \
-			-gravity Center  -composite -strip ../public/img/"$REPLY".png
-	done
-
diff --git a/data_tol_old/tolData.txt b/data_tol_old/tolData.txt
deleted file mode 100644
index f73a064..0000000
--- a/data_tol_old/tolData.txt
+++ /dev/null
@@ -1,388 +0,0 @@
-Tree of Life
-	Viruses
-		Caudovirales
-		Herpesvirales
-		Ligamenvirales
-		Mononegavirales
-		Nidovirales
-		Picornavirales
-		Tymovirales
-	Archaea
-		Crenarchaeota
-		Euryarchaeota
-	Bacteria
-		Acidobacteria
-		Actinobacteria
-		Aquificae
-		Armatimonadetes
-		Bacteroidetes
-		Caldiserica
-		Chlamydiae
-		Chlorobi
-		Chloroflexi
-		Chrysiogenetes
-		Cyanobacteria
-		Deferribacteres
-		Deinococcus-thermus
-		Dictyoglomi
-		Elusimicrobia
-		Fibrobacteres
-		Firmicutes
-		Fusobacteria
-		Gemmatimonadetes
-		Lentisphaerae
-		Nitrospira
-		Planctomycetes
-		Proteobacteria
-		Spirochaetae
-		Synergistetes
-		Tenericutes
-		Thermodesulfobacteria
-		Thermotogae
-		Verrucomicrobia
-	Eukaryota
-		Diatoms
-		Amoebozoa
-		Plantae
-			Rhodopyhta
-			Viridiplantae
-				Prasinophytes
-				Ulvophyceae
-				Streptophyta
-					Charales
-					Embryophytes
-						Marchantiomorpha
-						Anthocerotophyta
-						Bryophyta
-						Lycopodiopsida
-							Lycopodiidae
-							Selaginellales
-						Polypodiopsida
-							Polypodiidae
-								Polypodiales
-							Equisetidae
-						Spermatopsida
-							Cycads
-							Conifers
-								Taxaceae
-								Cupressaceae
-								Pinaceae
-									Pinus
-									Picea
-									Larix
-									Cedrus
-									Abies
-							Ginkgos
-							Angiosperms
-								Illicium
-								magnoliids
-									Piperales
-										Piperaceae
-									Magnoliales
-										Annonaceae
-										Myristicaceae
-									Laurales
-										Lauraceae
-								Monocotyledons
-									Alismatanae
-									Aranae
-									Liliaceae
-									Asparagales
-										Amaryllidaceae
-										Asparagaceae
-										Asphodelaceae
-										Iridaceae
-										Orchidaceae
-									Dioscoreaceae
-									Arecanae
-										Cocoeae
-										Phoeniceae
-									Zingiberanae
-										Musaceae
-										Strelitziaceae
-										Zingiberaceae
-									Commelinanae
-										Bromeliaceae
-										Cyperaceae
-										Typhaceae
-										Poaceae
-											Zea mays
-											Triticum
-											Bambusoideae
-								eudicots
-									Ranunculales
-										Papaveraceae
-										Ranunculaceae
-									Proteales
-										Proteaceae
-										Nelumbo
-									Core Eudicots
-										Saxifragales
-										Rosids
-											Fabaceae
-												Mimosoideae
-												IRLC (Inverted Repat-lacking clade)
-													Trifolieae
-													Fabeae
-											Rosales
-												Rosaceae
-													Rosa
-													Malus pumila
-												Ulmaceae
-												Urticaceae
-												Moraceae
-												Cannabaceae
-											Fagales
-												Fagaceae
-												Betulaceae
-												Juglandaceae
-											Cucurbitales
-												Cucurbitaceae
-											Malpighiales
-												Salicaceae
-												Violaceae
-												Passifloraceae
-												Erythroxylaceae
-												Rhizophoraceae
-												Euphorbiaceae
-												Linaceae
-												Rafflesiaceae
-											Myrtales
-												Myrtaceae
-												Onagraceae
-												Lythraceae
-											Brassicales
-												Caricaceae
-												Brassicaceae
-											Malvales
-												Core Malvales
-													Malvoideae
-													Bombacoideae
-													Sterculioideae
-													Helicteroideae
-													Byttnerioideae
-											Sapindales
-												Anacardiaceae
-												Burseraceae
-												Meliaceae
-												Rutaceae
-												Sapindaceae
-										Vitaceae
-										Caryophyllales
-											Polygonaceae
-											Droseraceae
-											Nepenthaceae
-											core Caryophyllales
-												Cactaceae
-												Amaranthaceae
-										Asterids
-											Ericales
-												Actinidiaceae
-												Ericaceae
-												Lecythidaceae
-												Sapotaceae
-												Ebenaceae
-												Theaceae
-											Solanales
-												Solanaceae
-												Convolvulaceae
-											Lamiales
-												Oleaceae
-													Fraxinus
-												Bignoniaceae
-												Pedaliaceae
-												Lentibulariaceae
-												Lamiaceae
-											Gentianales
-												Rubiaceae
-											Asterales
-												Campanulaceae
-												Asteraceae
-													Carduoideae
-														Cardueae
-													Cichorioideae
-														Cichorieae
-													Asteroideae
-														Asterodae
-														Helianthodae
-											Apiales
-												Apiaceae
-												Araliaceae
-											Aquifoliaceae
-		Fungi
-			Fungi 1
-				Dikarya
-					Basidiomycota
-						Agaricomycotina
-							Agaricomycetes
-								Agaricomycetes 1
-									Agaricomycetidae
-										Agaricales
-											Strophariaceae strict-sense
-											Psathyrellaceae
-											Agaricaceae
-											Nidulariaceae
-											Marasmiaceae
-											Physalacriaceae
-											Pleurotaceae
-											Amanitaceae
-											Podoserpula
-										Boletales
-											Serpulaceae
-											Sclerodermataceae
-											Boletaceae
-									Russulales
-									Hymenochaetales
-								Phallomycetidae
-									Geastrales
-									Gomphales
-									Phallales
-								Cantharellales
-								Auriculariales
-							Tremellomycetes
-						Ustilaginomycotina
-						Pucciniomycotina
-							Pucciniomycetes
-								Septobasidiales
-								Pucciniales
-							Mixiomycetes
-							Tritirachiomycetes
-						Entorrhizomycetes
-						Wallemiomycetes
-					Ascomycota
-						Pezizomycotina
-							Pezizomycetes
-							'Leotiomyceta'
-								Eurotiomycetes
-								Geoglossaceae
-								Sordariomycetes
-									Hypocreomycetidae
-									Sordariomycetidae
-								Laboulbeniomycetes
-								Pleosporomycetidae
-						Saccharomycotina
-						Taphrinomycotina
-							Schizosaccharomycetes
-							Pneumocystidiomycetes
-							Taphrinomycetes
-				Glomeromycota
-			Zygomycota
-				Endogonales
-				Mucorales
-			Blastocladiomycota
-			Chytridiomycota
-			Neocallimastigomycota
-			Microsporidia
-		Animalia
-			Porifera
-			Cnidaria
-			Tardigrada
-			Annelida
-			Mollusca
-				Bivalvia
-				Gastropoda
-				Cephalopoda
-			Arthropoda
-				Arachnida
-					Araneae
-					Opiliones
-					Scorpiones
-					Heterostigmata
-				Crustacea
-					Euphausiacea
-					Brachyura
-					Isopoda
-					Cirripedia
-				Insecta
-					Anisoptera
-					Mantodea
-					Cicadoidea
-					Siphonaptera
-					Cucujoidea
-					Phengodidae
-					Drosophilidae
-					Culicidae
-					Lepidoptera
-					Apini
-					Formicidae
-			Deuterostomia
-				Echinodermata
-					Crinoidea
-					Asteroidea
-					Echinoidea
-					Holothuroidea
-				Vertebrata
-					Chondrichthyes
-						Carcharodon carcharias
-						Rhinocodon typus
-						Batoidea
-						Pristidae
-					Actinopterygii
-						Clupeomorpha
-						Xiphias gladius
-						Siluriformes
-						Carassius auratus
-						Tetraodontidae
-						Molidae
-						Gymnotiformes
-						Lophiiformes
-						Exocoetidae
-						'mudskipper'
-						Hippocampus
-						Psudoliparis swirei
-					Sarcopterygii
-					Tetrapoda
-						Amphibia
-							Gymnophiona
-							Caudata
-								Salamandra
-								Cryptobranchidae
-								Ambystomatidae
-							Anura
-						Reptilia
-							Testudines
-							Plesiosauria
-							Chamaeleonidae
-							Serpentes
-							Crocodilia
-							Dinosauria
-								Triceratops
-								Sauropoda
-								Tyrannosauroidea
-								Aves
-									magpie
-									parrot
-									eagle
-									owl
-									swan
-									chicken
-									penguin
-									hummingbird
-						Synapsida
-							monotreme
-							marsupial
-								kangaroo
-								possum
-								wombat
-							rodent
-								mouse
-								beaver
-							rabbit
-							feline
-							canine
-							bear
-							walrus
-							Artiodactyla
-								pig
-								camel
-								deer
-								giraffe
-							horse
-							elephant
-							cetacean
-							armadillo
-							bat
-							monkey
-							gorilla
-							chimpanzee
-							homo sapien
diff --git a/data_tol_old/txtTreeToJSON.py b/data_tol_old/txtTreeToJSON.py
deleted file mode 100755
index 3b77622..0000000
--- a/data_tol_old/txtTreeToJSON.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/python3
-
-import sys, re
-
-usageInfo =  f"usage: {sys.argv[0]}\n"
-usageInfo += "Reads, from stdin, tab-indented lines representing trees, and outputs corresponding JSON.\n"
-
-if len(sys.argv) > 1:
-	print(usageInfo, file=sys.stderr)
-	sys.exit(1)
-
-lineNum = 0
-trees = [] #each node is a pair holding a name and an array of child nodes
-nodeList = []
-while True: 
-	#read line
-	line = sys.stdin.readline()
-	if line == "": break
-	line = line.rstrip()
-	lineNum += 1
-	#create node
-	match = re.match(r"^\t*", line)
-	indent = len(match.group())
-	newNode = [line[indent:], []]
-	#add node
-	if indent == len(nodeList): #sibling or new tree
-		if len(nodeList) == 0:
-			nodeList.append(newNode)
-			trees.append(newNode)
-		else:
-			nodeList[-1] = newNode
-			if len(nodeList) == 1:
-				trees[-1][1].append(newNode)
-			else:
-				nodeList[-2][1].append(newNode)
-	elif indent == len(nodeList) + 1: #direct child
-		if len(nodeList) == 0:
-			print(f"ERROR: Child without preceding root (line {lineNum})")
-			sys.exit(1)
-		nodeList.append(newNode)
-		nodeList[-2][1].append(newNode)
-	elif indent < len(nodeList): #ancestor sibling or new tree
-		nodeList = nodeList[:indent]
-		if len(nodeList) == 0:
-			nodeList.append(newNode)
-			trees.append(newNode)
-		else:
-			nodeList[-1] = newNode
-			if len(nodeList) == 1:
-				trees[-1][1].append(newNode)
-			else:
-				nodeList[-2][1].append(newNode)
-	else:
-		print(f"ERROR: Child with invalid relative indent (line {lineNum})")
-		sys.exit(1)
-#print as JSON
-if len(trees) > 1:
-	print("[")
-def printNode(node, indent):
-	if len(node[1]) == 0:
-		print(indent + "{\"name\": \"" + node[0] + "\"}", end="")
-	else:
-		print(indent + "{\"name\": \"" + node[0] + "\", \"children\": [")
-		for i in range(len(node[1])):
-			printNode(node[1][i], indent + "\t")
-			if i < len(node[1])-1:
-				print(",", end="")
-			print()
-		print(indent + "]}", end="")
-for i in range(len(trees)):
-	printNode(trees[i], "")
-	if i < len(trees)-1:
-		print(",", end="")
-	print()
-if len(trees) > 1:
-	print("]")
diff --git a/src/App.vue b/src/App.vue
index cb2b1f4..f1e6e2a 100644
--- a/src/App.vue
+++ b/src/App.vue
@@ -39,9 +39,8 @@ function getReverseAction(action: Action): Action | null {
 }
 
 // Get tree-of-life data
-import data from './tolData.json';
-let tolMap: TolMap = data;
-const rootName = "[Elaeocarpus williamsianus + Brunellia mexicana]";
+const rootName = "cellular organisms";
+const tolMap: TolMap = {[rootName]: new TolNode()};
 
 // Configurable options
 const defaultLytOpts: LayoutOptions = {
@@ -117,6 +116,8 @@ export default defineComponent({
 			height: document.documentElement.clientHeight,
 			resizeThrottled: false,
 			resizeDelay: 50, //ms (increasing to 100 seems to cause resize-skipping when opening browser mobile-view)
+			// Other
+			excessTolNodeThreshold: 1000, // Threshold where excess tolMap entries are removed (done on tile collapse)
 		};
 	},
 	computed: {
@@ -173,15 +174,32 @@ export default defineComponent({
 	methods: {
 		// For tile expand/collapse events
 		onLeafClick(layoutNode: LayoutNode){
-			let success = tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts, {
-				allowCollapse: false,
-				chg: {type: 'expand', node: layoutNode, tolMap: this.tolMap},
-				layoutMap: this.layoutMap
-			});
-			if (!success){
-				layoutNode.failFlag = !layoutNode.failFlag; // Triggers failure animation
+			let doExpansion = () => {
+				let success = tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts, {
+					allowCollapse: false,
+					chg: {type: 'expand', node: layoutNode, tolMap: this.tolMap},
+					layoutMap: this.layoutMap
+				});
+				if (!success){
+					layoutNode.failFlag = !layoutNode.failFlag; // Triggers failure animation
+				}
+				return success;
+			};
+			// Check if data for node-to-expand exists, getting from server if needed
+			let tolNode = this.tolMap[layoutNode.name];
+			if (tolNode.children[0] in this.tolMap == false){
+				return fetch('/tolnode/' + layoutNode.name + '?type=children')
+					.then(response => response.json())
+					.then(obj => {
+						Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap[key] = obj[key]});
+						doExpansion();
+					})
+					.catch(error => {
+						console.log('ERROR loading tolnode data', error);
+					});
+			} else {
+				return new Promise((resolve, reject) => resolve(doExpansion()));
 			}
-			return success;
 		},
 		onNonleafClick(layoutNode: LayoutNode){
 			let success = tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts, {
@@ -191,6 +209,19 @@ export default defineComponent({
 			});
 			if (!success){
 				layoutNode.failFlag = !layoutNode.failFlag; // Triggers failure animation
+			} else {
+				// Clear out excess nodes when a threshold is reached
+				let tolNodeNames = Object.getOwnPropertyNames(this.tolMap)
+				let extraNodes = tolNodeNames.length - this.layoutMap.size;
+				if (extraNodes > this.excessTolNodeThreshold){
+					for (let n of tolNodeNames){
+						if (!this.layoutMap.has(n)){
+							delete this.tolMap[n];
+						}
+					}
+					let numRemovedNodes = tolNodeNames.length - Object.getOwnPropertyNames(this.tolMap).length;
+					console.log(`Cleaned up tolMap (removed ${numRemovedNodes} out of ${tolNodeNames.length})`);
+				}
 			}
 			return success;
 		},
@@ -271,26 +302,27 @@ export default defineComponent({
 				return;
 			}
 			// Attempt tile-expand
-			let success = this.onLeafClick(layoutNode);
-			if (success){
-				setTimeout(() => this.expandToNode(name), this.uiOpts.tileChgDuration);
-				return;
-			}
-			// Attempt expand-to-view on ancestor just below activeRoot
-			if (layoutNode == this.activeRoot){
-				console.log('Unable to complete search (not enough room to expand active root)');
-					// Note: Only happens if screen is significantly small or node has significantly many children
-				this.modeRunning = false;
-				return;
-			}
-			while (true){
-				if (layoutNode.parent! == this.activeRoot){
-					break;
+			this.onLeafClick(layoutNode).then(success => {
+				if (success){
+					setTimeout(() => this.expandToNode(name), this.uiOpts.tileChgDuration);
+					return;
 				}
-				layoutNode = layoutNode.parent!;
-			}
-			this.onNonleafClickHeld(layoutNode);
-			setTimeout(() => this.expandToNode(name), this.uiOpts.tileChgDuration);
+				// Attempt expand-to-view on ancestor just below activeRoot
+				if (layoutNode == this.activeRoot){
+					console.log('Unable to complete search (not enough room to expand active root)');
+						// Note: Only happens if screen is significantly small or node has significantly many children
+					this.modeRunning = false;
+					return;
+				}
+				while (true){
+					if (layoutNode.parent! == this.activeRoot){
+						break;
+					}
+					layoutNode = layoutNode.parent!;
+				}
+				this.onNonleafClickHeld(layoutNode);
+				setTimeout(() => this.expandToNode(name), this.uiOpts.tileChgDuration);
+			});
 		},
 		// For auto-mode events
 		onPlayIconClick(){
@@ -385,7 +417,9 @@ export default defineComponent({
 						this.setLastFocused(node.parent!);
 						break;
 					case 'expand':
-						this.autoPrevActionFail = !this.onLeafClick(node);
+						this.onLeafClick(node)
+							.then(success => this.autoPrevActionFail = !success)
+							.catch(error => this.autoPrevActionFail = true);
 						break;
 					case 'collapse':
 						this.autoPrevActionFail = !this.onNonleafClick(node);
@@ -457,6 +491,20 @@ export default defineComponent({
 		window.addEventListener('keyup', this.onKeyUp);
 		tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
 			{allowCollapse: true, layoutMap: this.layoutMap});
+		// Get initial tol node data
+		fetch('/tolnode/' + rootName)
+			.then(response => response.json())
+			.then(obj => {
+				Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap[key] = obj[key]});
+				this.layoutTree = initLayoutTree(this.tolMap, this.layoutTree.name, 0);
+				this.activeRoot = this.layoutTree;
+				this.layoutMap = initLayoutMap(this.layoutTree);
+				tryLayout(this.activeRoot, this.tileAreaPos, this.tileAreaDims, this.lytOpts,
+					{allowCollapse: true, layoutMap: this.layoutMap});
+			})
+			.catch(error => {
+				console.log('ERROR loading initial tolnode data', error);
+			});
 	},
 	unmounted(){
 		window.removeEventListener('resize', this.onResize);
diff --git a/src/components/SearchModal.vue b/src/components/SearchModal.vue
index 22b6896..5accd62 100644
--- a/src/components/SearchModal.vue
+++ b/src/components/SearchModal.vue
@@ -12,21 +12,36 @@ export default defineComponent({
 	},
 	methods: {
 		onCloseClick(evt: Event){
-			if (evt.target == this.$el || (this.$refs.searchInput as typeof SearchIcon).$el.contains(evt.target)){
+			if (evt.target == this.$el || (this.$refs.searchIcon as typeof SearchIcon).$el.contains(evt.target)){
 				this.$emit('search-close');
 			}
 		},
 		onSearchEnter(){
 			let input = this.$refs.searchInput as HTMLInputElement;
-			if (this.tolMap.hasOwnProperty(input.value)){
-				this.$emit('search-node', input.value);
-			} else {
-				input.value = '';
-				// Trigger failure animation
-				input.classList.remove('animate-red-then-fade');
-				input.offsetWidth; // Triggers reflow
-				input.classList.add('animate-red-then-fade');
-			}
+			// Query server
+			let url = new URL(window.location.href);
+			url.pathname = '/tolnode/' + input.value;
+			fetch(url)
+				.then(response => {
+					// Search successful. Get nodes in parent-chain, add to tolMap, then emit event.
+					url.search = '?type=chain';
+					fetch(url)
+						.then(response => response.json())
+						.then(obj => {
+							Object.getOwnPropertyNames(obj).forEach(key => {this.tolMap[key] = obj[key]});
+							this.$emit('search-node', input.value);
+						})
+						.catch(error => {
+							console.log('ERROR loading tolnode chain', error);
+						});
+				})
+				.catch(error => {
+					input.value = '';
+					// Trigger failure animation
+					input.classList.remove('animate-red-then-fade');
+					input.offsetWidth; // Triggers reflow
+					input.classList.add('animate-red-then-fade');
+				});
 		},
 		focusInput(){
 			(this.$refs.searchInput as HTMLInputElement).focus();
@@ -46,7 +61,7 @@ export default defineComponent({
 		bg-stone-50 rounded-md shadow shadow-black flex gap-1">
 		<input type="text" class="block border"
 			@keyup.enter="onSearchEnter" @keyup.esc="onCloseClick" ref="searchInput"/>
-		<search-icon @click.stop="onSearchEnter"
+		<search-icon @click.stop="onSearchEnter" ref="searchIcon"
 			class="block w-6 h-6 ml-1 hover:cursor-pointer hover:bg-stone-200" />
 	</div>
 </div>
diff --git a/vite.config.js b/vite.config.js
index e1200f9..7b02d3f 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -3,5 +3,10 @@ import vue from '@vitejs/plugin-vue'
 
 export default defineConfig({
 	plugins: [vue()],
+	server: {
+		proxy: {
+			'/tolnode': 'http://localhost:8000',
+		}
+	},
 	//server: {open: true} //open browser when dev server starts
 })