aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTerry Truong <terry06890@gmail.com>2022-06-11 18:54:37 +1000
committerTerry Truong <terry06890@gmail.com>2022-06-11 18:54:37 +1000
commit1c6280a9d070a6ad44919ce42ed283ab8376e3f7 (patch)
tree8f84467badce2812f319e6edbb57f36a1458ac9f
parent5dc1ef81d352ac90f6ad1092ccbaa21dc074146d (diff)
Adapted linked-images code to handle merged image sets
-rw-r--r--backend/data/README.md3
-rwxr-xr-xbackend/data/genLinkedImgs.py52
-rwxr-xr-xbackend/server.py36
3 files changed, 44 insertions, 47 deletions
diff --git a/backend/data/README.md b/backend/data/README.md
index 174c262..7c03d9e 100644
--- a/backend/data/README.md
+++ b/backend/data/README.md
@@ -36,9 +36,8 @@ File Generation Process
and outputs choice information into mergedImgList.txt.
7 Run genImgsForWeb.py, which creates cropped/resized images in img/, using
mergedImgList.txt, and adds 'images' and 'node_imgs' tables to data.db.
-
8 Run genLinkedImgs.py to add a 'linked_imgs' table to data.db,
- which uses 'nodes', 'edges', 'eol\_ids', and 'images', to associate
+ which uses 'nodes', 'edges', 'eol\_ids', and 'node_imgs', to associate
nodes without images to child images.
5 Reduced Tree Structure Data
1 Run genReducedTreeData.py, which adds 'r_nodes' and 'r_edges' tables to
diff --git a/backend/data/genLinkedImgs.py b/backend/data/genLinkedImgs.py
index 6e26ca4..4ad1a8d 100755
--- a/backend/data/genLinkedImgs.py
+++ b/backend/data/genLinkedImgs.py
@@ -17,16 +17,14 @@ upPropagateCompoundImgs = False
# Open db
dbCon = sqlite3.connect(dbFile)
dbCur = dbCon.cursor()
-dbCur.execute("CREATE TABLE linked_imgs (name TEXT PRIMARY KEY, eol_id INT, eol_id2 INT)")
- # The eol_id2 column is used to provide the second part of a compound-image
+dbCur.execute("CREATE TABLE linked_imgs (name TEXT PRIMARY KEY, otol_id TEXT, otol_id2 TEXT)")
+ # Associates a node with one (or two) otol-ids with usable images
# Get nodes with images
print("Getting nodes with images")
-resolvedNodes = {} # Will map node names to eol IDs with a usable image
-query = "SELECT nodes.name, eol_ids.id FROM" \
- " nodes INNER JOIN eol_ids ON nodes.name = eol_ids.name" \
- " INNER JOIN images ON eol_ids.id = images.eol_id"
-for (name, eolId) in dbCur.execute(query):
- resolvedNodes[name] = eolId
+resolvedNodes = {} # Will map node names to otol IDs with a usable image
+query = "SELECT name, nodes.id FROM nodes INNER JOIN node_imgs ON nodes.id = node_imgs.id"
+for (name, otolId) in dbCur.execute(query):
+ resolvedNodes[name] = otolId
print(f"Got {len(resolvedNodes)} nodes")
# Iterate through resolved nodes, resolving ancestors where able
print("Resolving ancestor nodes")
@@ -39,8 +37,8 @@ while len(resolvedNodes) > 0:
if iterNum % 1e3 == 0:
print(f"At iteration {iterNum}")
# Get next node
- (nodeName, eolId) = resolvedNodes.popitem()
- processedNodes[nodeName] = eolId
+ (nodeName, otolId) = resolvedNodes.popitem()
+ processedNodes[nodeName] = otolId
# Traverse upwards, resolving ancestors if able
while True:
# Get parent
@@ -52,7 +50,7 @@ while len(resolvedNodes) > 0:
if parent not in nodesToResolve:
childNames = [row[0] for row in dbCur.execute("SELECT child FROM edges WHERE node = ?", (parent,))]
query = "SELECT name, tips FROM nodes WHERE name IN ({})".format(",".join(["?"] * len(childNames)))
- childObjs = [{"name": row[0], "tips": row[1], "eolId": None} for row in dbCur.execute(query, childNames)]
+ childObjs = [{"name": row[0], "tips": row[1], "otolId": None} for row in dbCur.execute(query, childNames)]
childObjs.sort(key=lambda x: x["tips"], reverse=True)
nodesToResolve[parent] = childObjs
else:
@@ -60,24 +58,24 @@ while len(resolvedNodes) > 0:
# Check if highest-tips child
if (childObjs[0]["name"] == nodeName):
# Resolve parent, and continue from it
- dbCur.execute("INSERT INTO linked_imgs VALUES (?, ?, ?)", (parent, eolId, None))
+ dbCur.execute("INSERT INTO linked_imgs VALUES (?, ?, ?)", (parent, otolId, None))
del nodesToResolve[parent]
- processedNodes[parent] = eolId
+ processedNodes[parent] = otolId
parentToChosenTips[parent] = childObjs[0]["tips"]
nodeName = parent
continue
else:
- # Add potential EOL ID to parent
+ # Add potential otol-id
childObj = next(c for c in childObjs if c["name"] == nodeName)
- childObj["eolId"] = eolId
+ childObj["otolId"] = otolId
break
- # When out of resolved nodes, resolve any nodesToResolve nodes
+ # When out of resolved nodes, resolve nodesToResolve nodes, possibly adding more nodes to resolve
if len(resolvedNodes) == 0:
for (name, childObjs) in nodesToResolve.items():
- childObj = next(c for c in childObjs if c["eolId"] != None)
- resolvedNodes[name] = childObj["eolId"]
+ childObj = next(c for c in childObjs if c["otolId"] != None)
+ resolvedNodes[name] = childObj["otolId"]
parentToChosenTips[name] = childObj["tips"]
- dbCur.execute("INSERT INTO linked_imgs VALUES (?, ?, ?)", (name, childObj["eolId"], None))
+ dbCur.execute("INSERT INTO linked_imgs VALUES (?, ?, ?)", (name, childObj["otolId"], None))
nodesToResolve.clear()
# Iterate through processed nodes with compound names
print("Replacing images for compound-name nodes")
@@ -91,18 +89,18 @@ for nodeName in processedNodes.keys():
if match != None:
# Replace associated image with subname images
(subName1, subName2) = match.group(1,2)
- eolIdPair = [0, 0]
+ otolIdPair = [0, 0]
if subName1 in processedNodes:
- eolIdPair[0] = processedNodes[subName1]
+ otolIdPair[0] = processedNodes[subName1]
if subName2 in processedNodes:
- eolIdPair[1] = processedNodes[subName2]
+ otolIdPair[1] = processedNodes[subName2]
# Use no image if both subimages not found
- if eolIdPair[0] == 0 and eolIdPair[1] == 0:
+ if otolIdPair[0] == 0 and otolIdPair[1] == 0:
dbCur.execute("DELETE FROM linked_imgs WHERE name = ?", (nodeName,))
continue
# Add to db
- dbCur.execute("UPDATE linked_imgs SET eol_id = ?, eol_id2 = ? WHERE name = ?",
- (eolIdPair[0], eolIdPair[1], nodeName,))
+ dbCur.execute("UPDATE linked_imgs SET otol_id = ?, otol_id2 = ? WHERE name = ?",
+ (otolIdPair[0], otolIdPair[1], nodeName))
# Possibly repeat operation upon parent/ancestors
if upPropagateCompoundImgs:
while True:
@@ -114,8 +112,8 @@ for nodeName in processedNodes.keys():
(numTips,) = dbCur.execute("SELECT tips from nodes WHERE name = ?", (nodeName,)).fetchone()
if parent in parentToChosenTips and parentToChosenTips[parent] <= numTips:
# Replace associated image
- dbCur.execute("UPDATE linked_imgs SET eol_id = ?, eol_id2 = ? WHERE name = ?",
- (eolIdPair[0], eolIdPair[1], parent))
+ dbCur.execute("UPDATE linked_imgs SET otol_id = ?, otol_id2 = ? WHERE name = ?",
+ (otolIdPair[0], otolIdPair[1], parent))
nodeName = parent
continue
break
diff --git a/backend/server.py b/backend/server.py
index f85c475..999da25 100755
--- a/backend/server.py
+++ b/backend/server.py
@@ -58,18 +58,18 @@ def lookupNodes(names, useReducedTree):
query = "SELECT id FROM node_imgs WHERE id IN ({})".format(",".join(["?"] * len(idsToNames)))
for (otolId,) in cur.execute(query, list(idsToNames.keys())):
nodeObjs[idsToNames[otolId]]["imgName"] = otolId + ".jpg"
- ## Get 'linked' images for unresolved names
- #unresolvedNames = [n for n in nodeObjs if nodeObjs[n]["imgName"] == None]
- #query = "SELECT name, eol_id, eol_id2 from linked_imgs WHERE name IN ({})"
- #query = query.format(",".join(["?"] * len(unresolvedNames)))
- #for (name, eolId, eolId2) in cur.execute(query, unresolvedNames):
- # if eolId2 == None:
- # nodeObjs[name]["imgName"] = str(eolId) + ".jpg"
- # else:
- # nodeObjs[name]["imgName"] = [
- # str(eolId) + ".jpg" if eolId != 0 else None,
- # str(eolId2) + ".jpg" if eolId2 != 0 else None,
- # ]
+ # Get 'linked' images for unresolved names
+ unresolvedNames = [n for n in nodeObjs if nodeObjs[n]["imgName"] == None]
+ query = "SELECT name, otol_id, otol_id2 from linked_imgs WHERE name IN ({})"
+ query = query.format(",".join(["?"] * len(unresolvedNames)))
+ for (name, otolId, otolId2) in cur.execute(query, unresolvedNames):
+ if otolId2 == None:
+ nodeObjs[name]["imgName"] = otolId + ".jpg"
+ else:
+ nodeObjs[name]["imgName"] = [
+ otolId + ".jpg" if otolId != 0 else None,
+ otolId2 + ".jpg" if otolId2 != 0 else None,
+ ]
# Get preferred-name info
query = f"SELECT name, alt_name FROM names WHERE pref_alt = 1 AND name IN ({queryParamStr})"
for (name, altName) in cur.execute(query, names):
@@ -145,17 +145,17 @@ def lookupNodeInfo(name, useReducedTree):
# Get info for compound-image parts
imgData = [None, None]
idsToLookup = [n[:-4] for n in nodeObj["imgName"] if n != None]
- query = "SELECT images.img_id, images.src, url, license, artist, credit FROM" \
+ query = "SELECT node_imgs.id, images.id, images.src, url, license, artist, credit FROM" \
" node_imgs INNER JOIN images ON node_imgs.img_id = images.id AND node_imgs.src = images.src" \
" WHERE node_imgs.id IN ({})".format(",".join(["?"] * len(idsToLookup)))
- for (imgId, imgSrc, url, license, artist, credit) in cur.execute(query, idsToLookup):
- imgData = {"imgId": imgId, "imgSrc": imgSrc,
+ for (imgOtolId, imgId, imgSrc, url, license, artist, credit) in cur.execute(query, idsToLookup):
+ imgDataVal = {"imgId": imgId, "imgSrc": imgSrc,
"url": url, "license": license, "artist": artist, "credit": credit}
imgName1 = nodeObj["imgName"][0]
- if imgName1 != None and str(imgId) == imgName1[:-4]:
- imgData[0] = imgData
+ if imgName1 != None and imgOtolId == imgName1[:-4]:
+ imgData[0] = imgDataVal
else:
- imgData[1] = imgData
+ imgData[1] = imgDataVal
#
return {"descData": descData, "imgData": imgData, "nodeObj": nodeObj}