From d87bb9bc0991d7ce4eeb895da61c63a204edaa4d Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Sat, 30 Apr 2022 13:24:26 +1000 Subject: Add scripts for downloading/reviewing/cropping_and_resizing images Also adjust client code to handle new format, and add backend/data/README.md explaining image production process. --- backend/data/README.md | 13 ++ backend/data/downloadImgsForReview.py | 143 +++++++++++++++++++++ backend/data/eol/README.md | 15 ++- backend/data/eolNamesToSqlite.py | 93 -------------- backend/data/genEolNameData.py | 104 ++++++++++++++++ backend/data/genImgsForWeb.py | 94 ++++++++++++++ backend/data/genOtolData.py | 224 +++++++++++++++++++++++++++++++++ backend/data/otol/README.md | 4 +- backend/data/otolToSqlite.py | 227 ---------------------------------- backend/data/reviewImgs.py | 176 ++++++++++++++++++++++++++ backend/server.py | 111 ++++++++++------- package.json | 1 + src/App.vue | 2 +- src/components/Tile.vue | 10 +- src/components/TileInfoModal.vue | 10 +- src/layout.ts | 1 - src/tol.ts | 2 + 17 files changed, 847 insertions(+), 383 deletions(-) create mode 100644 backend/data/README.md create mode 100755 backend/data/downloadImgsForReview.py delete mode 100755 backend/data/eolNamesToSqlite.py create mode 100755 backend/data/genEolNameData.py create mode 100755 backend/data/genImgsForWeb.py create mode 100755 backend/data/genOtolData.py delete mode 100755 backend/data/otolToSqlite.py create mode 100755 backend/data/reviewImgs.py diff --git a/backend/data/README.md b/backend/data/README.md new file mode 100644 index 0000000..3cbeb03 --- /dev/null +++ b/backend/data/README.md @@ -0,0 +1,13 @@ +File Generation Process +======================= +1 Obtain data in otol/ and eol/, as specified in their README files. +2 Run genOtolData.py, which creates data.db, and adds a 'nodes' + table using data in otol/*. +3 Run genEolNameData.py, which adds a 'names' table to data.db, + using data in eol/vernacularNames.csv and the 'nodes' table. +4 Use downloadImgsForReview.py to download EOL images into imgsForReview/. + It uses data in eol/imagesList.db, and the 'nodes' table. +5 Use reviewImgs.py to filter images in imgsForReview/ into EOL-id-unique + images in imgsReviewed/. +6 Use genImgsForWeb.py to create cropped/resized images in img/, using + images in imgsReviewed, and also to add an 'images' table to data.db. diff --git a/backend/data/downloadImgsForReview.py b/backend/data/downloadImgsForReview.py new file mode 100755 index 0000000..12b52ff --- /dev/null +++ b/backend/data/downloadImgsForReview.py @@ -0,0 +1,143 @@ +#!/usr/bin/python3 + +import sys, re, os, random +import sqlite3 +import urllib.parse, requests +import time +from threading import Thread +import signal + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Downloads images from URLs specified in an image-list database, using\n" +usageInfo += "EOL IDs obtained from another database. Downloaded images get names of\n" +usageInfo += "the form 'eolId1 contentId1.ext1'\n" +usageInfo += "\n" +usageInfo += "SIGINT causes the program to finish ongoing downloads and exit.\n" +usageInfo += "The program can be re-run to continue downloading, and uses\n" +usageInfo += "existing downloaded files to decide where to continue from.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +imagesListDb = "eol/imagesList.db" +dbFile = "data.db" +outDir = "imgsForReview/" +LICENSE_REGEX = r"cc-by((-nc)?(-sa)?(-[234]\.[05])?)|cc-publicdomain|cc-0-1\.0|public domain" +POST_DL_DELAY_MIN = 2 # Minimum delay in seconds to pause after download before starting another (for each thread) +POST_DL_DELAY_MAX = 3 + +# Get eol-ids from data db +eolIds = set() +print("Reading in EOL IDs") +dbCon = sqlite3.connect(dbFile) +dbCur = dbCon.cursor() +for row in dbCur.execute("SELECT DISTINCT eol_id FROM names"): + eolIds.add(row[0]) +dbCon.close() +# Get eol-ids from images db +imgDbCon = sqlite3.connect(imagesListDb) +imgCur = imgDbCon.cursor() +imgListIds = set() +for row in imgCur.execute("SELECT DISTINCT page_id FROM images"): + imgListIds.add(row[0]) +# Get eol-id intersection, and sort into list +eolIds = eolIds.intersection(imgListIds) +eolIds = sorted(eolIds) + +MAX_IMGS_PER_ID = 3 +MAX_THREADS = 10 +numThreads = 0 +threadException = None # Used for ending main thread after a non-main thread exception +def downloadImg(url, outFile): + global numThreads, threadException + try: + data = requests.get(url) + with open(outFile, 'wb') as file: + file.write(data.content) + time.sleep(random.random() * (POST_DL_DELAY_MAX - POST_DL_DELAY_MIN) + POST_DL_DELAY_MIN) + except Exception as e: + print("Error while downloading to {}: {}".format(outFile, str(e)), file=sys.stderr) + threadException = e + numThreads -= 1 +# Create output directory if not present +if not os.path.exists(outDir): + os.mkdir(outDir) +# Find next eol ID to download for +print("Finding next ID to download for") +nextIdx = 0 +fileList = os.listdir(outDir) +ids = list(map(lambda filename: int(filename.split(" ")[0]), fileList)) +if len(ids) > 0: + ids.sort() + nextIdx = eolIds.index(ids[-1]) +if nextIdx == len(eolIds): + print("No IDs left. Exiting...") + sys.exit(0) +# Detect SIGINT signals +interrupted = False +oldHandler = None +def onSigint(sig, frame): + global interrupted + interrupted = True + signal.signal(signal.SIGINT, oldHandler) +oldHandler = signal.signal(signal.SIGINT, onSigint) +# Manage downloading +for idx in range(nextIdx, len(eolIds)): + eolId = eolIds[idx] + # Get image urls + imgDataList = [] + ownerSet = set() # Used to get images from different owners, for variety + for row in imgCur.execute( + "SELECT content_id, page_id, copy_url, license, copyright_owner FROM images WHERE page_id = ?", (eolId,)): + license = row[3] + copyrightOwner = row[4] + if re.fullmatch(LICENSE_REGEX, license) == None: + continue + if len(copyrightOwner) > 100: # Ignore certain copyrightOwner fields that seem long and problematic + continue + if copyrightOwner not in ownerSet: + ownerSet.add(copyrightOwner) + imgDataList.append(row) + if len(ownerSet) == MAX_IMGS_PER_ID: + break + if len(imgDataList) == 0: + continue + # Determine output filenames + outFiles = [] + urls = [] + for row in imgDataList: + contentId = row[0] + url = row[2] + if url.startswith("data/"): + url = "https://content.eol.org/" + url + urlParts = urllib.parse.urlparse(url) + extension = os.path.splitext(urlParts.path)[1] + if len(extension) <= 1: + print("WARNING: No filename extension found in URL {}".format(url), file=sys.stderr) + continue + outFiles.append(str(eolId) + " " + str(contentId) + extension) + urls.append(url) + # Start downloads + exitLoop = False + for i in range(len(outFiles)): + outPath = outDir + outFiles[i] + if not os.path.exists(outPath): + # Enforce thread limit + while numThreads == MAX_THREADS: + time.sleep(1) + # Wait for threads after an interrupt or thread-exception + if interrupted or threadException != None: + print("Waiting for existing threads to end") + while numThreads > 0: + time.sleep(1) + exitLoop = True + break + print("Downloading image to {}".format(outPath)) + # Perform download + numThreads += 1 + thread = Thread(target=downloadImg, args=(urls[i], outPath), daemon=True) + thread.start() + if exitLoop: + break +# Close images-list db +imgDbCon.close() diff --git a/backend/data/eol/README.md b/backend/data/eol/README.md index ed970d2..3ce9799 100644 --- a/backend/data/eol/README.md +++ b/backend/data/eol/README.md @@ -1,8 +1,15 @@ -Files -===== -- images\_list.tgz +Downloaded Files +================ +- imagesList.tgz Obtained from https://opendata.eol.org/dataset/images-list on 24/04/2022 Listed as being last updated on 05/02/2020 -- vernacular\_names.csv +- vernacularNames.csv Obtained from https://opendata.eol.org/dataset/vernacular-names on 24/04/2022 Listed as being last updated on 27/10/2020 + +Generated Files +=============== +- imagesList/ + Obtained by extracting imagesList.tgz +- imagesList.db + Represents data from eol/imagesList/*, and is created by genImagesListDb.sh diff --git a/backend/data/eolNamesToSqlite.py b/backend/data/eolNamesToSqlite.py deleted file mode 100755 index 1df1c23..0000000 --- a/backend/data/eolNamesToSqlite.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/python3 - -import sys, re -import csv, sqlite3 - -vnamesFile = "eol/vernacular_names.csv" -dbFile = "data.db" - -# Read in vernacular-names data - # Note: Canonical-names may have multiple pids - # Note: A canonical-name's associated pids might all have other associated names -nameToPids = {} -pidToNames = {} -canonicalNameToPids = {} -def updateMaps(name, pid, canonical): - if name not in nameToPids: - nameToPids[name] = {pid} - else: - nameToPids[name].add(pid) - if canonical: - if name not in canonicalNameToPids: - canonicalNameToPids[name] = {pid} - else: - canonicalNameToPids[name].add(pid) - if pid not in pidToNames: - pidToNames[pid] = {name} - else: - pidToNames[pid].add(name) -with open(vnamesFile, newline="") as csvfile: - reader = csv.reader(csvfile) - lineNum = 0 - for row in reader: - lineNum += 1 - if lineNum == 1: - continue - pid = int(row[0]) - name1 = re.sub(r"<[^>]+>", "", row[1].lower()) - name2 = row[2].lower() - # Add to maps - updateMaps(name1, pid, True) - updateMaps(name2, pid, False) -# Open db connection -dbCon = sqlite3.connect(dbFile) -cur = dbCon.cursor() -# Create 'names' table -cur.execute("CREATE TABLE names(name TEXT, alt_name TEXT, eol_id INT, PRIMARY KEY(name, alt_name))") -# Iterate through 'nodes' table, resolving to canonical-names -usedPids = set() -unresolvedNodeNames = set() -cur2 = dbCon.cursor() -iterationNum = 0 -for row in cur2.execute("SELECT name FROM nodes"): - name = row[0] - iterationNum += 1 - if iterationNum % 10000 == 0: - print("Loop 1 iteration {}".format(iterationNum)) - # If name matches a canonical-name, add alt-name entries to 'names' table - if name in canonicalNameToPids: - pidToUse = 0 - for pid in canonicalNameToPids[name]: - if pid not in usedPids: - pidToUse = pid - break - if pidToUse > 0: - usedPids.add(pidToUse) - altNames = {name} - for n in pidToNames[pidToUse]: - altNames.add(n) - for n in altNames: - cur.execute("INSERT INTO names VALUES (?, ?, ?)", (name, n, pidToUse)) - elif name in nameToPids: - unresolvedNodeNames.add(name) -# Iterate through unresolved nodes, resolving to vernacular-names -iterationNum = 0 -for name in unresolvedNodeNames: - iterationNum += 1 - if iterationNum % 10000 == 0: - print("Loop 2 iteration {}".format(iterationNum)) - # Add alt-name entries to 'names' table for first corresponding pid - pidToUse = 0 - for pid in nameToPids[name]: - if pid not in usedPids: - pidToUse = pid - break - if pidToUse > 0: - usedPids.add(pidToUse) - altNames = {name} - for n in pidToNames[pidToUse]: - altNames.add(n) - for n in altNames: - cur.execute("INSERT INTO names VALUES (?, ?, ?)", (name, n, pidToUse)) -dbCon.commit() -dbCon.close() diff --git a/backend/data/genEolNameData.py b/backend/data/genEolNameData.py new file mode 100755 index 0000000..ce887b3 --- /dev/null +++ b/backend/data/genEolNameData.py @@ -0,0 +1,104 @@ +#!/usr/bin/python3 + +# + +import sys, re +import csv, sqlite3 + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Reads vernacular-names CSV data (from the Encyclopedia of Life site),\n" +usageInfo += "makes associations with node data in a sqlite database, and writes\n" +usageInfo += "name data to that database.\n" +usageInfo += "\n" +usageInfo += "Expects a CSV header describing lines with format:\n" +usageInfo += " page_id, canonical_form, vernacular_string, language_code,\n" +usageInfo += " resource_name, is_preferred_by_resource, is_preferred_by_eol\n" + +vnamesFile = "eol/vernacularNames.csv" +dbFile = "data.db" + +# Read in vernacular-names data + # Note: Canonical-names may have multiple pids + # Note: A canonical-name's associated pids might all have other associated names +nameToPids = {} +pidToNames = {} +canonicalNameToPids = {} +def updateMaps(name, pid, canonical): + if name not in nameToPids: + nameToPids[name] = {pid} + else: + nameToPids[name].add(pid) + if canonical: + if name not in canonicalNameToPids: + canonicalNameToPids[name] = {pid} + else: + canonicalNameToPids[name].add(pid) + if pid not in pidToNames: + pidToNames[pid] = {name} + else: + pidToNames[pid].add(name) +with open(vnamesFile, newline="") as csvfile: + reader = csv.reader(csvfile) + lineNum = 0 + for row in reader: + lineNum += 1 + if lineNum == 1: + continue + pid = int(row[0]) + name1 = re.sub(r"<[^>]+>", "", row[1].lower()) # Remove tags + name2 = row[2].lower() + # Add to maps + updateMaps(name1, pid, True) + updateMaps(name2, pid, False) +# Open db connection +dbCon = sqlite3.connect(dbFile) +cur = dbCon.cursor() +# Create 'names' table +cur.execute("CREATE TABLE names(name TEXT, alt_name TEXT, eol_id INT, PRIMARY KEY(name, alt_name))") +# Iterate through 'nodes' table, resolving to canonical-names +usedPids = set() +unresolvedNodeNames = set() +cur2 = dbCon.cursor() +iterationNum = 0 +for row in cur2.execute("SELECT name FROM nodes"): + name = row[0] + iterationNum += 1 + if iterationNum % 10000 == 0: + print("Loop 1 iteration {}".format(iterationNum)) + # If name matches a canonical-name, add alt-name entries to 'names' table + if name in canonicalNameToPids: + pidToUse = 0 + for pid in canonicalNameToPids[name]: + if pid not in usedPids: + pidToUse = pid + break + if pidToUse > 0: + usedPids.add(pidToUse) + altNames = {name} + for n in pidToNames[pidToUse]: + altNames.add(n) + for n in altNames: + cur.execute("INSERT INTO names VALUES (?, ?, ?)", (name, n, pidToUse)) + elif name in nameToPids: + unresolvedNodeNames.add(name) +# Iterate through unresolved nodes, resolving to vernacular-names +iterationNum = 0 +for name in unresolvedNodeNames: + iterationNum += 1 + if iterationNum % 10000 == 0: + print("Loop 2 iteration {}".format(iterationNum)) + # Add alt-name entries to 'names' table for first corresponding pid + pidToUse = 0 + for pid in nameToPids[name]: + if pid not in usedPids: + pidToUse = pid + break + if pidToUse > 0: + usedPids.add(pidToUse) + altNames = {name} + for n in pidToNames[pidToUse]: + altNames.add(n) + for n in altNames: + cur.execute("INSERT INTO names VALUES (?, ?, ?)", (name, n, pidToUse)) +dbCon.commit() +dbCon.close() diff --git a/backend/data/genImgsForWeb.py b/backend/data/genImgsForWeb.py new file mode 100755 index 0000000..14583d6 --- /dev/null +++ b/backend/data/genImgsForWeb.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 + +import sys, os, subprocess +import sqlite3 +import signal + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Creates web-usable copies of reviewed images.\n" +usageInfo += "Looks in a reviewed-images directory for images named 'eolId1 contentId1.ext1', \n" +usageInfo += "and places copied/resized versions in another directory, with name 'eolId1.jpg'.\n" +usageInfo += "Also adds image metadata to a database, making use of an images-list database.\n" +usageInfo += "\n" +usageInfo += "SIGINT can be used to stop conversion, and the program can be re-run to\n" +usageInfo += "continue processing. It uses existing output files to decide where from.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +imgDir = "imgsReviewed/" +outDir = "img/" +imagesListDb = "eol/imagesList.db" +dbFile = "data.db" +IMG_OUT_SZ = 200 + +# Create output directory if not present +if not os.path.exists(outDir): + os.mkdir(outDir) +# Open images-list db +imagesListDbCon = sqlite3.connect(imagesListDb) +imagesListCur = imagesListDbCon.cursor() +# Create/open data db +dbCon = sqlite3.connect(dbFile) +dbCur = dbCon.cursor() +if dbCur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='images'").fetchone() == None: + dbCur.execute("CREATE TABLE images (eol_id INT PRIMARY KEY, source_url TEXT, license TEXT, copyright_owner TEXT)") +def closeDb(): + dbCon.commit() + dbCon.close() +# Get list of input images +print("Reading input image list") +inputImgList = os.listdir(imgDir) +inputImgList.sort(key=lambda s: int(s.split(" ")[0])) +if len(inputImgList) == 0: + print("No input images found") + closeDb() + sys.exit(0) +# Get next image to convert +inputImgIdx = 0 +print("Checking for existing output files") +outputImgList = os.listdir(outDir) +if len(outputImgList) > 0: + latestOutputId = 0 + for filename in outputImgList: + latestOutputId = max(latestOutputId, int(filename.split(".")[0])) + while int(inputImgList[inputImgIdx].split(" ")[0]) <= latestOutputId: + inputImgIdx += 1 + if inputImgIdx == len(inputImgList): + print("No unprocessed input images found") + closeDb() + sys.exit(0) +# Detect SIGINT signals +interrupted = False +def onSigint(sig, frame): + global interrupted + interrupted = True +signal.signal(signal.SIGINT, onSigint) +# Convert input images + # There are two interrupt checks because the subprocess exits on a SIGINT (not prevented by the handler above). + # The second check prevents adding a db entry for a non-created file. + # The first check prevents starting a new subprocess after a sigint occurs while adding to db +print("Converting images") +for i in range(inputImgIdx, len(inputImgList)): + if interrupted: + print("Exiting") + break + imgName = inputImgList[i] + [eolIdStr, otherStr] = imgName.split(" ") + contentId = int(otherStr.split(".")[0]) + print("Converting {}".format(imgName)) + subprocess.run( + ['npx', 'smartcrop-cli', + '--width', str(IMG_OUT_SZ), + '--height', str(IMG_OUT_SZ), + imgDir + imgName, + outDir + eolIdStr + ".jpg"], + stdout=subprocess.DEVNULL) + if interrupted: + print("Exiting") + break + # Add entry to db + imagesListQuery = "SELECT content_id, source_url, license, copyright_owner FROM images WHERE content_id = ?" + row = imagesListCur.execute(imagesListQuery, (contentId,)).fetchone() + dbCur.execute("INSERT INTO images VALUES (?, ?, ?, ?)", (int(eolIdStr), row[1], row[2], row[3])) +closeDb() diff --git a/backend/data/genOtolData.py b/backend/data/genOtolData.py new file mode 100755 index 0000000..a7d9c03 --- /dev/null +++ b/backend/data/genOtolData.py @@ -0,0 +1,224 @@ +#!/usr/bin/python3 + +import sys, re, json, sqlite3 +import os.path + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Reads labelled_supertree_ottnames.tre & annotations.json (from an Open Tree of Life release), \n" +usageInfo += "and creates a sqlite database, which holds entries of the form (name text, data text).\n" +usageInfo += "Each row holds a tree-of-life node name, and a JSON string with the form \n" +usageInfo += "{\"children\": [name1, ...], \"parent\": name1, \"tips\": int1, \"pSupport\": bool1}, holding \n" +usageInfo += "child names, a parent name or null, descendant 'tips', and a phylogeny-support indicator\n" +usageInfo += "\n" +usageInfo += "Expected labelled_supertree_ottnames.tre format:\n" +usageInfo += " Represents a tree-of-life in Newick format, roughly like (n1,n2,(n3,n4)n5)n6,\n" +usageInfo += " where root node is named n6, and has children n1, n2, and n5.\n" +usageInfo += " Name forms include Homo_sapiens_ott770315, mrcaott6ott22687, and 'Oxalis san-miguelii ott5748753'\n" +usageInfo += " Some names can be split up into a 'simple' name (like Homo_sapiens) and an id (like ott770315)\n" +usageInfo += "Expected annotations.json format:\n" +usageInfo += " JSON object holding information about the tree-of-life release.\n" +usageInfo += " The object's 'nodes' field maps node IDs to objects holding information about that node,\n" +usageInfo += " such as phylogenetic trees that support/conflict with it's placement.\n" + +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +treeFile = "otol/labelled_supertree_ottnames.tre" +annFile = "otol/annotations.json" +dbFile = "data.db" +nodeMap = {} # Maps node names to node objects +idToName = {} # Maps node IDs to names + +# Check for existing db +if os.path.exists(dbFile): + print("ERROR: Existing {} db".format(dbFile), file=sys.stderr) + sys.exit(1) + +# Parse treeFile +data = None +with open(treeFile) as file: + data = file.read() +dataIdx = 0 +def parseNewick(): + """Parses a node using 'data' and 'dataIdx', updates nodeMap accordingly, and returns the node name or None""" + global dataIdx + # Check for EOF + if dataIdx == len(data): + print("ERROR: Unexpected EOF at index " + str(dataIdx), file=sys.stderr) + return None + # Check for node + if data[dataIdx] == "(": # parse inner node + dataIdx += 1 + childNames = [] + while True: + # Read child + childName = parseNewick() + if childName == None: + return None + childNames.append(childName) + if (dataIdx == len(data)): + print("ERROR: Unexpected EOF", file=sys.stderr) + return None + # Check for next child + if (data[dataIdx] == ","): + dataIdx += 1 + continue + else: + # Get node name + dataIdx += 1 # Consume an expected ')' + [name, id] = parseNewickName() + idToName[id] = name + # Get child num-tips total + tips = 0 + for childName in childNames: + tips += nodeMap[childName]["tips"] + # Add node to nodeMap + if name in nodeMap: # Turns out the names might not actually be unique + count = 2 + name2 = name + " [" + str(count) + "]" + while name2 in nodeMap: + count += 1 + name2 = name + " [" + str(count) + "]" + name = name2 + nodeMap[name] = { + "n": name, "id": id, "children": childNames, "parent": None, "tips": tips, "pSupport": False + } + # Update childrens' parent reference + for childName in childNames: + nodeMap[childName]["parent"] = name + return name + else: # Parse node name + [name, id] = parseNewickName() + idToName[id] = name + nodeMap[name] = {"n": name, "id": id, "children": [], "parent": None, "tips": 1, "pSupport": False} + return name +def parseNewickName(): + """Helper that parses an input node name, and returns a [name,id] pair""" + global data, dataIdx + name = None + end = dataIdx + # Get name + if (end < len(data) and data[end] == "'"): # Check for quoted name + end += 1 + inQuote = True + while end < len(data): + if (data[end] == "'"): + if end + 1 < len(data) and data[end+1] == "'": # Account for '' as escaped-quote + end += 2 + continue + else: + end += 1 + inQuote = False + break + end += 1 + if inQuote: + raise Exception("ERROR: Unexpected EOF") + name = data[dataIdx:end] + dataIdx = end + else: + while end < len(data) and not re.match(r"[(),]", data[end]): + end += 1 + if (end == dataIdx): + raise Exception("ERROR: Unexpected EOF") + name = data[dataIdx:end].rstrip() + if end == len(data): # Ignore trailing input semicolon + name = name[:-1] + dataIdx = end + # Convert to [name, id] + name = name.lower() + if name.startswith("mrca"): + return [name, name] + elif name[0] == "'": + match = re.fullmatch(r"'([^\\\"]+) (ott\d+)'", name) + if match == None: + raise Exception("ERROR: invalid name \"{}\"".format(name)) + name = match.group(1).replace("''", "'") + return [name, match.group(2)] + else: + match = re.fullmatch(r"([^\\\"]+)_(ott\d+)", name) + if match == None: + raise Exception("ERROR: invalid name \"{}\"".format(name)) + return [match.group(1).replace("_", " "), match.group(2)] +rootName = parseNewick() + +# Parse annFile +data = None +with open(annFile) as file: + data = file.read() +obj = json.loads(data) +nodeAnnsMap = obj['nodes'] + +# Change mrca* names +def applyMrcaNameConvert(name, namesToSwap): + """ + Given an mrca* name, makes namesToSwap map it to an expanded version with the form [childName1 + childName2]. + May recurse on child nodes with mrca* names. + Also returns the name of the highest-tips child (used when recursing). + """ + node = nodeMap[name] + childNames = node["children"] + if len(childNames) < 2: + print("WARNING: MRCA node \"{}\" has less than 2 children".format(name), file=sys.stderr) + return name + # Get 2 children with most tips + childTips = [] + for n in childNames: + childTips.append(nodeMap[n]["tips"]) + maxTips = max(childTips) + maxIdx = childTips.index(maxTips) + childTips[maxIdx] = 0 + maxTips2 = max(childTips) + maxIdx2 = childTips.index(maxTips2) + childName1 = node["children"][maxIdx] + childName2 = node["children"][maxIdx2] + # Check for composite child names + if childName1.startswith("mrca"): + childName1 = applyMrcaNameConvert(childName1, namesToSwap) + if childName2.startswith("mrca"): + childName2 = applyMrcaNameConvert(childName2, namesToSwap) + # Create composite name + namesToSwap[name] = "[{} + {}]".format(childName1, childName2) + return childName1 +namesToSwap = {} # Maps mrca* names to replacement names +for node in nodeMap.values(): + name = node["n"] + if (name.startswith("mrca") and name not in namesToSwap): + applyMrcaNameConvert(name, namesToSwap) +for [oldName, newName] in namesToSwap.items(): + nodeMap[newName] = nodeMap[oldName] + del nodeMap[oldName] +for node in nodeMap.values(): + parentName = node["parent"] + if (parentName in namesToSwap): + node["parent"] = namesToSwap[parentName] + childNames = node["children"] + for i in range(len(childNames)): + childName = childNames[i] + if (childName in namesToSwap): + childNames[i] = namesToSwap[childName] + +# Add annotations data, and delete certain fields +for node in nodeMap.values(): + # Set has-support value using annotations + id = node["id"] + if id in nodeAnnsMap: + nodeAnns = nodeAnnsMap[id] + supportQty = len(nodeAnns["supported_by"]) if "supported_by" in nodeAnns else 0 + conflictQty = len(nodeAnns["conflicts_with"]) if "conflicts_with" in nodeAnns else 0 + node["pSupport"] = supportQty > 0 and conflictQty == 0 + # Root node gets support + if node["parent"] == None: + node["pSupport"] = True + # Delete some no-longer-needed fields + del node["n"] + del node["id"] + +# Create db +dbCon = sqlite3.connect(dbFile) +dbCur = dbCon.cursor() +dbCur.execute("CREATE TABLE nodes (name TEXT PRIMARY KEY, data TEXT)") +for name in nodeMap.keys(): + dbCur.execute("INSERT INTO nodes VALUES (?, ?)", (name, json.dumps(nodeMap[name]))) +dbCon.commit() +dbCon.close() diff --git a/backend/data/otol/README.md b/backend/data/otol/README.md index f720772..58aad3c 100644 --- a/backend/data/otol/README.md +++ b/backend/data/otol/README.md @@ -1,5 +1,5 @@ -Files -===== +Downloaded Files +================ - labelled\_supertree\_ottnames.tre Obtained from https://tree.opentreeoflife.org/about/synthesis-release/v13.4 - annotations.json diff --git a/backend/data/otolToSqlite.py b/backend/data/otolToSqlite.py deleted file mode 100755 index 2ee47b7..0000000 --- a/backend/data/otolToSqlite.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python3 - -import sys, re, json, sqlite3 -import os.path - -usageInfo = f"usage: {sys.argv[0]}\n" -usageInfo += "Reads labelled_supertree_ottnames.tre & annotations.json (from an Open Tree of Life release), \n" -usageInfo += "and creates an sqlite database otol.db, which holds entries of the form (name text, data text).\n" -usageInfo += "Each row holds a tree-of-life node name, and a JSON string with the form \n" -usageInfo += "{\"children\": [name1, ...], \"parent\": name1, \"tips\": int1, \"pSupport\": bool1}, holding \n" -usageInfo += "child names, a parent name or null, descendant 'tips', and a phylogeny-support indicator\n" -usageInfo += "\n" -usageInfo += "This script was adapted to handle Open Tree of Life version 13.4.\n" -usageInfo += "Link: https://tree.opentreeoflife.org/about/synthesis-release/v13.4\n" -usageInfo += "\n" -usageInfo += "labelled_supertree_ottnames.tre format:\n" -usageInfo += " Represents a tree-of-life in Newick format, roughly like (n1,n2,(n3,n4)n5)n6,\n" -usageInfo += " where root node is named n6, and has children n1, n2, and n5.\n" -usageInfo += " Name forms include Homo_sapiens_ott770315, mrcaott6ott22687, and 'Oxalis san-miguelii ott5748753'\n" -usageInfo += " Some names can be split up into a 'simple' name (like Homo_sapiens) and an id (like ott770315)\n" -usageInfo += "annotations.json format:\n" -usageInfo += " JSON object holding information about the tree-of-life release.\n" -usageInfo += " The object's 'nodes' field maps node IDs to objects holding information about that node,\n" -usageInfo += " such as phylogenetic trees that support/conflict with it's placement.\n" - -if len(sys.argv) > 1: - print(usageInfo, file=sys.stderr) - sys.exit(1) - -treeFile = "otol/labelled_supertree_ottnames.tre" -annFile = "otol/annotations.json" -dbFile = "data.db" -nodeMap = {} # Maps node names to node objects -idToName = {} # Maps node IDs to names - -# Check for existing db -if os.path.exists(dbFile): - print("ERROR: Existing {} db".format(dbFile), file=sys.stderr) - sys.exit(1) - -# Parse treeFile -data = None -with open(treeFile) as file: - data = file.read() -dataIdx = 0 -def parseNewick(): - """Parses a node using 'data' and 'dataIdx', updates nodeMap accordingly, and returns the node name or None""" - global dataIdx - # Check for EOF - if dataIdx == len(data): - print("ERROR: Unexpected EOF at index " + str(dataIdx), file=sys.stderr) - return None - # Check for inner-node start - if data[dataIdx] == "(": - dataIdx += 1 - childNames = [] - while True: - # Read child - childName = parseNewick() - if childName == None: - return None - childNames.append(childName) - if (dataIdx == len(data)): - print("ERROR: Unexpected EOF", file=sys.stderr) - return None - # Check for next child - if (data[dataIdx] == ","): - dataIdx += 1 - continue - else: - # Get node name - dataIdx += 1 # Consume an expected ')' - [name, id] = parseNewickName() - idToName[id] = name - # Get child num-tips total - tips = 0 - for childName in childNames: - tips += nodeMap[childName]["tips"] - # Add node to nodeMap - if name in nodeMap: # Turns out the names might not actually be unique - count = 2 - name2 = name + " [" + str(count) + "]" - while name2 in nodeMap: - count += 1 - name2 = name + " [" + str(count) + "]" - name = name2 - nodeMap[name] = { - "n": name, "id": id, "children": childNames, "parent": None, "tips": tips, "pSupport": False - } - # Update childrens' parent reference - for childName in childNames: - nodeMap[childName]["parent"] = name - return name - else: - [name, id] = parseNewickName() - idToName[id] = name - nodeMap[name] = {"n": name, "id": id, "children": [], "parent": None, "tips": 1, "pSupport": False} - return name -def parseNewickName(): - """Helper that parses an input node name, and returns a [name,id] pair""" - global data, dataIdx - name = None - end = dataIdx - # Get name - if (end < len(data) and data[end] == "'"): # Check for quoted name - end += 1 - inQuote = True - while end < len(data): - if (data[end] == "'"): - if end + 1 < len(data) and data[end+1] == "'": # Account for '' as escaped-quote - end += 2 - continue - else: - end += 1 - inQuote = False - break - end += 1 - if inQuote: - raise Exception("ERROR: Unexpected EOF") - name = data[dataIdx:end] - dataIdx = end - else: - while end < len(data) and not re.match(r"[(),]", data[end]): - end += 1 - if (end == dataIdx): - raise Exception("ERROR: Unexpected EOF") - name = data[dataIdx:end].rstrip() - if end == len(data): # Ignore trailing input semicolon - name = name[:-1] - dataIdx = end - # Convert to [name, id] - name = name.lower() - if name.startswith("mrca"): - return [name, name] - elif name[0] == "'": - match = re.fullmatch(r"'([^\\\"]+) (ott\d+)'", name) - if match == None: - raise Exception("ERROR: invalid name \"{}\"".format(name)) - name = match.group(1).replace("''", "'") - return [name, match.group(2)] - else: - match = re.fullmatch(r"([^\\\"]+)_(ott\d+)", name) - if match == None: - raise Exception("ERROR: invalid name \"{}\"".format(name)) - return [match.group(1).replace("_", " "), match.group(2)] -rootName = parseNewick() - -# Parse annFile -data = None -with open(annFile) as file: - data = file.read() -obj = json.loads(data) -nodeAnnsMap = obj['nodes'] - -# Change mrca* names -def applyMrcaNameConvert(name, namesToSwap): - """ - Given an mrca* name, makes namesToSwap map it to an expanded version with the form [childName1 + childName2]. - May recurse on child nodes with mrca* names. - Also returns the name of the highest-tips child (used when recursing). - """ - node = nodeMap[name] - childNames = node["children"] - if len(childNames) < 2: - print("WARNING: MRCA node \"{}\" has less than 2 children".format(name), file=sys.stderr) - return name - # Get 2 children with most tips - childTips = [] - for n in childNames: - childTips.append(nodeMap[n]["tips"]) - maxTips = max(childTips) - maxIdx = childTips.index(maxTips) - childTips[maxIdx] = 0 - maxTips2 = max(childTips) - maxIdx2 = childTips.index(maxTips2) - # - childName1 = node["children"][maxIdx] - childName2 = node["children"][maxIdx2] - if childName1.startswith("mrca"): - childName1 = applyMrcaNameConvert(childName1, namesToSwap) - if childName2.startswith("mrca"): - childName2 = applyMrcaNameConvert(childName2, namesToSwap) - # Create composite name - namesToSwap[name] = "[{} + {}]".format(childName1, childName2) - return childName1 -namesToSwap = {} # Maps mrca* names to replacement names -for node in nodeMap.values(): - name = node["n"] - if (name.startswith("mrca") and name not in namesToSwap): - applyMrcaNameConvert(name, namesToSwap) -for [oldName, newName] in namesToSwap.items(): - nodeMap[newName] = nodeMap[oldName] - del nodeMap[oldName] -for node in nodeMap.values(): - parentName = node["parent"] - if (parentName in namesToSwap): - node["parent"] = namesToSwap[parentName] - childNames = node["children"] - for i in range(len(childNames)): - childName = childNames[i] - if (childName in namesToSwap): - childNames[i] = namesToSwap[childName] - -# Add annotations data, and delete certain fields -for node in nodeMap.values(): - # Set has-support value using annotations - id = node["id"] - if id in nodeAnnsMap: - nodeAnns = nodeAnnsMap[id] - supportQty = len(nodeAnns["supported_by"]) if "supported_by" in nodeAnns else 0 - conflictQty = len(nodeAnns["conflicts_with"]) if "conflicts_with" in nodeAnns else 0 - node["pSupport"] = supportQty > 0 and conflictQty == 0 - # Root node gets support - if node["parent"] == None: - node["pSupport"] = True - # Delete some no-longer-needed fields - del node["n"] - del node["id"] - -# Create db -con = sqlite3.connect(dbFile) -cur = con.cursor() -cur.execute("CREATE TABLE nodes (name TEXT PRIMARY KEY, data TEXT)") -for name in nodeMap.keys(): - cur.execute("INSERT INTO nodes VALUES (?, ?)", (name, json.dumps(nodeMap[name]))) -con.commit() -con.close() diff --git a/backend/data/reviewImgs.py b/backend/data/reviewImgs.py new file mode 100755 index 0000000..3df7809 --- /dev/null +++ b/backend/data/reviewImgs.py @@ -0,0 +1,176 @@ +#!/usr/bin/python3 + +import sys, re, os +import tkinter as tki +from tkinter import ttk +import PIL +from PIL import ImageTk, Image + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Provides a GUI for reviewing images. Looks in a for-review directory for\n" +usageInfo += "images named 'eolId1 contentId1.ext1', and, for each EOL ID, enables the user to\n" +usageInfo += "choose an image to keep, or reject all. Also provides image rotation.\n" +usageInfo += "Chosen images are placed in another directory, and rejected ones are deleted.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +imgDir = "imgsForReview/" +outDir = "imgsReviewed/" +IMG_DISPLAY_SZ = 400 +MAX_IMGS_PER_ID = 3 +PLACEHOLDER_IMG = Image.new("RGB", (IMG_DISPLAY_SZ, IMG_DISPLAY_SZ), (88, 28, 135)) + +# Create output directory if not present +if not os.path.exists(outDir): + os.mkdir(outDir) +# Get images for review +print("Reading input image list") +imgList = os.listdir(imgDir) +imgList.sort(key=lambda s: int(s.split(" ")[0])) +if len(imgList) == 0: + print("No input images found", file=sys.stderr) + sys.exit(1) + +class EolImgReviewer: + """ Provides the GUI for reviewing images """ + def __init__(self, root, imgList): + self.root = root + root.title("EOL Image Reviewer") + # Setup main frame + mainFrame = ttk.Frame(root, padding="5 5 5 5") + mainFrame.grid(column=0, row=0, sticky=(tki.N, tki.W, tki.E, tki.S)) + root.columnconfigure(0, weight=1) + root.rowconfigure(0, weight=1) + # Set up images-to-be-reviewed frames + self.imgs = [PLACEHOLDER_IMG] * MAX_IMGS_PER_ID # Stored as fields for use in rotation + self.photoImgs = list(map(lambda img: ImageTk.PhotoImage(img), self.imgs)) # Image objects usable by tkinter + # These need a persistent reference for some reason (doesn't display otherwise) + self.labels = [] + for i in range(MAX_IMGS_PER_ID): + frame = ttk.Frame(mainFrame, width=IMG_DISPLAY_SZ, height=IMG_DISPLAY_SZ) + frame.grid(column=i, row=0) + label = ttk.Label(frame, image=self.photoImgs[i]) + label.grid(column=0, row=0) + self.labels.append(label) + # Add padding + for child in mainFrame.winfo_children(): + child.grid_configure(padx=5, pady=5) + # Add bindings + root.bind("", self.quit) + root.bind("", lambda evt: self.accept(0)) + root.bind("", lambda evt: self.accept(1)) + root.bind("", lambda evt: self.accept(2)) + root.bind("", lambda evt: self.reject()) + root.bind("", lambda evt: self.rotate(0)) + root.bind("", lambda evt: self.rotate(1)) + root.bind("", lambda evt: self.rotate(2)) + root.bind("", lambda evt: self.rotate(0, True)) + root.bind("", lambda evt: self.rotate(1, True)) + root.bind("", lambda evt: self.rotate(2, True)) + # Initialise images to review + self.imgList = imgList + self.imgListIdx = 0 + self.nextEolId = 0 + self.nextImgNames = [] + self.rotations = [] + self.getNextImgs() + def getNextImgs(self): + """ Updates display with new images to review, or ends program """ + # Gather names of next images to review + for i in range(MAX_IMGS_PER_ID): + if self.imgListIdx == len(self.imgList): + if i == 0: + self.quit() + return + break + imgName = self.imgList[self.imgListIdx] + eolId = int(re.match(r"(\d+) (\d+)", imgName).group(1)) + if i == 0: + self.nextEolId = eolId + self.nextImgNames = [imgName] + self.rotations = [0] + else: + if self.nextEolId != eolId: + break + self.nextImgNames.append(imgName) + self.rotations.append(0) + self.imgListIdx += 1 + # Update displayed images + idx = 0 + while idx < MAX_IMGS_PER_ID: + if idx < len(self.nextImgNames): + try: + img = Image.open(imgDir + self.nextImgNames[idx]) + except PIL.UnidentifiedImageError: + os.remove(imgDir + self.nextImgNames[idx]) + del self.nextImgNames[idx] + del self.rotations[idx] + continue + self.imgs[idx] = self.resizeForDisplay(img) + else: + self.imgs[idx] = PLACEHOLDER_IMG + self.photoImgs[idx] = ImageTk.PhotoImage(self.imgs[idx]) + self.labels[idx].config(image=self.photoImgs[idx]) + idx += 1 + # Restart if all image files non-recognisable + if len(self.nextImgNames) == 0: + self.getNextImgs() + return + # Update title + firstImgIdx = self.imgListIdx - len(self.nextImgNames) + 1 + lastImgIdx = self.imgListIdx + self.root.title("Reviewing EOL ID {} (imgs {} to {} out of {})".format( + self.nextEolId, firstImgIdx, lastImgIdx, len(self.imgList))) + def accept(self, imgIdx): + """ React to a user selecting an image """ + if imgIdx >= len(self.nextImgNames): + print("Invalid selection") + return + for i in range(len(self.nextImgNames)): + inFile = imgDir + self.nextImgNames[i] + if i == imgIdx: # Move accepted image, rotating if needed + outFile = outDir + self.nextImgNames[i] + if self.rotations[i] == 0: + os.replace(inFile, outFile) + else: + img = Image.open(inFile) + img = img.rotate(self.rotations[i], expand=True) + img.save(outFile) + os.remove(inFile) + else: # Delete non-accepted image + os.remove(inFile) + self.getNextImgs() + def reject(self): + """ React to a user rejecting all images of a set """ + for i in range(len(self.nextImgNames)): + os.remove(imgDir + self.nextImgNames[i]) + self.getNextImgs() + def rotate(self, imgIdx, anticlockwise = False): + """ Respond to a user rotating an image """ + deg = -90 if not anticlockwise else 90 + self.imgs[imgIdx] = self.imgs[imgIdx].rotate(deg) + self.photoImgs[imgIdx] = ImageTk.PhotoImage(self.imgs[imgIdx]) + self.labels[imgIdx].config(image=self.photoImgs[imgIdx]) + self.rotations[imgIdx] = (self.rotations[imgIdx] + deg) % 360 + def quit(self, e = None): + self.root.destroy() + def resizeForDisplay(self, img): + """ Returns a copy of an image, shrunk to fit the display (keeps aspect ratio), and with a background """ + if max(img.width, img.height) > IMG_DISPLAY_SZ: + if (img.width > img.height): + newHeight = int(img.height * IMG_DISPLAY_SZ/img.width) + img = img.resize((IMG_DISPLAY_SZ, newHeight)) + else: + newWidth = int(img.width * IMG_DISPLAY_SZ / img.height) + img = img.resize((newWidth, IMG_DISPLAY_SZ)) + bgImg = PLACEHOLDER_IMG.copy() + bgImg.paste(img, box=( + int((IMG_DISPLAY_SZ - img.width) / 2), + int((IMG_DISPLAY_SZ - img.height) / 2))) + return bgImg +# Create GUI and defer control +root = tki.Tk() +EolImgReviewer(root, imgList) +root.mainloop() + diff --git a/backend/server.py b/backend/server.py index ded74d6..178d95c 100755 --- a/backend/server.py +++ b/backend/server.py @@ -1,13 +1,15 @@ #!/usr/bin/python3 import sys, re, sqlite3, json +import os.path from http.server import HTTPServer, BaseHTTPRequestHandler import urllib.parse hostname = "localhost" port = 8000 dbFile = "data/data.db" -tolnodeReqDepth = 1 +imgDir = "../public/img/" +NODE_REQ_DEPTH = 1 # For a /node?name=name1 request, respond with name1's node, and descendent nodes in a subtree to some depth > 0 usageInfo = f"usage: {sys.argv[0]}\n" @@ -16,17 +18,40 @@ usageInfo += "Responds to path+query /data/type1?name=name1 with JSON data.\n" usageInfo += "\n" usageInfo += "If type1 is 'node': \n" usageInfo += " Responds with a map from names to node objects, representing\n" -usageInfo += " nodes name1, and child nodes up to depth " + str(tolnodeReqDepth) + ".\n" +usageInfo += " nodes name1, and child nodes up to depth " + str(NODE_REQ_DEPTH) + ".\n" usageInfo += "If type1 is 'children': Like 'node', but excludes node name1.\n" usageInfo += "If type1 is 'chain': Like 'node', but gets nodes from name1 up to the root, and their direct children.\n" usageInfo += "If type1 is 'search': Responds with a tolnode name that has alt-name name1, or null.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) dbCon = sqlite3.connect(dbFile) def lookupNode(name): + # Get from db cur = dbCon.cursor() cur.execute("SELECT name, data FROM nodes WHERE name = ?", (name,)) row = cur.fetchone() - return row[1] if row != None else None + if row == None: + return None + nodeObj = json.loads(row[1]) + # Check for image file + match = re.fullmatch(r"\[(.+) \+ (.+)]", name) + if match == None: + nodeObj["imgFile"] = nodeNameToFile(name, cur) + else: + nodeObj["imgFile"] = nodeNameToFile(match.group(1), cur) + if nodeObj["imgFile"] == None: + nodeObj["imgFile"] = nodeNameToFile(match.group(2), cur) + return nodeObj; +def nodeNameToFile(name, cur): + row = cur.execute("SELECT name, eol_id FROM names WHERE name = ?", (name,)).fetchone() + if row == None: + return None + imgFile = str(row[1]) + ".jpg" + if not os.path.exists(imgDir + imgFile): + return None + return imgFile def lookupName(name): cur = dbCon.cursor() cur.execute("SELECT name, alt_name FROM names WHERE alt_name = ?", (name,)) @@ -44,57 +69,57 @@ class DbServer(BaseHTTPRequestHandler): if match != None and match.group(1) == "data" and "name" in queryDict: reqType = match.group(2) name = queryDict["name"][0] - print(name) # Check query string if reqType == "node": - nodeJson = lookupNode(name) - if nodeJson != None: - results = [] - getResultsUntilDepth(name, nodeJson, tolnodeReqDepth, results) - self.respondJson(nodeResultsToJSON(results)) + nodeObj = lookupNode(name) + if nodeObj != None: + results = {} + getResultsUntilDepth(name, nodeObj, NODE_REQ_DEPTH, results) + self.respondJson(json.dumps(results)) return elif reqType == "children": - nodeJson = lookupNode(name) - if nodeJson != None: - obj = json.loads(nodeJson) - results = [] - for childName in obj["children"]: - nodeJson = lookupNode(childName) - if nodeJson != None: - getResultsUntilDepth(childName, nodeJson, tolnodeReqDepth, results) - self.respondJson(nodeResultsToJSON(results)) + nodeObj = lookupNode(name) + if nodeObj != None: + results = {} + for childName in nodeObj["children"]: + nodeObj = lookupNode(childName) + if nodeObj != None: + getResultsUntilDepth(childName, nodeObj, NODE_REQ_DEPTH, results) + self.respondJson(json.dumps(results)) return elif reqType == "chain": - results = [] + results = {} ranOnce = False while True: - jsonResult = lookupNode(name) - if jsonResult == None: + # Get node + nodeObj = lookupNode(name) + if nodeObj == None: if ranOnce: print("ERROR: Parent-chain node {} not found".format(name), file=sys.stderr) break - results.append([name, jsonResult]) - obj = json.loads(jsonResult) - # Add children + results[name] = nodeObj + # Conditionally add children if not ranOnce: ranOnce = True else: internalFail = False - for childName in obj["children"]: - jsonResult = lookupNode(childName) - if jsonResult == None: + for childName in nodeObj["children"]: + if childName in results: + continue + nodeObj = lookupNode(childName) + if nodeObj == None: print("ERROR: Parent-chain-child node {} not found".format(name), file=sys.stderr) internalFail = True break - results.append([childName, jsonResult]) + results[childName] = nodeObj if internalFail: break # Check if root - if obj["parent"] == None: - self.respondJson(nodeResultsToJSON(results)) + if nodeObj["parent"] == None: + self.respondJson(json.dumps(results)) return else: - name = obj["parent"] + name = nodeObj["parent"] elif reqType == "search": nameJson = lookupName(name) if nameJson != None: @@ -108,24 +133,14 @@ class DbServer(BaseHTTPRequestHandler): self.send_header("Content-type", "application/json") self.end_headers() self.wfile.write(jsonStr.encode("utf-8")) -def getResultsUntilDepth(name, nodeJson, depth, results): - """Given a node [name, nodeJson] pair, adds child node pairs to 'results', up until 'depth'""" - results.append([name, nodeJson]) +def getResultsUntilDepth(name, nodeObj, depth, results): + """Given a node [name, nodeObj] pair, adds child node pairs to 'results', up until 'depth'""" + results[name] = nodeObj if depth > 0: - obj = json.loads(nodeJson) - for childName in obj["children"]: - childJson = lookupNode(childName) - if childJson != None: - getResultsUntilDepth(childName, childJson, depth-1, results) -def nodeResultsToJSON(results): - """Given a list of [name, nodeJson] pairs, returns a representative JSON string""" - jsonStr = "{" - for i in range(len(results)): - jsonStr += json.dumps(results[i][0]) + ":" + results[i][1] - if i < len(results) - 1: - jsonStr += "," - jsonStr += "}" - return jsonStr + for childName in nodeObj["children"]: + childObj = lookupNode(childName) + if childObj != None: + getResultsUntilDepth(childName, childObj, depth-1, results) server = HTTPServer((hostname, port), DbServer) print("Server started at http://{}:{}".format(hostname, port)) diff --git a/package.json b/package.json index b251046..1f73d3c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "author": "Terry Truong", "license": "MIT", "dependencies": { + "smartcrop-cli": "^2.0.3", "vue": "^3.2.25" }, "devDependencies": { diff --git a/src/App.vue b/src/App.vue index 89d227d..b683b21 100644 --- a/src/App.vue +++ b/src/App.vue @@ -541,7 +541,7 @@ export default defineComponent({ text-white/40 hover:text-white hover:cursor-pointer"/> - diff --git a/src/components/Tile.vue b/src/components/Tile.vue index 1087a58..2ab1df1 100644 --- a/src/components/Tile.vue +++ b/src/components/Tile.vue @@ -88,11 +88,11 @@ export default defineComponent({ leafStyles(): Record { return { // Image (and scrims) - //backgroundImage: - // 'linear-gradient(to bottom, rgba(0,0,0,0.4), #0000 40%, #0000 60%, rgba(0,0,0,0.4) 100%),' + - // 'url(\'/img/' + this.layoutNode.name.replaceAll('\'', '\\\'') + '.png\')', - backgroundImage: - 'linear-gradient(to bottom, rgba(0,0,0,0.4), #0000 40%, #0000 60%, rgba(0,0,0,0.4) 100%)', + backgroundImage: this.tolNode.imgFile != null ? + 'linear-gradient(to bottom, rgba(0,0,0,0.4), #0000 40%, #0000 60%, rgba(0,0,0,0.4) 100%),' + + 'url(\'/img/' + this.tolNode.imgFile.replaceAll('\'', '\\\'') + '\')' : + 'none', + backgroundColor: '#1c1917', backgroundSize: 'cover', // Other borderRadius: this.uiOpts.borderRadius + 'px', diff --git a/src/components/TileInfoModal.vue b/src/components/TileInfoModal.vue index 0e2fc94..9805b48 100644 --- a/src/components/TileInfoModal.vue +++ b/src/components/TileInfoModal.vue @@ -7,13 +7,19 @@ import {LayoutNode} from '../layout'; export default defineComponent({ props: { node: {type: Object as PropType, required: true}, + tolMap: {type: Object as PropType, required: true}, uiOpts: {type: Object, required: true}, }, computed: { + tolNode(): TolNode { + return this.tolMap.get(this.node.name)!; + }, imgStyles(): Record { return { - //backgroundImage: 'url(\'/img/' + this.node.name.replaceAll('\'', '\\\'') + '.png\')', - background: 'black', + backgroundImage: this.tolNode.imgFile != null ? + 'linear-gradient(to bottom, rgba(0,0,0,0.4), #0000 40%, #0000 60%, rgba(0,0,0,0.4) 100%),' + + 'url(\'/img/' + this.tolNode.imgFile.replaceAll('\'', '\\\'') + '\')' : + 'none', width: this.uiOpts.infoModalImgSz + 'px', height: this.uiOpts.infoModalImgSz + 'px', backgroundSize: 'cover', diff --git a/src/layout.ts b/src/layout.ts index fd65b87..e023619 100644 --- a/src/layout.ts +++ b/src/layout.ts @@ -6,7 +6,6 @@ * find a tile-based layout, filling in node fields to represent placement. */ -import {TolNode} from './tol'; import type {TolMap} from './tol'; import {range, arraySum, linspace, limitVals, updateAscSeq} from './util'; diff --git a/src/tol.ts b/src/tol.ts index daa3339..bfc778f 100644 --- a/src/tol.ts +++ b/src/tol.ts @@ -10,10 +10,12 @@ export class TolNode { parent: string | null; tips: number; pSupport: boolean; + imgFile: string | null; constructor(children: string[] = [], parent = null, tips = 0, pSupport = false){ this.children = children; this.parent = parent; this.tips = tips; this.pSupport = pSupport; + this.imgFile = null; } } -- cgit v1.2.3