aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTerry Truong <terry06890@gmail.com>2022-05-10 19:52:47 +1000
committerTerry Truong <terry06890@gmail.com>2022-05-11 11:44:14 +1000
commit6c61612564b9a30f747207c43729c3e7e8cbf0d3 (patch)
tree3f0f3b4cc35162e73542942ef30da3f1a0bf6276
parent3d895370a608d4f51726b74e2560dcf5f4ec43a8 (diff)
Use prefix-search with ranking-by-length
-rw-r--r--backend/data/README.md19
-rwxr-xr-xbackend/data/genSpellfixNameData.py32
-rwxr-xr-xbackend/data/spellfix.sobin86456 -> 0 bytes
-rwxr-xr-xbackend/server.py22
-rw-r--r--src/components/SearchModal.vue2
5 files changed, 6 insertions, 69 deletions
diff --git a/backend/data/README.md b/backend/data/README.md
index 9f0ea82..27619de 100644
--- a/backend/data/README.md
+++ b/backend/data/README.md
@@ -9,8 +9,6 @@ File Generation Process
1 Obtain data in eol/, as specified in it's README.
2 Run genEolNameData.py, which adds 'names' and 'eol\_ids' tables to data.db,
using data in eol/vernacularNames.csv and the 'nodes' table.
- 3 Run genSpellfixNameData.py, which adds a 'spellfix\_alt\_names'
- table to data.db, using data in the 'names' table.
3 Image Data
1 Use downloadImgsForReview.py to download EOL images into imgsForReview/.
It uses data in eol/imagesList.db, and the 'eol\_ids' table.
@@ -31,24 +29,7 @@ data.db tables
name TEXT, alt\_name TEXT, pref\_alt INT, PRIMARY KEY(name, alt\_name)
- eol\_ids <br>
id INT PRIMARY KEY, name TEXT
-- spellfix\_alt\_names
- images <br>
eol\_id INT PRIMARY KEY, source\_url TEXT, license TEXT, copyright\_owner TEXT
- descs <br>
name TEXT PRIMARY KEY, desc TEXT, redirected INT
-
-spellfix.so
-===========
-
-This file provides the spellfix1 extension for Sqlite, and
-is used for responding to fuzzy-search requests.
-
-It was obtained by:
-1 Downloading the sqlite source tree from
- the github mirror at <https://github.com/sqlite/sqlite>,
- into a directory sqlite/
-2 After making sure autoconf 2.61+ and libtool are installed,
- running `mkdir bld; cd bld; ../sqlite/configure;`
-3 Running `make`
-4 Running `cp ../sqlite/ext/misc/spellfix.c .`
-5 Running `gcc -fPIC -shared spellfix.c -o spellfix.so`
diff --git a/backend/data/genSpellfixNameData.py b/backend/data/genSpellfixNameData.py
deleted file mode 100755
index 9a3a7a1..0000000
--- a/backend/data/genSpellfixNameData.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/python3
-
-import sys
-import sqlite3
-
-usageInfo = f"usage: {sys.argv[0]}\n"
-usageInfo += "Reads alt-names from a 'names' table in a database, and adds a spellfix \n"
-usageInfo += "table 'spellfix_alt_names' usable for fuzzy-searching those names.\n"
-if len(sys.argv) > 1:
- print(usageInfo, file=sys.stderr)
- sys.exit(1)
-
-dbFile = "data.db"
-
-# Connect to db, and load spellfix extension
-dbCon = sqlite3.connect(dbFile)
-dbCon.enable_load_extension(True)
-dbCon.load_extension('./spellfix')
-# Create spellfix table, and insert alt-names
-spellfixCur = dbCon.cursor()
-spellfixCur.execute("CREATE VIRTUAL TABLE spellfix_alt_names USING spellfix1")
-namesCur = dbCon.cursor()
-iterationNum = 0
-for row in namesCur.execute("SELECT DISTINCT alt_name FROM names"):
- iterationNum += 1
- if iterationNum % 10000 == 0:
- print("Loop {}: {}".format(iterationNum, row[0]))
- # Insert alt-name
- spellfixCur.execute("INSERT INTO spellfix_alt_names(word) VALUES (?)", (row[0],))
-# Close db
-dbCon.commit()
-dbCon.close()
diff --git a/backend/data/spellfix.so b/backend/data/spellfix.so
deleted file mode 100755
index 0bc985c..0000000
--- a/backend/data/spellfix.so
+++ /dev/null
Binary files differ
diff --git a/backend/server.py b/backend/server.py
index 8e4a90f..e778c2f 100755
--- a/backend/server.py
+++ b/backend/server.py
@@ -24,8 +24,6 @@ if len(sys.argv) > 1:
# Connect to db, and load spellfix extension
dbCon = sqlite3.connect(dbFile)
-dbCon.enable_load_extension(True)
-dbCon.load_extension('./data/spellfix')
# Some functions
def lookupNodes(names):
nodeObjs = {}
@@ -95,22 +93,12 @@ def lookupName(name):
cur = dbCon.cursor()
results = []
hasMore = False
- #for row in cur.execute(
- # "SELECT DISTINCT name, alt_name FROM names WHERE alt_name LIKE ? LIMIT ?",
- # (name, SEARCH_SUGG_LIMIT)):
- # results.append({"name": row[0], "altName": row[1]})
- #for row in cur.execute(
- # "SELECT DISTINCT names.name, names.alt_name, nodes.tips FROM" \
- # " names INNER JOIN nodes ON names.name = nodes.name " \
- # " WHERE alt_name LIKE ? ORDER BY nodes.tips DESC LIMIT ?",
- # (name + "%", SEARCH_SUGG_LIMIT)):
- # results.append({"name": row[0], "altName": row[1]})
for row in cur.execute(
- "SELECT word, alt_name, name FROM" \
- " spellfix_alt_names INNER JOIN names ON alt_name = word" \
- " WHERE word MATCH ? LIMIT ?",
- (name, SEARCH_SUGG_LIMIT)):
- results.append({"name": row[2], "altName": row[0]})
+ "SELECT DISTINCT names.name, names.alt_name, nodes.tips FROM" \
+ " names INNER JOIN nodes ON names.name = nodes.name " \
+ " WHERE alt_name LIKE ? ORDER BY length(alt_name) LIMIT ?",
+ (name + "%", SEARCH_SUGG_LIMIT)):
+ results.append({"name": row[0], "altName": row[1]})
if len(results) > SEARCH_SUGG_LIMIT:
hasMore = True
del results[-1]
diff --git a/src/components/SearchModal.vue b/src/components/SearchModal.vue
index 4ba0c1a..dbe47af 100644
--- a/src/components/SearchModal.vue
+++ b/src/components/SearchModal.vue
@@ -85,7 +85,7 @@ export default defineComponent({
let reqDelay = 0;
if (this.pendingSearchSuggReq != 0){
clearTimeout(this.pendingSearchSuggReq);
- reqDelay = 300;
+ reqDelay = 500;
}
this.pendingSearchSuggReq = setTimeout(() =>
fetch(url.toString())