Add wikipedia-link to tile-info display

Add 'wiki_id' and 'from_dbp' columns to 'descs' table, adjust dbpedia
data to include wikipedia IDs, adjust data generations scripts, make
server send extra data, and make TileInfo display it.

1 files changed, 24 insertions, 1 deletions

diff --git a/backend/data/dbpedia/genData.py b/backend/data/dbpedia/genData.py
index e147641..3df1961 100755
--- a/backend/data/dbpedia/genData.py
+++ b/backend/data/dbpedia/genData.py
@@ -4,13 +4,14 @@ import sys, re
 import bz2, sqlite3
 
 usageInfo =  f"usage: {sys.argv[0]}\n"
-usageInfo += "Reads DBpedia labels+types+redirects+abstracts data,\n"
+usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n"
 usageInfo += "and creates a sqlite db containing that data.\n"
 if len(sys.argv) > 1:
 	print(usageInfo, file=sys.stderr)
 	sys.exit(1)
 
 labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines
+idsFile = "page_lang=en_ids.ttl.bz2"
 redirectsFile = "redirects_lang=en_transitive.ttl.bz2"
 disambigFile = "disambiguations_lang=en.ttl.bz2"
 typesFile = "instance-types_lang=en_specific.ttl.bz2"
@@ -39,6 +40,28 @@ with bz2.open(labelsFile, mode='rt') as file:
 		else:
 			dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2)))
 dbCon.commit()
+# Read/store wiki page ids
+print("Reading/storing wiki page ids")
+dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)")
+idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n')
+lineNum = 0
+with bz2.open(idsFile, mode='rt') as file:
+	for line in file:
+		lineNum += 1
+		if lineNum % 1e5 == 0:
+			print("Processing line {}".format(lineNum))
+		#
+		match = idLineRegex.fullmatch(line)
+		if match == None:
+			print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr)
+			sys.exit(1)
+		else:
+			try:
+				dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2))))
+			except sqlite3.IntegrityError as e:
+				# Accounts for certain lines that have the same IRI
+				print("Failed to add entry with IRI \"{}\": {}".format(match.group(1), e))
+dbCon.commit()
 # Read/store redirects
 print("Reading/storing redirection data")
 dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)")