From 52560266b585e63742a81e27a3b6f1ef194470c6 Mon Sep 17 00:00:00 2001
From: Terry Truong <terry06890@gmail.com>
Date: Wed, 18 May 2022 21:09:05 +1000
Subject: Add wikipedia-link to tile-info display

Add 'wiki_id' and 'from_dbp' columns to 'descs' table, adjust dbpedia
data to include wikipedia IDs, adjust data generations scripts, make
server send extra data, and make TileInfo display it.
---
 backend/data/dbpedia/README.md  |  3 +++
 backend/data/dbpedia/genData.py | 25 ++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

(limited to 'backend/data/dbpedia')

diff --git a/backend/data/dbpedia/README.md b/backend/data/dbpedia/README.md
index 0e7c266..78e2a90 100644
--- a/backend/data/dbpedia/README.md
+++ b/backend/data/dbpedia/README.md
@@ -3,6 +3,8 @@ Downloaded Files
 -   labels\_lang=en.ttl.bz2 <br>
     Obtained via https://databus.dbpedia.org/dbpedia/collections/latest-core, 
     using the link <https://databus.dbpedia.org/dbpedia/generic/labels/2022.03.01/labels_lang=en.ttl.bz2>.
+-   page\_lang=en\_ids.ttl.bz2 <br>
+    Downloaded from <https://databus.dbpedia.org/dbpedia/generic/page/2022.03.01/page_lang=en_ids.ttl.bz2>
 -   redirects\_lang=en\_transitive.ttl.bz2 <br>
     Downloaded from <https://databus.dbpedia.org/dbpedia/generic/redirects/2022.03.01/redirects_lang=en_transitive.ttl.bz2>.
 -   disambiguations\_lang=en.ttl.bz2 <br>
@@ -19,6 +21,7 @@ Generated Files
     Generated by running genData.py.
     Tables
     -   labels:          iri TEXT PRIMARY KEY, label TEXT 
+    -   ids:             iri TEXT PRIMARY KEY, id INT
     -   redirects:       iri TEXT PRIMARY KEY, target TEXT
     -   disambiguations: iri TEXT PRIMARY KEY
     -   types:           iri TEXT, type TEXT
diff --git a/backend/data/dbpedia/genData.py b/backend/data/dbpedia/genData.py
index e147641..3df1961 100755
--- a/backend/data/dbpedia/genData.py
+++ b/backend/data/dbpedia/genData.py
@@ -4,13 +4,14 @@ import sys, re
 import bz2, sqlite3
 
 usageInfo =  f"usage: {sys.argv[0]}\n"
-usageInfo += "Reads DBpedia labels+types+redirects+abstracts data,\n"
+usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n"
 usageInfo += "and creates a sqlite db containing that data.\n"
 if len(sys.argv) > 1:
 	print(usageInfo, file=sys.stderr)
 	sys.exit(1)
 
 labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines
+idsFile = "page_lang=en_ids.ttl.bz2"
 redirectsFile = "redirects_lang=en_transitive.ttl.bz2"
 disambigFile = "disambiguations_lang=en.ttl.bz2"
 typesFile = "instance-types_lang=en_specific.ttl.bz2"
@@ -39,6 +40,28 @@ with bz2.open(labelsFile, mode='rt') as file:
 		else:
 			dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2)))
 dbCon.commit()
+# Read/store wiki page ids
+print("Reading/storing wiki page ids")
+dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)")
+idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n')
+lineNum = 0
+with bz2.open(idsFile, mode='rt') as file:
+	for line in file:
+		lineNum += 1
+		if lineNum % 1e5 == 0:
+			print("Processing line {}".format(lineNum))
+		#
+		match = idLineRegex.fullmatch(line)
+		if match == None:
+			print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr)
+			sys.exit(1)
+		else:
+			try:
+				dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2))))
+			except sqlite3.IntegrityError as e:
+				# Accounts for certain lines that have the same IRI
+				print("Failed to add entry with IRI \"{}\": {}".format(match.group(1), e))
+dbCon.commit()
 # Read/store redirects
 print("Reading/storing redirection data")
 dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)")
-- 
cgit v1.2.3