From c97acf8852e2017fd4776d65069f707121405f43 Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Sat, 14 May 2022 19:30:43 +1000 Subject: Use DBpedia data for node descriptions Add backend/data/dbpedia/ directory containing scripts and README for obtaining DBpedia data, storing it into a db, converting/adding description data to data.db, and for resolving tol-node DBpedia-node association conflicts (via DBpedia relations, manual listing, etc). Resulted in less (about 3/4 as many) descriptions as with using enwiki, but with notably less mis-associations (eg: node Thor is described as a shrimp instead of a god). --- backend/data/dbpedia/README.md | 25 ++++++++ backend/data/dbpedia/genData.py | 122 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 backend/data/dbpedia/README.md create mode 100755 backend/data/dbpedia/genData.py (limited to 'backend/data/dbpedia') diff --git a/backend/data/dbpedia/README.md b/backend/data/dbpedia/README.md new file mode 100644 index 0000000..0e7c266 --- /dev/null +++ b/backend/data/dbpedia/README.md @@ -0,0 +1,25 @@ +Downloaded Files +================ +- labels\_lang=en.ttl.bz2
+ Obtained via https://databus.dbpedia.org/dbpedia/collections/latest-core, + using the link . +- redirects\_lang=en\_transitive.ttl.bz2
+ Downloaded from . +- disambiguations\_lang=en.ttl.bz2
+ Downloaded from . +- instance-types\_lang=en\_specific.ttl.bz2
+ Downloaded from . +- short-abstracts\_lang=en.ttl.bz2
+ Downloaded from . + +Generated Files +=============== +- dbpData.db
+ An sqlite database representing data from the ttl files. + Generated by running genData.py. + Tables + - labels: iri TEXT PRIMARY KEY, label TEXT + - redirects: iri TEXT PRIMARY KEY, target TEXT + - disambiguations: iri TEXT PRIMARY KEY + - types: iri TEXT, type TEXT + - abstracts: iri TEXT PRIMARY KEY, abstract TEXT diff --git a/backend/data/dbpedia/genData.py b/backend/data/dbpedia/genData.py new file mode 100755 index 0000000..e147641 --- /dev/null +++ b/backend/data/dbpedia/genData.py @@ -0,0 +1,122 @@ +#!/usr/bin/python3 + +import sys, re +import bz2, sqlite3 + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Reads DBpedia labels+types+redirects+abstracts data,\n" +usageInfo += "and creates a sqlite db containing that data.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines +redirectsFile = "redirects_lang=en_transitive.ttl.bz2" +disambigFile = "disambiguations_lang=en.ttl.bz2" +typesFile = "instance-types_lang=en_specific.ttl.bz2" +abstractsFile = "short-abstracts_lang=en.ttl.bz2" +dbFile = "dbpData.db" + +# Open db +dbCon = sqlite3.connect(dbFile) +dbCur = dbCon.cursor() +# Read/store labels +print("Reading/storing label data") +dbCur.execute("CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)") +dbCur.execute("CREATE INDEX labels_idx ON labels(label COLLATE NOCASE)") +labelLineRegex = re.compile(r'<([^>]+)> <[^>]+> "((?:[^"]|\\")+)"@en \.\n') +lineNum = 0 +with bz2.open(labelsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + match = labelLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store redirects +print("Reading/storing redirection data") +dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") +redirLineRegex = re.compile(r'<([^>]+)> <[^>]+> <([^>]+)> \.\n') +lineNum = 0 +with bz2.open(redirectsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + match = redirLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store diambiguation-page data +print("Reading/storing diambiguation-page data") +disambigNames = set() +disambigLineRegex = redirLineRegex +lineNum = 0 +with bz2.open(disambigFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + match = disambigLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + disambigNames.add(match.group(1)) +dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") +for name in disambigNames: + dbCur.execute("INSERT INTO disambiguations VALUES (?)", (name,)) +dbCon.commit() +# Read/store instance-type +print("Reading/storing instance-type data") +dbCur.execute("CREATE TABLE types (iri TEXT, type TEXT)") +dbCur.execute("CREATE INDEX types_iri_idx ON types(iri)") +typeLineRegex = redirLineRegex +lineNum = 0 +with bz2.open(typesFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + match = typeLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store abstracts +print("Reading/storing abstracts") +dbCur.execute("CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)") +descLineRegex = labelLineRegex +lineNum = 0 +with bz2.open(abstractsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + if line[0] == "#": + continue + match = descLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", + (match.group(1), match.group(2).replace(r'\"', '"'))) +# Close db +dbCon.commit() +dbCon.close() -- cgit v1.2.3