diff options
Diffstat (limited to 'backend/data/dbpedia')
| -rwxr-xr-x | backend/data/dbpedia/genDescData.py | 96 |
1 files changed, 40 insertions, 56 deletions
diff --git a/backend/data/dbpedia/genDescData.py b/backend/data/dbpedia/genDescData.py index bba3ff5..d9e8a80 100755 --- a/backend/data/dbpedia/genDescData.py +++ b/backend/data/dbpedia/genDescData.py @@ -3,25 +3,28 @@ import sys, re import bz2, sqlite3 -usageInfo = f"usage: {sys.argv[0]}\n" -usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n" -usageInfo += "and creates a sqlite db containing that data.\n" +usageInfo = f""" +Usage: {sys.argv[0]} + +Adds DBpedia labels/types/abstracts/etc data into a database. +""" if len(sys.argv) > 1: print(usageInfo, file=sys.stderr) sys.exit(1) -labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines +labelsFile = "labels_lang=en.ttl.bz2" # Had about 16e6 entries idsFile = "page_lang=en_ids.ttl.bz2" redirectsFile = "redirects_lang=en_transitive.ttl.bz2" disambigFile = "disambiguations_lang=en.ttl.bz2" typesFile = "instance-types_lang=en_specific.ttl.bz2" abstractsFile = "short-abstracts_lang=en.ttl.bz2" dbFile = "descData.db" +# In testing, this script took a few hours to run, and generated about 10GB -# Open db +print("Creating database") dbCon = sqlite3.connect(dbFile) dbCur = dbCon.cursor() -# Read/store labels + print("Reading/storing label data") dbCur.execute("CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)") dbCur.execute("CREATE INDEX labels_idx ON labels(label)") @@ -32,16 +35,13 @@ with bz2.open(labelsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # match = labelLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store wiki page ids + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) + print("Reading/storing wiki page ids") dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)") idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n') @@ -50,20 +50,17 @@ with bz2.open(idsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # match = idLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - try: - dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) - except sqlite3.IntegrityError as e: - # Accounts for certain lines that have the same IRI - print(f"Failed to add entry with IRI \"{match.group(1)}\": {e}") -dbCon.commit() -# Read/store redirects + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + try: + dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) + except sqlite3.IntegrityError as e: + # Accounts for certain lines that have the same IRI + print(f"WARNING: Failed to add entry with IRI \"{match.group(1)}\": {e}") + print("Reading/storing redirection data") dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") redirLineRegex = re.compile(r'<([^>]+)> <[^>]+> <([^>]+)> \.\n') @@ -72,37 +69,28 @@ with bz2.open(redirectsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # match = redirLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store diambiguation-page data + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) + print("Reading/storing diambiguation-page data") -disambigNames = set() +dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") disambigLineRegex = redirLineRegex lineNum = 0 with bz2.open(disambigFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # match = disambigLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - disambigNames.add(match.group(1)) -dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") -for name in disambigNames: - dbCur.execute("INSERT INTO disambiguations VALUES (?)", (name,)) -dbCon.commit() -# Read/store instance-type + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + dbCur.execute("INSERT OR IGNORE INTO disambiguations VALUES (?)", (match.group(1),)) + print("Reading/storing instance-type data") dbCur.execute("CREATE TABLE types (iri TEXT, type TEXT)") dbCur.execute("CREATE INDEX types_iri_idx ON types(iri)") @@ -112,16 +100,13 @@ with bz2.open(typesFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # match = typeLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store abstracts + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) + print("Reading/storing abstracts") dbCur.execute("CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)") descLineRegex = labelLineRegex @@ -130,17 +115,16 @@ with bz2.open(abstractsFile, mode='rt') as file: for line in file: lineNum += 1 if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") + print(f"At line {lineNum}") # if line[0] == "#": continue match = descLineRegex.fullmatch(line) if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", - (match.group(1), match.group(2).replace(r'\"', '"'))) -# Close db + raise Exception(f"ERROR: Line {lineNum} has unexpected format") + dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", + (match.group(1), match.group(2).replace(r'\"', '"'))) + +print("Closing database") dbCon.commit() dbCon.close() |