diff options
Diffstat (limited to 'backend/data/dbpedia')
| -rw-r--r-- | backend/data/dbpedia/README.md | 3 | ||||
| -rwxr-xr-x | backend/data/dbpedia/genData.py | 25 |
2 files changed, 27 insertions, 1 deletions
diff --git a/backend/data/dbpedia/README.md b/backend/data/dbpedia/README.md index 0e7c266..78e2a90 100644 --- a/backend/data/dbpedia/README.md +++ b/backend/data/dbpedia/README.md @@ -3,6 +3,8 @@ Downloaded Files - labels\_lang=en.ttl.bz2 <br> Obtained via https://databus.dbpedia.org/dbpedia/collections/latest-core, using the link <https://databus.dbpedia.org/dbpedia/generic/labels/2022.03.01/labels_lang=en.ttl.bz2>. +- page\_lang=en\_ids.ttl.bz2 <br> + Downloaded from <https://databus.dbpedia.org/dbpedia/generic/page/2022.03.01/page_lang=en_ids.ttl.bz2> - redirects\_lang=en\_transitive.ttl.bz2 <br> Downloaded from <https://databus.dbpedia.org/dbpedia/generic/redirects/2022.03.01/redirects_lang=en_transitive.ttl.bz2>. - disambiguations\_lang=en.ttl.bz2 <br> @@ -19,6 +21,7 @@ Generated Files Generated by running genData.py. Tables - labels: iri TEXT PRIMARY KEY, label TEXT + - ids: iri TEXT PRIMARY KEY, id INT - redirects: iri TEXT PRIMARY KEY, target TEXT - disambiguations: iri TEXT PRIMARY KEY - types: iri TEXT, type TEXT diff --git a/backend/data/dbpedia/genData.py b/backend/data/dbpedia/genData.py index e147641..3df1961 100755 --- a/backend/data/dbpedia/genData.py +++ b/backend/data/dbpedia/genData.py @@ -4,13 +4,14 @@ import sys, re import bz2, sqlite3 usageInfo = f"usage: {sys.argv[0]}\n" -usageInfo += "Reads DBpedia labels+types+redirects+abstracts data,\n" +usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n" usageInfo += "and creates a sqlite db containing that data.\n" if len(sys.argv) > 1: print(usageInfo, file=sys.stderr) sys.exit(1) labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines +idsFile = "page_lang=en_ids.ttl.bz2" redirectsFile = "redirects_lang=en_transitive.ttl.bz2" disambigFile = "disambiguations_lang=en.ttl.bz2" typesFile = "instance-types_lang=en_specific.ttl.bz2" @@ -39,6 +40,28 @@ with bz2.open(labelsFile, mode='rt') as file: else: dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) dbCon.commit() +# Read/store wiki page ids +print("Reading/storing wiki page ids") +dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)") +idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n') +lineNum = 0 +with bz2.open(idsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print("Processing line {}".format(lineNum)) + # + match = idLineRegex.fullmatch(line) + if match == None: + print("ERROR: Line {} has unexpected format".format(lineNum), file=sys.stderr) + sys.exit(1) + else: + try: + dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) + except sqlite3.IntegrityError as e: + # Accounts for certain lines that have the same IRI + print("Failed to add entry with IRI \"{}\": {}".format(match.group(1), e)) +dbCon.commit() # Read/store redirects print("Reading/storing redirection data") dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") |