From e78c4df403e5f98afa08f7a0841ff233d5f6d05b Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Wed, 22 Jun 2022 01:42:41 +1000 Subject: Update backend READMEs, rename some files for consistency --- backend/data/dbpedia/README.md | 45 +++++------ backend/data/dbpedia/genData.py | 146 ------------------------------------ backend/data/dbpedia/genDescData.py | 146 ++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 168 deletions(-) delete mode 100755 backend/data/dbpedia/genData.py create mode 100755 backend/data/dbpedia/genDescData.py (limited to 'backend/data/dbpedia') diff --git a/backend/data/dbpedia/README.md b/backend/data/dbpedia/README.md index 78e2a90..8a08f20 100644 --- a/backend/data/dbpedia/README.md +++ b/backend/data/dbpedia/README.md @@ -1,28 +1,29 @@ -Downloaded Files -================ -- labels\_lang=en.ttl.bz2
- Obtained via https://databus.dbpedia.org/dbpedia/collections/latest-core, - using the link . -- page\_lang=en\_ids.ttl.bz2
+This directory holds files obtained from/using [Dbpedia](https://www.dbpedia.org). + +# Downloaded Files +- `labels_lang=en.ttl.bz2`
+ Obtained via https://databus.dbpedia.org/dbpedia/collections/latest-core. + Downloaded from . +- `page_lang=en_ids.ttl.bz2`
Downloaded from -- redirects\_lang=en\_transitive.ttl.bz2
+- `redirects_lang=en_transitive.ttl.bz2`
Downloaded from . -- disambiguations\_lang=en.ttl.bz2
+- `disambiguations_lang=en.ttl.bz2`
Downloaded from . -- instance-types\_lang=en\_specific.ttl.bz2
+- `instance-types_lang=en_specific.ttl.bz2`
Downloaded from . -- short-abstracts\_lang=en.ttl.bz2
+- `short-abstracts_lang=en.ttl.bz2`
Downloaded from . -Generated Files -=============== -- dbpData.db
- An sqlite database representing data from the ttl files. - Generated by running genData.py. - Tables - - labels: iri TEXT PRIMARY KEY, label TEXT - - ids: iri TEXT PRIMARY KEY, id INT - - redirects: iri TEXT PRIMARY KEY, target TEXT - - disambiguations: iri TEXT PRIMARY KEY - - types: iri TEXT, type TEXT - - abstracts: iri TEXT PRIMARY KEY, abstract TEXT +# Other Files +- genDescData.py
+ Used to generate a database representing data from the ttl files. +- descData.db
+ Generated by genDescData.py.
+ Tables:
+ - `labels`: `iri TEXT PRIMARY KEY, label TEXT ` + - `ids`: `iri TEXT PRIMARY KEY, id INT` + - `redirects`: `iri TEXT PRIMARY KEY, target TEXT` + - `disambiguations`: `iri TEXT PRIMARY KEY` + - `types`: `iri TEXT, type TEXT` + - `abstracts`: `iri TEXT PRIMARY KEY, abstract TEXT` diff --git a/backend/data/dbpedia/genData.py b/backend/data/dbpedia/genData.py deleted file mode 100755 index 41c48a8..0000000 --- a/backend/data/dbpedia/genData.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/python3 - -import sys, re -import bz2, sqlite3 - -usageInfo = f"usage: {sys.argv[0]}\n" -usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n" -usageInfo += "and creates a sqlite db containing that data.\n" -if len(sys.argv) > 1: - print(usageInfo, file=sys.stderr) - sys.exit(1) - -labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines -idsFile = "page_lang=en_ids.ttl.bz2" -redirectsFile = "redirects_lang=en_transitive.ttl.bz2" -disambigFile = "disambiguations_lang=en.ttl.bz2" -typesFile = "instance-types_lang=en_specific.ttl.bz2" -abstractsFile = "short-abstracts_lang=en.ttl.bz2" -dbFile = "dbpData.db" - -# Open db -dbCon = sqlite3.connect(dbFile) -dbCur = dbCon.cursor() -# Read/store labels -print("Reading/storing label data") -dbCur.execute("CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)") -dbCur.execute("CREATE INDEX labels_idx ON labels(label)") -dbCur.execute("CREATE INDEX labels_idx_nc ON labels(label COLLATE NOCASE)") -labelLineRegex = re.compile(r'<([^>]+)> <[^>]+> "((?:[^"]|\\")+)"@en \.\n') -lineNum = 0 -with bz2.open(labelsFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - match = labelLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store wiki page ids -print("Reading/storing wiki page ids") -dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)") -idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n') -lineNum = 0 -with bz2.open(idsFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - match = idLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - try: - dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) - except sqlite3.IntegrityError as e: - # Accounts for certain lines that have the same IRI - print(f"Failed to add entry with IRI \"{match.group(1)}\": {e}") -dbCon.commit() -# Read/store redirects -print("Reading/storing redirection data") -dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") -redirLineRegex = re.compile(r'<([^>]+)> <[^>]+> <([^>]+)> \.\n') -lineNum = 0 -with bz2.open(redirectsFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - match = redirLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store diambiguation-page data -print("Reading/storing diambiguation-page data") -disambigNames = set() -disambigLineRegex = redirLineRegex -lineNum = 0 -with bz2.open(disambigFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - match = disambigLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - disambigNames.add(match.group(1)) -dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") -for name in disambigNames: - dbCur.execute("INSERT INTO disambiguations VALUES (?)", (name,)) -dbCon.commit() -# Read/store instance-type -print("Reading/storing instance-type data") -dbCur.execute("CREATE TABLE types (iri TEXT, type TEXT)") -dbCur.execute("CREATE INDEX types_iri_idx ON types(iri)") -typeLineRegex = redirLineRegex -lineNum = 0 -with bz2.open(typesFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - match = typeLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) -dbCon.commit() -# Read/store abstracts -print("Reading/storing abstracts") -dbCur.execute("CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)") -descLineRegex = labelLineRegex -lineNum = 0 -with bz2.open(abstractsFile, mode='rt') as file: - for line in file: - lineNum += 1 - if lineNum % 1e5 == 0: - print(f"Processing line {lineNum}") - # - if line[0] == "#": - continue - match = descLineRegex.fullmatch(line) - if match == None: - print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) - sys.exit(1) - else: - dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", - (match.group(1), match.group(2).replace(r'\"', '"'))) -# Close db -dbCon.commit() -dbCon.close() diff --git a/backend/data/dbpedia/genDescData.py b/backend/data/dbpedia/genDescData.py new file mode 100755 index 0000000..bba3ff5 --- /dev/null +++ b/backend/data/dbpedia/genDescData.py @@ -0,0 +1,146 @@ +#!/usr/bin/python3 + +import sys, re +import bz2, sqlite3 + +usageInfo = f"usage: {sys.argv[0]}\n" +usageInfo += "Reads DBpedia labels/types/abstracts/etc data,\n" +usageInfo += "and creates a sqlite db containing that data.\n" +if len(sys.argv) > 1: + print(usageInfo, file=sys.stderr) + sys.exit(1) + +labelsFile = "labels_lang=en.ttl.bz2" # Has about 16e6 lines +idsFile = "page_lang=en_ids.ttl.bz2" +redirectsFile = "redirects_lang=en_transitive.ttl.bz2" +disambigFile = "disambiguations_lang=en.ttl.bz2" +typesFile = "instance-types_lang=en_specific.ttl.bz2" +abstractsFile = "short-abstracts_lang=en.ttl.bz2" +dbFile = "descData.db" + +# Open db +dbCon = sqlite3.connect(dbFile) +dbCur = dbCon.cursor() +# Read/store labels +print("Reading/storing label data") +dbCur.execute("CREATE TABLE labels (iri TEXT PRIMARY KEY, label TEXT)") +dbCur.execute("CREATE INDEX labels_idx ON labels(label)") +dbCur.execute("CREATE INDEX labels_idx_nc ON labels(label COLLATE NOCASE)") +labelLineRegex = re.compile(r'<([^>]+)> <[^>]+> "((?:[^"]|\\")+)"@en \.\n') +lineNum = 0 +with bz2.open(labelsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + match = labelLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO labels VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store wiki page ids +print("Reading/storing wiki page ids") +dbCur.execute("CREATE TABLE ids (iri TEXT PRIMARY KEY, id INT)") +idLineRegex = re.compile(r'<([^>]+)> <[^>]+> "(\d+)".*\n') +lineNum = 0 +with bz2.open(idsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + match = idLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + try: + dbCur.execute("INSERT INTO ids VALUES (?, ?)", (match.group(1), int(match.group(2)))) + except sqlite3.IntegrityError as e: + # Accounts for certain lines that have the same IRI + print(f"Failed to add entry with IRI \"{match.group(1)}\": {e}") +dbCon.commit() +# Read/store redirects +print("Reading/storing redirection data") +dbCur.execute("CREATE TABLE redirects (iri TEXT PRIMARY KEY, target TEXT)") +redirLineRegex = re.compile(r'<([^>]+)> <[^>]+> <([^>]+)> \.\n') +lineNum = 0 +with bz2.open(redirectsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + match = redirLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO redirects VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store diambiguation-page data +print("Reading/storing diambiguation-page data") +disambigNames = set() +disambigLineRegex = redirLineRegex +lineNum = 0 +with bz2.open(disambigFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + match = disambigLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + disambigNames.add(match.group(1)) +dbCur.execute("CREATE TABLE disambiguations (iri TEXT PRIMARY KEY)") +for name in disambigNames: + dbCur.execute("INSERT INTO disambiguations VALUES (?)", (name,)) +dbCon.commit() +# Read/store instance-type +print("Reading/storing instance-type data") +dbCur.execute("CREATE TABLE types (iri TEXT, type TEXT)") +dbCur.execute("CREATE INDEX types_iri_idx ON types(iri)") +typeLineRegex = redirLineRegex +lineNum = 0 +with bz2.open(typesFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + match = typeLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO types VALUES (?, ?)", (match.group(1), match.group(2))) +dbCon.commit() +# Read/store abstracts +print("Reading/storing abstracts") +dbCur.execute("CREATE TABLE abstracts (iri TEXT PRIMARY KEY, abstract TEXT)") +descLineRegex = labelLineRegex +lineNum = 0 +with bz2.open(abstractsFile, mode='rt') as file: + for line in file: + lineNum += 1 + if lineNum % 1e5 == 0: + print(f"Processing line {lineNum}") + # + if line[0] == "#": + continue + match = descLineRegex.fullmatch(line) + if match == None: + print(f"ERROR: Line {lineNum} has unexpected format", file=sys.stderr) + sys.exit(1) + else: + dbCur.execute("INSERT INTO abstracts VALUES (?, ?)", + (match.group(1), match.group(2).replace(r'\"', '"'))) +# Close db +dbCon.commit() +dbCon.close() -- cgit v1.2.3