diff options
Diffstat (limited to 'backend/data')
| -rw-r--r-- | backend/data/README.md | 6 | ||||
| -rwxr-xr-x | backend/data/genEnwikiData.py | 44 |
2 files changed, 25 insertions, 25 deletions
diff --git a/backend/data/README.md b/backend/data/README.md index b568f90..e2b5db7 100644 --- a/backend/data/README.md +++ b/backend/data/README.md @@ -21,10 +21,10 @@ File Generation Process 1 Obtain data in dbpedia/, as specified in it's README. 2 Run genDbpData.py, which adds a 'descs' table to data.db, using data in dbpedia/dbpData.db, dbpPickedLabels.txt, and the 'nodes' table. - - Using wikipedia dump (old method) + - Supplementing with Wikipedia dump 1 Obtain data in enwiki/, as specified in it's README. - 2 Run genEnwikiData.py, which adds a 'descs' table to data.db, - using data in enwiki/enwikiData.db, and the 'nodes' table. + 2 Run genEnwikiData.py, which adds to the 'descs' table, using data in + enwiki/enwikiData.db, reducedTol/names.txt, and the 'nodes' table. 5 Reduced Tree Structure Data 1 Run genReducedTreeData.py, which adds a 'reduced_nodes' table to data.db, using reducedTol/names.txt, and the 'nodes' and 'names' tables. diff --git a/backend/data/genEnwikiData.py b/backend/data/genEnwikiData.py index f1490b6..48fd2c6 100755 --- a/backend/data/genEnwikiData.py +++ b/backend/data/genEnwikiData.py @@ -4,14 +4,15 @@ import sys, re import sqlite3 usageInfo = f"usage: {sys.argv[0]}\n" -usageInfo += "Reads Wikimedia enwiki data from enwiki/, along with node and name data\n" -usageInfo += "from a sqlite database, associates nodes with enwiki pages, and adds\n" -usageInfo += "alt-name and description information for those nodes.\n" +usageInfo += "Reads Wikimedia enwiki data from enwiki/, a list of node names," +usageInfo += "and node and name data from a sqlite database, and adds\n" +usageInfo += "description data for names that don't have them\n" if len(sys.argv) > 1: print(usageInfo, file=sys.stderr) sys.exit(1) enwikiDb = "enwiki/enwikiData.db" +namesFile = "reducedTol/names.txt" dbFile = "data.db" # Open dbs @@ -19,41 +20,40 @@ enwikiCon = sqlite3.connect(enwikiDb) enwikiCur = enwikiCon.cursor() dbCon = sqlite3.connect(dbFile) dbCur = dbCon.cursor() +# Read in names to check +print("Getting names to check") +nodeNames = set() +with open(namesFile) as file: + for line in file: + nodeNames.add(line.rstrip()) +print("Found {} names".format(len(nodeNames))) +# Remove names that have descriptions +print("Checking for existing name descriptions") +namesWithDescs = set() +for name in nodeNames: + row = dbCur.execute("SELECT name FROM descs where name = ?", (name,)).fetchone() + if row != None: + namesWithDescs.add(name) +nodeNames.difference_update(namesWithDescs) +print("Remaining nodes: {}".format(len(nodeNames))) # Find page id for each node name nodeToPageId = {} print("Getting node page-ids") -iterationNum = 0 -for row in dbCur.execute("SELECT name from nodes"): - iterationNum += 1 - if iterationNum % 1e4 == 0: - print("At iteration {}".format(iterationNum)) - # - name = row[0] +for name in nodeNames: row = enwikiCur.execute("SELECT id FROM pages where pages.title = ? COLLATE nocase", (name,)).fetchone() if row != None: nodeToPageId[name] = row[0] # Resolve redirects print("Resolving redirects") redirectingNames = set() -iterationNum = 0 for (name, pageId) in nodeToPageId.items(): - iterationNum += 1 - if iterationNum % 1e4 == 0: - print("At iteration {}".format(iterationNum)) - # row = enwikiCur.execute("SELECT target_id FROM redirects where redirects.id = ?", (pageId,)).fetchone() if row != None: nodeToPageId[name] = row[0] redirectingNames.add(name) # Add descriptions for each node -print("Adding node description data") -dbCur.execute("CREATE TABLE descs (name TEXT PRIMARY KEY, desc TEXT, redirected INT)") -iterationNum = 0 +print("Adding description data") for (name, pageId) in nodeToPageId.items(): - iterationNum += 1 - if iterationNum % 1e4 == 0: - print("At iteration {}".format(iterationNum)) - # row = enwikiCur.execute("SELECT desc FROM descs where descs.id = ?", (pageId,)).fetchone() if row != None: dbCur.execute("INSERT INTO descs VALUES (?, ?, ?)", (name, row[0], 1 if name in redirectingNames else 0)) |