From 5de5fb93e50fe9006221b30ac4a66f1be0db82e7 Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Sun, 11 Sep 2022 14:55:42 +1000 Subject: Add backend unit tests - Add unit testing code in backend/tests/ - Change to snake-case for script/file/directory names - Use os.path.join() instead of '/' - Refactor script code into function defs and a main-guard - Make global vars all-caps Some fixes: - For getting descriptions, some wiki redirects weren't properly resolved - Linked images were sub-optimally propagated - Generation of reduced trees assumed a wiki-id association implied a description - Tilo.py had potential null dereferences by not always using a reduced node set - EOL image downloading didn't properly wait for all threads to end when finishing --- backend/tolData/genPopData.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 backend/tolData/genPopData.py (limited to 'backend/tolData/genPopData.py') diff --git a/backend/tolData/genPopData.py b/backend/tolData/genPopData.py deleted file mode 100755 index 3bb1325..0000000 --- a/backend/tolData/genPopData.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python3 - -import sqlite3 - -import argparse -parser = argparse.ArgumentParser(description=""" -Reads enwiki page view info from a database, and stores it -as node popularity values in the database. -""", formatter_class=argparse.RawDescriptionHelpFormatter) -args = parser.parse_args() - -pageviewsDb = 'enwiki/pageviewData.db' -dbFile = 'data.db' - -dbCon = sqlite3.connect(dbFile) -dbCur = dbCon.cursor() - -print('Getting view counts') -pdbCon = sqlite3.connect(pageviewsDb) -pdbCur = pdbCon.cursor() -nodeToViews: dict[str, int] = {} # Maps node names to counts -iterNum = 0 -for wikiId, views in pdbCur.execute('SELECT id, views from views'): - iterNum += 1 - if iterNum % 1e4 == 0: - print(f'At iteration {iterNum}') # Reached 1.6e6 - # - row = dbCur.execute('SELECT name FROM wiki_ids WHERE id = ?', (wikiId,)).fetchone() - if row is not None: - nodeToViews[row[0]] = views -pdbCon.close() - -print(f'Writing {len(nodeToViews)} entries to db') -dbCur.execute('CREATE TABLE node_pop (name TEXT PRIMARY KEY, pop INT)') -for nodeName, views in nodeToViews.items(): - dbCur.execute('INSERT INTO node_pop VALUES (?, ?)', (nodeName, views)) - -dbCon.commit() -dbCon.close() -- cgit v1.2.3