Add backend unit tests

- Add unit testing code in backend/tests/
- Change to snake-case for script/file/directory names
- Use os.path.join() instead of '/'
- Refactor script code into function defs and a main-guard
- Make global vars all-caps

Some fixes:
- For getting descriptions, some wiki redirects weren't properly resolved
- Linked images were sub-optimally propagated
- Generation of reduced trees assumed a wiki-id association implied a description
- Tilo.py had potential null dereferences by not always using a reduced node set
- EOL image downloading didn't properly wait for all threads to end when finishing

1 files changed, 44 insertions, 0 deletions

diff --git a/backend/tests/enwiki/test_gen_pageview_data.py b/backend/tests/enwiki/test_gen_pageview_data.py
new file mode 100644
index 0000000..5002eb0
--- /dev/null
+++ b/backend/tests/enwiki/test_gen_pageview_data.py
@@ -0,0 +1,44 @@
+import unittest
+import tempfile, os
+
+from tests.common import createTestBz2, createTestDbTable, readTestDbTable
+from tol_data.enwiki.gen_pageview_data import genData
+
+class TestGenData(unittest.TestCase):
+	def test_gen(self):
+		with tempfile.TemporaryDirectory() as tempDir:
+			# Create temp pageview files
+			pageviewFiles = [os.path.join(tempDir, 'pageviews1.bz2'), os.path.join(tempDir, 'pageviews2.bz2')]
+			createTestBz2(pageviewFiles[0], (
+				'aa.wikibooks One null desktop 1 W1\n'
+				'en.wikipedia Two null mobile-web 10 A9B1\n'
+				'en.wikipedia Three null desktop 4 D3\n'
+			))
+			createTestBz2(pageviewFiles[1], (
+				'fr.wikipedia Four null desktop 12 T6U6\n'
+				'en.wikipedia Three null desktop 10 E4G5Z61\n'
+			))
+			# Create temp dump-index db
+			dumpIndexDb = os.path.join(tempDir, 'dump_index.db')
+			createTestDbTable(
+				dumpIndexDb,
+				'CREATE TABLE offsets (title TEXT PRIMARY KEY, id INT UNIQUE, offset INT, next_offset INT)',
+				'INSERT INTO offsets VALUES (?, ?, ?, ?)',
+				{
+					('One', 1, 0, -1),
+					('Two', 2, 0, -1),
+					('Three', 3, 0, -1),
+					('Four', 4, 0, -1),
+				}
+			)
+			# Run
+			dbFile = os.path.join(tempDir, 'data.db')
+			genData(pageviewFiles, dumpIndexDb, dbFile)
+			# Check
+			self.assertEqual(
+				readTestDbTable(dbFile, 'SELECT title, id, views from views'),
+				{
+					('Two', 2, 5),
+					('Three', 3, 7),
+				}
+			)