Add backend unit tests

- Add unit testing code in backend/tests/
- Change to snake-case for script/file/directory names
- Use os.path.join() instead of '/'
- Refactor script code into function defs and a main-guard
- Make global vars all-caps

Some fixes:
- For getting descriptions, some wiki redirects weren't properly resolved
- Linked images were sub-optimally propagated
- Generation of reduced trees assumed a wiki-id association implied a description
- Tilo.py had potential null dereferences by not always using a reduced node set
- EOL image downloading didn't properly wait for all threads to end when finishing

1 files changed, 39 insertions, 0 deletions

diff --git a/backend/tests/enwiki/test_gen_dump_index_db.py b/backend/tests/enwiki/test_gen_dump_index_db.py
new file mode 100644
index 0000000..e0715f3
--- /dev/null
+++ b/backend/tests/enwiki/test_gen_dump_index_db.py
@@ -0,0 +1,39 @@
+import unittest
+import tempfile, os
+
+from tests.common import createTestBz2, readTestDbTable
+from tol_data.enwiki.gen_dump_index_db import genData
+
+def runGenData(indexFileContents: str):
+	""" Sets up index file to be read by genData(), runs it, reads the output database, and returns offset info. """
+	with tempfile.TemporaryDirectory() as tempDir:
+		# Create temp index file
+		indexFile = os.path.join(tempDir, 'index.txt.bz2')
+		createTestBz2(indexFile, indexFileContents)
+		# Run
+		dbFile = os.path.join(tempDir, 'data.db')
+		genData(indexFile, dbFile)
+		# Read db
+		return readTestDbTable(dbFile, 'SELECT title, id, offset, next_offset FROM offsets')
+
+class TestGenData(unittest.TestCase):
+	def setUp(self):
+		self.maxDiff = None # Remove output-diff size limit
+	def test_index_file(self):
+		indexFileContents = (
+			'100:10:apple\n'
+			'100:11:ant\n'
+			'300:99:banana ice-cream\n'
+			'1000:2030:Custard!\n'
+		)
+		offsetsMap = runGenData(indexFileContents)
+		self.assertEqual(offsetsMap, {
+			('apple', 10, 100, 300),
+			('ant', 11, 100, 300),
+			('banana ice-cream', 99, 300, 1000),
+			('Custard!', 2030, 1000, -1),
+		})
+	def test_emp_index(self):
+		offsetsMap = runGenData('')
+		self.assertEqual(offsetsMap, set())
+		pass