diff options
| -rw-r--r-- | backend/hist_data/enwiki/README.md | 2 | ||||
| -rwxr-xr-x | backend/hist_data/enwiki/gen_img_data.py | 23 | ||||
| -rw-r--r-- | backend/tests/enwiki/test_gen_img_data.py | 14 | ||||
| -rw-r--r-- | backend/tests/wikidata/__init__.py | 0 |
4 files changed, 21 insertions, 18 deletions
diff --git a/backend/hist_data/enwiki/README.md b/backend/hist_data/enwiki/README.md index 95795f3..29fc2ff 100644 --- a/backend/hist_data/enwiki/README.md +++ b/backend/hist_data/enwiki/README.md @@ -40,7 +40,7 @@ This directory holds files obtained/derived from [English Wikipedia](https://en. Used to hold metadata about infobox images for a set of page IDs. Generated using `get_enwiki_img_data.py` and `download_img_license_info.py`. <br> Tables: <br> - - `page_imgs`: `page_id INT PRIMAY KEY, img_name TEXT` <br> + - `page_imgs`: `page_id INT PRIMARY KEY, title TEXT UNIQUE, img_name TEXT` <br> `img_name` may be NULL, which means 'none found', and is used to avoid re-processing page IDs. - `imgs`: `id INT PRIMARY KEY, name TEXT UNIQUE, license TEXT, artist TEXT, credit TEXT, restrictions TEXT, url TEXT` diff --git a/backend/hist_data/enwiki/gen_img_data.py b/backend/hist_data/enwiki/gen_img_data.py index 762952e..b4ade9f 100755 --- a/backend/hist_data/enwiki/gen_img_data.py +++ b/backend/hist_data/enwiki/gen_img_data.py @@ -34,7 +34,8 @@ def genData(pageIds: set[int], dumpFile: str, indexDb: str, imgDb: str) -> None: print('Checking tables') if imgDbCur.execute('SELECT name FROM sqlite_master WHERE type="table" AND name="page_imgs"').fetchone() is None: # Create tables if not present - imgDbCur.execute('CREATE TABLE page_imgs (page_id INT PRIMARY KEY, img_name TEXT)') # img_name may be NULL + imgDbCur.execute('CREATE TABLE page_imgs (page_id INT PRIMARY KEY, title TEXT UNIQUE, img_name TEXT)') + # 'img_name' may be NULL imgDbCur.execute('CREATE INDEX page_imgs_idx ON page_imgs(img_name)') else: # Check for already-processed page IDs @@ -48,24 +49,26 @@ def genData(pageIds: set[int], dumpFile: str, indexDb: str, imgDb: str) -> None: print(f'Will skip {numSkipped} already-processed page IDs') # print('Getting dump-file offsets') - offsetToPageids: dict[int, list[int]] = {} + offsetToPageId: dict[int, list[int]] = {} offsetToEnd: dict[int, int] = {} # Maps chunk-start offsets to their chunk-end offsets + pageIdToTitle: dict[int, str] = {} iterNum = 0 for pageId in pageIds: iterNum += 1 if iterNum % 1e4 == 0: print(f'At iteration {iterNum}') # - query = 'SELECT offset, next_offset FROM offsets WHERE id = ?' - row: tuple[int, int] | None = indexDbCur.execute(query, (pageId,)).fetchone() + query = 'SELECT offset, next_offset, title FROM offsets WHERE id = ?' + row = indexDbCur.execute(query, (pageId,)).fetchone() if row is None: print(f'WARNING: Page ID {pageId} not found') continue - chunkOffset, endOffset = row + chunkOffset, endOffset, title = row offsetToEnd[chunkOffset] = endOffset - if chunkOffset not in offsetToPageids: - offsetToPageids[chunkOffset] = [] - offsetToPageids[chunkOffset].append(pageId) + if chunkOffset not in offsetToPageId: + offsetToPageId[chunkOffset] = [] + offsetToPageId[chunkOffset].append(pageId) + pageIdToTitle[pageId] = title print(f'Found {len(offsetToEnd)} chunks to check') # print('Iterating through chunks in dump file') @@ -76,7 +79,7 @@ def genData(pageIds: set[int], dumpFile: str, indexDb: str, imgDb: str) -> None: if iterNum % 100 == 0: print(f'At iteration {iterNum}') # - chunkPageIds = offsetToPageids[pageOffset] + chunkPageIds = offsetToPageId[pageOffset] # Jump to chunk file.seek(pageOffset) compressedData = file.read(None if endOffset == -1 else endOffset - pageOffset) @@ -121,7 +124,7 @@ def genData(pageIds: set[int], dumpFile: str, indexDb: str, imgDb: str) -> None: content.append(line[:line.rfind('</text>')]) # Look for image-filename imageName = getImageName(content) - imgDbCur.execute('INSERT into page_imgs VALUES (?, ?)', (pageId, imageName)) + imgDbCur.execute('INSERT into page_imgs VALUES (?, ?, ?)', (pageId, None if imageName is None else pageIdToTitle[pageId], imageName)) break if not foundTextEnd: print(f'WARNING: Did not find </text> for page id {pageId}') diff --git a/backend/tests/enwiki/test_gen_img_data.py b/backend/tests/enwiki/test_gen_img_data.py index dc77fe6..93bb196 100644 --- a/backend/tests/enwiki/test_gen_img_data.py +++ b/backend/tests/enwiki/test_gen_img_data.py @@ -76,20 +76,20 @@ class TestGenData(unittest.TestCase): genData({10, 25}, TEST_DUMP_FILE, indexDb, imgDb) # Check self.assertEqual( - readTestDbTable(imgDb, 'SELECT page_id, img_name from page_imgs'), + readTestDbTable(imgDb, 'SELECT page_id, title, img_name from page_imgs'), { - (10, None), - (25, 'Autism-stacking-cans 2nd edit.jpg'), + (10, None, None), + (25, 'Autism', 'Autism-stacking-cans 2nd edit.jpg'), } ) # Run with updated page-ids set genData({13, 10}, TEST_DUMP_FILE, indexDb, imgDb) # Check self.assertEqual( - readTestDbTable(imgDb, 'SELECT page_id, img_name from page_imgs'), + readTestDbTable(imgDb, 'SELECT page_id, title, img_name from page_imgs'), { - (10, None), - (13, None), - (25, 'Autism-stacking-cans 2nd edit.jpg'), + (10, None, None), + (13, None, None), + (25, 'Autism', 'Autism-stacking-cans 2nd edit.jpg'), } ) diff --git a/backend/tests/wikidata/__init__.py b/backend/tests/wikidata/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/backend/tests/wikidata/__init__.py +++ /dev/null |