1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
#!/usr/bin/python3
"""
Looks up a page with title title1 in the wiki dump, using the dump-index
db, and prints the corresponding <page>.
"""
import sys
import bz2
import sqlite3
DUMP_FILE = 'enwiki-20220501-pages-articles-multistream.xml.bz2'
INDEX_DB = 'dumpIndex.db'
def lookupPage(dumpFile: str, indexDb: str, pageTitle: str) -> None:
print('Looking up offset in index db')
dbCon = sqlite3.connect(indexDb)
dbCur = dbCon.cursor()
query = 'SELECT title, offset, next_offset FROM offsets WHERE title = ?'
row = dbCur.execute(query, (pageTitle,)).fetchone()
if row is None:
print('Title not found')
sys.exit(0)
_, pageOffset, endOffset = row
dbCon.close()
print(f'Found chunk at offset {pageOffset}')
#
print('Reading from wiki dump')
content: list[str] = []
with open(dumpFile, mode='rb') as file:
# Get uncompressed chunk
file.seek(pageOffset)
compressedData = file.read(None if endOffset == -1 else endOffset - pageOffset)
data = bz2.BZ2Decompressor().decompress(compressedData).decode()
# Look in chunk for page
lines = data.splitlines()
lineIdx = 0
found = False
pageNum = 0
while not found:
line = lines[lineIdx]
if line.lstrip() == '<page>':
pageNum += 1
if pageNum > 100:
print('ERROR: Did not find title after 100 pages')
break
lineIdx += 1
titleLine = lines[lineIdx]
if titleLine.lstrip() == '<title>' + pageTitle + '</title>':
found = True
print(f'Found title in chunk as page {pageNum}')
content.append(line)
content.append(titleLine)
while True:
lineIdx += 1
line = lines[lineIdx]
content.append(line)
if line.lstrip() == '</page>':
break
lineIdx += 1
#
print('Content: ')
print('\n'.join(content))
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('title', help='The title to look up')
args = parser.parse_args()
#
lookupPage(DUMP_FILE, INDEX_DB, args.title.replace('_', ' '))
|
