1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
#!/usr/bin/python3
"""
For some set of EOL IDs, downloads associated images from URLs in
an image-list database. Uses multiple downloading threads.
May obtain multiple images per ID. The images will get names
with the form 'eolId1 contentId1.ext1'.
SIGINT causes the program to finish ongoing downloads and exit.
The program can be re-run to continue downloading. It looks for
already-downloaded files, and continues after the one with
highest EOL ID.
"""
import argparse
import sys
import re
import os
import random
import sqlite3
import requests
import urllib.parse
import time
from threading import Thread
import signal
IMAGES_LIST_DB = 'images_list.db'
OUT_DIR = 'imgs_for_review'
DB_FILE = os.path.join('..', 'data.db')
MAX_IMGS_PER_ID = 3
MAX_THREADS = 5
POST_DL_DELAY_MIN = 2 # Minimum delay in seconds to pause after download before starting another (for each thread)
POST_DL_DELAY_MAX = 3
LICENSE_REGEX = r'cc-by((-nc)?(-sa)?(-[234]\.[05])?)|cc-publicdomain|cc-0-1\.0|public domain'
def downloadImgs(eolIds, imagesListDb, outDir):
print('Getting EOL IDs to download for')
# Get IDs from images-list db
imgDbCon = sqlite3.connect(imagesListDb)
imgCur = imgDbCon.cursor()
imgListIds: set[int] = set()
for (pageId,) in imgCur.execute('SELECT DISTINCT page_id FROM images'):
imgListIds.add(pageId)
# Get set intersection, and sort into list
eolIds = eolIds.intersection(imgListIds)
eolIdList = sorted(eolIds)
nextIdx = 0
print(f'Result: {len(eolIdList)} EOL IDs')
print('Checking output directory')
if not os.path.exists(outDir):
os.mkdir(outDir)
else:
print('Finding next ID to download for')
fileList = os.listdir(outDir)
ids = [int(filename.split(' ')[0]) for filename in fileList]
if ids:
ids.sort()
nextIdx = eolIdList.index(ids[-1]) + 1
if nextIdx == len(eolIdList):
print('No IDs left. Exiting...')
return
print('Starting download threads')
numThreads = 0
threadException: Exception | None = None # Used for ending main thread after a non-main thread exception
# Handle SIGINT signals
interrupted = False
oldHandler = None
def onSigint(sig, frame):
nonlocal interrupted
interrupted = True
signal.signal(signal.SIGINT, oldHandler)
oldHandler = signal.signal(signal.SIGINT, onSigint)
# Function for threads to execute
def downloadImg(url, outFile):
nonlocal numThreads, threadException
try:
data = requests.get(url)
with open(outFile, 'wb') as file:
file.write(data.content)
time.sleep(random.random() * (POST_DL_DELAY_MAX - POST_DL_DELAY_MIN) + POST_DL_DELAY_MIN)
except Exception as e:
print(f'Error while downloading to {outFile}: {str(e)}', file=sys.stderr)
threadException = e
numThreads -= 1
# Manage downloading
for idx in range(nextIdx, len(eolIdList)):
eolId = eolIdList[idx]
# Get image urls
ownerSet: set[str] = set() # Used to get images from different owners, for variety
exitLoop = False
query = 'SELECT content_id, copy_url, license, copyright_owner FROM images WHERE page_id = ?'
for contentId, url, license, copyrightOwner in imgCur.execute(query, (eolId,)):
if url.startswith('data/'):
url = 'https://content.eol.org/' + url
urlParts = urllib.parse.urlparse(url)
extension = os.path.splitext(urlParts.path)[1]
if len(extension) <= 1:
print(f'WARNING: No filename extension found in URL {url}', file=sys.stderr)
continue
# Check image-quantity limit
if len(ownerSet) == MAX_IMGS_PER_ID:
break
# Check for skip conditions
if re.fullmatch(LICENSE_REGEX, license) is None:
continue
if len(copyrightOwner) > 100: # Avoid certain copyrightOwner fields that seem long and problematic
continue
if copyrightOwner in ownerSet:
continue
ownerSet.add(copyrightOwner)
# Determine output filename
outPath = os.path.join(outDir, f'{eolId} {contentId}{extension}')
if os.path.exists(outPath):
print(f'WARNING: {outPath} already exists. Skipping download.')
continue
# Check thread limit
while numThreads == MAX_THREADS:
time.sleep(1)
# Wait for threads after an interrupt or thread-exception
if interrupted or threadException is not None:
print('Waiting for existing threads to end')
while numThreads > 0:
time.sleep(1)
exitLoop = True
break
# Perform download
print(f'Downloading image to {outPath}')
numThreads += 1
thread = Thread(target=downloadImg, args=(url, outPath), daemon=True)
thread.start()
if exitLoop:
break
# Close images-list db
while numThreads > 0:
time.sleep(1)
print('Finished downloading')
imgDbCon.close()
def getEolIdsFromDb(dbFile) -> set[int]:
eolIds: set[int] = set()
dbCon = sqlite3.connect(dbFile)
dbCur = dbCon.cursor()
for (id,) in dbCur.execute('SELECT id FROM eol_ids'):
eolIds.add(id)
dbCon.close()
return eolIds
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.parse_args()
eolIds = getEolIdsFromDb(DB_FILE)
downloadImgs(eolIds, IMAGES_LIST_DB, OUT_DIR)
|
