From e8e58a3bb9dc233dacf573973457c5b48d369503 Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Tue, 30 Aug 2022 12:27:42 +1000 Subject: Add scripts for generating eol/enwiki mappings - New data sources: OTOL taxonomy, EOL provider-ids, Wikidata dump - Add 'node_iucn' table - Remove 'redirected' field from 'wiki_ids' table - Make 'eol_ids' table have 'name' as the primary key - Combine name-generation scripts into genNameData.py - Combine description-generation scripts into genDescData.py --- backend/tolData/enwiki/README.md | 2 +- backend/tolData/enwiki/genImgData.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'backend/tolData/enwiki') diff --git a/backend/tolData/enwiki/README.md b/backend/tolData/enwiki/README.md index dfced94..7df21c9 100644 --- a/backend/tolData/enwiki/README.md +++ b/backend/tolData/enwiki/README.md @@ -11,7 +11,7 @@ This directory holds files obtained/derived from [English Wikipedia](https://en. providing, for each page, an offset into the dump file of a chunk of 100 pages that includes it. -# Generated Dump-Index Files +# Dump-Index Files - genDumpIndexDb.py
Creates an sqlite-database version of the enwiki-dump index file. - dumpIndex.db
diff --git a/backend/tolData/enwiki/genImgData.py b/backend/tolData/enwiki/genImgData.py index 97e696f..b5d546d 100755 --- a/backend/tolData/enwiki/genImgData.py +++ b/backend/tolData/enwiki/genImgData.py @@ -30,7 +30,6 @@ imageLineRegex = re.compile(r".*\| *image *= *([^|]*)") bracketImageRegex = re.compile(r"\[\[(File:[^|]*).*]]") imageNameRegex = re.compile(r".*\.(jpg|jpeg|png|gif|tiff|tif)", flags=re.IGNORECASE) cssImgCropRegex = re.compile(r"{{css image crop\|image *= *(.*)", flags=re.IGNORECASE) -# In testing, got about 360k image names print("Getting input page-ids") pageIds = getInputPageIds() -- cgit v1.2.3