diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | backend/hist_data/README.md | 44 | ||||
| -rwxr-xr-x | backend/hist_data/gen_events_data.py (renamed from backend/hist_data/wikidata/gen_events_data.py) | 12 | ||||
| -rw-r--r-- | backend/hist_data/wikidata/README.md | 19 | ||||
| -rw-r--r-- | backend/tests/test_gen_events_data.py (renamed from backend/tests/wikidata/test_gen_events_data.py) | 2 |
5 files changed, 54 insertions, 26 deletions
@@ -4,5 +4,8 @@ __pycache__ .venv +/backend/hist_data/data.db /backend/hist_data/wikidata/events.db /backend/hist_data/wikidata/*.json.bz2 +/backend/hist_data/enwiki/*.db +/backend/hist_data/enwiki/pageviews/ diff --git a/backend/hist_data/README.md b/backend/hist_data/README.md new file mode 100644 index 0000000..c55549e --- /dev/null +++ b/backend/hist_data/README.md @@ -0,0 +1,44 @@ +This directory holds files used to generate the history database data.db. + +# Database Tables +- `events` <br> + Format: + `id INT PRIMARY KEY, title TEXT UNIQUE, start INT, start_upper INT, end INT, end_upper INT, fmt INT, ctg TEXT` + <br> + Each row has a Wikidata ID, Wikipedia title, start and end dates, and an event category. + - `start*` and `end*` specify start and end dates. + `start_upper`, `end`, and `end_upper`, are optional. + If `start_upper` is present, it and `start` denote an uncertain range of start times. + Similarly for 'end' and 'end_upper'. + - `fmt` indicates format info for `start`, `start_upper`, `end`, and `end_upper`. + - If 1, they denote a Julian date (with 0.5 removed to align with midnight). + This allows simple comparison of events with day-level precision, but only goes back to 4713 BCE. + - If 2, same as 1, but dates are preferably displayed using the Gregorian calendar, not the Julian calendar. + For example, William Shakespeare's birth appears 'preferably Julian', but Samuel Johnson's does not. + - If 3, same as 1, but 'end' and 'end_upper' are 'preferably Gregorian'. + For example, Galileo Galilei's birth date appears 'preferably Julian', but his death date does not. + - If 0, they denote a number of years CE (if positive) or BCE (if negative). + +# Generating the Database + +## Generate Event Data +1. Obtain a Wikidata JSON dump in wikidata/, as specified in it's README. +1. Run `gen_events_data.py`, which creates `data.db`, and adds the `events` table. + +## Generate Description Data +1. Obtain an enwiki dump in enwiki/, as specified in the README. +1. In enwiki/, run `gen_dump_index.db.py`, which generates a database for indexing the dump. +1. In enwiki/, run `gen_desc_data.py`, which extracts page descriptions into a database. +1. Run + +## Generate Popularity Data +1. Obtain 'page view files' in enwiki/, as specified in it's README. +1. Run + +## Generate Image Data and Popularity Data +1. In enwiki/, run `gen_img_data.py` which looks at pages in the dump that match entries in `events`, + looks for infobox image names, and stores them in an image database. +1. In enwiki/, run `download_img_license_info.py`, which downloads licensing info for found + images, and adds them to the image database. +1. In enwiki/, run `download_imgs.py`, which downloads images into enwiki/imgs/. +1. Run diff --git a/backend/hist_data/wikidata/gen_events_data.py b/backend/hist_data/gen_events_data.py index f4766f0..1f990d0 100755 --- a/backend/hist_data/wikidata/gen_events_data.py +++ b/backend/hist_data/gen_events_data.py @@ -64,10 +64,10 @@ import bz2, json, sqlite3 import multiprocessing, indexed_bzip2, pickle, tempfile from jdcal import gcal2jd, jcal2jd -WIKIDATA_FILE = 'latest-all.json.bz2' +WIKIDATA_FILE = os.path.join('wikidata', 'latest-all.json.bz2') DUMP_YEAR = 2022 # Used for converting 'age' values into dates -OFFSETS_FILE = 'offsets.dat' -DB_FILE = 'events.db' +OFFSETS_FILE = os.path.join('wikidata', 'offsets.dat') +DB_FILE = 'data.db' N_PROCS = 6 # For getting Wikidata entity IDs @@ -164,10 +164,8 @@ def genData(wikidataFile: str, offsetsFile: str, dbFile: str, nProcs: int) -> No print('Writing to db') dbCon = sqlite3.connect(dbFile) dbCur = dbCon.cursor() - dbCur.execute('CREATE TABLE events (' \ - 'id INT NOT NULL PRIMARY KEY, title TEXT NOT NULL UNIQUE,' \ - 'start INT NOT NULL, start_upper INT, end INT, end_upper INT,' \ - 'fmt INT, ctg TEXT NOT NULL)') + dbCur.execute('CREATE TABLE events (id INT PRIMARY KEY, title TEXT UNIQUE, ' \ + 'start INT, start_upper INT, end INT, end_upper INT, fmt INT, ctg TEXT)') if nProcs == 1: with bz2.open(wikidataFile, mode='rb') as file: for lineNum, line in enumerate(file, 1): diff --git a/backend/hist_data/wikidata/README.md b/backend/hist_data/wikidata/README.md index 35dad34..d5b2c5e 100644 --- a/backend/hist_data/wikidata/README.md +++ b/backend/hist_data/wikidata/README.md @@ -6,23 +6,6 @@ This directory holds files obtained/derived from [Wikidata](https://www.wikidata Format info can be found at <https://doc.wikimedia.org/Wikibase/master/php/md_docs_topics_json.html>. # Other Files -- `gen_events_data.py` <br> - Used to generate a database holding event information from the dump. - `offsets.dat` <br> Holds bzip2 block offsets for the dump. Generated and used by - gen_events_data.py for parallel processing of the dump. -- `events.db` <br> - Generated by `gen_events_data.py`. <br> - Has one table `events`: - - Columns: `id INT, title TEXT, start INT, start_upper INT, end INT, end_upper INT, fmt INT, ctg TEXT` - - Contains a Wikidata ID, Wikipedia title, start and end dates, and an event category. - - If `start_upper` is not NULL, `start` and `start_upper` denote an uncertain range of start times. - And similarly for 'end' and 'end_upper'. - - `fmt` indicates format info for `start`, `start_upper`, `end`, and `end_upper`. - - If 1, they denote a Julian date (with 0.5 removed to align with midnight). - This allows simple comparison of events with day-level precision, but only goes back to 4713 BCE. - - If 2, same as 1, but dates are preferably displayed using the Gregorian calendar, not the Julian calendar. - For example, William Shakespeare's birth appears 'preferably Julian', but Samuel Johnson's does not. - - If 3, same as 1, but 'end' and 'end_upper' are 'preferably Gregorian'. - For example, Galileo Galilei's birth date appears 'preferably Julian', but his death date does not. - - If 0, they denote a number of years CE (if positive) or BCE (if negative). + `../gen_events_data.py` for parallel processing of the dump. diff --git a/backend/tests/wikidata/test_gen_events_data.py b/backend/tests/test_gen_events_data.py index faa19c9..37b24a3 100644 --- a/backend/tests/wikidata/test_gen_events_data.py +++ b/backend/tests/test_gen_events_data.py @@ -2,7 +2,7 @@ import unittest import tempfile, os, json, bz2, pickle, indexed_bzip2 from tests.common import readTestDbTable -from hist_data.wikidata.gen_events_data import genData +from hist_data.gen_events_data import genData def runGenData(wikiItemArray: str, preGenOffsets: bool, nProcs: int): """ Sets up wikidata file to be read by genData(), runs it, and returns the output database's contents. |