diff options
| author | Terry Truong <terry06890@gmail.com> | 2022-10-02 12:23:19 +1100 |
|---|---|---|
| committer | Terry Truong <terry06890@gmail.com> | 2022-10-02 12:23:19 +1100 |
| commit | d70b96295d768aa0c80bf66639ad7a56bdef92a8 (patch) | |
| tree | 3f960ad83e4158fff1c0931d838033392a3391ec /backend/hist_data/enwiki/gen_pageview_data.py | |
| parent | 1b4fc8667714ef4ce9f326bd14f795fc2417ecb9 (diff) | |
Add gen_pop_data.py
Diffstat (limited to 'backend/hist_data/enwiki/gen_pageview_data.py')
| -rwxr-xr-x | backend/hist_data/enwiki/gen_pageview_data.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/backend/hist_data/enwiki/gen_pageview_data.py b/backend/hist_data/enwiki/gen_pageview_data.py index b37a107..90ec925 100755 --- a/backend/hist_data/enwiki/gen_pageview_data.py +++ b/backend/hist_data/enwiki/gen_pageview_data.py @@ -5,7 +5,7 @@ Reads through wikimedia files containing pageview counts, computes average counts, and adds them to a database """ -# Took about 15min per file (each had about 180e6 lines) +# Took about 10min per file (each had about 180e6 lines) import sys, os, glob, math, re from collections import defaultdict @@ -42,6 +42,7 @@ def genData(pageviewFiles: list[str], dumpIndexDb: str, dbFile: str) -> None: if namespaceRegex.match(title) is not None: continue # Update map + title = title.replace('_', ' ') titleToViews[title] += viewCount print(f'Found {len(titleToViews)} titles') # |
