diff options
| author | Terry Truong <terry06890@gmail.com> | 2023-01-14 17:06:55 +1100 |
|---|---|---|
| committer | Terry Truong <terry06890@gmail.com> | 2023-01-14 17:06:55 +1100 |
| commit | a8ea534f13cc23dfab25c6d856f9d9833dd000ba (patch) | |
| tree | 2bacfd988d29a7359d46a7826b61fb438d36f926 /backend/hist_data | |
| parent | b5cf828824bcc6a9518ad66e91d91a2eae2ed5e1 (diff) | |
Add 'imgonly' server query param
Remove script for deleting events in db without images
Diffstat (limited to 'backend/hist_data')
| -rw-r--r-- | backend/hist_data/README.md | 3 | ||||
| -rwxr-xr-x | backend/hist_data/reduce_event_data.py | 64 |
2 files changed, 0 insertions, 67 deletions
diff --git a/backend/hist_data/README.md b/backend/hist_data/README.md index 2a9475a..c947aa6 100644 --- a/backend/hist_data/README.md +++ b/backend/hist_data/README.md @@ -84,6 +84,3 @@ Some of the scripts use third-party packages: 1. Additional events can be described in `picked/events.json`, with images for them put in `picked` (see the README for details). 1. Can run `gen_picked_data.py` to add those described events to the database. - -## Remove Events Without Images/Descs -1. Run `reduce_event_data.py` to remove data for events that have no image. diff --git a/backend/hist_data/reduce_event_data.py b/backend/hist_data/reduce_event_data.py deleted file mode 100755 index 5801f4d..0000000 --- a/backend/hist_data/reduce_event_data.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python3 - -""" -Delete events from the database that have no image -""" - -# Code used in unit testing (for resolving imports of modules within this directory) -import os, sys -parentDir = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(parentDir) -# Standard imports -import argparse -import sqlite3 -# Local imports -from cal import SCALES, dbDateToHistDate, dateToUnit - -DB_FILE = 'data.db' - -def reduceData(dbFile: str, scales: list[int]) -> None: - dbCon = sqlite3.connect(dbFile) - dbCur = dbCon.cursor() - # - print('Getting events to delete') - eventsToDel: list[int] = [] - scaleUnitToDelCount: dict[tuple[int, int], int] = {} # Stores counts to subtract from entries in 'dist' - query = 'SELECT events.id, events.start, events.fmt FROM events' \ - ' LEFT JOIN event_imgs ON events.id = event_imgs.id WHERE event_imgs.id IS NULL' - iterNum = 0 - for (eventId, start, fmt) in dbCur.execute(query): - if iterNum % 1e5 == 0: - print(f'At iteration {iterNum}') - # - eventsToDel.append(eventId) - date = dbDateToHistDate(start, fmt) - for scale in scales: - unit = dateToUnit(date, scale) - if (scale, unit) not in scaleUnitToDelCount: - scaleUnitToDelCount[(scale, unit)] = 1 - else: - scaleUnitToDelCount[(scale, unit)] += 1 - print(f'Found {len(eventsToDel)}') - # - print('Deleting events') - iterNum = 0 - for eventId in eventsToDel: - iterNum += 1 - if iterNum % 1e5 == 0: - print(f'At iteration {iterNum}') - # - dbCur.execute('DELETE FROM events WHERE id = ?', (eventId,)) - dbCur.execute('DELETE FROM pop WHERE id = ?', (eventId,)) - dbCur.execute('DELETE FROM event_disp WHERE id = ?', (eventId,)) - for (scale, unit), delCount in scaleUnitToDelCount.items(): - dbCur.execute('UPDATE dist SET count = count - ? WHERE scale = ? AND unit = ?', (delCount, scale, unit)) - dbCur.execute('DELETE FROM dist WHERE count < 1') - # - dbCon.commit() - dbCon.close() - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - args = parser.parse_args() - # - reduceData(DB_FILE, SCALES) |
