aboutsummaryrefslogtreecommitdiff
path: root/backend/hist_data
diff options
context:
space:
mode:
authorTerry Truong <terry06890@gmail.com>2023-01-14 17:06:55 +1100
committerTerry Truong <terry06890@gmail.com>2023-01-14 17:06:55 +1100
commita8ea534f13cc23dfab25c6d856f9d9833dd000ba (patch)
tree2bacfd988d29a7359d46a7826b61fb438d36f926 /backend/hist_data
parentb5cf828824bcc6a9518ad66e91d91a2eae2ed5e1 (diff)
Add 'imgonly' server query param
Remove script for deleting events in db without images
Diffstat (limited to 'backend/hist_data')
-rw-r--r--backend/hist_data/README.md3
-rwxr-xr-xbackend/hist_data/reduce_event_data.py64
2 files changed, 0 insertions, 67 deletions
diff --git a/backend/hist_data/README.md b/backend/hist_data/README.md
index 2a9475a..c947aa6 100644
--- a/backend/hist_data/README.md
+++ b/backend/hist_data/README.md
@@ -84,6 +84,3 @@ Some of the scripts use third-party packages:
1. Additional events can be described in `picked/events.json`, with images for them put
in `picked` (see the README for details).
1. Can run `gen_picked_data.py` to add those described events to the database.
-
-## Remove Events Without Images/Descs
-1. Run `reduce_event_data.py` to remove data for events that have no image.
diff --git a/backend/hist_data/reduce_event_data.py b/backend/hist_data/reduce_event_data.py
deleted file mode 100755
index 5801f4d..0000000
--- a/backend/hist_data/reduce_event_data.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/python3
-
-"""
-Delete events from the database that have no image
-"""
-
-# Code used in unit testing (for resolving imports of modules within this directory)
-import os, sys
-parentDir = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(parentDir)
-# Standard imports
-import argparse
-import sqlite3
-# Local imports
-from cal import SCALES, dbDateToHistDate, dateToUnit
-
-DB_FILE = 'data.db'
-
-def reduceData(dbFile: str, scales: list[int]) -> None:
- dbCon = sqlite3.connect(dbFile)
- dbCur = dbCon.cursor()
- #
- print('Getting events to delete')
- eventsToDel: list[int] = []
- scaleUnitToDelCount: dict[tuple[int, int], int] = {} # Stores counts to subtract from entries in 'dist'
- query = 'SELECT events.id, events.start, events.fmt FROM events' \
- ' LEFT JOIN event_imgs ON events.id = event_imgs.id WHERE event_imgs.id IS NULL'
- iterNum = 0
- for (eventId, start, fmt) in dbCur.execute(query):
- if iterNum % 1e5 == 0:
- print(f'At iteration {iterNum}')
- #
- eventsToDel.append(eventId)
- date = dbDateToHistDate(start, fmt)
- for scale in scales:
- unit = dateToUnit(date, scale)
- if (scale, unit) not in scaleUnitToDelCount:
- scaleUnitToDelCount[(scale, unit)] = 1
- else:
- scaleUnitToDelCount[(scale, unit)] += 1
- print(f'Found {len(eventsToDel)}')
- #
- print('Deleting events')
- iterNum = 0
- for eventId in eventsToDel:
- iterNum += 1
- if iterNum % 1e5 == 0:
- print(f'At iteration {iterNum}')
- #
- dbCur.execute('DELETE FROM events WHERE id = ?', (eventId,))
- dbCur.execute('DELETE FROM pop WHERE id = ?', (eventId,))
- dbCur.execute('DELETE FROM event_disp WHERE id = ?', (eventId,))
- for (scale, unit), delCount in scaleUnitToDelCount.items():
- dbCur.execute('UPDATE dist SET count = count - ? WHERE scale = ? AND unit = ?', (delCount, scale, unit))
- dbCur.execute('DELETE FROM dist WHERE count < 1')
- #
- dbCon.commit()
- dbCon.close()
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- args = parser.parse_args()
- #
- reduceData(DB_FILE, SCALES)