From 20d69469a4c80a196de23625d0420487b0ed04a6 Mon Sep 17 00:00:00 2001 From: Terry Truong Date: Thu, 29 Dec 2022 16:17:39 +1100 Subject: Show event-count data on timeline Backend: Send event-count data to client in EventResponse instance Fix certain usages of gregorian calendar instead of julian Move HistDate, SCALES, etc, into cal.py Frontend: Make App update a unitCountMaps object using event-count data from server Make TimeLine show visual indication of unit counts Add showEventCounts option to store Update unit tests --- backend/hist_data/cal.py | 53 ++++++++++++++++++++++- backend/hist_data/gen_disp_data.py | 19 +-------- backend/histplorer.py | 84 +++++++++++++++++-------------------- backend/tests/test_cal.py | 13 +++++- backend/tests/test_gen_disp_data.py | 18 ++++---- backend/tests/test_histplorer.py | 26 +++++++++--- 6 files changed, 134 insertions(+), 79 deletions(-) (limited to 'backend') diff --git a/backend/hist_data/cal.py b/backend/hist_data/cal.py index c0e8772..3b65205 100644 --- a/backend/hist_data/cal.py +++ b/backend/hist_data/cal.py @@ -1,6 +1,6 @@ """ -Provides functions for converting between Julian calendar, Gregorian calendar, -and Julian day number values. Algorithms were obtained from +Provides date conversion functions, HistDate, and date scales. +Algorithms for converting between calendars and Julian day number values were obtained from https://en.wikipedia.org/wiki/Julian_day#Converting_Gregorian_calendar_date_to_Julian_Day_Number. """ @@ -69,3 +69,52 @@ def julianToGregorian(year: int, month: int, day: int) -> tuple[int, int, int]: def gregorianToJulian(year: int, month: int, day: int) -> tuple[int, int, int]: return jdnToJulian(gregorianToJdn(year, month, day)) + +MIN_CAL_YEAR = -4713 # Disallow within-year dates before this year +MONTH_SCALE = -1; +DAY_SCALE = -2; +SCALES: list[int] = [int(x) for x in [1e9, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 100, 10, 1, MONTH_SCALE, DAY_SCALE]]; +class HistDate: + """ + Represents a historical date + - 'year' may be negative (-1 means 1 BCE) + - 'month' and 'day' are at least 1, if given + - 'gcal' may be: + - True: Indicates a Gregorian calendar date + - False: Means the date should, for display, be converted to a Julian calendar date + - None: 'month' and 'day' are 1 (used for dates before the Julian period starting year 4713 BCE) + """ + def __init__(self, gcal: bool | None, year: int, month=1, day=1): + self.gcal = gcal + self.year = year + self.month = month + self.day = day + # Used in unit testing + def __eq__(self, other): + return isinstance(other, HistDate) and \ + (self.gcal, self.year, self.month, self.day) == (other.gcal, other.year, other.month, other.day) + def __repr__(self): + return str(self.__dict__) +def dbDateToHistDate(n: int, fmt: int, end=False) -> HistDate: + if fmt == 0: # year + if n >= MIN_CAL_YEAR: + return HistDate(True, n, 1, 1) + else: + return HistDate(None, n) + elif fmt == 1 or fmt == 3 and not end: # jdn for julian calendar + return HistDate(False, *jdnToJulian(n)) + else: # fmt == 2 or fmt == 3 and end + return HistDate(True, *jdnToGregorian(n)) +def dateToUnit(date: HistDate, scale: int) -> int: + if scale >= 1: + return date.year // scale + elif scale == MONTH_SCALE: + if date.gcal == False: + return julianToJdn(date.year, date.month, 1) + else: + return gregorianToJdn(date.year, date.month, 1) + else: # scale == DAY_SCALE + if date.gcal == False: + return julianToJdn(date.year, date.month, date.day) + else: + return gregorianToJdn(date.year, date.month, date.day) diff --git a/backend/hist_data/gen_disp_data.py b/backend/hist_data/gen_disp_data.py index e425efc..a81263f 100644 --- a/backend/hist_data/gen_disp_data.py +++ b/backend/hist_data/gen_disp_data.py @@ -10,13 +10,9 @@ parentDir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(parentDir) import sqlite3 -from cal import gregorianToJdn, jdnToGregorian +from cal import SCALES, dbDateToHistDate, dateToUnit -MONTH_SCALE = -1; -DAY_SCALE = -2; -SCALES: list[int] = [int(x) for x in [1e9, 1e8, 1e7, 1e6, 1e5, 1e4, 1e3, 100, 10, 1, MONTH_SCALE, DAY_SCALE]]; MAX_DISPLAYED_PER_UNIT = 4 -# DB_FILE = 'data.db' def genData(dbFile: str, scales: list[int], maxDisplayedPerUnit: int) -> None: @@ -36,18 +32,7 @@ def genData(dbFile: str, scales: list[int], maxDisplayedPerUnit: int) -> None: print(f'At iteration {iterNum}') # For each scale for scale in scales: - # Get unit - unit: int - if scale >= 1: - unit = (eventStart if fmt == 0 else jdnToGregorian(eventStart)[0]) // scale - elif scale == MONTH_SCALE: - if fmt == 0: - unit = gregorianToJdn(eventStart, 1, 1) - else: - year, month, day = jdnToGregorian(eventStart) - unit = eventStart if day == 1 else gregorianToJdn(year, month, 1) - else: # scale == DAY_SCALE - unit = eventStart if fmt != 0 else gregorianToJdn(eventStart, 1, 1) + unit = dateToUnit(dbDateToHistDate(eventStart, fmt), scale) # Update maps counts: list[int] if (scale, unit) in scaleUnitToCounts: diff --git a/backend/histplorer.py b/backend/histplorer.py index 72ef88e..edd675f 100755 --- a/backend/histplorer.py +++ b/backend/histplorer.py @@ -23,39 +23,17 @@ from typing import Iterable import sys, re import urllib.parse, sqlite3 import gzip, jsonpickle -from hist_data.cal import gregorianToJdn, jdnToGregorian, jdnToJulian +from hist_data.cal import gregorianToJdn, HistDate, dbDateToHistDate, dateToUnit DB_FILE = 'hist_data/data.db' MAX_REQ_EVENTS = 500 DEFAULT_REQ_EVENTS = 20 MAX_REQ_SUGGS = 50 DEFAULT_REQ_SUGGS = 5 -MIN_CAL_YEAR = -4713 # Disallow within-year dates before this year # Classes for objects sent as responses -class HistDate: - """ - Represents a historical date - - 'year' may be negative (-1 means 1 BCE) - - 'month' and 'day' are at least 1, if given - - 'gcal' may be: - - True: Indicates a Gregorian calendar date - - False: Means the date should, for display, be converted to a Julian calendar date - - None: 'month' and 'day' are 1 (used for dates before the Julian period starting year 4713 BCE) - """ - def __init__(self, gcal: bool | None, year: int, month=1, day=1): - self.gcal = gcal - self.year = year - self.month = month - self.day = day - # Used in unit testing - def __eq__(self, other): - return isinstance(other, HistDate) and \ - (self.gcal, self.year, self.month, self.day) == (other.gcal, other.year, other.month, other.day) - def __repr__(self): - return str(self.__dict__) class Event: - """ Used when responding to type=events requests """ + """ Represents an historical event """ def __init__( self, id: int, @@ -85,6 +63,17 @@ class Event: other.ctg, other.pop, other.imgId) def __repr__(self): return str(self.__dict__) +class EventResponse: + """ Used when responding to type=events requests """ + def __init__(self, events: list[Event], unitCounts: dict[int, int]): + self.events = events + self.unitCounts = unitCounts + # Used in unit testing + def __eq__(self, other): + return isinstance(other, EventResponse) and \ + (self.events, self.unitCounts) == (other.events, other.unitCounts) + def __repr__(self): + return str(self.__dict__) class ImgInfo: """ Represents an event's associated image """ def __init__(self, url: str, license: str, artist: str, credit: str): @@ -138,7 +127,7 @@ def application(environ: dict[str, str], start_response) -> Iterable[bytes]: headers.append(('Content-Length', str(len(data)))) start_response('200 OK', headers) return [data] -def handleReq(dbFile: str, environ: dict[str, str]) -> None | list[Event] | EventInfo | SuggResponse: +def handleReq(dbFile: str, environ: dict[str, str]) -> None | EventResponse | EventInfo | SuggResponse: """ Queries the database, and constructs a response object """ # Open db dbCon = sqlite3.connect(dbFile) @@ -169,7 +158,7 @@ def reqParamToHistDate(s: str): return HistDate(True, int(m.group(1)), int(m.group(2)), int(m.group(3))) # For type=events -def handleEventsReq(params: dict[str, str], dbCur: sqlite3.Cursor): +def handleEventsReq(params: dict[str, str], dbCur: sqlite3.Cursor) -> EventResponse | None: """ Generates a response for a type=events request """ # Get dates dateRange = params['range'] if 'range' in params else '.' @@ -208,9 +197,11 @@ def handleEventsReq(params: dict[str, str], dbCur: sqlite3.Cursor): print(f'INFO: Invalid results limit {resultLimit}', file=sys.stderr) return None # - return lookupEvents(start, end, scale, ctg, incl, resultLimit, dbCur) + events = lookupEvents(start, end, scale, ctg, incl, resultLimit, dbCur) + unitCounts = lookupUnitCounts(start, end, scale, dbCur) + return EventResponse(events, unitCounts) def lookupEvents(start: HistDate | None, end: HistDate | None, scale: int, ctg: str | None, - incl: int | None, resultLimit: int, dbCur: sqlite3.Cursor) -> list[Event] | None: + incl: int | None, resultLimit: int, dbCur: sqlite3.Cursor) -> list[Event]: """ Looks for events within a date range, in given scale, restricted by event category, an optional particular inclusion, and a result limit """ query = \ @@ -270,31 +261,34 @@ def lookupEvents(start: HistDate | None, end: HistDate | None, scale: int, ctg: # return results def eventEntryToResults( - row: tuple[int, str, int, int | None, int | None, int | None, int, str, int, int | None]) -> Event: + row: tuple[int, str, int, int | None, int | None, int | None, int, str, int, int]) -> Event: eventId, title, start, startUpper, end, endUpper, fmt, ctg, imageId, pop = row """ Helper for converting an 'events' db entry into an Event object """ # Convert dates dateVals: list[int | None] = [start, startUpper, end, endUpper] newDates: list[HistDate | None] = [None for n in dateVals] for i, n in enumerate(dateVals): - if n is None: - continue - elif fmt == 0: - if n >= MIN_CAL_YEAR: - newDates[i] = HistDate(True, n, 1, 1) - else: - newDates[i] = HistDate(None, n) - elif fmt == 1: - newDates[i] = HistDate(False, *jdnToJulian(n)) - elif fmt == 2: - newDates[i] = HistDate(True, *jdnToGregorian(n)) - elif fmt == 3: - if i in [0, 2]: - newDates[i] = HistDate(False, *jdnToJulian(n)) - else: - newDates[i] = HistDate(True, *jdnToGregorian(n)) + if n: + newDates[i] = dbDateToHistDate(n, fmt, i < 2) # return Event(eventId, title, newDates[0], newDates[1], newDates[2], newDates[3], ctg, imageId, pop) +def lookupUnitCounts( + start: HistDate | None, end: HistDate | None, scale: int, dbCur: sqlite3.Cursor) -> dict[int, int]: + # Build query + query = 'SELECT unit, count FROM dist WHERE scale = ?' + params = [scale] + if start: + query += ' AND unit >= ?' + params.append(dateToUnit(start, scale)) + if end: + query += ' AND unit <= ?' + params.append(dateToUnit(end, scale)) + query += ' ORDER BY unit ASC LIMIT ' + str(MAX_REQ_EVENTS) + # Get results + unitCounts: dict[int, int] = {} + for unit, count in dbCur.execute(query, params): + unitCounts[unit] = count + return unitCounts # For type=info def handleInfoReq(params: dict[str, str], dbCur: sqlite3.Cursor): diff --git a/backend/tests/test_cal.py b/backend/tests/test_cal.py index 7f2aa41..d5f2860 100644 --- a/backend/tests/test_cal.py +++ b/backend/tests/test_cal.py @@ -2,7 +2,8 @@ import unittest from hist_data.cal import \ gregorianToJdn, julianToJdn, jdnToGregorian, jdnToJulian, \ - julianToGregorian, gregorianToJulian + julianToGregorian, gregorianToJulian, \ + MONTH_SCALE, DAY_SCALE, HistDate, dbDateToHistDate, dateToUnit class TestCal(unittest.TestCase): def test_gregorian_to_jdn(self): @@ -27,3 +28,13 @@ class TestCal(unittest.TestCase): def test_julian_to_gregorian(self): self.assertEqual(julianToGregorian(2022, 9, 17), (2022, 9, 30)) self.assertEqual(julianToGregorian(1616, 4, 23), (1616, 5, 3)) + def test_db_to_hist_date(self): + self.assertEqual(dbDateToHistDate(2001, 0), HistDate(True, 2001, 1, 1)) + self.assertEqual(dbDateToHistDate(1721455, 1), HistDate(False, 1, 2, 1)) + self.assertEqual(dbDateToHistDate(1356438, 2), HistDate(True, -1000, 9, 13)) + self.assertEqual(dbDateToHistDate(2268942, 3, False), HistDate(False, 1500, 1, 10)) + self.assertEqual(dbDateToHistDate(2268933, 3, True), HistDate(True, 1500, 1, 10)) + def test_date_to_unit(self): + self.assertEqual(dateToUnit(HistDate(None, 1914, 1, 1), 10), 191) + self.assertEqual(dateToUnit(HistDate(True, 1500, 10, 5), MONTH_SCALE), 2269197) + self.assertEqual(dateToUnit(HistDate(False, 1500, 1, 10), DAY_SCALE), 2268942) diff --git a/backend/tests/test_gen_disp_data.py b/backend/tests/test_gen_disp_data.py index b806958..464405a 100644 --- a/backend/tests/test_gen_disp_data.py +++ b/backend/tests/test_gen_disp_data.py @@ -2,8 +2,8 @@ import unittest import tempfile, os from tests.common import createTestDbTable, readTestDbTable -from hist_data.gen_disp_data import genData, MONTH_SCALE, DAY_SCALE -from hist_data.cal import gregorianToJdn +from hist_data.gen_disp_data import genData +from hist_data.cal import gregorianToJdn, julianToJdn, MONTH_SCALE, DAY_SCALE class TestGenData(unittest.TestCase): def test_gen(self): @@ -17,11 +17,11 @@ class TestGenData(unittest.TestCase): 'INSERT INTO events VALUES (?, ?, ?, ?, ?, ?, ?, ?)', { (1, 'event one', 1900, None, None, None, 0, 'event'), - (2, 'event two', 2452594, None, 2455369, None, 3, 'human'), # 15/11/2002 to 21/06/2010 + (2, 'event two', 2452607, None, 2455369, None, 3, 'human'), # 15/11/2002 to 21/06/2010 (3, 'event three', 1900, None, 2000, None, 0, 'event'), (4, 'event four', 1901, None, 2000, 2010, 0, 'event'), - (5, 'event five', 2415294, None, None, None, 1, 'event'), # 01/10/1900 - (6, 'event six', 2415030, None, None, None, 1, 'event'), # 10/01/1900 + (5, 'event five', 2415307, None, None, None, 1, 'event'), # 01/10/1900 + (6, 'event six', 2415030, None, None, None, 2, 'event'), # 10/01/1900 } ) createTestDbTable( @@ -49,13 +49,13 @@ class TestGenData(unittest.TestCase): (1, 2002, 1), (MONTH_SCALE, gregorianToJdn(1900, 1, 1), 2), (MONTH_SCALE, gregorianToJdn(1901, 1, 1), 1), - (MONTH_SCALE, gregorianToJdn(1900, 10, 1), 1), - (MONTH_SCALE, gregorianToJdn(2002, 11, 1), 1), + (MONTH_SCALE, julianToJdn(1900, 10, 1), 1), + (MONTH_SCALE, julianToJdn(2002, 11, 1), 1), (DAY_SCALE, gregorianToJdn(1900, 1, 1), 1), (DAY_SCALE, gregorianToJdn(1900, 1, 10), 1), - (DAY_SCALE, gregorianToJdn(1900, 10, 1), 1), + (DAY_SCALE, julianToJdn(1900, 10, 1), 1), (DAY_SCALE, gregorianToJdn(1901, 1, 1), 1), - (DAY_SCALE, gregorianToJdn(2002, 11, 15), 1), + (DAY_SCALE, julianToJdn(2002, 11, 15), 1), } ) self.assertEqual( diff --git a/backend/tests/test_histplorer.py b/backend/tests/test_histplorer.py index a2b4623..be01a90 100644 --- a/backend/tests/test_histplorer.py +++ b/backend/tests/test_histplorer.py @@ -2,7 +2,7 @@ import unittest import tempfile, os from tests.common import createTestDbTable -from histplorer import handleReq, HistDate, Event, ImgInfo, EventInfo, SuggResponse +from histplorer import handleReq, HistDate, Event, EventResponse, ImgInfo, EventInfo, SuggResponse def initTestDb(dbFile: str) -> None: createTestDbTable( @@ -13,10 +13,24 @@ def initTestDb(dbFile: str) -> None: { (1, 'event one', 1900, None, None, None, 0, 'event'), (2, 'event two', 2452594, None, 2455369, None, 3, 'human'), # 2/11/2002 to 21/06/2010 - (3, 'event three', 2448175, 2451828, None, None, 2, 'discovery'), # 10/10/1990 to 10/10/2000 + (3, 'event three', 2448175, 2451828, None, None, 2, 'discovery'), # 10/10/1990 til 10/10/2000 (4, 'event four', 991206, None, 1721706, None, 1, 'event'), # 10/10/-2000 to 10/10/1 (5, 'event five', 2000, None, 2001, None, 0, 'event'), - (6, 'event six', 1500, None, 2000, None, 0, 'event'), + (6, 'event six', 1900, None, 2000, None, 0, 'event'), + } + ) + createTestDbTable( + dbFile, + 'CREATE TABLE dist (scale INT, unit INT, count INT, PRIMARY KEY (scale, unit))', + 'INSERT INTO dist VALUES (?, ?, ?)', + { + (1, -2000, 1), + (1, 1900, 2), + (1, 1990, 1), + (1, 2000, 1), + (1, 2001, 1), + (1, 2002, 1), + (10, 190, 2), } ) createTestDbTable( @@ -95,18 +109,20 @@ class TestHandleReq(unittest.TestCase): self.tempDir.cleanup() def test_events_req(self): response = handleReq(self.dbFile, {'QUERY_STRING': 'type=events&range=-1999.2002-11-1&scale=1&incl=3&limit=2'}) - self.assertEqual(response, [ + self.assertEqual(response.events, [ Event(5, 'event five', HistDate(True, 2000, 1, 1), None, HistDate(True, 2001, 1, 1), None, 'event', 50, 51), Event(3, 'event three', HistDate(True, 1990, 10, 10), HistDate(True, 2000, 10, 10), None, None, 'discovery', 30, 0), ]) + self.assertEqual(response.unitCounts, {1900: 2, 1990: 1, 2000: 1, 2001: 1, 2002: 1}) response = handleReq(self.dbFile, {'QUERY_STRING': 'type=events&range=.1999-11-27&scale=1&ctg=event'}) - self.assertEqual(response, [ + self.assertEqual(response.events, [ Event(4, 'event four', HistDate(False, -2000, 10, 10), None, HistDate(False, 1, 10, 10), None, 'event', 20, 1000), Event(1, 'event one', HistDate(True, 1900, 1, 1), None, None, None, 'event', 10, 11), ]) + self.assertEqual(response.unitCounts, {-2000: 1, 1900: 2, 1990: 1}) def test_info_req(self): response = handleReq(self.dbFile, {'QUERY_STRING': 'type=info&event=3'}) self.assertEqual(response, -- cgit v1.2.3