Skip to content

Commit

Permalink
#196 Add raw_json option and bump version from 0.13.0 to 0.14.0
Browse files Browse the repository at this point in the history
  • Loading branch information
alexhad6 committed Jun 18, 2024
1 parent e76d3ea commit 5172f38
Show file tree
Hide file tree
Showing 9 changed files with 93 additions and 73 deletions.
17 changes: 16 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,20 @@ project adheres to clauses 1–8 of [Semantic Versioning](https://semver.org/spe

## [Unreleased]

## [0.14.0] (Jun 18 2024)

### Changed

- The option `decode_json` in `ParamDB.load()` was replaced with `raw_json`, which
allows loading the raw JSON string from the database.
- The order of data for `ParamData` objects in the underlying JSON representation was
changed; see `ParamDB.load()` for the new order.

### Removed

- `ParamDBKey.WRAPPER` was removed in favor of encoding these values using
`ParamDBKey.PARAM` with a class name of `None`.

## [0.13.0] (Jun 14 2024)

### Added
Expand Down Expand Up @@ -191,7 +205,8 @@ project adheres to clauses 1–8 of [Semantic Versioning](https://semver.org/spe
- Database class `ParamDB` to store parameters in a SQLite file
- Ability to retrieve the commit history as `CommitEntry` objects

[unreleased]: https://github.com/PainterQubits/paramdb/compare/v0.13.0...develop
[unreleased]: https://github.com/PainterQubits/paramdb/compare/v0.14.0...develop
[0.14.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.14.0
[0.13.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.13.0
[0.12.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.12.0
[0.11.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.11.0
Expand Down
4 changes: 2 additions & 2 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ authors:
- family-names: "Hadley"
given-names: "Alex"
title: "ParamDB"
version: 0.13.0
date-released: 2024-06-14
version: 0.14.0
date-released: 2024-06-18
url: "https://github.com/PainterQubits/paramdb"
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
project = "ParamDB"
copyright = "2023–2024, California Institute of Technology"
author = "Alex Hadley"
release = "0.13.0"
release = "0.14.0"

# General configuration
extensions = [
Expand Down
76 changes: 38 additions & 38 deletions paramdb/_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ class ParamDBKey:
"""Key for ordinary lists."""
DICT = "d"
"""Key for ordinary dictionaries."""
WRAPPER = "w"
PARAM = "p"
"""
Key for non-:py:class:`ParamData` children of :py:class:`ParamData` objects, since
they are wrapped with additional metadata, such as a last updated time.
Key for :py:class:`ParamData` objects.
The JSON object should either include a parameter class name, or be None if wrapping
a non-:py:class:`ParamData` with parameter metadata (e.g. a last updated time).
"""
PARAM = "p"
"""Key for :py:class:`ParamData` objects."""


def _compress(text: str) -> bytes:
Expand Down Expand Up @@ -80,13 +80,12 @@ def _encode_json(obj: Any) -> Any:
{key: _encode_json(value) for key, value in obj.items()},
]
if isinstance(obj, ParamData):
timestamp_and_json = [
obj.last_updated.timestamp(),
return [
ParamDBKey.PARAM,
_encode_json(obj.to_json()),
None if isinstance(obj, _ParamWrapper) else type(obj).__name__,
obj.last_updated.timestamp(),
]
if isinstance(obj, _ParamWrapper):
return [ParamDBKey.WRAPPER, *timestamp_and_json]
return [ParamDBKey.PARAM, type(obj).__name__, *timestamp_and_json]
raise TypeError(
f"'{type(obj).__name__}' object {repr(obj)} is not JSON serializable, so the"
" commit failed"
Expand All @@ -105,13 +104,13 @@ def _decode_json(json_data: Any) -> Any:
return [_decode_json(item) for item in data[0]]
if key == ParamDBKey.DICT:
return {key: _decode_json(value) for key, value in data[0].items()}
if key == ParamDBKey.WRAPPER:
return _ParamWrapper.from_json(data[0], _decode_json(data[1]))
if key == ParamDBKey.PARAM:
class_name = data[0]
param_class = get_param_class(class_name)
json_data, class_name, timestamp = data
param_class = (
_ParamWrapper if class_name is None else get_param_class(class_name)
)
if param_class is not None:
return param_class.from_json(data[1], _decode_json(data[2]))
return param_class.from_json(_decode_json(json_data), timestamp)
raise ValueError(
f"ParamData class '{class_name}' is not known to ParamDB, so the load"
" failed"
Expand All @@ -122,16 +121,19 @@ def _decode_json(json_data: Any) -> Any:
def _encode(obj: Any) -> bytes:
"""Encode the given object into bytes that will be stored in the database."""
# pylint: disable=no-member
return _compress(json.dumps(_encode_json(obj)))
return _compress(json.dumps(_encode_json(obj), separators=(",", ":")))


def _decode(data: bytes, decode_json: bool) -> Any:
def _decode(data: bytes, raw_json: bool) -> Any:
"""
Decode an object from the given data from the database. Classes will be loaded in
if ``load_classes`` is True; otherwise, classes will be loaded as dictionaries.
Decode an object from the given data from the database.
If ``raw_json`` is True, the raw JSON string will from the database will be
returned; otherwise, the JSON data will be parsed and decoded into the corresponding
classes.
"""
json_data = json.loads(_decompress(data))
return _decode_json(json_data) if decode_json else json_data
json_str = _decompress(data)
return json_str if raw_json else _decode_json(json.loads(json_str))


class _Base(MappedAsDataclass, DeclarativeBase):
Expand Down Expand Up @@ -283,24 +285,23 @@ def num_commits(self) -> int:

@overload
def load(
self, commit_id: int | None = None, *, decode_json: Literal[True] = True
self, commit_id: int | None = None, *, raw_json: Literal[False] = False
) -> DataT: ...

@overload
def load(
self, commit_id: int | None = None, *, decode_json: Literal[False]
) -> Any: ...
def load(self, commit_id: int | None = None, *, raw_json: Literal[True]) -> str: ...

def load(self, commit_id: int | None = None, *, decode_json: bool = True) -> Any:
def load(self, commit_id: int | None = None, *, raw_json: bool = False) -> Any:
"""
Load and return data from the database. If a commit ID is given, load from that
commit; otherwise, load from the most recent commit. Raise an ``IndexError`` if
the specified commit does not exist. Note that commit IDs begin at 1.
By default, objects are reconstructed, which requires the relevant parameter
data classes to be defined in the current program. However, if ``decode_json``
is False, the encoded JSON data is loaded directly from the database. The format
of the encoded data is as follows (see :py:class:`ParamDBKey` for key codes)::
data classes to be defined in the current program. However, if ``raw_json``
is True, the JSON data is returned directly from the database as a string.
The format of the JSON data is as follows (see :py:class:`ParamDBKey` for key
codes)::
json_data:
| int
Expand All @@ -312,15 +313,14 @@ def load(self, commit_id: int | None = None, *, decode_json: bool = True) -> Any
| [ParamDBKey.QUANTITY, float<value>, str<unit>]
| [ParamDBKey.LIST, [json_data<item>, ...]]
| [ParamDBKey.DICT, {str<key>: json_data<value>, ...}]
| [ParamDBKey.WRAPPED, float<timestamp>, json_data<data>]
| [ParamDBKey.PARAM, str<class_name>, float<timestamp>, json_data<data>]
"""
| [ParamDBKey.PARAM, json_data<data>, str<class_name> | None, float<timestamp>]
""" # noqa: E501
select_stmt = self._select_commit(select(_Snapshot.data), commit_id)
with self._Session() as session:
data = session.scalar(select_stmt)
if data is None:
raise self._index_error(commit_id)
return _decode(data, decode_json)
return _decode(data, raw_json)

def load_commit_entry(self, commit_id: int | None = None) -> CommitEntry:
"""
Expand Down Expand Up @@ -358,7 +358,7 @@ def commit_history_with_data(
start: int | None = None,
end: int | None = None,
*,
decode_json: Literal[True] = True,
raw_json: Literal[False] = False,
) -> list[CommitEntryWithData[DataT]]: ...

@overload
Expand All @@ -367,15 +367,15 @@ def commit_history_with_data(
start: int | None = None,
end: int | None = None,
*,
decode_json: Literal[False],
) -> list[CommitEntryWithData[Any]]: ...
raw_json: Literal[True],
) -> list[CommitEntryWithData[str]]: ...

def commit_history_with_data(
self,
start: int | None = None,
end: int | None = None,
*,
decode_json: bool = True,
raw_json: bool = False,
) -> list[CommitEntryWithData[Any]]:
"""
Retrieve the commit history with data as a list of
Expand All @@ -392,7 +392,7 @@ def commit_history_with_data(
snapshot.id,
snapshot.message,
snapshot.timestamp,
_decode(snapshot.data, decode_json),
_decode(snapshot.data, raw_json),
)
for snapshot in snapshots
]
Expand Down
2 changes: 1 addition & 1 deletion paramdb/_param_data/_param_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def _init_from_json(self, json_data: Any) -> None:
self.__init__(json_data) # type: ignore[misc]

@classmethod
def from_json(cls, last_updated_timestamp: float, json_data: list[Any]) -> Self:
def from_json(cls, json_data: list[Any], last_updated_timestamp: float) -> Self:
"""
Construct a parameter data object from the given last updated timestamp and JSON
data originally constructed by :py:meth:`to_json`.
Expand Down
34 changes: 17 additions & 17 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "paramdb"
version = "0.13.0"
version = "0.14.0"
description = "Python package for storing and retrieving experiment parameters."
authors = ["Alex Hadley <[email protected]>"]
license = "BSD-3-Clause"
Expand All @@ -29,7 +29,7 @@ pydantic = ["pydantic", "eval-type-backport"]

[tool.poetry.group.dev.dependencies]
mypy = "^1.10.0"
flake8 = "^7.0.0"
flake8 = "^7.1.0"
pylint = "^3.2.3"
black = "^24.4.2"
pytest = "^8.2.2"
Expand Down
4 changes: 2 additions & 2 deletions tests/_param_data/test_param_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,10 @@ def test_child_does_not_change(param_data: ParamData[Any]) -> None:

def test_to_and_from_json(param_data: ParamData[Any]) -> None:
"""Parameter data can be converted to and from JSON data."""
timestamp = param_data.last_updated.timestamp()
json_data = param_data.to_json()
timestamp = param_data.last_updated.timestamp()
with capture_start_end_times():
param_data_from_json = param_data.from_json(timestamp, json_data)
param_data_from_json = param_data.from_json(json_data, timestamp)
assert param_data_from_json == param_data
assert param_data_from_json.last_updated == param_data.last_updated

Expand Down
Loading

0 comments on commit 5172f38

Please sign in to comment.