Skip to content

Commit

Permalink
Use a single key to store deltas
Browse files Browse the repository at this point in the history
To improve the distribution of the stored deltas, we removed the deep
key value maps and use a custom key to encode the `context` and
`timestamp` of a stored delta. Thus, improving the amount of data
that needs to be loaded for each operation.

Added additional end-to-end tests and benchmarks to validate the
behavior.
  • Loading branch information
aleics committed Jan 22, 2025
1 parent d244b89 commit 3e7164a
Show file tree
Hide file tree
Showing 7 changed files with 397 additions and 96 deletions.
33 changes: 33 additions & 0 deletions benches/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,36 @@ fn bench_apply_deltas(b: &mut Bencher) {
});
});
}

#[bench]
fn bench_apply_deltas_with_multiple_dates(b: &mut Bencher) {
let mut deltas = Vec::new();

deltas.extend(decrease_score_deltas(&PLAYERS, COUNT));
deltas.extend(switch_sports_deltas(&PLAYERS, COUNT));

let mut date: Date = *DATE;

tokio_test::block_on(async {
for delta_chunk in deltas.chunks(10) {
ENGINE
.store_deltas(&NAME, &DeltaScope::date(date), delta_chunk)
.await
.unwrap();

date = date.next_day().unwrap();
}
});

b.iter(move || {
tokio_test::block_on(async {
let scope = DeltaScope::date(date.next_day().unwrap());

let query = QueryExecution::new()
.with_scope(scope)
.with_pagination(*PAGINATION);

ENGINE.query(&NAME, query).await.unwrap();
});
});
}
46 changes: 39 additions & 7 deletions examples/simple.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use delta_search::data::FieldValue;
use delta_search::fixtures::{
create_players_storage, cristiano_ronaldo, david, lionel_messi, michael_jordan, roger, Sport,
SwitchSportsDelta,
create_players_storage, cristiano_ronaldo, david, lionel_messi, michael_jordan, roger,
DecreaseScoreDelta, Sport, SwitchSportsDelta,
};
use delta_search::query::{
CompositeFilter, DeltaScope, OptionsQueryExecution, QueryExecution, Sort, SortDirection,
Expand Down Expand Up @@ -50,7 +50,7 @@ async fn main() {
.with_sort(Sort::new("score").with_direction(SortDirection::DESC));
let players = engine.query(name, query).await;

println!("Basketball players sorted by score: {:?}", players);
println!("Basketball players sorted by score:\n{:?}\n", players);

let players = engine
.query(
Expand All @@ -65,7 +65,7 @@ async fn main() {
)
.await;

println!("Players born in the 80s: {:?}", players);
println!("Players born in the 80s:\n{:?}\n", players);

let switch_sports = vec![
SwitchSportsDelta::create(michael_jordan_id, Sport::Basketball, Sport::Football),
Expand All @@ -92,8 +92,37 @@ async fn main() {
let players = engine.query(name, query).await;

println!(
"Basketball players sorted by score after switching sports in 2023: {:?}",
players
"Basketball players sorted by score after switching sports in 2023:\n{:?}\n",
players.unwrap()
);

let lower_scores = vec![
DecreaseScoreDelta::create(michael_jordan_id, 10.0),
DecreaseScoreDelta::create(lionel_messi_id, 9.0),
];

let delta_scope = DeltaScope::context(
0,
Date::from_calendar_date(2023, Month::January, 1).unwrap(),
);

engine
.store_deltas(name, &delta_scope, &lower_scores)
.await
.unwrap();

let query = QueryExecution::new()
.with_sort(Sort::new("score").with_direction(SortDirection::DESC))
.with_scope(DeltaScope::context(
0,
Date::from_calendar_date(2024, Month::January, 1).unwrap(),
));

let players = engine.query(name, query).await;

println!(
"Players sorted by score after decreasing their score by 1:\n{:?}\n",
players.unwrap()
);

engine.remove(name, &david_id).await.unwrap();
Expand All @@ -108,5 +137,8 @@ async fn main() {
)
.await;

println!("Players playing basketball after deletion: {:?}", players);
println!(
"Players playing basketball after deletion:\n{:?}\n",
players.unwrap()
);
}
2 changes: 1 addition & 1 deletion src/fixtures.rs
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ impl Player {
pub struct DecreaseScoreDelta;

impl DecreaseScoreDelta {
pub(crate) fn create(id: DataItemId, score: f64) -> DeltaChange {
pub fn create(id: DataItemId, score: f64) -> DeltaChange {
DeltaChange::new(
id,
"score".to_string(),
Expand Down
2 changes: 1 addition & 1 deletion src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ fn parse_date(string: &str) -> Result<Date, time::error::Parse> {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct DeltaScopeInput {
context: Option<u64>,
context: Option<u32>,
date: String,
}

Expand Down
6 changes: 3 additions & 3 deletions src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@ impl FilterOption {

#[derive(Debug, PartialEq, Deserialize)]
pub struct DeltaScope {
pub(crate) context: Option<u64>,
pub(crate) context: Option<u32>,
pub(crate) date: Date,
}

impl DeltaScope {
pub fn new(context: Option<u64>, date: Date) -> Self {
pub fn new(context: Option<u32>, date: Date) -> Self {
Self { context, date }
}

Expand All @@ -41,7 +41,7 @@ impl DeltaScope {
}
}

pub fn context(context: u64, date: Date) -> Self {
pub fn context(context: u32, date: Date) -> Self {
DeltaScope {
context: Some(context),
date,
Expand Down
Loading

0 comments on commit 3e7164a

Please sign in to comment.