diff --git a/src/data.rs b/src/data.rs index b0eb7f7..b254f28 100644 --- a/src/data.rs +++ b/src/data.rs @@ -791,6 +791,26 @@ impl GetUnitType for ExerciseManifest { } } +/// An exercise score along with additional heuristics for filtering exercises into the final batch. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ExerciseScore { + /// The value of the mastery score of the exercise computed from all previous trials, as a value + /// between 0.0 and 5.0. + #[serde(default)] + pub value: f32, + + /// A value between 0 and 1 representing the urgency to review the exercise. + #[serde(default)] + pub urgency: f32, + + /// A value representing the trend at which the exercise's score is changing. A positive value + /// means the score is increasing, a negative value means the score is decreasing, and a value + /// close to zero means the score is stable. A value of None means there is not enough data to + /// compute a reliable velocity. + #[serde(default)] + pub velocity: Option, +} + /// The score at which fractional selection reaches 100% of lesson candidates. pub const FULL_CANDIDATES_SCORE: f32 = 4.0; diff --git a/src/exercise_scorer.rs b/src/exercise_scorer.rs index b6b18de..41d3d6c 100644 --- a/src/exercise_scorer.rs +++ b/src/exercise_scorer.rs @@ -9,7 +9,7 @@ use anyhow::{Result, anyhow}; -use crate::data::{ExerciseDelta, ExerciseTrial, ExerciseType}; +use crate::data::{ExerciseDelta, ExerciseScore, ExerciseTrial, ExerciseType}; /// A trait exposing a function to score an exercise based on the results of previous trials. pub trait ExerciseScorer { @@ -22,12 +22,7 @@ pub trait ExerciseScorer { previous_trials: &[ExerciseTrial], previous_deltas: &[ExerciseDelta], now: i64, - ) -> Result; - - /// Returns the velocity of learning for exercise with the given trials. The velocity is a - /// measure of how quickly the score is improving or worsening over trials. A value of None - /// indicates that there are too few trials to compute a reliable velocity. - fn velocity(&self, previous_trials: &[ExerciseTrial]) -> Option; + ) -> Result; } // Adjustable constants: these can be tuned to calibrate the scorer. @@ -414,6 +409,35 @@ impl PowerLawScorer { let avg_delta = Self::compute_weighted_avg(previous_deltas); avg_delta * retrievability / 4.0 } + + fn velocity(previous_trials: &[ExerciseTrial]) -> Option { + // Need at least 2 trials for a meaningful slope. + if previous_trials.len() < 2 { + return None; + } + + // Compute the velocity using the ordinary least squares regression method. The oldest trial + // is used as the reference point and other trials are converted to days from it. + let oldest_timestamp = previous_trials.last().unwrap().timestamp; + let n = previous_trials.len() as f32; + let mut sum_t = 0.0_f32; + let mut sum_scores = 0.0_f32; + let mut sum_t_scores = 0.0_f32; + let mut sum_t_sq = 0.0_f32; + for trial in previous_trials { + let t = (trial.timestamp.saturating_sub(oldest_timestamp)) as f32 / SECONDS_PER_DAY; + sum_t += t; + sum_scores += trial.score; + sum_t_scores += t * trial.score; + sum_t_sq += t * t; + } + let denominator = n * sum_t_sq - sum_t * sum_t; + if denominator.abs() < f32::EPSILON { + return Some(0.0); + } + let slope = (n * sum_t_scores - sum_t * sum_scores) / denominator; + Some(slope) + } } impl ExerciseScorer for PowerLawScorer { @@ -423,10 +447,15 @@ impl ExerciseScorer for PowerLawScorer { previous_trials: &[ExerciseTrial], previous_deltas: &[ExerciseDelta], now: i64, - ) -> Result { + ) -> Result { // Guard input ordering and missing-history edge cases. if previous_trials.is_empty() { - return Ok(0.0); + // Set urgency to 1.0 for exercises with no history to prioritize them for review. + return Ok(ExerciseScore { + value: 0.0, + urgency: 1.0, + velocity: None, + }); } if previous_trials .windows(2) @@ -460,36 +489,12 @@ impl ExerciseScorer for PowerLawScorer { // Compute the effective delta and add it to the score to compensate for differences between // predicted and actual performance. let delta = Self::compute_delta(previous_deltas, effective_retrievability); - Ok((adjusted_score + delta).clamp(0.0, 5.0)) - } - - fn velocity(&self, previous_trials: &[ExerciseTrial]) -> Option { - // Need at least 2 trials for a meaningful slope. - if previous_trials.len() < 2 { - return None; - } - - // Compute the velocity using the ordinary least squares regression method. The oldest trial - // is used as the reference point and other trials are converted to days from it. - let oldest_timestamp = previous_trials.last().unwrap().timestamp; - let n = previous_trials.len() as f32; - let mut sum_t = 0.0_f32; - let mut sum_scores = 0.0_f32; - let mut sum_t_scores = 0.0_f32; - let mut sum_t_sq = 0.0_f32; - for trial in previous_trials { - let t = (trial.timestamp.saturating_sub(oldest_timestamp)) as f32 / SECONDS_PER_DAY; - sum_t += t; - sum_scores += trial.score; - sum_t_scores += t * trial.score; - sum_t_sq += t * t; - } - let denominator = n * sum_t_sq - sum_t * sum_t; - if denominator.abs() < f32::EPSILON { - return Some(0.0); - } - let slope = (n * sum_t_scores - sum_t * sum_scores) / denominator; - Some(slope) + let final_score = (adjusted_score + delta).clamp(0.0, 5.0); + Ok(ExerciseScore { + value: final_score, + urgency: 1.0 - retrievability, + velocity: Self::velocity(previous_trials), + }) } } @@ -508,6 +513,18 @@ mod test { now - num_days * SECONDS_PER_DAY as i64 } + /// Computes and unwraps an exercise score estimate for assertions. + fn score_helper( + exercise_type: ExerciseType, + previous_trials: &[ExerciseTrial], + previous_deltas: &[ExerciseDelta], + now: i64, + ) -> ExerciseScore { + SCORER + .score(exercise_type, previous_trials, previous_deltas, now) + .unwrap() + } + /// Verifies that difficulty is estimated correctly from failure rates. #[test] fn estimate_difficulty() { @@ -598,12 +615,10 @@ mod test { /// Verifies the score for an exercise with no previous trials is 0.0. #[test] fn no_previous_trials() { - assert_eq!( - 0.0, - SCORER - .score(ExerciseType::Declarative, &[], &[], Utc::now().timestamp()) - .unwrap() - ); + let score = score_helper(ExerciseType::Declarative, &[], &[], Utc::now().timestamp()); + assert_eq!(score.value, 0.0); + assert_eq!(score.urgency, 1.0); + assert_eq!(score.velocity, None); } /// Verifies running the full scoring algorithm on a set of trials produces a reasonable score. @@ -624,22 +639,22 @@ mod test { }, ]; - let score = SCORER - .score( - ExerciseType::Declarative, - &trials, - &[], - Utc::now().timestamp(), - ) - .unwrap(); - assert!(score > 0.0 && score <= 5.0); - assert!(score > 2.0); // Decent due to good recent performance + let score = score_helper( + ExerciseType::Declarative, + &trials, + &[], + Utc::now().timestamp(), + ); + assert!(score.value > 0.0 && score.value <= 5.0); + assert!(score.value > 2.0); // Decent due to good recent performance + assert!((0.0..=1.0).contains(&score.urgency)); + assert!(score.velocity.unwrap() < 0.0); } /// Verifies scoring an exercise with an invalid timestamp still returns a sane score. #[test] fn invalid_timestamp() -> Result<()> { - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &[ExerciseTrial { score: 5.0, @@ -647,16 +662,17 @@ mod test { }], &[], Utc::now().timestamp(), - )?; - assert!(score >= 0.0 && score <= 5.0); - assert!(score < 1.0); // Low due to long time elapsed + ); + assert!(score.value >= 0.0 && score.value <= 5.0); + assert!(score.value < 1.0); // Low due to long time elapsed + assert!((0.0..=1.0).contains(&score.urgency)); Ok(()) } /// Verifies extreme timestamp gaps do not overflow elapsed-time calculations. #[test] fn extreme_timestamp_gap_does_not_overflow() -> Result<()> { - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &[ ExerciseTrial { @@ -670,8 +686,9 @@ mod test { ], &[], Utc::now().timestamp(), - )?; - assert!(score >= 0.0 && score <= 5.0); + ); + assert!(score.value >= 0.0 && score.value <= 5.0); + assert!((0.0..=1.0).contains(&score.urgency)); Ok(()) } @@ -1092,13 +1109,13 @@ mod test { timestamp: generate_timestamp(13), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score < 2.0); + ); + assert!(score.value < 2.0); Ok(()) } @@ -1147,13 +1164,13 @@ mod test { timestamp: generate_timestamp(25), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score > 1.0 && score < 4.0); + ); + assert!(score.value > 1.0 && score.value < 4.0); Ok(()) } @@ -1181,7 +1198,7 @@ mod test { /// Verifies that trials with old timestamp result in a low score. #[test] fn score_old_timestamp() -> Result<()> { - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &[ExerciseTrial { score: 5.0, @@ -1189,8 +1206,8 @@ mod test { }], &[], Utc::now().timestamp(), - )?; - assert!(score < 3.0); + ); + assert!(score.value < 3.0); Ok(()) } @@ -1231,13 +1248,13 @@ mod test { timestamp: generate_timestamp(7), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score > 4.0); + ); + assert!(score.value > 4.0); Ok(()) } @@ -1278,13 +1295,13 @@ mod test { timestamp: generate_timestamp(27), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score < 2.0); + ); + assert!(score.value < 2.0); Ok(()) } @@ -1318,20 +1335,20 @@ mod test { timestamp: generate_timestamp(270), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Procedural, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score >= 3.5); - let score = SCORER.score( + ); + assert!(score.value >= 3.5); + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score >= 3.5); + ); + assert!(score.value >= 3.5); Ok(()) } @@ -1365,33 +1382,61 @@ mod test { timestamp: generate_timestamp(431), }, ]; - let score = SCORER.score( + let score = score_helper( ExerciseType::Procedural, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score >= 3.5); - let score = SCORER.score( + ); + assert!(score.value >= 3.5); + let score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - assert!(score >= 3.5); + ); + assert!(score.value >= 3.5); Ok(()) } + /// Verifies that urgency increases as an otherwise identical review becomes older. + #[test] + fn urgency_increases_with_elapsed_time() { + let recent_trials = vec![ExerciseTrial { + score: 5.0, + timestamp: generate_timestamp(1), + }]; + let old_trials = vec![ExerciseTrial { + score: 5.0, + timestamp: generate_timestamp(30), + }]; + + let recent = score_helper( + ExerciseType::Declarative, + &recent_trials, + &[], + Utc::now().timestamp(), + ); + let old = score_helper( + ExerciseType::Declarative, + &old_trials, + &[], + Utc::now().timestamp(), + ); + + assert!(old.urgency > recent.urgency); + } + /// Verifies that velocity returns None for 0 or 1 trials. #[test] fn velocity_empty_trials() { - assert_eq!(SCORER.velocity(&[]), None); + assert_eq!(PowerLawScorer::velocity(&[]), None); let trials = vec![ExerciseTrial { score: 3.0, timestamp: generate_timestamp(0), }]; - assert_eq!(SCORER.velocity(&trials), None); + assert_eq!(PowerLawScorer::velocity(&trials), None); } /// Verifies that improving scores (most recent is highest) produce positive velocity. @@ -1420,7 +1465,7 @@ mod test { timestamp: generate_timestamp(4), }, ]; - let velocity = SCORER.velocity(&trials).unwrap(); + let velocity = PowerLawScorer::velocity(&trials).unwrap(); assert!(velocity > 0.0); } @@ -1450,7 +1495,7 @@ mod test { timestamp: generate_timestamp(4), }, ]; - let velocity = SCORER.velocity(&trials).unwrap(); + let velocity = PowerLawScorer::velocity(&trials).unwrap(); assert!(velocity < 0.0); } @@ -1471,7 +1516,7 @@ mod test { timestamp: generate_timestamp(2), }, ]; - let velocity = SCORER.velocity(&trials).unwrap(); + let velocity = PowerLawScorer::velocity(&trials).unwrap(); assert!(velocity.abs() < 1e-6); } @@ -1503,19 +1548,19 @@ mod test { }, ]; - let base_score = SCORER.score( + let base_score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - let delta_score = SCORER.score( + ); + let delta_score = score_helper( ExerciseType::Declarative, &trials, &deltas, Utc::now().timestamp(), - )?; - assert!(delta_score > base_score); + ); + assert!(delta_score.value > base_score.value); Ok(()) } @@ -1547,19 +1592,19 @@ mod test { }, ]; - let base_score = SCORER.score( + let base_score = score_helper( ExerciseType::Declarative, &trials, &[], Utc::now().timestamp(), - )?; - let delta_score = SCORER.score( + ); + let delta_score = score_helper( ExerciseType::Declarative, &trials, &deltas, Utc::now().timestamp(), - )?; - assert!(delta_score < base_score); + ); + assert!(delta_score.value < base_score.value); Ok(()) } } diff --git a/src/scheduler/unit_scorer.rs b/src/scheduler/unit_scorer.rs index 1cd0a29..13bec57 100644 --- a/src/scheduler/unit_scorer.rs +++ b/src/scheduler/unit_scorer.rs @@ -18,11 +18,15 @@ use crate::{ }; /// Stores information about a cached score. -#[derive(Clone)] +#[derive(Clone, Default)] pub(super) struct CachedScore { /// The computed score. score: f32, + /// The urgency of scheduling the unit, as a value between 0.0 and 1.0. + #[allow(dead_code)] + urgency: f32, + /// The velocity of learning, a measure of how quickly the score is improving or worsening over /// trials. velocity: Option, @@ -234,15 +238,16 @@ impl UnitScorer { // Apply the reward if it meets the criteria and cache the final score. let final_score = if self.reward_scorer.apply_reward(reward, &scores) { - (score + reward).clamp(0.0, 5.0) + (score.value + reward).clamp(0.0, 5.0) } else { - score + score.value }; self.exercise_cache.borrow_mut().insert( exercise_id, CachedScore { score: final_score, - velocity: self.exercise_scorer.velocity(&scores), + urgency: score.urgency, + velocity: score.velocity, num_trials: scores.len(), last_seen, }, @@ -250,6 +255,30 @@ impl UnitScorer { Ok(final_score) } + /// Returns the urgency of scheduling the given exercise, as a value between 0.0 and 1.0. + #[allow(dead_code)] + pub(super) fn get_exercise_urgency(&self, exercise_id: Ustr) -> Result> { + // Return the cached value if it exists. + let cached_urgency = self + .exercise_cache + .borrow() + .get(&exercise_id) + .map(|c| c.urgency); + if let Some(urgency) = cached_urgency { + return Ok(Some(urgency)); + } + + // Compute the exercise's score, which populates the cache. Then, retrieve the urgency from + // the cache. + self.get_exercise_score(exercise_id)?; + let cached_urgency = self + .exercise_cache + .borrow() + .get(&exercise_id) + .map(|s| s.urgency); + Ok(cached_urgency) + } + /// Returns the velocity of learning for the given exercise. pub(super) fn get_exercise_velocity(&self, exercise_id: Ustr) -> Result> { // Return the cached value if it exists. @@ -793,6 +822,7 @@ mod test { Ustr::from("a"), CachedScore { score: 5.0, + urgency: 0.0, velocity: None, num_trials: 1, last_seen: 0.0, @@ -802,6 +832,7 @@ mod test { Ustr::from("b::a"), CachedScore { score: 5.0, + urgency: 0.0, velocity: None, num_trials: 1, last_seen: 0.0,