Skip to content

Commit

Permalink
sqlite: add size based retention through webapp
Browse files Browse the repository at this point in the history
  • Loading branch information
@jasonish
jasonish committed Feb 28, 2025
1 parent 7c03c07 commit 694f611
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 26 deletions.
6 changes: 3 additions & 3 deletions src/elastic/retention.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use chrono::TimeZone;

use crate::elastic::{HistoryEntryBuilder, TAG_ARCHIVED, TAG_AUTO_ARCHIVED};
use crate::server::metrics::Metrics;
use crate::sqlite::configdb::{AutoArchiveConfig, ConfigDb};
use crate::sqlite::configdb::{ConfigDb, EnabledWithValue};
use crate::{elastic::DateTime, prelude::*};

use super::client::SimpleIndexStats;
Expand All @@ -32,7 +32,7 @@ async fn run(metrics: Arc<Metrics>, configdb: ConfigDb, repo: ElasticEventRepo)
}

async fn delete_indices(configdb: &ConfigDb, repo: &ElasticEventRepo) -> Result<()> {
let config: Option<AutoArchiveConfig> = configdb.kv_get_config_as_t("config.retention").await?;
let config: Option<EnabledWithValue> = configdb.kv_get_config_as_t("config.retention").await?;
let config = match config {
Some(config) => config,
None => return Ok(()),
Expand Down Expand Up @@ -86,7 +86,7 @@ async fn auto_archive(
configdb: &ConfigDb,
repo: &ElasticEventRepo,
) -> Result<()> {
let config: Option<AutoArchiveConfig> =
let config: Option<EnabledWithValue> =
configdb.kv_get_config_as_t("config.autoarchive").await?;
let config = match config {
Some(config) => config,
Expand Down
2 changes: 1 addition & 1 deletion src/sqlite/configdb.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ pub(crate) struct User {
}

#[derive(Debug, Deserialize)]
pub(crate) struct AutoArchiveConfig {
pub(crate) struct EnabledWithValue {
pub enabled: bool,
pub value: u64,
}
Expand Down
48 changes: 32 additions & 16 deletions src/sqlite/retention.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};

use super::configdb::{AutoArchiveConfig, ConfigDb};
use super::configdb::{ConfigDb, EnabledWithValue};
use super::info::Info;
use crate::config::Config;
use crate::datetime::DateTime;
Expand All @@ -30,19 +30,26 @@ const REPEAT_INTERVAL: u64 = 1;
/// Number of events to delete per run.
const LIMIT: usize = 1000;

fn get_size(config: &Config) -> Result<usize> {
// Size as a number.
if let Ok(Some(size)) = config.get::<usize>("database.retention.size") {
Ok(size)
} else if let Ok(Some(size)) = config.get::<String>("database.retention.size") {
async fn get_size(configdb: &ConfigDb, config: &Config) -> Result<usize> {
if let Ok(Some(size)) = config.get::<String>("database.retention.size") {
if let Ok(size) = size.parse::<usize>() {
Ok(size)
} else {
crate::util::parse_humansize(&size)
return Ok(size);
}
} else {
Ok(0)
if let Ok(size) = crate::util::parse_humansize(&size) {
return Ok(size);
}
warn!("Invalid database.retention.size: {}", size);
}

let retention_size_config: Result<Option<EnabledWithValue>> =
configdb.kv_get_config_as_t("config.retention.size").await;
if let Ok(Some(config)) = retention_size_config {
if config.enabled {
return Ok(config.value as usize * 1000000000);
}
}

Ok(0)
}

async fn get_days(configdb: &ConfigDb, config: &Config) -> Result<Option<usize>> {
Expand All @@ -59,7 +66,7 @@ async fn get_days(configdb: &ConfigDb, config: &Config) -> Result<Option<usize>>
);
days
} else {
let retention_config: Result<Option<AutoArchiveConfig>> =
let retention_config: Result<Option<EnabledWithValue>> =
configdb.kv_get_config_as_t("config.retention").await;
match retention_config {
Ok(None) => {
Expand Down Expand Up @@ -143,24 +150,29 @@ async fn retention_task(
let default_delay = Duration::from_secs(INTERVAL);
let report_interval = Duration::from_secs(60);

// Delay on startup.
tokio::time::sleep(default_delay).await;
// Short delay on startup.
tokio::time::sleep(Duration::from_secs(3)).await;

let mut last_report = Instant::now();
let mut count: u64 = 0;

loop {
let mut delay = default_delay;

if !size_enabled {
debug!("Size based database retention not available.");
}
if size_enabled {
match get_size(&config) {
match get_size(&configdb, &config).await {
Ok(size) => {
dbg!(size);
if size > 0 {
match delete_to_size(conn.clone(), &filename, size).await {
Err(err) => {
error!("Failed to delete database to max size: {:?}", err);
}
Ok(n) => {
dbg!(n);
if n > 0 {
debug!(
"Deleted {n} events to reduce database size to {} bytes",
Expand Down Expand Up @@ -216,7 +228,7 @@ async fn auto_archive(
configdb: &ConfigDb,
conn: Arc<tokio::sync::Mutex<SqliteConnection>>,
) -> Result<()> {
let config: Option<AutoArchiveConfig> =
let config: Option<EnabledWithValue> =
configdb.kv_get_config_as_t("config.autoarchive").await?;
if let Some(config) = config {
if config.enabled {
Expand Down Expand Up @@ -257,6 +269,10 @@ async fn delete_to_size(
loop {
let file_size = crate::file::file_size(filename)? as usize;
if file_size < bytes {
debug!(
"File size {} less than retention size limit of {}",
file_size, bytes
);
break;
}

Expand Down
136 changes: 130 additions & 6 deletions webapp/src/pages/admin/Admin.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,30 @@ async function fetchRetentionSettings(): Promise<AutoArchiveSettings> {
}
}

async function fetchRetentionSizeSettings(): Promise<AutoArchiveSettings> {
const json = await api.API.getJson("api/admin/kv/config");
const config = json["config.retention.size"];
if (config) {
return config;
} else {
return defaultRetentionSizeSettings();
}
}

function defaultRetentionSettings(): RetentionSettings {
return {
enabled: false,
value: 365,
};
}

function defaultRetentionSizeSettings(): RetentionSettings {
return {
enabled: false,
value: 20,
};
}

export function Admin() {
const [state, setState] = createStore({
ja4: {
Expand All @@ -60,18 +77,24 @@ export function Admin() {
},
});

// Auto archive.
const [autoArchiveSettings, { refetch: refetchAutoArchiveSettings }] =
createResource<AutoArchiveSettings>(fetchAutoArchiveSettings);

const [localAutoArchiveSettings, setLocalAutoArchiveSettings] =
createStore<AutoArchiveSettings>(defaultAutoArchiveSettings());

// Retention by age.
const [retentionSettings, { refetch: refetchRetentionSettings }] =
createResource<RetentionSettings>(fetchRetentionSettings);

const [localRetentionSettings, setLocalRetentionSettings] =
createStore<RetentionSettings>(defaultRetentionSettings());

// Retention by size.
const [retentionSizeSettings, { refetch: refetchRetentionSizeSettings }] =
createResource<RetentionSettings>(fetchRetentionSizeSettings);
const [localRetentionSizeSettings, setLocalRetentionSizeSettings] =
createStore<RetentionSettings>(defaultRetentionSizeSettings());

createEffect(() => {
if (autoArchiveSettings()) {
setLocalAutoArchiveSettings(autoArchiveSettings()!);
Expand All @@ -84,6 +107,12 @@ export function Admin() {
}
});

createEffect(() => {
if (retentionSizeSettings()) {
setLocalRetentionSizeSettings(retentionSizeSettings()!);
}
});

const archiveSettingsModified = createMemo(() => {
return (
JSON.stringify(localAutoArchiveSettings) !=
Expand All @@ -98,6 +127,13 @@ export function Admin() {
);
});

const retentionSizeSettingsModified = createMemo(() => {
return (
JSON.stringify(localRetentionSizeSettings) !=
JSON.stringify(retentionSizeSettings.latest)
);
});

const saveAutoArchiveSettings = async () => {
await api.API.postJson(
"api/admin/kv/config/config.autoarchive",
Expand All @@ -114,6 +150,14 @@ export function Admin() {
refetchRetentionSettings();
};

const saveRetentionSizeSettings = async () => {
await api.API.postJson(
"api/admin/kv/config/config.retention.size",
localRetentionSizeSettings,
);
refetchRetentionSizeSettings();
};

const updateJa4Db = async (e: any) => {
e.preventDefault();
try {
Expand Down Expand Up @@ -248,7 +292,7 @@ export function Admin() {
</div>
</div>

{/* Retention settings. */}
{/* Retention by age. */}
<div class="row mt-2">
<div class="col">
<div class="card">
Expand Down Expand Up @@ -324,9 +368,7 @@ export function Admin() {
<button
class="btn btn-danger"
onClick={() => {
setLocalAutoArchiveSettings(
retentionSettings.latest!,
);
setLocalRetentionSettings(retentionSettings.latest!);
}}
>
Reset
Expand All @@ -338,6 +380,88 @@ export function Admin() {
</div>
</div>
</div>

{/* Retention by disk size. */}
<Show when={serverConfig?.datastore === "sqlite"}>
<div class="row mt-2">
<div class="col">
<div class="card">
<div class="card-body">
<div class="row mt-2">
<div class="col">
Warning: This setting will not be effective if size
retention is set in the configuration file.
</div>
</div>
<div class="row mt-2">
<label class="col col-form-label">
<div class="form-check form-switch">
<input
class="form-check-input"
type="checkbox"
role="switch"
checked={localRetentionSizeSettings.enabled}
onChange={(e) => {
setLocalRetentionSizeSettings({
enabled: e.target.checked,
});
}}
/>
<label class="form-check-label">
Limit event database to size:
</label>
</div>
</label>
<div class="col">
<div class="input-group">
<input
type="number"
class="form-control"
value={localRetentionSizeSettings.value}
onInput={(e) => {
setLocalRetentionSizeSettings(
"value",
+e.target.value,
);
}}
onChange={(e) => {
setLocalRetentionSizeSettings(
"value",
+e.target.value,
);
}}
/>
<span class="input-group-text">Gigabytes</span>
</div>
</div>
<div class="col text-end">
<Show when={retentionSizeSettingsModified()}>
<button
class="btn btn-success me-2"
onClick={() => {
saveRetentionSizeSettings();
}}
>
Save
</button>
<button
class="btn btn-danger"
onClick={() => {
setLocalRetentionSizeSettings(
retentionSizeSettings.latest!,
);
}}
>
Reset
</button>
</Show>
</div>
</div>
</div>
</div>
</div>
</div>
</Show>
</div>
</>
);
Expand Down

0 comments on commit 694f611

Please sign in to comment.